Hugging Face's logo

anhtld
/
vla

Model card Files
xet
 Community
anhtld commited on
Commit
0d1de44
·
verified ·
1 Parent(s): 8621c85

auto-sync 2026-07-03T01:42:26Z workspace (part 4)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. workspace/runs/ctt_residual_rollout_val16_seed0/measured_metrics/report.md +20 -4
  2. workspace/runs/ctt_residual_rollout_val16_seed0/measured_metrics/table.tex +20 -4
  3. workspace/runs/ctt_residual_rollout_val16_seed0/metrics.json +1183 -0
  4. workspace/runs/ctt_residual_rollout_val16_seed0/metrics_by_seed.json +324 -0
  5. workspace/runs/ctt_residual_rollout_val16_seed0/metrics_by_task.json +71 -0
  6. workspace/runs/ctt_residual_rollout_val16_seed0/source_data_hash.txt +1 -0
  7. workspace/runs/ctt_residual_rollout_val16_seed0/source_split_hash.txt +1 -0
  8. workspace/runs/ctt_residual_rollout_val16_seed0/split_hash.txt +1 -0
  9. workspace/runs/ctt_residual_rollout_val16_seed0/table.tex +28 -0
  10. workspace/runs/ctt_residual_rollout_val16_seed0/train.log +1 -0
  11. workspace/runs/ctt_residual_rollout_val69_seed0/data_hash.txt +1 -0
  12. workspace/runs/ctt_residual_rollout_val69_seed0/eval.log +668 -0
  13. workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics.json +0 -0
  14. workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics_by_seed.json +799 -47
  15. workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics_by_task.json +85 -5
  16. workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/report.md +16 -0
  17. workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/table.tex +16 -0
  18. workspace/runs/ctt_residual_rollout_val69_seed0/metrics.json +1918 -0
  19. workspace/runs/ctt_residual_rollout_val69_seed0/metrics_by_seed.json +1083 -0
  20. workspace/runs/ctt_residual_rollout_val69_seed0/metrics_by_task.json +117 -0
  21. workspace/runs/ctt_residual_rollout_val69_seed0/source_data_hash.txt +1 -0
  22. workspace/runs/ctt_residual_rollout_val69_seed0/source_split_hash.txt +1 -0
  23. workspace/runs/ctt_residual_rollout_val69_seed0/split_hash.txt +1 -0
  24. workspace/runs/ctt_residual_rollout_val69_seed0/table.tex +28 -0
  25. workspace/runs/ctt_residual_rollout_val69_seed0/train.log +1 -0
  26. workspace/runs/ctt_residual_rollout_val69_seed1/data_hash.txt +1 -0
  27. workspace/runs/ctt_residual_rollout_val69_seed1/eval.log +668 -0
  28. workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics.json +0 -0
  29. workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics_by_seed.json +799 -47
  30. workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics_by_task.json +85 -5
  31. workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/report.md +16 -0
  32. workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/table.tex +16 -0
  33. workspace/runs/ctt_residual_rollout_val69_seed1/metrics.json +1918 -0
  34. workspace/runs/ctt_residual_rollout_val69_seed1/metrics_by_seed.json +1083 -0
  35. workspace/runs/ctt_residual_rollout_val69_seed1/metrics_by_task.json +117 -0
  36. workspace/runs/ctt_residual_rollout_val69_seed1/source_data_hash.txt +1 -0
  37. workspace/runs/ctt_residual_rollout_val69_seed1/source_split_hash.txt +1 -0
  38. workspace/runs/ctt_residual_rollout_val69_seed1/split_hash.txt +1 -0
  39. workspace/runs/ctt_residual_rollout_val69_seed1/table.tex +28 -0
  40. workspace/runs/ctt_residual_rollout_val69_seed1/train.log +1 -0
  41. workspace/runs/ctt_residual_rollout_val69_seed2/data_hash.txt +1 -0
  42. workspace/runs/ctt_residual_rollout_val69_seed2/eval.log +668 -0
  43. workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics.json +0 -0
  44. workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics_by_seed.json +799 -47
  45. workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics_by_task.json +85 -5
  46. workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/report.md +16 -0
  47. workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/table.tex +16 -0
  48. workspace/runs/ctt_residual_rollout_val69_seed2/metrics.json +1918 -0
  49. workspace/runs/ctt_residual_rollout_val69_seed2/metrics_by_seed.json +1083 -0
  50. workspace/runs/ctt_residual_rollout_val69_seed2/metrics_by_task.json +117 -0
workspace/runs/ctt_residual_rollout_val16_seed0/measured_metrics/report.md CHANGED
@@ -4,8 +4,24 @@ K: `8`
4
 
5
  | Metric | N | Micro mean | 95% CI | Task macro | Seed macro |
6
  | --- | ---: | ---: | ---: | ---: | ---: |
7
- | branch_car_at_8 | 16 | 0.2645 | [0.0178, 0.6268] | 0.2315 | 0.3012 |
 
 
 
 
8
  | outcome_ptr_at_8 | 16 | 0.3750 | [0.1250, 0.6250] | 0.2963 | 0.3929 |
9
- | pairwise_causal_calibration_ece | 16 | 0.1330 | [0.0823, 0.1798] | 0.1366 | 0.1312 |
10
- | selector_regret_at_8 | 16 | 0.2645 | [0.0178, 0.6268] | 0.2315 | 0.3012 |
11
- | support_gap_at_8 | 16 | 0.5139 | [0.1567, 0.8696] | 0.9286 | 0.5714 |
 
 
 
 
 
 
 
 
 
 
 
 
 
4
 
5
  | Metric | N | Micro mean | 95% CI | Task macro | Seed macro |
6
  | --- | ---: | ---: | ---: | ---: | ---: |
7
+ | base_success | 16 | 0.3125 | [0.1250, 0.5625] | 0.2222 | 0.3571 |
8
+ | base_utility | 16 | 0.6871 | [0.3147, 1.1643] | 0.5332 | 0.7765 |
9
+ | branch_car_at_8 | 16 | 0.2645 | [0.0149, 0.6270] | 0.2315 | 0.3012 |
10
+ | hidden_chart_oracle_success_at_8 | 16 | 0.6250 | [0.3750, 0.8125] | 0.7778 | 0.7143 |
11
+ | hidden_chart_oracle_utility_at_8 | 16 | 1.3220 | [0.8644, 1.6665] | 1.5982 | 1.4848 |
12
  | outcome_ptr_at_8 | 16 | 0.3750 | [0.1250, 0.6250] | 0.2963 | 0.3929 |
13
+ | pairwise_causal_calibration_ece | 16 | 0.1330 | [0.0830, 0.1813] | 0.1366 | 0.1312 |
14
+ | proposal_oracle_success_at_8 | 16 | 0.3750 | [0.1250, 0.6250] | 0.2963 | 0.4286 |
15
+ | proposal_oracle_success_gain_over_base_at_8 | 16 | 0.0625 | [-0.1250, 0.2500] | 0.0741 | 0.0714 |
16
+ | proposal_oracle_utility_at_8 | 16 | 0.8081 | [0.3368, 1.2765] | 0.6696 | 0.9134 |
17
+ | proposal_oracle_utility_gain_over_base_at_8 | 16 | 0.1210 | [-0.2626, 0.4792] | 0.1364 | 0.1369 |
18
+ | selected_success_at_8 | 16 | 0.2500 | [0.0625, 0.5000] | 0.1852 | 0.2857 |
19
+ | selected_success_gain_over_base_at_8 | 16 | -0.0625 | [-0.3125, 0.1250] | -0.0370 | -0.0714 |
20
+ | selected_utility_at_8 | 16 | 0.5436 | [0.1771, 1.0157] | 0.4381 | 0.6122 |
21
+ | selected_utility_gain_over_base_at_8 | 16 | -0.1435 | [-0.6171, 0.2281] | -0.0950 | -0.1643 |
22
+ | selector_regret_at_8 | 16 | 0.2645 | [0.0149, 0.6270] | 0.2315 | 0.3012 |
23
+ | success_selector_gap_at_8 | 16 | 0.1250 | [0.0000, 0.3125] | 0.1111 | 0.1429 |
24
+ | success_support_gap_at_8 | 16 | 0.2500 | [0.0625, 0.5000] | 0.4815 | 0.2857 |
25
+ | success_total_car_to_hidden_at_8 | 16 | 0.3750 | [0.1250, 0.6250] | 0.5926 | 0.4286 |
26
+ | support_gap_at_8 | 16 | 0.5139 | [0.1583, 0.9626] | 0.9286 | 0.5714 |
27
+ | total_car_to_hidden_at_8 | 16 | 0.7784 | [0.3386, 1.2434] | 1.1601 | 0.8726 |
workspace/runs/ctt_residual_rollout_val16_seed0/measured_metrics/table.tex CHANGED
@@ -3,10 +3,26 @@
3
  \toprule
4
  Metric & N & Micro mean & CI low & CI high \\
5
  \midrule
6
- branch\_car\_at\_8 & 16 & 0.2645 & 0.0178 & 0.6268 \\
 
 
 
 
7
  outcome\_ptr\_at\_8 & 16 & 0.3750 & 0.1250 & 0.6250 \\
8
- pairwise\_causal\_calibration\_ece & 16 & 0.1330 & 0.0823 & 0.1798 \\
9
- selector\_regret\_at\_8 & 16 & 0.2645 & 0.0178 & 0.6268 \\
10
- support\_gap\_at\_8 & 16 & 0.5139 & 0.1567 & 0.8696 \\
 
 
 
 
 
 
 
 
 
 
 
 
11
  \bottomrule
12
  \end{tabular}
 
3
  \toprule
4
  Metric & N & Micro mean & CI low & CI high \\
5
  \midrule
6
+ base\_success & 16 & 0.3125 & 0.1250 & 0.5625 \\
7
+ base\_utility & 16 & 0.6871 & 0.3147 & 1.1643 \\
8
+ branch\_car\_at\_8 & 16 & 0.2645 & 0.0149 & 0.6270 \\
9
+ hidden\_chart\_oracle\_success\_at\_8 & 16 & 0.6250 & 0.3750 & 0.8125 \\
10
+ hidden\_chart\_oracle\_utility\_at\_8 & 16 & 1.3220 & 0.8644 & 1.6665 \\
11
  outcome\_ptr\_at\_8 & 16 & 0.3750 & 0.1250 & 0.6250 \\
12
+ pairwise\_causal\_calibration\_ece & 16 & 0.1330 & 0.0830 & 0.1813 \\
13
+ proposal\_oracle\_success\_at\_8 & 16 & 0.3750 & 0.1250 & 0.6250 \\
14
+ proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 16 & 0.0625 & -0.1250 & 0.2500 \\
15
+ proposal\_oracle\_utility\_at\_8 & 16 & 0.8081 & 0.3368 & 1.2765 \\
16
+ proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 16 & 0.1210 & -0.2626 & 0.4792 \\
17
+ selected\_success\_at\_8 & 16 & 0.2500 & 0.0625 & 0.5000 \\
18
+ selected\_success\_gain\_over\_base\_at\_8 & 16 & -0.0625 & -0.3125 & 0.1250 \\
19
+ selected\_utility\_at\_8 & 16 & 0.5436 & 0.1771 & 1.0157 \\
20
+ selected\_utility\_gain\_over\_base\_at\_8 & 16 & -0.1435 & -0.6171 & 0.2281 \\
21
+ selector\_regret\_at\_8 & 16 & 0.2645 & 0.0149 & 0.6270 \\
22
+ success\_selector\_gap\_at\_8 & 16 & 0.1250 & 0.0000 & 0.3125 \\
23
+ success\_support\_gap\_at\_8 & 16 & 0.2500 & 0.0625 & 0.5000 \\
24
+ success\_total\_car\_to\_hidden\_at\_8 & 16 & 0.3750 & 0.1250 & 0.6250 \\
25
+ support\_gap\_at\_8 & 16 & 0.5139 & 0.1583 & 0.9626 \\
26
+ total\_car\_to\_hidden\_at\_8 & 16 & 0.7784 & 0.3386 & 1.2434 \\
27
  \bottomrule
28
  \end{tabular}
workspace/runs/ctt_residual_rollout_val16_seed0/metrics.json ADDED
@@ -0,0 +1,1183 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "runs/ctt_residual_full_seed0/model.pt",
3
+ "k": 8,
4
+ "measured_metric_summary": {
5
+ "base_success": {
6
+ "macro_by_seed": {
7
+ "ci": {
8
+ "confidence": 0.95,
9
+ "high": 0.6428571428571429,
10
+ "low": 0.14285714285714285,
11
+ "mean": 0.35714285714285715,
12
+ "n": 14,
13
+ "num_samples": 500
14
+ },
15
+ "mean": 0.35714285714285715,
16
+ "per_seed": {
17
+ "1189392917": 0.0,
18
+ "1271718007": 1.0,
19
+ "1273247072": 0.0,
20
+ "1375492786": 1.0,
21
+ "141984615": 0.0,
22
+ "1549324805": 1.0,
23
+ "1881223807": 0.0,
24
+ "1997090716": 0.0,
25
+ "2015956957": 0.0,
26
+ "2141802816": 0.0,
27
+ "565369392": 0.0,
28
+ "565621826": 1.0,
29
+ "767747424": 0.0,
30
+ "77147264": 1.0
31
+ }
32
+ },
33
+ "macro_by_task": {
34
+ "ci": {
35
+ "confidence": 0.95,
36
+ "high": 0.3333333333333333,
37
+ "low": 0.0,
38
+ "mean": 0.2222222222222222,
39
+ "n": 3,
40
+ "num_samples": 500
41
+ },
42
+ "mean": 0.2222222222222222,
43
+ "per_task": {
44
+ "PickCube-v1": 0.0,
45
+ "PullCube-v1": 0.3333333333333333,
46
+ "PushCube-v1": 0.3333333333333333
47
+ }
48
+ },
49
+ "metric": "base_success",
50
+ "micro": {
51
+ "confidence": 0.95,
52
+ "high": 0.5625,
53
+ "low": 0.125,
54
+ "mean": 0.3125,
55
+ "n": 16,
56
+ "num_samples": 500
57
+ },
58
+ "n": 16
59
+ },
60
+ "base_utility": {
61
+ "macro_by_seed": {
62
+ "ci": {
63
+ "confidence": 0.95,
64
+ "high": 1.3199879928891147,
65
+ "low": 0.34861662624669926,
66
+ "mean": 0.7764950924153838,
67
+ "n": 14,
68
+ "num_samples": 500
69
+ },
70
+ "mean": 0.7764950924153838,
71
+ "per_seed": {
72
+ "1189392917": 0.10841864347457886,
73
+ "1271718007": 2.0,
74
+ "1273247072": 0.15342965722084045,
75
+ "1375492786": 2.0,
76
+ "141984615": 0.050487834960222244,
77
+ "1549324805": 2.0,
78
+ "1881223807": 0.0320269912481308,
79
+ "1997090716": 0.13927479088306427,
80
+ "2015956957": 0.058304885402321815,
81
+ "2141802816": 0.07080724835395813,
82
+ "565369392": 0.19400137662887573,
83
+ "565621826": 2.0,
84
+ "767747424": 0.06417986564338207,
85
+ "77147264": 2.0
86
+ }
87
+ },
88
+ "macro_by_task": {
89
+ "ci": {
90
+ "confidence": 0.95,
91
+ "high": 0.7250879456599554,
92
+ "low": 0.15342965722084045,
93
+ "mean": 0.5331895236891729,
94
+ "n": 3,
95
+ "num_samples": 500
96
+ },
97
+ "mean": 0.5331895236891729,
98
+ "per_task": {
99
+ "PickCube-v1": 0.15342965722084045,
100
+ "PullCube-v1": 0.7210509681867229,
101
+ "PushCube-v1": 0.7250879456599554
102
+ }
103
+ },
104
+ "metric": "base_utility",
105
+ "micro": {
106
+ "confidence": 0.95,
107
+ "high": 1.1643496139440686,
108
+ "low": 0.3147284146398306,
109
+ "mean": 0.6870885028038174,
110
+ "n": 16,
111
+ "num_samples": 500
112
+ },
113
+ "n": 16
114
+ },
115
+ "branch_car_at_8": {
116
+ "macro_by_seed": {
117
+ "ci": {
118
+ "confidence": 0.95,
119
+ "high": 0.7187583449163607,
120
+ "low": 0.014021761948242784,
121
+ "mean": 0.3011761402262242,
122
+ "n": 14,
123
+ "num_samples": 500
124
+ },
125
+ "mean": 0.3011761402262242,
126
+ "per_seed": {
127
+ "1189392917": 0.0,
128
+ "1271718007": 0.0,
129
+ "1273247072": 0.0,
130
+ "1375492786": 0.0002854267368093133,
131
+ "141984615": 0.09784513339400291,
132
+ "1549324805": 0.0,
133
+ "1881223807": 0.020236089825630188,
134
+ "1997090716": 1.9691698104143143,
135
+ "2015956957": 0.003587027546018362,
136
+ "2141802816": 0.07543906569480896,
137
+ "565369392": 0.0671776756644249,
138
+ "565621826": 0.0,
139
+ "767747424": 0.012708732858300209,
140
+ "77147264": 1.9700170010328293
141
+ }
142
+ },
143
+ "macro_by_task": {
144
+ "ci": {
145
+ "confidence": 0.95,
146
+ "high": 0.6724769944945971,
147
+ "low": 0.0,
148
+ "mean": 0.231488618779824,
149
+ "n": 3,
150
+ "num_samples": 500
151
+ },
152
+ "mean": 0.231488618779824,
153
+ "per_task": {
154
+ "PickCube-v1": 0.0,
155
+ "PullCube-v1": 0.02198886184487492,
156
+ "PushCube-v1": 0.6724769944945971
157
+ }
158
+ },
159
+ "metric": "branch_car_at_8",
160
+ "micro": {
161
+ "confidence": 0.95,
162
+ "high": 0.6269801619346254,
163
+ "low": 0.014924341201549396,
164
+ "mean": 0.26454760772321606,
165
+ "n": 16,
166
+ "num_samples": 500
167
+ },
168
+ "n": 16
169
+ },
170
+ "hidden_chart_oracle_success_at_8": {
171
+ "macro_by_seed": {
172
+ "ci": {
173
+ "confidence": 0.95,
174
+ "high": 0.9285714285714286,
175
+ "low": 0.5,
176
+ "mean": 0.7142857142857143,
177
+ "n": 14,
178
+ "num_samples": 500
179
+ },
180
+ "mean": 0.7142857142857143,
181
+ "per_seed": {
182
+ "1189392917": 1.0,
183
+ "1271718007": 1.0,
184
+ "1273247072": 1.0,
185
+ "1375492786": 0.0,
186
+ "141984615": 0.0,
187
+ "1549324805": 1.0,
188
+ "1881223807": 1.0,
189
+ "1997090716": 1.0,
190
+ "2015956957": 0.0,
191
+ "2141802816": 1.0,
192
+ "565369392": 1.0,
193
+ "565621826": 1.0,
194
+ "767747424": 0.0,
195
+ "77147264": 1.0
196
+ }
197
+ },
198
+ "macro_by_task": {
199
+ "ci": {
200
+ "confidence": 0.95,
201
+ "high": 1.0,
202
+ "low": 0.3333333333333333,
203
+ "mean": 0.7777777777777777,
204
+ "n": 3,
205
+ "num_samples": 500
206
+ },
207
+ "mean": 0.7777777777777777,
208
+ "per_task": {
209
+ "PickCube-v1": 1.0,
210
+ "PullCube-v1": 0.3333333333333333,
211
+ "PushCube-v1": 1.0
212
+ }
213
+ },
214
+ "metric": "hidden_chart_oracle_success_at_8",
215
+ "micro": {
216
+ "confidence": 0.95,
217
+ "high": 0.8125,
218
+ "low": 0.375,
219
+ "mean": 0.625,
220
+ "n": 16,
221
+ "num_samples": 500
222
+ },
223
+ "n": 16
224
+ },
225
+ "hidden_chart_oracle_utility_at_8": {
226
+ "macro_by_seed": {
227
+ "ci": {
228
+ "confidence": 0.95,
229
+ "high": 1.8699864170381002,
230
+ "low": 1.0907963934753622,
231
+ "mean": 1.4848211647144385,
232
+ "n": 14,
233
+ "num_samples": 500
234
+ },
235
+ "mean": 1.4848211647144385,
236
+ "per_seed": {
237
+ "1189392917": 2.0,
238
+ "1271718007": 2.0,
239
+ "1273247072": 2.0,
240
+ "1375492786": 0.24345096945762634,
241
+ "141984615": 0.1798098385334015,
242
+ "1549324805": 2.0,
243
+ "1881223807": 2.0,
244
+ "1997090716": 2.0,
245
+ "2015956957": 0.16121773421764374,
246
+ "2141802816": 2.0,
247
+ "565369392": 2.0,
248
+ "565621826": 2.0,
249
+ "767747424": 0.20301776379346848,
250
+ "77147264": 2.0
251
+ }
252
+ },
253
+ "macro_by_task": {
254
+ "ci": {
255
+ "confidence": 0.95,
256
+ "high": 2.0,
257
+ "low": 0.7946368671125836,
258
+ "mean": 1.5982122890375277,
259
+ "n": 3,
260
+ "num_samples": 500
261
+ },
262
+ "mean": 1.5982122890375277,
263
+ "per_task": {
264
+ "PickCube-v1": 2.0,
265
+ "PullCube-v1": 0.7946368671125836,
266
+ "PushCube-v1": 2.0
267
+ }
268
+ },
269
+ "metric": "hidden_chart_oracle_utility_at_8",
270
+ "micro": {
271
+ "confidence": 0.95,
272
+ "high": 1.666475722566247,
273
+ "low": 0.8644023519009352,
274
+ "mean": 1.3219832377508283,
275
+ "n": 16,
276
+ "num_samples": 500
277
+ },
278
+ "n": 16
279
+ },
280
+ "outcome_ptr_at_8": {
281
+ "macro_by_seed": {
282
+ "ci": {
283
+ "confidence": 0.95,
284
+ "high": 0.6428571428571429,
285
+ "low": 0.14285714285714285,
286
+ "mean": 0.39285714285714285,
287
+ "n": 14,
288
+ "num_samples": 500
289
+ },
290
+ "mean": 0.39285714285714285,
291
+ "per_seed": {
292
+ "1189392917": 1.0,
293
+ "1271718007": 0.0,
294
+ "1273247072": 0.0,
295
+ "1375492786": 0.0,
296
+ "141984615": 1.0,
297
+ "1549324805": 0.0,
298
+ "1881223807": 1.0,
299
+ "1997090716": 1.0,
300
+ "2015956957": 0.0,
301
+ "2141802816": 1.0,
302
+ "565369392": 0.0,
303
+ "565621826": 0.0,
304
+ "767747424": 0.5,
305
+ "77147264": 0.0
306
+ }
307
+ },
308
+ "macro_by_task": {
309
+ "ci": {
310
+ "confidence": 0.95,
311
+ "high": 0.6666666666666666,
312
+ "low": 0.0,
313
+ "mean": 0.2962962962962963,
314
+ "n": 3,
315
+ "num_samples": 500
316
+ },
317
+ "mean": 0.2962962962962963,
318
+ "per_task": {
319
+ "PickCube-v1": 0.0,
320
+ "PullCube-v1": 0.2222222222222222,
321
+ "PushCube-v1": 0.6666666666666666
322
+ }
323
+ },
324
+ "metric": "outcome_ptr_at_8",
325
+ "micro": {
326
+ "confidence": 0.95,
327
+ "high": 0.625,
328
+ "low": 0.125,
329
+ "mean": 0.375,
330
+ "n": 16,
331
+ "num_samples": 500
332
+ },
333
+ "n": 16
334
+ },
335
+ "pairwise_causal_calibration_ece": {
336
+ "macro_by_seed": {
337
+ "ci": {
338
+ "confidence": 0.95,
339
+ "high": 0.19134660629025252,
340
+ "low": 0.07903649816149518,
341
+ "mean": 0.13118677987364852,
342
+ "n": 14,
343
+ "num_samples": 500
344
+ },
345
+ "mean": 0.13118677987364852,
346
+ "per_seed": {
347
+ "1189392917": 0.1628408162009477,
348
+ "1271718007": 0.0,
349
+ "1273247072": 0.15040783499409596,
350
+ "1375492786": 0.19928218350064877,
351
+ "141984615": 0.0825986241056758,
352
+ "1549324805": 0.2212803648424329,
353
+ "1881223807": 0.019060791998604134,
354
+ "1997090716": 0.4044963838715546,
355
+ "2015956957": 0.18582613278258653,
356
+ "2141802816": 0.05242798758366053,
357
+ "565369392": 0.1760029309478162,
358
+ "565621826": 0.0,
359
+ "767747424": 0.10628866977652043,
360
+ "77147264": 0.07610219762653536
361
+ }
362
+ },
363
+ "macro_by_task": {
364
+ "ci": {
365
+ "confidence": 0.95,
366
+ "high": 0.15040783499409596,
367
+ "low": 0.11915469621355039,
368
+ "mean": 0.13664653664383722,
369
+ "n": 3,
370
+ "num_samples": 500
371
+ },
372
+ "mean": 0.13664653664383722,
373
+ "per_task": {
374
+ "PickCube-v1": 0.15040783499409596,
375
+ "PullCube-v1": 0.1403770787238653,
376
+ "PushCube-v1": 0.11915469621355039
377
+ }
378
+ },
379
+ "metric": "pairwise_causal_calibration_ece",
380
+ "micro": {
381
+ "confidence": 0.95,
382
+ "high": 0.18134943609804377,
383
+ "low": 0.08303117149093137,
384
+ "mean": 0.13304560754938663,
385
+ "n": 16,
386
+ "num_samples": 500
387
+ },
388
+ "n": 16
389
+ },
390
+ "proposal_oracle_success_at_8": {
391
+ "macro_by_seed": {
392
+ "ci": {
393
+ "confidence": 0.95,
394
+ "high": 0.7142857142857143,
395
+ "low": 0.21428571428571427,
396
+ "mean": 0.42857142857142855,
397
+ "n": 14,
398
+ "num_samples": 500
399
+ },
400
+ "mean": 0.42857142857142855,
401
+ "per_seed": {
402
+ "1189392917": 1.0,
403
+ "1271718007": 1.0,
404
+ "1273247072": 0.0,
405
+ "1375492786": 0.0,
406
+ "141984615": 0.0,
407
+ "1549324805": 1.0,
408
+ "1881223807": 0.0,
409
+ "1997090716": 1.0,
410
+ "2015956957": 0.0,
411
+ "2141802816": 0.0,
412
+ "565369392": 0.0,
413
+ "565621826": 1.0,
414
+ "767747424": 0.0,
415
+ "77147264": 1.0
416
+ }
417
+ },
418
+ "macro_by_task": {
419
+ "ci": {
420
+ "confidence": 0.95,
421
+ "high": 0.6666666666666666,
422
+ "low": 0.0,
423
+ "mean": 0.2962962962962963,
424
+ "n": 3,
425
+ "num_samples": 500
426
+ },
427
+ "mean": 0.2962962962962963,
428
+ "per_task": {
429
+ "PickCube-v1": 0.0,
430
+ "PullCube-v1": 0.2222222222222222,
431
+ "PushCube-v1": 0.6666666666666666
432
+ }
433
+ },
434
+ "metric": "proposal_oracle_success_at_8",
435
+ "micro": {
436
+ "confidence": 0.95,
437
+ "high": 0.625,
438
+ "low": 0.125,
439
+ "mean": 0.375,
440
+ "n": 16,
441
+ "num_samples": 500
442
+ },
443
+ "n": 16
444
+ },
445
+ "proposal_oracle_success_gain_over_base_at_8": {
446
+ "macro_by_seed": {
447
+ "ci": {
448
+ "confidence": 0.95,
449
+ "high": 0.2857142857142857,
450
+ "low": -0.21428571428571427,
451
+ "mean": 0.07142857142857142,
452
+ "n": 14,
453
+ "num_samples": 500
454
+ },
455
+ "mean": 0.07142857142857142,
456
+ "per_seed": {
457
+ "1189392917": 1.0,
458
+ "1271718007": 0.0,
459
+ "1273247072": 0.0,
460
+ "1375492786": -1.0,
461
+ "141984615": 0.0,
462
+ "1549324805": 0.0,
463
+ "1881223807": 0.0,
464
+ "1997090716": 1.0,
465
+ "2015956957": 0.0,
466
+ "2141802816": 0.0,
467
+ "565369392": 0.0,
468
+ "565621826": 0.0,
469
+ "767747424": 0.0,
470
+ "77147264": 0.0
471
+ }
472
+ },
473
+ "macro_by_task": {
474
+ "ci": {
475
+ "confidence": 0.95,
476
+ "high": 0.3333333333333333,
477
+ "low": -0.1111111111111111,
478
+ "mean": 0.07407407407407407,
479
+ "n": 3,
480
+ "num_samples": 500
481
+ },
482
+ "mean": 0.07407407407407407,
483
+ "per_task": {
484
+ "PickCube-v1": 0.0,
485
+ "PullCube-v1": -0.1111111111111111,
486
+ "PushCube-v1": 0.3333333333333333
487
+ }
488
+ },
489
+ "metric": "proposal_oracle_success_gain_over_base_at_8",
490
+ "micro": {
491
+ "confidence": 0.95,
492
+ "high": 0.25,
493
+ "low": -0.125,
494
+ "mean": 0.0625,
495
+ "n": 16,
496
+ "num_samples": 500
497
+ },
498
+ "n": 16
499
+ },
500
+ "proposal_oracle_utility_at_8": {
501
+ "macro_by_seed": {
502
+ "ci": {
503
+ "confidence": 0.95,
504
+ "high": 1.456283768015315,
505
+ "low": 0.4903542116434047,
506
+ "mean": 0.9134039957092942,
507
+ "n": 14,
508
+ "num_samples": 500
509
+ },
510
+ "mean": 0.9134039957092942,
511
+ "per_seed": {
512
+ "1189392917": 2.0,
513
+ "1271718007": 2.0,
514
+ "1273247072": 0.13278456032276154,
515
+ "1375492786": 0.0016440550098195672,
516
+ "141984615": 0.1541295200586319,
517
+ "1549324805": 2.0,
518
+ "1881223807": 0.0513521283864975,
519
+ "1997090716": 2.0,
520
+ "2015956957": 0.028452674858272076,
521
+ "2141802816": 0.12472955882549286,
522
+ "565369392": 0.18084704875946045,
523
+ "565621826": 2.0,
524
+ "767747424": 0.11371639370918274,
525
+ "77147264": 2.0
526
+ }
527
+ },
528
+ "macro_by_task": {
529
+ "ci": {
530
+ "confidence": 0.95,
531
+ "high": 1.3626802812019985,
532
+ "low": 0.13278456032276154,
533
+ "mean": 0.6696349012846542,
534
+ "n": 3,
535
+ "num_samples": 500
536
+ },
537
+ "mean": 0.6696349012846542,
538
+ "per_task": {
539
+ "PickCube-v1": 0.13278456032276154,
540
+ "PullCube-v1": 0.5134398623292024,
541
+ "PushCube-v1": 1.3626802812019985
542
+ }
543
+ },
544
+ "metric": "proposal_oracle_utility_at_8",
545
+ "micro": {
546
+ "confidence": 0.95,
547
+ "high": 1.2764708213871927,
548
+ "low": 0.3368462507642107,
549
+ "mean": 0.8081140630310983,
550
+ "n": 16,
551
+ "num_samples": 500
552
+ },
553
+ "n": 16
554
+ },
555
+ "proposal_oracle_utility_gain_over_base_at_8": {
556
+ "macro_by_seed": {
557
+ "ci": {
558
+ "confidence": 0.95,
559
+ "high": 0.5669070603325963,
560
+ "low": -0.4169755119780478,
561
+ "mean": 0.1369089032939103,
562
+ "n": 14,
563
+ "num_samples": 500
564
+ },
565
+ "mean": 0.1369089032939103,
566
+ "per_seed": {
567
+ "1189392917": 1.8915813565254211,
568
+ "1271718007": 0.0,
569
+ "1273247072": -0.02064509689807892,
570
+ "1375492786": -1.9983559449901804,
571
+ "141984615": 0.10364168509840965,
572
+ "1549324805": 0.0,
573
+ "1881223807": 0.0193251371383667,
574
+ "1997090716": 1.8607252091169357,
575
+ "2015956957": -0.02985221054404974,
576
+ "2141802816": 0.05392231047153473,
577
+ "565369392": -0.013154327869415283,
578
+ "565621826": 0.0,
579
+ "767747424": 0.04953652806580067,
580
+ "77147264": 0.0
581
+ }
582
+ },
583
+ "macro_by_task": {
584
+ "ci": {
585
+ "confidence": 0.95,
586
+ "high": 0.6375923355420431,
587
+ "low": -0.20761110585752043,
588
+ "mean": 0.13644537759548123,
589
+ "n": 3,
590
+ "num_samples": 500
591
+ },
592
+ "mean": 0.13644537759548123,
593
+ "per_task": {
594
+ "PickCube-v1": -0.02064509689807892,
595
+ "PullCube-v1": -0.20761110585752046,
596
+ "PushCube-v1": 0.6375923355420431
597
+ }
598
+ },
599
+ "metric": "proposal_oracle_utility_gain_over_base_at_8",
600
+ "micro": {
601
+ "confidence": 0.95,
602
+ "high": 0.47923582210205495,
603
+ "low": -0.26257677190733375,
604
+ "mean": 0.12102556022728095,
605
+ "n": 16,
606
+ "num_samples": 500
607
+ },
608
+ "n": 16
609
+ },
610
+ "selected_success_at_8": {
611
+ "macro_by_seed": {
612
+ "ci": {
613
+ "confidence": 0.95,
614
+ "high": 0.5,
615
+ "low": 0.07142857142857142,
616
+ "mean": 0.2857142857142857,
617
+ "n": 14,
618
+ "num_samples": 500
619
+ },
620
+ "mean": 0.2857142857142857,
621
+ "per_seed": {
622
+ "1189392917": 1.0,
623
+ "1271718007": 1.0,
624
+ "1273247072": 0.0,
625
+ "1375492786": 0.0,
626
+ "141984615": 0.0,
627
+ "1549324805": 1.0,
628
+ "1881223807": 0.0,
629
+ "1997090716": 0.0,
630
+ "2015956957": 0.0,
631
+ "2141802816": 0.0,
632
+ "565369392": 0.0,
633
+ "565621826": 1.0,
634
+ "767747424": 0.0,
635
+ "77147264": 0.0
636
+ }
637
+ },
638
+ "macro_by_task": {
639
+ "ci": {
640
+ "confidence": 0.95,
641
+ "high": 0.3333333333333333,
642
+ "low": 0.0,
643
+ "mean": 0.1851851851851852,
644
+ "n": 3,
645
+ "num_samples": 500
646
+ },
647
+ "mean": 0.1851851851851852,
648
+ "per_task": {
649
+ "PickCube-v1": 0.0,
650
+ "PullCube-v1": 0.2222222222222222,
651
+ "PushCube-v1": 0.3333333333333333
652
+ }
653
+ },
654
+ "metric": "selected_success_at_8",
655
+ "micro": {
656
+ "confidence": 0.95,
657
+ "high": 0.5,
658
+ "low": 0.0625,
659
+ "mean": 0.25,
660
+ "n": 16,
661
+ "num_samples": 500
662
+ },
663
+ "n": 16
664
+ },
665
+ "selected_success_gain_over_base_at_8": {
666
+ "macro_by_seed": {
667
+ "ci": {
668
+ "confidence": 0.95,
669
+ "high": 0.14285714285714285,
670
+ "low": -0.2857142857142857,
671
+ "mean": -0.07142857142857142,
672
+ "n": 14,
673
+ "num_samples": 500
674
+ },
675
+ "mean": -0.07142857142857142,
676
+ "per_seed": {
677
+ "1189392917": 1.0,
678
+ "1271718007": 0.0,
679
+ "1273247072": 0.0,
680
+ "1375492786": -1.0,
681
+ "141984615": 0.0,
682
+ "1549324805": 0.0,
683
+ "1881223807": 0.0,
684
+ "1997090716": 0.0,
685
+ "2015956957": 0.0,
686
+ "2141802816": 0.0,
687
+ "565369392": 0.0,
688
+ "565621826": 0.0,
689
+ "767747424": 0.0,
690
+ "77147264": -1.0
691
+ }
692
+ },
693
+ "macro_by_task": {
694
+ "ci": {
695
+ "confidence": 0.95,
696
+ "high": 0.0,
697
+ "low": -0.1111111111111111,
698
+ "mean": -0.037037037037037035,
699
+ "n": 3,
700
+ "num_samples": 500
701
+ },
702
+ "mean": -0.037037037037037035,
703
+ "per_task": {
704
+ "PickCube-v1": 0.0,
705
+ "PullCube-v1": -0.1111111111111111,
706
+ "PushCube-v1": 0.0
707
+ }
708
+ },
709
+ "metric": "selected_success_gain_over_base_at_8",
710
+ "micro": {
711
+ "confidence": 0.95,
712
+ "high": 0.125,
713
+ "low": -0.3125,
714
+ "mean": -0.0625,
715
+ "n": 16,
716
+ "num_samples": 500
717
+ },
718
+ "n": 16
719
+ },
720
+ "selected_utility_at_8": {
721
+ "macro_by_seed": {
722
+ "ci": {
723
+ "confidence": 0.95,
724
+ "high": 1.0413506987305092,
725
+ "low": 0.18699326519189136,
726
+ "mean": 0.61222785548307,
727
+ "n": 14,
728
+ "num_samples": 500
729
+ },
730
+ "mean": 0.61222785548307,
731
+ "per_seed": {
732
+ "1189392917": 2.0,
733
+ "1271718007": 2.0,
734
+ "1273247072": 0.13278456032276154,
735
+ "1375492786": 0.001358628273010254,
736
+ "141984615": 0.05628438666462898,
737
+ "1549324805": 2.0,
738
+ "1881223807": 0.03111603856086731,
739
+ "1997090716": 0.03083018958568573,
740
+ "2015956957": 0.024865647312253714,
741
+ "2141802816": 0.0492904931306839,
742
+ "565369392": 0.11366937309503555,
743
+ "565621826": 2.0,
744
+ "767747424": 0.10100766085088253,
745
+ "77147264": 0.029982998967170715
746
+ }
747
+ },
748
+ "macro_by_task": {
749
+ "ci": {
750
+ "confidence": 0.95,
751
+ "high": 0.6902032867074013,
752
+ "low": 0.13278456032276154,
753
+ "mean": 0.4381462825048301,
754
+ "n": 3,
755
+ "num_samples": 500
756
+ },
757
+ "mean": 0.4381462825048301,
758
+ "per_task": {
759
+ "PickCube-v1": 0.13278456032276154,
760
+ "PullCube-v1": 0.4914510004843275,
761
+ "PushCube-v1": 0.6902032867074013
762
+ }
763
+ },
764
+ "metric": "selected_utility_at_8",
765
+ "micro": {
766
+ "confidence": 0.95,
767
+ "high": 1.015705294383224,
768
+ "low": 0.17710547981550917,
769
+ "mean": 0.5435664553078823,
770
+ "n": 16,
771
+ "num_samples": 500
772
+ },
773
+ "n": 16
774
+ },
775
+ "selected_utility_gain_over_base_at_8": {
776
+ "macro_by_seed": {
777
+ "ci": {
778
+ "confidence": 0.95,
779
+ "high": 0.2521127736129399,
780
+ "low": -0.5916834228805133,
781
+ "mean": -0.16426723693231388,
782
+ "n": 14,
783
+ "num_samples": 500
784
+ },
785
+ "mean": -0.16426723693231388,
786
+ "per_seed": {
787
+ "1189392917": 1.8915813565254211,
788
+ "1271718007": 0.0,
789
+ "1273247072": -0.02064509689807892,
790
+ "1375492786": -1.9986413717269897,
791
+ "141984615": 0.005796551704406738,
792
+ "1549324805": 0.0,
793
+ "1881223807": -0.0009109526872634888,
794
+ "1997090716": -0.10844460129737854,
795
+ "2015956957": -0.0334392380900681,
796
+ "2141802816": -0.02151675522327423,
797
+ "565369392": -0.08033200353384018,
798
+ "565621826": 0.0,
799
+ "767747424": 0.03682779520750046,
800
+ "77147264": -1.9700170010328293
801
+ }
802
+ },
803
+ "macro_by_task": {
804
+ "ci": {
805
+ "confidence": 0.95,
806
+ "high": -0.02064509689807892,
807
+ "low": -0.22959996770239535,
808
+ "mean": -0.09504324118434278,
809
+ "n": 3,
810
+ "num_samples": 500
811
+ },
812
+ "mean": -0.09504324118434278,
813
+ "per_task": {
814
+ "PickCube-v1": -0.02064509689807892,
815
+ "PullCube-v1": -0.22959996770239538,
816
+ "PushCube-v1": -0.03488465895255407
817
+ }
818
+ },
819
+ "metric": "selected_utility_gain_over_base_at_8",
820
+ "micro": {
821
+ "confidence": 0.95,
822
+ "high": 0.22805603814776987,
823
+ "low": -0.617137425870169,
824
+ "mean": -0.1435220474959351,
825
+ "n": 16,
826
+ "num_samples": 500
827
+ },
828
+ "n": 16
829
+ },
830
+ "selector_regret_at_8": {
831
+ "macro_by_seed": {
832
+ "ci": {
833
+ "confidence": 0.95,
834
+ "high": 0.7187583449163607,
835
+ "low": 0.014021761948242784,
836
+ "mean": 0.3011761402262242,
837
+ "n": 14,
838
+ "num_samples": 500
839
+ },
840
+ "mean": 0.3011761402262242,
841
+ "per_seed": {
842
+ "1189392917": 0.0,
843
+ "1271718007": 0.0,
844
+ "1273247072": 0.0,
845
+ "1375492786": 0.0002854267368093133,
846
+ "141984615": 0.09784513339400291,
847
+ "1549324805": 0.0,
848
+ "1881223807": 0.020236089825630188,
849
+ "1997090716": 1.9691698104143143,
850
+ "2015956957": 0.003587027546018362,
851
+ "2141802816": 0.07543906569480896,
852
+ "565369392": 0.0671776756644249,
853
+ "565621826": 0.0,
854
+ "767747424": 0.012708732858300209,
855
+ "77147264": 1.9700170010328293
856
+ }
857
+ },
858
+ "macro_by_task": {
859
+ "ci": {
860
+ "confidence": 0.95,
861
+ "high": 0.6724769944945971,
862
+ "low": 0.0,
863
+ "mean": 0.231488618779824,
864
+ "n": 3,
865
+ "num_samples": 500
866
+ },
867
+ "mean": 0.231488618779824,
868
+ "per_task": {
869
+ "PickCube-v1": 0.0,
870
+ "PullCube-v1": 0.02198886184487492,
871
+ "PushCube-v1": 0.6724769944945971
872
+ }
873
+ },
874
+ "metric": "selector_regret_at_8",
875
+ "micro": {
876
+ "confidence": 0.95,
877
+ "high": 0.6269801619346254,
878
+ "low": 0.014924341201549396,
879
+ "mean": 0.26454760772321606,
880
+ "n": 16,
881
+ "num_samples": 500
882
+ },
883
+ "n": 16
884
+ },
885
+ "success_selector_gap_at_8": {
886
+ "macro_by_seed": {
887
+ "ci": {
888
+ "confidence": 0.95,
889
+ "high": 0.35714285714285715,
890
+ "low": 0.0,
891
+ "mean": 0.14285714285714285,
892
+ "n": 14,
893
+ "num_samples": 500
894
+ },
895
+ "mean": 0.14285714285714285,
896
+ "per_seed": {
897
+ "1189392917": 0.0,
898
+ "1271718007": 0.0,
899
+ "1273247072": 0.0,
900
+ "1375492786": 0.0,
901
+ "141984615": 0.0,
902
+ "1549324805": 0.0,
903
+ "1881223807": 0.0,
904
+ "1997090716": 1.0,
905
+ "2015956957": 0.0,
906
+ "2141802816": 0.0,
907
+ "565369392": 0.0,
908
+ "565621826": 0.0,
909
+ "767747424": 0.0,
910
+ "77147264": 1.0
911
+ }
912
+ },
913
+ "macro_by_task": {
914
+ "ci": {
915
+ "confidence": 0.95,
916
+ "high": 0.3333333333333333,
917
+ "low": 0.0,
918
+ "mean": 0.1111111111111111,
919
+ "n": 3,
920
+ "num_samples": 500
921
+ },
922
+ "mean": 0.1111111111111111,
923
+ "per_task": {
924
+ "PickCube-v1": 0.0,
925
+ "PullCube-v1": 0.0,
926
+ "PushCube-v1": 0.3333333333333333
927
+ }
928
+ },
929
+ "metric": "success_selector_gap_at_8",
930
+ "micro": {
931
+ "confidence": 0.95,
932
+ "high": 0.3125,
933
+ "low": 0.0,
934
+ "mean": 0.125,
935
+ "n": 16,
936
+ "num_samples": 500
937
+ },
938
+ "n": 16
939
+ },
940
+ "success_support_gap_at_8": {
941
+ "macro_by_seed": {
942
+ "ci": {
943
+ "confidence": 0.95,
944
+ "high": 0.5,
945
+ "low": 0.07142857142857142,
946
+ "mean": 0.2857142857142857,
947
+ "n": 14,
948
+ "num_samples": 500
949
+ },
950
+ "mean": 0.2857142857142857,
951
+ "per_seed": {
952
+ "1189392917": 0.0,
953
+ "1271718007": 0.0,
954
+ "1273247072": 1.0,
955
+ "1375492786": 0.0,
956
+ "141984615": 0.0,
957
+ "1549324805": 0.0,
958
+ "1881223807": 1.0,
959
+ "1997090716": 0.0,
960
+ "2015956957": 0.0,
961
+ "2141802816": 1.0,
962
+ "565369392": 1.0,
963
+ "565621826": 0.0,
964
+ "767747424": 0.0,
965
+ "77147264": 0.0
966
+ }
967
+ },
968
+ "macro_by_task": {
969
+ "ci": {
970
+ "confidence": 0.95,
971
+ "high": 1.0,
972
+ "low": 0.1111111111111111,
973
+ "mean": 0.48148148148148145,
974
+ "n": 3,
975
+ "num_samples": 500
976
+ },
977
+ "mean": 0.48148148148148145,
978
+ "per_task": {
979
+ "PickCube-v1": 1.0,
980
+ "PullCube-v1": 0.1111111111111111,
981
+ "PushCube-v1": 0.3333333333333333
982
+ }
983
+ },
984
+ "metric": "success_support_gap_at_8",
985
+ "micro": {
986
+ "confidence": 0.95,
987
+ "high": 0.5,
988
+ "low": 0.0625,
989
+ "mean": 0.25,
990
+ "n": 16,
991
+ "num_samples": 500
992
+ },
993
+ "n": 16
994
+ },
995
+ "success_total_car_to_hidden_at_8": {
996
+ "macro_by_seed": {
997
+ "ci": {
998
+ "confidence": 0.95,
999
+ "high": 0.7142857142857143,
1000
+ "low": 0.14285714285714285,
1001
+ "mean": 0.42857142857142855,
1002
+ "n": 14,
1003
+ "num_samples": 500
1004
+ },
1005
+ "mean": 0.42857142857142855,
1006
+ "per_seed": {
1007
+ "1189392917": 0.0,
1008
+ "1271718007": 0.0,
1009
+ "1273247072": 1.0,
1010
+ "1375492786": 0.0,
1011
+ "141984615": 0.0,
1012
+ "1549324805": 0.0,
1013
+ "1881223807": 1.0,
1014
+ "1997090716": 1.0,
1015
+ "2015956957": 0.0,
1016
+ "2141802816": 1.0,
1017
+ "565369392": 1.0,
1018
+ "565621826": 0.0,
1019
+ "767747424": 0.0,
1020
+ "77147264": 1.0
1021
+ }
1022
+ },
1023
+ "macro_by_task": {
1024
+ "ci": {
1025
+ "confidence": 0.95,
1026
+ "high": 1.0,
1027
+ "low": 0.1111111111111111,
1028
+ "mean": 0.5925925925925926,
1029
+ "n": 3,
1030
+ "num_samples": 500
1031
+ },
1032
+ "mean": 0.5925925925925926,
1033
+ "per_task": {
1034
+ "PickCube-v1": 1.0,
1035
+ "PullCube-v1": 0.1111111111111111,
1036
+ "PushCube-v1": 0.6666666666666666
1037
+ }
1038
+ },
1039
+ "metric": "success_total_car_to_hidden_at_8",
1040
+ "micro": {
1041
+ "confidence": 0.95,
1042
+ "high": 0.625,
1043
+ "low": 0.125,
1044
+ "mean": 0.375,
1045
+ "n": 16,
1046
+ "num_samples": 500
1047
+ },
1048
+ "n": 16
1049
+ },
1050
+ "support_gap_at_8": {
1051
+ "macro_by_seed": {
1052
+ "ci": {
1053
+ "confidence": 0.95,
1054
+ "high": 1.00120393927708,
1055
+ "low": 0.16012767239174405,
1056
+ "mean": 0.5714171690051444,
1057
+ "n": 14,
1058
+ "num_samples": 500
1059
+ },
1060
+ "mean": 0.5714171690051444,
1061
+ "per_seed": {
1062
+ "1189392917": 0.0,
1063
+ "1271718007": 0.0,
1064
+ "1273247072": 1.8672154396772385,
1065
+ "1375492786": 0.24180691444780678,
1066
+ "141984615": 0.025680318474769592,
1067
+ "1549324805": 0.0,
1068
+ "1881223807": 1.9486478716135025,
1069
+ "1997090716": 0.0,
1070
+ "2015956957": 0.13276505935937166,
1071
+ "2141802816": 1.8752704411745071,
1072
+ "565369392": 1.8191529512405396,
1073
+ "565621826": 0.0,
1074
+ "767747424": 0.08930137008428574,
1075
+ "77147264": 0.0
1076
+ }
1077
+ },
1078
+ "macro_by_task": {
1079
+ "ci": {
1080
+ "confidence": 0.95,
1081
+ "high": 1.8672154396772385,
1082
+ "low": 0.28119700478338117,
1083
+ "mean": 0.9285773877528739,
1084
+ "n": 3,
1085
+ "num_samples": 500
1086
+ },
1087
+ "mean": 0.9285773877528739,
1088
+ "per_task": {
1089
+ "PickCube-v1": 1.8672154396772385,
1090
+ "PullCube-v1": 0.28119700478338117,
1091
+ "PushCube-v1": 0.6373197187980016
1092
+ }
1093
+ },
1094
+ "metric": "support_gap_at_8",
1095
+ "micro": {
1096
+ "confidence": 0.95,
1097
+ "high": 0.9626226120162755,
1098
+ "low": 0.1582781979450374,
1099
+ "mean": 0.5138691747197299,
1100
+ "n": 16,
1101
+ "num_samples": 500
1102
+ },
1103
+ "n": 16
1104
+ },
1105
+ "total_car_to_hidden_at_8": {
1106
+ "macro_by_seed": {
1107
+ "ci": {
1108
+ "confidence": 0.95,
1109
+ "high": 1.4018085617904685,
1110
+ "low": 0.3681092694667833,
1111
+ "mean": 0.8725933092313686,
1112
+ "n": 14,
1113
+ "num_samples": 500
1114
+ },
1115
+ "mean": 0.8725933092313686,
1116
+ "per_seed": {
1117
+ "1189392917": 0.0,
1118
+ "1271718007": 0.0,
1119
+ "1273247072": 1.8672154396772385,
1120
+ "1375492786": 0.2420923411846161,
1121
+ "141984615": 0.1235254518687725,
1122
+ "1549324805": 0.0,
1123
+ "1881223807": 1.9688839614391327,
1124
+ "1997090716": 1.9691698104143143,
1125
+ "2015956957": 0.13635208690539002,
1126
+ "2141802816": 1.950709506869316,
1127
+ "565369392": 1.8863306269049644,
1128
+ "565621826": 0.0,
1129
+ "767747424": 0.10201010294258595,
1130
+ "77147264": 1.9700170010328293
1131
+ }
1132
+ },
1133
+ "macro_by_task": {
1134
+ "ci": {
1135
+ "confidence": 0.95,
1136
+ "high": 1.8672154396772385,
1137
+ "low": 0.3031858666282561,
1138
+ "mean": 1.1600660065326978,
1139
+ "n": 3,
1140
+ "num_samples": 500
1141
+ },
1142
+ "mean": 1.1600660065326978,
1143
+ "per_task": {
1144
+ "PickCube-v1": 1.8672154396772385,
1145
+ "PullCube-v1": 0.3031858666282561,
1146
+ "PushCube-v1": 1.3097967132925987
1147
+ }
1148
+ },
1149
+ "metric": "total_car_to_hidden_at_8",
1150
+ "micro": {
1151
+ "confidence": 0.95,
1152
+ "high": 1.2434286201023497,
1153
+ "low": 0.338613064144738,
1154
+ "mean": 0.778416782442946,
1155
+ "n": 16,
1156
+ "num_samples": 500
1157
+ },
1158
+ "n": 16
1159
+ }
1160
+ },
1161
+ "num_rows": 16,
1162
+ "report_type": "ctt_generated_measured_rollout",
1163
+ "schema_version": 1,
1164
+ "source_content_hash": "90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957",
1165
+ "source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372",
1166
+ "success_summary": {
1167
+ "base_success_rate": 0.3125,
1168
+ "base_utility_mean": 0.6870885028038174,
1169
+ "hidden_chart_oracle_success_rate": 0.625,
1170
+ "hidden_chart_oracle_utility_mean": 1.3219832377508283,
1171
+ "max_restore_error": 1.4901161193847656e-08,
1172
+ "proposal_oracle_success_gain_over_base": 0.0625,
1173
+ "proposal_oracle_success_rate": 0.375,
1174
+ "proposal_oracle_utility_mean": 0.8081140630310983,
1175
+ "selected_success_gain_over_base": -0.0625,
1176
+ "selected_success_rate": 0.25,
1177
+ "selected_utility_mean": 0.5435664553078823,
1178
+ "success_selector_gap": 0.125,
1179
+ "success_support_gap": 0.25
1180
+ },
1181
+ "target_content_hash": "fa0fe0e3881ee70ce91492445871614d01b3a47fc7c576659c53ff17ddee4e42",
1182
+ "target_split_hash": "9d1dc1c10868f76fa080895df0f2730c173f12b6db14932599d55c2b60859381"
1183
+ }
workspace/runs/ctt_residual_rollout_val16_seed0/metrics_by_seed.json ADDED
@@ -0,0 +1,324 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "1189392917": {
3
+ "base_success": 0.0,
4
+ "base_utility": 0.10841864347457886,
5
+ "branch_car_at_8": 0.0,
6
+ "hidden_chart_oracle_success_at_8": 1.0,
7
+ "hidden_chart_oracle_utility_at_8": 2.0,
8
+ "outcome_ptr_at_8": 1.0,
9
+ "pairwise_causal_calibration_ece": 0.1628408162009477,
10
+ "proposal_oracle_success_at_8": 1.0,
11
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
12
+ "proposal_oracle_utility_at_8": 2.0,
13
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211,
14
+ "selected_success_at_8": 1.0,
15
+ "selected_success_gain_over_base_at_8": 1.0,
16
+ "selected_utility_at_8": 2.0,
17
+ "selected_utility_gain_over_base_at_8": 1.8915813565254211,
18
+ "selector_regret_at_8": 0.0,
19
+ "success_selector_gap_at_8": 0.0,
20
+ "success_support_gap_at_8": 0.0,
21
+ "success_total_car_to_hidden_at_8": 0.0,
22
+ "support_gap_at_8": 0.0,
23
+ "total_car_to_hidden_at_8": 0.0
24
+ },
25
+ "1271718007": {
26
+ "base_success": 1.0,
27
+ "base_utility": 2.0,
28
+ "branch_car_at_8": 0.0,
29
+ "hidden_chart_oracle_success_at_8": 1.0,
30
+ "hidden_chart_oracle_utility_at_8": 2.0,
31
+ "outcome_ptr_at_8": 0.0,
32
+ "pairwise_causal_calibration_ece": 0.0,
33
+ "proposal_oracle_success_at_8": 1.0,
34
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
35
+ "proposal_oracle_utility_at_8": 2.0,
36
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
37
+ "selected_success_at_8": 1.0,
38
+ "selected_success_gain_over_base_at_8": 0.0,
39
+ "selected_utility_at_8": 2.0,
40
+ "selected_utility_gain_over_base_at_8": 0.0,
41
+ "selector_regret_at_8": 0.0,
42
+ "success_selector_gap_at_8": 0.0,
43
+ "success_support_gap_at_8": 0.0,
44
+ "success_total_car_to_hidden_at_8": 0.0,
45
+ "support_gap_at_8": 0.0,
46
+ "total_car_to_hidden_at_8": 0.0
47
+ },
48
+ "1273247072": {
49
+ "base_success": 0.0,
50
+ "base_utility": 0.15342965722084045,
51
+ "branch_car_at_8": 0.0,
52
+ "hidden_chart_oracle_success_at_8": 1.0,
53
+ "hidden_chart_oracle_utility_at_8": 2.0,
54
+ "outcome_ptr_at_8": 0.0,
55
+ "pairwise_causal_calibration_ece": 0.15040783499409596,
56
+ "proposal_oracle_success_at_8": 0.0,
57
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
58
+ "proposal_oracle_utility_at_8": 0.13278456032276154,
59
+ "proposal_oracle_utility_gain_over_base_at_8": -0.02064509689807892,
60
+ "selected_success_at_8": 0.0,
61
+ "selected_success_gain_over_base_at_8": 0.0,
62
+ "selected_utility_at_8": 0.13278456032276154,
63
+ "selected_utility_gain_over_base_at_8": -0.02064509689807892,
64
+ "selector_regret_at_8": 0.0,
65
+ "success_selector_gap_at_8": 0.0,
66
+ "success_support_gap_at_8": 1.0,
67
+ "success_total_car_to_hidden_at_8": 1.0,
68
+ "support_gap_at_8": 1.8672154396772385,
69
+ "total_car_to_hidden_at_8": 1.8672154396772385
70
+ },
71
+ "1375492786": {
72
+ "base_success": 1.0,
73
+ "base_utility": 2.0,
74
+ "branch_car_at_8": 0.0002854267368093133,
75
+ "hidden_chart_oracle_success_at_8": 0.0,
76
+ "hidden_chart_oracle_utility_at_8": 0.24345096945762634,
77
+ "outcome_ptr_at_8": 0.0,
78
+ "pairwise_causal_calibration_ece": 0.19928218350064877,
79
+ "proposal_oracle_success_at_8": 0.0,
80
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
81
+ "proposal_oracle_utility_at_8": 0.0016440550098195672,
82
+ "proposal_oracle_utility_gain_over_base_at_8": -1.9983559449901804,
83
+ "selected_success_at_8": 0.0,
84
+ "selected_success_gain_over_base_at_8": -1.0,
85
+ "selected_utility_at_8": 0.001358628273010254,
86
+ "selected_utility_gain_over_base_at_8": -1.9986413717269897,
87
+ "selector_regret_at_8": 0.0002854267368093133,
88
+ "success_selector_gap_at_8": 0.0,
89
+ "success_support_gap_at_8": 0.0,
90
+ "success_total_car_to_hidden_at_8": 0.0,
91
+ "support_gap_at_8": 0.24180691444780678,
92
+ "total_car_to_hidden_at_8": 0.2420923411846161
93
+ },
94
+ "141984615": {
95
+ "base_success": 0.0,
96
+ "base_utility": 0.050487834960222244,
97
+ "branch_car_at_8": 0.09784513339400291,
98
+ "hidden_chart_oracle_success_at_8": 0.0,
99
+ "hidden_chart_oracle_utility_at_8": 0.1798098385334015,
100
+ "outcome_ptr_at_8": 1.0,
101
+ "pairwise_causal_calibration_ece": 0.0825986241056758,
102
+ "proposal_oracle_success_at_8": 0.0,
103
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
104
+ "proposal_oracle_utility_at_8": 0.1541295200586319,
105
+ "proposal_oracle_utility_gain_over_base_at_8": 0.10364168509840965,
106
+ "selected_success_at_8": 0.0,
107
+ "selected_success_gain_over_base_at_8": 0.0,
108
+ "selected_utility_at_8": 0.05628438666462898,
109
+ "selected_utility_gain_over_base_at_8": 0.005796551704406738,
110
+ "selector_regret_at_8": 0.09784513339400291,
111
+ "success_selector_gap_at_8": 0.0,
112
+ "success_support_gap_at_8": 0.0,
113
+ "success_total_car_to_hidden_at_8": 0.0,
114
+ "support_gap_at_8": 0.025680318474769592,
115
+ "total_car_to_hidden_at_8": 0.1235254518687725
116
+ },
117
+ "1549324805": {
118
+ "base_success": 1.0,
119
+ "base_utility": 2.0,
120
+ "branch_car_at_8": 0.0,
121
+ "hidden_chart_oracle_success_at_8": 1.0,
122
+ "hidden_chart_oracle_utility_at_8": 2.0,
123
+ "outcome_ptr_at_8": 0.0,
124
+ "pairwise_causal_calibration_ece": 0.2212803648424329,
125
+ "proposal_oracle_success_at_8": 1.0,
126
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
127
+ "proposal_oracle_utility_at_8": 2.0,
128
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
129
+ "selected_success_at_8": 1.0,
130
+ "selected_success_gain_over_base_at_8": 0.0,
131
+ "selected_utility_at_8": 2.0,
132
+ "selected_utility_gain_over_base_at_8": 0.0,
133
+ "selector_regret_at_8": 0.0,
134
+ "success_selector_gap_at_8": 0.0,
135
+ "success_support_gap_at_8": 0.0,
136
+ "success_total_car_to_hidden_at_8": 0.0,
137
+ "support_gap_at_8": 0.0,
138
+ "total_car_to_hidden_at_8": 0.0
139
+ },
140
+ "1881223807": {
141
+ "base_success": 0.0,
142
+ "base_utility": 0.0320269912481308,
143
+ "branch_car_at_8": 0.020236089825630188,
144
+ "hidden_chart_oracle_success_at_8": 1.0,
145
+ "hidden_chart_oracle_utility_at_8": 2.0,
146
+ "outcome_ptr_at_8": 1.0,
147
+ "pairwise_causal_calibration_ece": 0.019060791998604134,
148
+ "proposal_oracle_success_at_8": 0.0,
149
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
150
+ "proposal_oracle_utility_at_8": 0.0513521283864975,
151
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0193251371383667,
152
+ "selected_success_at_8": 0.0,
153
+ "selected_success_gain_over_base_at_8": 0.0,
154
+ "selected_utility_at_8": 0.03111603856086731,
155
+ "selected_utility_gain_over_base_at_8": -0.0009109526872634888,
156
+ "selector_regret_at_8": 0.020236089825630188,
157
+ "success_selector_gap_at_8": 0.0,
158
+ "success_support_gap_at_8": 1.0,
159
+ "success_total_car_to_hidden_at_8": 1.0,
160
+ "support_gap_at_8": 1.9486478716135025,
161
+ "total_car_to_hidden_at_8": 1.9688839614391327
162
+ },
163
+ "1997090716": {
164
+ "base_success": 0.0,
165
+ "base_utility": 0.13927479088306427,
166
+ "branch_car_at_8": 1.9691698104143143,
167
+ "hidden_chart_oracle_success_at_8": 1.0,
168
+ "hidden_chart_oracle_utility_at_8": 2.0,
169
+ "outcome_ptr_at_8": 1.0,
170
+ "pairwise_causal_calibration_ece": 0.4044963838715546,
171
+ "proposal_oracle_success_at_8": 1.0,
172
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
173
+ "proposal_oracle_utility_at_8": 2.0,
174
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357,
175
+ "selected_success_at_8": 0.0,
176
+ "selected_success_gain_over_base_at_8": 0.0,
177
+ "selected_utility_at_8": 0.03083018958568573,
178
+ "selected_utility_gain_over_base_at_8": -0.10844460129737854,
179
+ "selector_regret_at_8": 1.9691698104143143,
180
+ "success_selector_gap_at_8": 1.0,
181
+ "success_support_gap_at_8": 0.0,
182
+ "success_total_car_to_hidden_at_8": 1.0,
183
+ "support_gap_at_8": 0.0,
184
+ "total_car_to_hidden_at_8": 1.9691698104143143
185
+ },
186
+ "2015956957": {
187
+ "base_success": 0.0,
188
+ "base_utility": 0.058304885402321815,
189
+ "branch_car_at_8": 0.003587027546018362,
190
+ "hidden_chart_oracle_success_at_8": 0.0,
191
+ "hidden_chart_oracle_utility_at_8": 0.16121773421764374,
192
+ "outcome_ptr_at_8": 0.0,
193
+ "pairwise_causal_calibration_ece": 0.18582613278258653,
194
+ "proposal_oracle_success_at_8": 0.0,
195
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
196
+ "proposal_oracle_utility_at_8": 0.028452674858272076,
197
+ "proposal_oracle_utility_gain_over_base_at_8": -0.02985221054404974,
198
+ "selected_success_at_8": 0.0,
199
+ "selected_success_gain_over_base_at_8": 0.0,
200
+ "selected_utility_at_8": 0.024865647312253714,
201
+ "selected_utility_gain_over_base_at_8": -0.0334392380900681,
202
+ "selector_regret_at_8": 0.003587027546018362,
203
+ "success_selector_gap_at_8": 0.0,
204
+ "success_support_gap_at_8": 0.0,
205
+ "success_total_car_to_hidden_at_8": 0.0,
206
+ "support_gap_at_8": 0.13276505935937166,
207
+ "total_car_to_hidden_at_8": 0.13635208690539002
208
+ },
209
+ "2141802816": {
210
+ "base_success": 0.0,
211
+ "base_utility": 0.07080724835395813,
212
+ "branch_car_at_8": 0.07543906569480896,
213
+ "hidden_chart_oracle_success_at_8": 1.0,
214
+ "hidden_chart_oracle_utility_at_8": 2.0,
215
+ "outcome_ptr_at_8": 1.0,
216
+ "pairwise_causal_calibration_ece": 0.05242798758366053,
217
+ "proposal_oracle_success_at_8": 0.0,
218
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
219
+ "proposal_oracle_utility_at_8": 0.12472955882549286,
220
+ "proposal_oracle_utility_gain_over_base_at_8": 0.05392231047153473,
221
+ "selected_success_at_8": 0.0,
222
+ "selected_success_gain_over_base_at_8": 0.0,
223
+ "selected_utility_at_8": 0.0492904931306839,
224
+ "selected_utility_gain_over_base_at_8": -0.02151675522327423,
225
+ "selector_regret_at_8": 0.07543906569480896,
226
+ "success_selector_gap_at_8": 0.0,
227
+ "success_support_gap_at_8": 1.0,
228
+ "success_total_car_to_hidden_at_8": 1.0,
229
+ "support_gap_at_8": 1.8752704411745071,
230
+ "total_car_to_hidden_at_8": 1.950709506869316
231
+ },
232
+ "565369392": {
233
+ "base_success": 0.0,
234
+ "base_utility": 0.19400137662887573,
235
+ "branch_car_at_8": 0.0671776756644249,
236
+ "hidden_chart_oracle_success_at_8": 1.0,
237
+ "hidden_chart_oracle_utility_at_8": 2.0,
238
+ "outcome_ptr_at_8": 0.0,
239
+ "pairwise_causal_calibration_ece": 0.1760029309478162,
240
+ "proposal_oracle_success_at_8": 0.0,
241
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
242
+ "proposal_oracle_utility_at_8": 0.18084704875946045,
243
+ "proposal_oracle_utility_gain_over_base_at_8": -0.013154327869415283,
244
+ "selected_success_at_8": 0.0,
245
+ "selected_success_gain_over_base_at_8": 0.0,
246
+ "selected_utility_at_8": 0.11366937309503555,
247
+ "selected_utility_gain_over_base_at_8": -0.08033200353384018,
248
+ "selector_regret_at_8": 0.0671776756644249,
249
+ "success_selector_gap_at_8": 0.0,
250
+ "success_support_gap_at_8": 1.0,
251
+ "success_total_car_to_hidden_at_8": 1.0,
252
+ "support_gap_at_8": 1.8191529512405396,
253
+ "total_car_to_hidden_at_8": 1.8863306269049644
254
+ },
255
+ "565621826": {
256
+ "base_success": 1.0,
257
+ "base_utility": 2.0,
258
+ "branch_car_at_8": 0.0,
259
+ "hidden_chart_oracle_success_at_8": 1.0,
260
+ "hidden_chart_oracle_utility_at_8": 2.0,
261
+ "outcome_ptr_at_8": 0.0,
262
+ "pairwise_causal_calibration_ece": 0.0,
263
+ "proposal_oracle_success_at_8": 1.0,
264
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
265
+ "proposal_oracle_utility_at_8": 2.0,
266
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
267
+ "selected_success_at_8": 1.0,
268
+ "selected_success_gain_over_base_at_8": 0.0,
269
+ "selected_utility_at_8": 2.0,
270
+ "selected_utility_gain_over_base_at_8": 0.0,
271
+ "selector_regret_at_8": 0.0,
272
+ "success_selector_gap_at_8": 0.0,
273
+ "success_support_gap_at_8": 0.0,
274
+ "success_total_car_to_hidden_at_8": 0.0,
275
+ "support_gap_at_8": 0.0,
276
+ "total_car_to_hidden_at_8": 0.0
277
+ },
278
+ "767747424": {
279
+ "base_success": 0.0,
280
+ "base_utility": 0.06417986564338207,
281
+ "branch_car_at_8": 0.012708732858300209,
282
+ "hidden_chart_oracle_success_at_8": 0.0,
283
+ "hidden_chart_oracle_utility_at_8": 0.20301776379346848,
284
+ "outcome_ptr_at_8": 0.5,
285
+ "pairwise_causal_calibration_ece": 0.10628866977652043,
286
+ "proposal_oracle_success_at_8": 0.0,
287
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
288
+ "proposal_oracle_utility_at_8": 0.11371639370918274,
289
+ "proposal_oracle_utility_gain_over_base_at_8": 0.04953652806580067,
290
+ "selected_success_at_8": 0.0,
291
+ "selected_success_gain_over_base_at_8": 0.0,
292
+ "selected_utility_at_8": 0.10100766085088253,
293
+ "selected_utility_gain_over_base_at_8": 0.03682779520750046,
294
+ "selector_regret_at_8": 0.012708732858300209,
295
+ "success_selector_gap_at_8": 0.0,
296
+ "success_support_gap_at_8": 0.0,
297
+ "success_total_car_to_hidden_at_8": 0.0,
298
+ "support_gap_at_8": 0.08930137008428574,
299
+ "total_car_to_hidden_at_8": 0.10201010294258595
300
+ },
301
+ "77147264": {
302
+ "base_success": 1.0,
303
+ "base_utility": 2.0,
304
+ "branch_car_at_8": 1.9700170010328293,
305
+ "hidden_chart_oracle_success_at_8": 1.0,
306
+ "hidden_chart_oracle_utility_at_8": 2.0,
307
+ "outcome_ptr_at_8": 0.0,
308
+ "pairwise_causal_calibration_ece": 0.07610219762653536,
309
+ "proposal_oracle_success_at_8": 1.0,
310
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
311
+ "proposal_oracle_utility_at_8": 2.0,
312
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
313
+ "selected_success_at_8": 0.0,
314
+ "selected_success_gain_over_base_at_8": -1.0,
315
+ "selected_utility_at_8": 0.029982998967170715,
316
+ "selected_utility_gain_over_base_at_8": -1.9700170010328293,
317
+ "selector_regret_at_8": 1.9700170010328293,
318
+ "success_selector_gap_at_8": 1.0,
319
+ "success_support_gap_at_8": 0.0,
320
+ "success_total_car_to_hidden_at_8": 1.0,
321
+ "support_gap_at_8": 0.0,
322
+ "total_car_to_hidden_at_8": 1.9700170010328293
323
+ }
324
+ }
workspace/runs/ctt_residual_rollout_val16_seed0/metrics_by_task.json ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "PickCube-v1": {
3
+ "base_success": 0.0,
4
+ "base_utility": 0.15342965722084045,
5
+ "branch_car_at_8": 0.0,
6
+ "hidden_chart_oracle_success_at_8": 1.0,
7
+ "hidden_chart_oracle_utility_at_8": 2.0,
8
+ "outcome_ptr_at_8": 0.0,
9
+ "pairwise_causal_calibration_ece": 0.15040783499409596,
10
+ "proposal_oracle_success_at_8": 0.0,
11
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
12
+ "proposal_oracle_utility_at_8": 0.13278456032276154,
13
+ "proposal_oracle_utility_gain_over_base_at_8": -0.02064509689807892,
14
+ "selected_success_at_8": 0.0,
15
+ "selected_success_gain_over_base_at_8": 0.0,
16
+ "selected_utility_at_8": 0.13278456032276154,
17
+ "selected_utility_gain_over_base_at_8": -0.02064509689807892,
18
+ "selector_regret_at_8": 0.0,
19
+ "success_selector_gap_at_8": 0.0,
20
+ "success_support_gap_at_8": 1.0,
21
+ "success_total_car_to_hidden_at_8": 1.0,
22
+ "support_gap_at_8": 1.8672154396772385,
23
+ "total_car_to_hidden_at_8": 1.8672154396772385
24
+ },
25
+ "PullCube-v1": {
26
+ "base_success": 0.3333333333333333,
27
+ "base_utility": 0.7210509681867229,
28
+ "branch_car_at_8": 0.02198886184487492,
29
+ "hidden_chart_oracle_success_at_8": 0.3333333333333333,
30
+ "hidden_chart_oracle_utility_at_8": 0.7946368671125836,
31
+ "outcome_ptr_at_8": 0.2222222222222222,
32
+ "pairwise_causal_calibration_ece": 0.1403770787238653,
33
+ "proposal_oracle_success_at_8": 0.2222222222222222,
34
+ "proposal_oracle_success_gain_over_base_at_8": -0.1111111111111111,
35
+ "proposal_oracle_utility_at_8": 0.5134398623292024,
36
+ "proposal_oracle_utility_gain_over_base_at_8": -0.20761110585752046,
37
+ "selected_success_at_8": 0.2222222222222222,
38
+ "selected_success_gain_over_base_at_8": -0.1111111111111111,
39
+ "selected_utility_at_8": 0.4914510004843275,
40
+ "selected_utility_gain_over_base_at_8": -0.22959996770239538,
41
+ "selector_regret_at_8": 0.02198886184487492,
42
+ "success_selector_gap_at_8": 0.0,
43
+ "success_support_gap_at_8": 0.1111111111111111,
44
+ "success_total_car_to_hidden_at_8": 0.1111111111111111,
45
+ "support_gap_at_8": 0.28119700478338117,
46
+ "total_car_to_hidden_at_8": 0.3031858666282561
47
+ },
48
+ "PushCube-v1": {
49
+ "base_success": 0.3333333333333333,
50
+ "base_utility": 0.7250879456599554,
51
+ "branch_car_at_8": 0.6724769944945971,
52
+ "hidden_chart_oracle_success_at_8": 1.0,
53
+ "hidden_chart_oracle_utility_at_8": 2.0,
54
+ "outcome_ptr_at_8": 0.6666666666666666,
55
+ "pairwise_causal_calibration_ece": 0.11915469621355039,
56
+ "proposal_oracle_success_at_8": 0.6666666666666666,
57
+ "proposal_oracle_success_gain_over_base_at_8": 0.3333333333333333,
58
+ "proposal_oracle_utility_at_8": 1.3626802812019985,
59
+ "proposal_oracle_utility_gain_over_base_at_8": 0.6375923355420431,
60
+ "selected_success_at_8": 0.3333333333333333,
61
+ "selected_success_gain_over_base_at_8": 0.0,
62
+ "selected_utility_at_8": 0.6902032867074013,
63
+ "selected_utility_gain_over_base_at_8": -0.03488465895255407,
64
+ "selector_regret_at_8": 0.6724769944945971,
65
+ "success_selector_gap_at_8": 0.3333333333333333,
66
+ "success_support_gap_at_8": 0.3333333333333333,
67
+ "success_total_car_to_hidden_at_8": 0.6666666666666666,
68
+ "support_gap_at_8": 0.6373197187980016,
69
+ "total_car_to_hidden_at_8": 1.3097967132925987
70
+ }
71
+ }
workspace/runs/ctt_residual_rollout_val16_seed0/source_data_hash.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957
workspace/runs/ctt_residual_rollout_val16_seed0/source_split_hash.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372
workspace/runs/ctt_residual_rollout_val16_seed0/split_hash.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 9d1dc1c10868f76fa080895df0f2730c173f12b6db14932599d55c2b60859381
workspace/runs/ctt_residual_rollout_val16_seed0/table.tex ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ % Auto-generated by scripts/eval_metrics.py
2
+ \begin{tabular}{lrrrr}
3
+ \toprule
4
+ Metric & N & Micro mean & CI low & CI high \\
5
+ \midrule
6
+ base\_success & 16 & 0.3125 & 0.1250 & 0.5625 \\
7
+ base\_utility & 16 & 0.6871 & 0.3147 & 1.1643 \\
8
+ branch\_car\_at\_8 & 16 & 0.2645 & 0.0149 & 0.6270 \\
9
+ hidden\_chart\_oracle\_success\_at\_8 & 16 & 0.6250 & 0.3750 & 0.8125 \\
10
+ hidden\_chart\_oracle\_utility\_at\_8 & 16 & 1.3220 & 0.8644 & 1.6665 \\
11
+ outcome\_ptr\_at\_8 & 16 & 0.3750 & 0.1250 & 0.6250 \\
12
+ pairwise\_causal\_calibration\_ece & 16 & 0.1330 & 0.0830 & 0.1813 \\
13
+ proposal\_oracle\_success\_at\_8 & 16 & 0.3750 & 0.1250 & 0.6250 \\
14
+ proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 16 & 0.0625 & -0.1250 & 0.2500 \\
15
+ proposal\_oracle\_utility\_at\_8 & 16 & 0.8081 & 0.3368 & 1.2765 \\
16
+ proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 16 & 0.1210 & -0.2626 & 0.4792 \\
17
+ selected\_success\_at\_8 & 16 & 0.2500 & 0.0625 & 0.5000 \\
18
+ selected\_success\_gain\_over\_base\_at\_8 & 16 & -0.0625 & -0.3125 & 0.1250 \\
19
+ selected\_utility\_at\_8 & 16 & 0.5436 & 0.1771 & 1.0157 \\
20
+ selected\_utility\_gain\_over\_base\_at\_8 & 16 & -0.1435 & -0.6171 & 0.2281 \\
21
+ selector\_regret\_at\_8 & 16 & 0.2645 & 0.0149 & 0.6270 \\
22
+ success\_selector\_gap\_at\_8 & 16 & 0.1250 & 0.0000 & 0.3125 \\
23
+ success\_support\_gap\_at\_8 & 16 & 0.2500 & 0.0625 & 0.5000 \\
24
+ success\_total\_car\_to\_hidden\_at\_8 & 16 & 0.3750 & 0.1250 & 0.6250 \\
25
+ support\_gap\_at\_8 & 16 & 0.5139 & 0.1583 & 0.9626 \\
26
+ total\_car\_to\_hidden\_at\_8 & 16 & 0.7784 & 0.3386 & 1.2434 \\
27
+ \bottomrule
28
+ \end{tabular}
workspace/runs/ctt_residual_rollout_val16_seed0/train.log ADDED
@@ -0,0 +1 @@
 
 
1
+ CTT checkpoint trained separately; rollout used checkpoint=runs/ctt_residual_full_seed0/model.pt
workspace/runs/ctt_residual_rollout_val69_seed0/data_hash.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ fa0fe0e3881ee70ce91492445871614d01b3a47fc7c576659c53ff17ddee4e42
workspace/runs/ctt_residual_rollout_val69_seed0/eval.log ADDED
@@ -0,0 +1,668 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ start
2
+ importing gymnasium/mani_skill
3
+ imported gymnasium/mani_skill
4
+ loaded checkpoint=runs/ctt_residual_full_seed0/model.pt
5
+ loaded charts source=301 target=69
6
+ generated proposals rows=69 total=552
7
+ env init sequential task=LiftPegUpright-v1 batch=4 sim=physx_cpu render=cpu
8
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=0 shape=(1, 1, 16, 7)
9
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=1 shape=(1, 1, 16, 7)
10
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=2 shape=(1, 1, 16, 7)
11
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=3 shape=(1, 1, 16, 7)
12
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=4 shape=(1, 1, 16, 7)
13
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=5 shape=(1, 1, 16, 7)
14
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=6 shape=(1, 1, 16, 7)
15
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=7 shape=(1, 1, 16, 7)
16
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=8 shape=(1, 1, 16, 7)
17
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=0 shape=(1, 1, 16, 7)
18
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=1 shape=(1, 1, 16, 7)
19
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=2 shape=(1, 1, 16, 7)
20
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=3 shape=(1, 1, 16, 7)
21
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=4 shape=(1, 1, 16, 7)
22
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=5 shape=(1, 1, 16, 7)
23
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=6 shape=(1, 1, 16, 7)
24
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=7 shape=(1, 1, 16, 7)
25
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=8 shape=(1, 1, 16, 7)
26
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=0 shape=(1, 1, 16, 7)
27
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=1 shape=(1, 1, 16, 7)
28
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=2 shape=(1, 1, 16, 7)
29
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=3 shape=(1, 1, 16, 7)
30
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=4 shape=(1, 1, 16, 7)
31
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=5 shape=(1, 1, 16, 7)
32
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=6 shape=(1, 1, 16, 7)
33
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=7 shape=(1, 1, 16, 7)
34
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=8 shape=(1, 1, 16, 7)
35
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=0 shape=(1, 1, 16, 7)
36
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=1 shape=(1, 1, 16, 7)
37
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=2 shape=(1, 1, 16, 7)
38
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=3 shape=(1, 1, 16, 7)
39
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=4 shape=(1, 1, 16, 7)
40
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=5 shape=(1, 1, 16, 7)
41
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=6 shape=(1, 1, 16, 7)
42
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=7 shape=(1, 1, 16, 7)
43
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=8 shape=(1, 1, 16, 7)
44
+ batch done sequential task=LiftPegUpright-v1
45
+ env init sequential task=LiftPegUpright-v1 batch=2 sim=physx_cpu render=cpu
46
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=0 shape=(1, 1, 16, 7)
47
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=1 shape=(1, 1, 16, 7)
48
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=2 shape=(1, 1, 16, 7)
49
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=3 shape=(1, 1, 16, 7)
50
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=4 shape=(1, 1, 16, 7)
51
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=5 shape=(1, 1, 16, 7)
52
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=6 shape=(1, 1, 16, 7)
53
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=7 shape=(1, 1, 16, 7)
54
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=8 shape=(1, 1, 16, 7)
55
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=0 shape=(1, 1, 16, 7)
56
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=1 shape=(1, 1, 16, 7)
57
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=2 shape=(1, 1, 16, 7)
58
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=3 shape=(1, 1, 16, 7)
59
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=4 shape=(1, 1, 16, 7)
60
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=5 shape=(1, 1, 16, 7)
61
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=6 shape=(1, 1, 16, 7)
62
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=7 shape=(1, 1, 16, 7)
63
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=8 shape=(1, 1, 16, 7)
64
+ batch done sequential task=LiftPegUpright-v1
65
+ env init sequential task=PickCube-v1 batch=1 sim=physx_cpu render=cpu
66
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=0 shape=(1, 1, 16, 7)
67
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=1 shape=(1, 1, 16, 7)
68
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=2 shape=(1, 1, 16, 7)
69
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=3 shape=(1, 1, 16, 7)
70
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=4 shape=(1, 1, 16, 7)
71
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=5 shape=(1, 1, 16, 7)
72
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=6 shape=(1, 1, 16, 7)
73
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=7 shape=(1, 1, 16, 7)
74
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=8 shape=(1, 1, 16, 7)
75
+ batch done sequential task=PickCube-v1
76
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
77
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=0 shape=(1, 1, 16, 7)
78
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=1 shape=(1, 1, 16, 7)
79
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=2 shape=(1, 1, 16, 7)
80
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=3 shape=(1, 1, 16, 7)
81
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=4 shape=(1, 1, 16, 7)
82
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=5 shape=(1, 1, 16, 7)
83
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=6 shape=(1, 1, 16, 7)
84
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=7 shape=(1, 1, 16, 7)
85
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=8 shape=(1, 1, 16, 7)
86
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=0 shape=(1, 1, 16, 7)
87
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=1 shape=(1, 1, 16, 7)
88
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=2 shape=(1, 1, 16, 7)
89
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=3 shape=(1, 1, 16, 7)
90
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=4 shape=(1, 1, 16, 7)
91
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=5 shape=(1, 1, 16, 7)
92
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=6 shape=(1, 1, 16, 7)
93
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=7 shape=(1, 1, 16, 7)
94
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=8 shape=(1, 1, 16, 7)
95
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=0 shape=(1, 1, 16, 7)
96
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=1 shape=(1, 1, 16, 7)
97
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=2 shape=(1, 1, 16, 7)
98
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=3 shape=(1, 1, 16, 7)
99
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=4 shape=(1, 1, 16, 7)
100
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=5 shape=(1, 1, 16, 7)
101
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=6 shape=(1, 1, 16, 7)
102
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=7 shape=(1, 1, 16, 7)
103
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=8 shape=(1, 1, 16, 7)
104
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=0 shape=(1, 1, 16, 7)
105
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=1 shape=(1, 1, 16, 7)
106
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=2 shape=(1, 1, 16, 7)
107
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=3 shape=(1, 1, 16, 7)
108
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=4 shape=(1, 1, 16, 7)
109
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=5 shape=(1, 1, 16, 7)
110
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=6 shape=(1, 1, 16, 7)
111
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=7 shape=(1, 1, 16, 7)
112
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=8 shape=(1, 1, 16, 7)
113
+ batch done sequential task=PullCube-v1
114
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
115
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=0 shape=(1, 1, 16, 7)
116
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=1 shape=(1, 1, 16, 7)
117
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=2 shape=(1, 1, 16, 7)
118
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=3 shape=(1, 1, 16, 7)
119
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=4 shape=(1, 1, 16, 7)
120
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=5 shape=(1, 1, 16, 7)
121
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=6 shape=(1, 1, 16, 7)
122
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=7 shape=(1, 1, 16, 7)
123
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=8 shape=(1, 1, 16, 7)
124
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=0 shape=(1, 1, 16, 7)
125
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=1 shape=(1, 1, 16, 7)
126
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=2 shape=(1, 1, 16, 7)
127
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=3 shape=(1, 1, 16, 7)
128
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=4 shape=(1, 1, 16, 7)
129
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=5 shape=(1, 1, 16, 7)
130
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=6 shape=(1, 1, 16, 7)
131
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=7 shape=(1, 1, 16, 7)
132
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=8 shape=(1, 1, 16, 7)
133
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=0 shape=(1, 1, 16, 7)
134
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=1 shape=(1, 1, 16, 7)
135
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=2 shape=(1, 1, 16, 7)
136
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=3 shape=(1, 1, 16, 7)
137
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=4 shape=(1, 1, 16, 7)
138
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=5 shape=(1, 1, 16, 7)
139
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=6 shape=(1, 1, 16, 7)
140
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=7 shape=(1, 1, 16, 7)
141
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=8 shape=(1, 1, 16, 7)
142
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=0 shape=(1, 1, 16, 7)
143
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=1 shape=(1, 1, 16, 7)
144
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=2 shape=(1, 1, 16, 7)
145
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=3 shape=(1, 1, 16, 7)
146
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=4 shape=(1, 1, 16, 7)
147
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=5 shape=(1, 1, 16, 7)
148
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=6 shape=(1, 1, 16, 7)
149
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=7 shape=(1, 1, 16, 7)
150
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=8 shape=(1, 1, 16, 7)
151
+ batch done sequential task=PullCube-v1
152
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
153
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=0 shape=(1, 1, 16, 7)
154
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=1 shape=(1, 1, 16, 7)
155
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=2 shape=(1, 1, 16, 7)
156
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=3 shape=(1, 1, 16, 7)
157
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=4 shape=(1, 1, 16, 7)
158
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=5 shape=(1, 1, 16, 7)
159
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=6 shape=(1, 1, 16, 7)
160
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=7 shape=(1, 1, 16, 7)
161
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=8 shape=(1, 1, 16, 7)
162
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=0 shape=(1, 1, 16, 7)
163
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=1 shape=(1, 1, 16, 7)
164
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=2 shape=(1, 1, 16, 7)
165
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=3 shape=(1, 1, 16, 7)
166
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=4 shape=(1, 1, 16, 7)
167
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=5 shape=(1, 1, 16, 7)
168
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=6 shape=(1, 1, 16, 7)
169
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=7 shape=(1, 1, 16, 7)
170
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=8 shape=(1, 1, 16, 7)
171
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=0 shape=(1, 1, 16, 7)
172
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=1 shape=(1, 1, 16, 7)
173
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=2 shape=(1, 1, 16, 7)
174
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=3 shape=(1, 1, 16, 7)
175
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=4 shape=(1, 1, 16, 7)
176
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=5 shape=(1, 1, 16, 7)
177
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=6 shape=(1, 1, 16, 7)
178
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=7 shape=(1, 1, 16, 7)
179
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=8 shape=(1, 1, 16, 7)
180
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=0 shape=(1, 1, 16, 7)
181
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=1 shape=(1, 1, 16, 7)
182
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=2 shape=(1, 1, 16, 7)
183
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=3 shape=(1, 1, 16, 7)
184
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=4 shape=(1, 1, 16, 7)
185
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=5 shape=(1, 1, 16, 7)
186
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=6 shape=(1, 1, 16, 7)
187
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=7 shape=(1, 1, 16, 7)
188
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=8 shape=(1, 1, 16, 7)
189
+ batch done sequential task=PullCube-v1
190
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
191
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=0 shape=(1, 1, 16, 7)
192
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=1 shape=(1, 1, 16, 7)
193
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=2 shape=(1, 1, 16, 7)
194
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=3 shape=(1, 1, 16, 7)
195
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=4 shape=(1, 1, 16, 7)
196
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=5 shape=(1, 1, 16, 7)
197
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=6 shape=(1, 1, 16, 7)
198
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=7 shape=(1, 1, 16, 7)
199
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=8 shape=(1, 1, 16, 7)
200
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=0 shape=(1, 1, 16, 7)
201
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=1 shape=(1, 1, 16, 7)
202
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=2 shape=(1, 1, 16, 7)
203
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=3 shape=(1, 1, 16, 7)
204
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=4 shape=(1, 1, 16, 7)
205
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=5 shape=(1, 1, 16, 7)
206
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=6 shape=(1, 1, 16, 7)
207
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=7 shape=(1, 1, 16, 7)
208
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=8 shape=(1, 1, 16, 7)
209
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=0 shape=(1, 1, 16, 7)
210
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=1 shape=(1, 1, 16, 7)
211
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=2 shape=(1, 1, 16, 7)
212
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=3 shape=(1, 1, 16, 7)
213
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=4 shape=(1, 1, 16, 7)
214
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=5 shape=(1, 1, 16, 7)
215
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=6 shape=(1, 1, 16, 7)
216
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=7 shape=(1, 1, 16, 7)
217
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=8 shape=(1, 1, 16, 7)
218
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=0 shape=(1, 1, 16, 7)
219
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=1 shape=(1, 1, 16, 7)
220
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=2 shape=(1, 1, 16, 7)
221
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=3 shape=(1, 1, 16, 7)
222
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=4 shape=(1, 1, 16, 7)
223
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=5 shape=(1, 1, 16, 7)
224
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=6 shape=(1, 1, 16, 7)
225
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=7 shape=(1, 1, 16, 7)
226
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=8 shape=(1, 1, 16, 7)
227
+ batch done sequential task=PullCube-v1
228
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
229
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=0 shape=(1, 1, 16, 7)
230
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=1 shape=(1, 1, 16, 7)
231
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=2 shape=(1, 1, 16, 7)
232
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=3 shape=(1, 1, 16, 7)
233
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=4 shape=(1, 1, 16, 7)
234
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=5 shape=(1, 1, 16, 7)
235
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=6 shape=(1, 1, 16, 7)
236
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=7 shape=(1, 1, 16, 7)
237
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=8 shape=(1, 1, 16, 7)
238
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=0 shape=(1, 1, 16, 7)
239
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=1 shape=(1, 1, 16, 7)
240
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=2 shape=(1, 1, 16, 7)
241
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=3 shape=(1, 1, 16, 7)
242
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=4 shape=(1, 1, 16, 7)
243
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=5 shape=(1, 1, 16, 7)
244
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=6 shape=(1, 1, 16, 7)
245
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=7 shape=(1, 1, 16, 7)
246
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=8 shape=(1, 1, 16, 7)
247
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=0 shape=(1, 1, 16, 7)
248
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=1 shape=(1, 1, 16, 7)
249
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=2 shape=(1, 1, 16, 7)
250
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=3 shape=(1, 1, 16, 7)
251
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=4 shape=(1, 1, 16, 7)
252
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=5 shape=(1, 1, 16, 7)
253
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=6 shape=(1, 1, 16, 7)
254
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=7 shape=(1, 1, 16, 7)
255
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=8 shape=(1, 1, 16, 7)
256
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=0 shape=(1, 1, 16, 7)
257
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=1 shape=(1, 1, 16, 7)
258
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=2 shape=(1, 1, 16, 7)
259
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=3 shape=(1, 1, 16, 7)
260
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=4 shape=(1, 1, 16, 7)
261
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=5 shape=(1, 1, 16, 7)
262
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=6 shape=(1, 1, 16, 7)
263
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=7 shape=(1, 1, 16, 7)
264
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=8 shape=(1, 1, 16, 7)
265
+ batch done sequential task=PullCube-v1
266
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
267
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=0 shape=(1, 1, 16, 7)
268
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=1 shape=(1, 1, 16, 7)
269
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=2 shape=(1, 1, 16, 7)
270
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=3 shape=(1, 1, 16, 7)
271
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=4 shape=(1, 1, 16, 7)
272
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=5 shape=(1, 1, 16, 7)
273
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=6 shape=(1, 1, 16, 7)
274
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=7 shape=(1, 1, 16, 7)
275
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=8 shape=(1, 1, 16, 7)
276
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=0 shape=(1, 1, 16, 7)
277
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=1 shape=(1, 1, 16, 7)
278
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=2 shape=(1, 1, 16, 7)
279
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=3 shape=(1, 1, 16, 7)
280
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=4 shape=(1, 1, 16, 7)
281
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=5 shape=(1, 1, 16, 7)
282
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=6 shape=(1, 1, 16, 7)
283
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=7 shape=(1, 1, 16, 7)
284
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=8 shape=(1, 1, 16, 7)
285
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=0 shape=(1, 1, 16, 7)
286
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=1 shape=(1, 1, 16, 7)
287
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=2 shape=(1, 1, 16, 7)
288
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=3 shape=(1, 1, 16, 7)
289
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=4 shape=(1, 1, 16, 7)
290
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=5 shape=(1, 1, 16, 7)
291
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=6 shape=(1, 1, 16, 7)
292
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=7 shape=(1, 1, 16, 7)
293
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=8 shape=(1, 1, 16, 7)
294
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=0 shape=(1, 1, 16, 7)
295
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=1 shape=(1, 1, 16, 7)
296
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=2 shape=(1, 1, 16, 7)
297
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=3 shape=(1, 1, 16, 7)
298
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=4 shape=(1, 1, 16, 7)
299
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=5 shape=(1, 1, 16, 7)
300
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=6 shape=(1, 1, 16, 7)
301
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=7 shape=(1, 1, 16, 7)
302
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=8 shape=(1, 1, 16, 7)
303
+ batch done sequential task=PullCube-v1
304
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
305
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=0 shape=(1, 1, 16, 7)
306
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=1 shape=(1, 1, 16, 7)
307
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=2 shape=(1, 1, 16, 7)
308
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=3 shape=(1, 1, 16, 7)
309
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=4 shape=(1, 1, 16, 7)
310
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=5 shape=(1, 1, 16, 7)
311
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=6 shape=(1, 1, 16, 7)
312
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=7 shape=(1, 1, 16, 7)
313
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=8 shape=(1, 1, 16, 7)
314
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=0 shape=(1, 1, 16, 7)
315
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=1 shape=(1, 1, 16, 7)
316
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=2 shape=(1, 1, 16, 7)
317
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=3 shape=(1, 1, 16, 7)
318
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=4 shape=(1, 1, 16, 7)
319
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=5 shape=(1, 1, 16, 7)
320
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=6 shape=(1, 1, 16, 7)
321
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=7 shape=(1, 1, 16, 7)
322
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=8 shape=(1, 1, 16, 7)
323
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=0 shape=(1, 1, 16, 7)
324
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=1 shape=(1, 1, 16, 7)
325
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=2 shape=(1, 1, 16, 7)
326
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=3 shape=(1, 1, 16, 7)
327
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=4 shape=(1, 1, 16, 7)
328
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=5 shape=(1, 1, 16, 7)
329
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=6 shape=(1, 1, 16, 7)
330
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=7 shape=(1, 1, 16, 7)
331
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=8 shape=(1, 1, 16, 7)
332
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=0 shape=(1, 1, 16, 7)
333
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=1 shape=(1, 1, 16, 7)
334
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=2 shape=(1, 1, 16, 7)
335
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=3 shape=(1, 1, 16, 7)
336
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=4 shape=(1, 1, 16, 7)
337
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=5 shape=(1, 1, 16, 7)
338
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=6 shape=(1, 1, 16, 7)
339
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=7 shape=(1, 1, 16, 7)
340
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=8 shape=(1, 1, 16, 7)
341
+ batch done sequential task=PullCube-v1
342
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
343
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=0 shape=(1, 1, 16, 7)
344
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=1 shape=(1, 1, 16, 7)
345
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=2 shape=(1, 1, 16, 7)
346
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=3 shape=(1, 1, 16, 7)
347
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=4 shape=(1, 1, 16, 7)
348
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=5 shape=(1, 1, 16, 7)
349
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=6 shape=(1, 1, 16, 7)
350
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=7 shape=(1, 1, 16, 7)
351
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=8 shape=(1, 1, 16, 7)
352
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=0 shape=(1, 1, 16, 7)
353
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=1 shape=(1, 1, 16, 7)
354
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=2 shape=(1, 1, 16, 7)
355
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=3 shape=(1, 1, 16, 7)
356
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=4 shape=(1, 1, 16, 7)
357
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=5 shape=(1, 1, 16, 7)
358
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=6 shape=(1, 1, 16, 7)
359
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=7 shape=(1, 1, 16, 7)
360
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=8 shape=(1, 1, 16, 7)
361
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=0 shape=(1, 1, 16, 7)
362
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=1 shape=(1, 1, 16, 7)
363
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=2 shape=(1, 1, 16, 7)
364
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=3 shape=(1, 1, 16, 7)
365
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=4 shape=(1, 1, 16, 7)
366
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=5 shape=(1, 1, 16, 7)
367
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=6 shape=(1, 1, 16, 7)
368
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=7 shape=(1, 1, 16, 7)
369
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=8 shape=(1, 1, 16, 7)
370
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=0 shape=(1, 1, 16, 7)
371
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=1 shape=(1, 1, 16, 7)
372
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=2 shape=(1, 1, 16, 7)
373
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=3 shape=(1, 1, 16, 7)
374
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=4 shape=(1, 1, 16, 7)
375
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=5 shape=(1, 1, 16, 7)
376
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=6 shape=(1, 1, 16, 7)
377
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=7 shape=(1, 1, 16, 7)
378
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=8 shape=(1, 1, 16, 7)
379
+ batch done sequential task=PullCube-v1
380
+ env init sequential task=PullCube-v1 batch=2 sim=physx_cpu render=cpu
381
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=0 shape=(1, 1, 16, 7)
382
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=1 shape=(1, 1, 16, 7)
383
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=2 shape=(1, 1, 16, 7)
384
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=3 shape=(1, 1, 16, 7)
385
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=4 shape=(1, 1, 16, 7)
386
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=5 shape=(1, 1, 16, 7)
387
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=6 shape=(1, 1, 16, 7)
388
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=7 shape=(1, 1, 16, 7)
389
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=8 shape=(1, 1, 16, 7)
390
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=0 shape=(1, 1, 16, 7)
391
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=1 shape=(1, 1, 16, 7)
392
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=2 shape=(1, 1, 16, 7)
393
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=3 shape=(1, 1, 16, 7)
394
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=4 shape=(1, 1, 16, 7)
395
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=5 shape=(1, 1, 16, 7)
396
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=6 shape=(1, 1, 16, 7)
397
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=7 shape=(1, 1, 16, 7)
398
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=8 shape=(1, 1, 16, 7)
399
+ batch done sequential task=PullCube-v1
400
+ env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu
401
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=0 shape=(1, 1, 16, 7)
402
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=1 shape=(1, 1, 16, 7)
403
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=2 shape=(1, 1, 16, 7)
404
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=3 shape=(1, 1, 16, 7)
405
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=4 shape=(1, 1, 16, 7)
406
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=5 shape=(1, 1, 16, 7)
407
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=6 shape=(1, 1, 16, 7)
408
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=7 shape=(1, 1, 16, 7)
409
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=8 shape=(1, 1, 16, 7)
410
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=0 shape=(1, 1, 16, 7)
411
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=1 shape=(1, 1, 16, 7)
412
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=2 shape=(1, 1, 16, 7)
413
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=3 shape=(1, 1, 16, 7)
414
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=4 shape=(1, 1, 16, 7)
415
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=5 shape=(1, 1, 16, 7)
416
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=6 shape=(1, 1, 16, 7)
417
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=7 shape=(1, 1, 16, 7)
418
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=8 shape=(1, 1, 16, 7)
419
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=0 shape=(1, 1, 16, 7)
420
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=1 shape=(1, 1, 16, 7)
421
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=2 shape=(1, 1, 16, 7)
422
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=3 shape=(1, 1, 16, 7)
423
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=4 shape=(1, 1, 16, 7)
424
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=5 shape=(1, 1, 16, 7)
425
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=6 shape=(1, 1, 16, 7)
426
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=7 shape=(1, 1, 16, 7)
427
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=8 shape=(1, 1, 16, 7)
428
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=0 shape=(1, 1, 16, 7)
429
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=1 shape=(1, 1, 16, 7)
430
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=2 shape=(1, 1, 16, 7)
431
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=3 shape=(1, 1, 16, 7)
432
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=4 shape=(1, 1, 16, 7)
433
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=5 shape=(1, 1, 16, 7)
434
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=6 shape=(1, 1, 16, 7)
435
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=7 shape=(1, 1, 16, 7)
436
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=8 shape=(1, 1, 16, 7)
437
+ batch done sequential task=PushCube-v1
438
+ env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu
439
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=0 shape=(1, 1, 16, 7)
440
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=1 shape=(1, 1, 16, 7)
441
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=2 shape=(1, 1, 16, 7)
442
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=3 shape=(1, 1, 16, 7)
443
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=4 shape=(1, 1, 16, 7)
444
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=5 shape=(1, 1, 16, 7)
445
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=6 shape=(1, 1, 16, 7)
446
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=7 shape=(1, 1, 16, 7)
447
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=8 shape=(1, 1, 16, 7)
448
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=0 shape=(1, 1, 16, 7)
449
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=1 shape=(1, 1, 16, 7)
450
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=2 shape=(1, 1, 16, 7)
451
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=3 shape=(1, 1, 16, 7)
452
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=4 shape=(1, 1, 16, 7)
453
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=5 shape=(1, 1, 16, 7)
454
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=6 shape=(1, 1, 16, 7)
455
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=7 shape=(1, 1, 16, 7)
456
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=8 shape=(1, 1, 16, 7)
457
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=0 shape=(1, 1, 16, 7)
458
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=1 shape=(1, 1, 16, 7)
459
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=2 shape=(1, 1, 16, 7)
460
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=3 shape=(1, 1, 16, 7)
461
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=4 shape=(1, 1, 16, 7)
462
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=5 shape=(1, 1, 16, 7)
463
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=6 shape=(1, 1, 16, 7)
464
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=7 shape=(1, 1, 16, 7)
465
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=8 shape=(1, 1, 16, 7)
466
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=0 shape=(1, 1, 16, 7)
467
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=1 shape=(1, 1, 16, 7)
468
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=2 shape=(1, 1, 16, 7)
469
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=3 shape=(1, 1, 16, 7)
470
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=4 shape=(1, 1, 16, 7)
471
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=5 shape=(1, 1, 16, 7)
472
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=6 shape=(1, 1, 16, 7)
473
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=7 shape=(1, 1, 16, 7)
474
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=8 shape=(1, 1, 16, 7)
475
+ batch done sequential task=PushCube-v1
476
+ env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu
477
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=0 shape=(1, 1, 16, 7)
478
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=1 shape=(1, 1, 16, 7)
479
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=2 shape=(1, 1, 16, 7)
480
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=3 shape=(1, 1, 16, 7)
481
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=4 shape=(1, 1, 16, 7)
482
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=5 shape=(1, 1, 16, 7)
483
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=6 shape=(1, 1, 16, 7)
484
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=7 shape=(1, 1, 16, 7)
485
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=8 shape=(1, 1, 16, 7)
486
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=0 shape=(1, 1, 16, 7)
487
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=1 shape=(1, 1, 16, 7)
488
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=2 shape=(1, 1, 16, 7)
489
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=3 shape=(1, 1, 16, 7)
490
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=4 shape=(1, 1, 16, 7)
491
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=5 shape=(1, 1, 16, 7)
492
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=6 shape=(1, 1, 16, 7)
493
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=7 shape=(1, 1, 16, 7)
494
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=8 shape=(1, 1, 16, 7)
495
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=0 shape=(1, 1, 16, 7)
496
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=1 shape=(1, 1, 16, 7)
497
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=2 shape=(1, 1, 16, 7)
498
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=3 shape=(1, 1, 16, 7)
499
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=4 shape=(1, 1, 16, 7)
500
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=5 shape=(1, 1, 16, 7)
501
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=6 shape=(1, 1, 16, 7)
502
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=7 shape=(1, 1, 16, 7)
503
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=8 shape=(1, 1, 16, 7)
504
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=0 shape=(1, 1, 16, 7)
505
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=1 shape=(1, 1, 16, 7)
506
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=2 shape=(1, 1, 16, 7)
507
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=3 shape=(1, 1, 16, 7)
508
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=4 shape=(1, 1, 16, 7)
509
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=5 shape=(1, 1, 16, 7)
510
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=6 shape=(1, 1, 16, 7)
511
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=7 shape=(1, 1, 16, 7)
512
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=8 shape=(1, 1, 16, 7)
513
+ batch done sequential task=PushCube-v1
514
+ env init sequential task=PushCube-v1 batch=1 sim=physx_cpu render=cpu
515
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=0 shape=(1, 1, 16, 7)
516
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=1 shape=(1, 1, 16, 7)
517
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=2 shape=(1, 1, 16, 7)
518
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=3 shape=(1, 1, 16, 7)
519
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=4 shape=(1, 1, 16, 7)
520
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=5 shape=(1, 1, 16, 7)
521
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=6 shape=(1, 1, 16, 7)
522
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=7 shape=(1, 1, 16, 7)
523
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=8 shape=(1, 1, 16, 7)
524
+ batch done sequential task=PushCube-v1
525
+ env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu
526
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=0 shape=(1, 1, 16, 7)
527
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=1 shape=(1, 1, 16, 7)
528
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=2 shape=(1, 1, 16, 7)
529
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=3 shape=(1, 1, 16, 7)
530
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=4 shape=(1, 1, 16, 7)
531
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=5 shape=(1, 1, 16, 7)
532
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=6 shape=(1, 1, 16, 7)
533
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=7 shape=(1, 1, 16, 7)
534
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=8 shape=(1, 1, 16, 7)
535
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=0 shape=(1, 1, 16, 7)
536
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=1 shape=(1, 1, 16, 7)
537
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=2 shape=(1, 1, 16, 7)
538
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=3 shape=(1, 1, 16, 7)
539
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=4 shape=(1, 1, 16, 7)
540
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=5 shape=(1, 1, 16, 7)
541
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=6 shape=(1, 1, 16, 7)
542
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=7 shape=(1, 1, 16, 7)
543
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=8 shape=(1, 1, 16, 7)
544
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=0 shape=(1, 1, 16, 7)
545
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=1 shape=(1, 1, 16, 7)
546
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=2 shape=(1, 1, 16, 7)
547
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=3 shape=(1, 1, 16, 7)
548
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=4 shape=(1, 1, 16, 7)
549
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=5 shape=(1, 1, 16, 7)
550
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=6 shape=(1, 1, 16, 7)
551
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=7 shape=(1, 1, 16, 7)
552
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=8 shape=(1, 1, 16, 7)
553
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=0 shape=(1, 1, 16, 7)
554
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=1 shape=(1, 1, 16, 7)
555
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=2 shape=(1, 1, 16, 7)
556
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=3 shape=(1, 1, 16, 7)
557
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=4 shape=(1, 1, 16, 7)
558
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=5 shape=(1, 1, 16, 7)
559
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=6 shape=(1, 1, 16, 7)
560
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=7 shape=(1, 1, 16, 7)
561
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=8 shape=(1, 1, 16, 7)
562
+ batch done sequential task=StackCube-v1
563
+ env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu
564
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=0 shape=(1, 1, 16, 7)
565
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=1 shape=(1, 1, 16, 7)
566
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=2 shape=(1, 1, 16, 7)
567
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=3 shape=(1, 1, 16, 7)
568
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=4 shape=(1, 1, 16, 7)
569
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=5 shape=(1, 1, 16, 7)
570
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=6 shape=(1, 1, 16, 7)
571
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=7 shape=(1, 1, 16, 7)
572
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=8 shape=(1, 1, 16, 7)
573
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=0 shape=(1, 1, 16, 7)
574
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=1 shape=(1, 1, 16, 7)
575
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=2 shape=(1, 1, 16, 7)
576
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=3 shape=(1, 1, 16, 7)
577
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=4 shape=(1, 1, 16, 7)
578
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=5 shape=(1, 1, 16, 7)
579
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=6 shape=(1, 1, 16, 7)
580
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=7 shape=(1, 1, 16, 7)
581
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=8 shape=(1, 1, 16, 7)
582
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=0 shape=(1, 1, 16, 7)
583
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=1 shape=(1, 1, 16, 7)
584
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=2 shape=(1, 1, 16, 7)
585
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=3 shape=(1, 1, 16, 7)
586
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=4 shape=(1, 1, 16, 7)
587
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=5 shape=(1, 1, 16, 7)
588
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=6 shape=(1, 1, 16, 7)
589
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=7 shape=(1, 1, 16, 7)
590
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=8 shape=(1, 1, 16, 7)
591
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=0 shape=(1, 1, 16, 7)
592
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=1 shape=(1, 1, 16, 7)
593
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=2 shape=(1, 1, 16, 7)
594
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=3 shape=(1, 1, 16, 7)
595
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=4 shape=(1, 1, 16, 7)
596
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=5 shape=(1, 1, 16, 7)
597
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=6 shape=(1, 1, 16, 7)
598
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=7 shape=(1, 1, 16, 7)
599
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=8 shape=(1, 1, 16, 7)
600
+ batch done sequential task=StackCube-v1
601
+ env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu
602
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=0 shape=(1, 1, 16, 7)
603
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=1 shape=(1, 1, 16, 7)
604
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=2 shape=(1, 1, 16, 7)
605
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=3 shape=(1, 1, 16, 7)
606
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=4 shape=(1, 1, 16, 7)
607
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=5 shape=(1, 1, 16, 7)
608
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=6 shape=(1, 1, 16, 7)
609
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=7 shape=(1, 1, 16, 7)
610
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=8 shape=(1, 1, 16, 7)
611
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=0 shape=(1, 1, 16, 7)
612
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=1 shape=(1, 1, 16, 7)
613
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=2 shape=(1, 1, 16, 7)
614
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=3 shape=(1, 1, 16, 7)
615
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=4 shape=(1, 1, 16, 7)
616
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=5 shape=(1, 1, 16, 7)
617
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=6 shape=(1, 1, 16, 7)
618
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=7 shape=(1, 1, 16, 7)
619
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=8 shape=(1, 1, 16, 7)
620
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=0 shape=(1, 1, 16, 7)
621
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=1 shape=(1, 1, 16, 7)
622
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=2 shape=(1, 1, 16, 7)
623
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=3 shape=(1, 1, 16, 7)
624
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=4 shape=(1, 1, 16, 7)
625
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=5 shape=(1, 1, 16, 7)
626
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=6 shape=(1, 1, 16, 7)
627
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=7 shape=(1, 1, 16, 7)
628
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=8 shape=(1, 1, 16, 7)
629
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=0 shape=(1, 1, 16, 7)
630
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=1 shape=(1, 1, 16, 7)
631
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=2 shape=(1, 1, 16, 7)
632
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=3 shape=(1, 1, 16, 7)
633
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=4 shape=(1, 1, 16, 7)
634
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=5 shape=(1, 1, 16, 7)
635
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=6 shape=(1, 1, 16, 7)
636
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=7 shape=(1, 1, 16, 7)
637
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=8 shape=(1, 1, 16, 7)
638
+ batch done sequential task=StackCube-v1
639
+ env init sequential task=StackCube-v1 batch=3 sim=physx_cpu render=cpu
640
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=0 shape=(1, 1, 16, 7)
641
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=1 shape=(1, 1, 16, 7)
642
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=2 shape=(1, 1, 16, 7)
643
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=3 shape=(1, 1, 16, 7)
644
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=4 shape=(1, 1, 16, 7)
645
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=5 shape=(1, 1, 16, 7)
646
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=6 shape=(1, 1, 16, 7)
647
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=7 shape=(1, 1, 16, 7)
648
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=8 shape=(1, 1, 16, 7)
649
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=0 shape=(1, 1, 16, 7)
650
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=1 shape=(1, 1, 16, 7)
651
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=2 shape=(1, 1, 16, 7)
652
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=3 shape=(1, 1, 16, 7)
653
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=4 shape=(1, 1, 16, 7)
654
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=5 shape=(1, 1, 16, 7)
655
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=6 shape=(1, 1, 16, 7)
656
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=7 shape=(1, 1, 16, 7)
657
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=8 shape=(1, 1, 16, 7)
658
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=0 shape=(1, 1, 16, 7)
659
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=1 shape=(1, 1, 16, 7)
660
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=2 shape=(1, 1, 16, 7)
661
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=3 shape=(1, 1, 16, 7)
662
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=4 shape=(1, 1, 16, 7)
663
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=5 shape=(1, 1, 16, 7)
664
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=6 shape=(1, 1, 16, 7)
665
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=7 shape=(1, 1, 16, 7)
666
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=8 shape=(1, 1, 16, 7)
667
+ batch done sequential task=StackCube-v1
668
+ rollout complete rows=69
workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics.json CHANGED
The diff for this file is too large to render. See raw diff
 
workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics_by_seed.json CHANGED
@@ -1,331 +1,1083 @@
1
  {
2
  "1025740212": {
 
 
3
  "branch_car_at_8": 0.26188503205776215,
 
 
4
  "outcome_ptr_at_8": 0.0,
5
  "pairwise_causal_calibration_ece": 0.08963387331564551,
 
 
 
 
 
 
 
 
6
  "selector_regret_at_8": 0.26188503205776215,
7
- "support_gap_at_8": 1.5238737165927887
 
 
 
 
8
  },
9
  "1055157354": {
 
 
10
  "branch_car_at_8": 1.8516457676887512,
 
 
11
  "outcome_ptr_at_8": 1.0,
12
  "pairwise_causal_calibration_ece": 0.2973966626262834,
 
 
 
 
 
 
 
 
13
  "selector_regret_at_8": 1.8516457676887512,
14
- "support_gap_at_8": 0.0
 
 
 
 
15
  },
16
  "1055567618": {
 
 
17
  "branch_car_at_8": 1.183156132698059,
 
 
18
  "outcome_ptr_at_8": 0.0,
19
  "pairwise_causal_calibration_ece": 0.07486610854656817,
 
 
 
 
 
 
 
 
20
  "selector_regret_at_8": 1.183156132698059,
21
- "support_gap_at_8": 0.0
 
 
 
 
22
  },
23
  "1113268940": {
 
 
24
  "branch_car_at_8": 0.38470593094825745,
 
 
25
  "outcome_ptr_at_8": 1.0,
26
  "pairwise_causal_calibration_ece": 0.1616344352768096,
 
 
 
 
 
 
 
 
27
  "selector_regret_at_8": 0.38470593094825745,
28
- "support_gap_at_8": 1.3353878855705261
 
 
 
 
29
  },
30
  "1113371261": {
 
 
31
  "branch_car_at_8": 0.0,
 
 
32
  "outcome_ptr_at_8": 0.0,
33
  "pairwise_causal_calibration_ece": 0.3179396489681421,
 
 
 
 
 
 
 
 
34
  "selector_regret_at_8": 0.0,
35
- "support_gap_at_8": 0.0
 
 
 
 
36
  },
37
  "1176943004": {
 
 
38
  "branch_car_at_8": 0.0,
 
 
39
  "outcome_ptr_at_8": 1.0,
40
  "pairwise_causal_calibration_ece": 0.2176350935010736,
 
 
 
 
 
 
 
 
41
  "selector_regret_at_8": 0.0,
42
- "support_gap_at_8": 0.0
 
 
 
 
43
  },
44
  "1189392917": {
 
 
45
  "branch_car_at_8": 0.0,
 
 
46
  "outcome_ptr_at_8": 1.0,
47
  "pairwise_causal_calibration_ece": 0.1628408162009477,
 
 
 
 
 
 
 
 
48
  "selector_regret_at_8": 0.0,
49
- "support_gap_at_8": 0.0
 
 
 
 
50
  },
51
  "1231780997": {
 
 
52
  "branch_car_at_8": 0.4897426515817642,
 
 
53
  "outcome_ptr_at_8": 0.0,
54
  "pairwise_causal_calibration_ece": 0.18322944379343714,
 
 
 
 
 
 
 
 
55
  "selector_regret_at_8": 0.4897426515817642,
56
- "support_gap_at_8": 1.2812588810920715
 
 
 
 
57
  },
58
  "1271718007": {
 
 
59
  "branch_car_at_8": 0.0,
 
 
60
  "outcome_ptr_at_8": 0.5,
61
  "pairwise_causal_calibration_ece": 0.07543261691159965,
 
 
 
 
 
 
 
 
62
  "selector_regret_at_8": 0.0,
63
- "support_gap_at_8": 0.0
 
 
 
 
64
  },
65
  "1273247072": {
 
 
66
  "branch_car_at_8": 0.0,
 
 
67
  "outcome_ptr_at_8": 0.0,
68
  "pairwise_causal_calibration_ece": 0.15040783499409596,
 
 
 
 
 
 
 
 
69
  "selector_regret_at_8": 0.0,
70
- "support_gap_at_8": 1.8672154396772385
 
 
 
 
71
  },
72
  "1358421325": {
 
 
73
  "branch_car_at_8": 0.5775325745344162,
 
 
74
  "outcome_ptr_at_8": 0.0,
75
  "pairwise_causal_calibration_ece": 0.13502516765102715,
 
 
 
 
 
 
 
 
76
  "selector_regret_at_8": 0.5775325745344162,
77
- "support_gap_at_8": 1.1857540607452393
 
 
 
 
78
  },
79
  "1375492786": {
 
 
80
  "branch_car_at_8": 0.0002854267368093133,
 
 
81
  "outcome_ptr_at_8": 0.0,
82
  "pairwise_causal_calibration_ece": 0.19928218350064877,
 
 
 
 
 
 
 
 
83
  "selector_regret_at_8": 0.0002854267368093133,
84
- "support_gap_at_8": 0.24180691444780678
 
 
 
 
85
  },
86
  "141984615": {
 
 
87
  "branch_car_at_8": 0.03316789741317431,
 
 
88
  "outcome_ptr_at_8": 0.3333333333333333,
89
  "pairwise_causal_calibration_ece": 0.10926310602682258,
 
 
 
 
 
 
 
 
90
  "selector_regret_at_8": 0.03316789741317431,
91
- "support_gap_at_8": 0.12070147196451823
 
 
 
 
92
  },
93
  "1475440973": {
 
 
94
  "branch_car_at_8": 0.0,
 
 
95
  "outcome_ptr_at_8": 0.0,
96
  "pairwise_causal_calibration_ece": 0.09930428958964055,
 
 
 
 
 
 
 
 
97
  "selector_regret_at_8": 0.0,
98
- "support_gap_at_8": 0.0
 
 
 
 
99
  },
100
  "1483645336": {
 
 
101
  "branch_car_at_8": 0.0,
 
 
102
  "outcome_ptr_at_8": 1.0,
103
  "pairwise_causal_calibration_ece": 0.0,
 
 
 
 
 
 
 
 
104
  "selector_regret_at_8": 0.0,
105
- "support_gap_at_8": 0.0
 
 
 
 
106
  },
107
  "1522272686": {
 
 
108
  "branch_car_at_8": 0.009524375200271606,
 
 
109
  "outcome_ptr_at_8": 1.0,
110
  "pairwise_causal_calibration_ece": 0.08257819446483214,
 
 
 
 
 
 
 
 
111
  "selector_regret_at_8": 0.009524375200271606,
112
- "support_gap_at_8": 1.7564445286989212
 
 
 
 
113
  },
114
  "1528740417": {
 
 
115
  "branch_car_at_8": 0.0,
 
 
116
  "outcome_ptr_at_8": 0.0,
117
  "pairwise_causal_calibration_ece": 0.015990866806051107,
 
 
 
 
 
 
 
 
118
  "selector_regret_at_8": 0.0,
119
- "support_gap_at_8": 0.0
 
 
 
 
120
  },
121
  "1549324805": {
 
 
122
  "branch_car_at_8": 0.0034310482442378998,
 
 
123
  "outcome_ptr_at_8": 0.0,
124
  "pairwise_causal_calibration_ece": 0.19382954770507574,
 
 
 
 
 
 
 
 
125
  "selector_regret_at_8": 0.0034310482442378998,
126
- "support_gap_at_8": 0.05998400536676248
 
 
 
 
127
  },
128
  "1607773860": {
 
 
129
  "branch_car_at_8": 1.779732346534729,
 
 
130
  "outcome_ptr_at_8": 0.0,
131
  "pairwise_causal_calibration_ece": 0.08588083120555855,
 
 
 
 
 
 
 
 
132
  "selector_regret_at_8": 1.779732346534729,
133
- "support_gap_at_8": 0.0
 
 
 
 
134
  },
135
  "1665739511": {
 
 
136
  "branch_car_at_8": 0.007551252841949463,
 
 
137
  "outcome_ptr_at_8": 0.0,
138
  "pairwise_causal_calibration_ece": 0.050940808914706356,
 
 
 
 
 
 
 
 
139
  "selector_regret_at_8": 0.007551252841949463,
140
- "support_gap_at_8": 1.17837792634964
 
 
 
 
141
  },
142
  "1747178030": {
 
 
143
  "branch_car_at_8": 0.0,
 
 
144
  "outcome_ptr_at_8": 1.0,
145
  "pairwise_causal_calibration_ece": 0.15825371494561297,
 
 
 
 
 
 
 
 
146
  "selector_regret_at_8": 0.0,
147
- "support_gap_at_8": 0.0
 
 
 
 
148
  },
149
  "1881223807": {
 
 
150
  "branch_car_at_8": 0.020236089825630188,
 
 
151
  "outcome_ptr_at_8": 1.0,
152
  "pairwise_causal_calibration_ece": 0.019060791998604134,
 
 
 
 
 
 
 
 
153
  "selector_regret_at_8": 0.020236089825630188,
154
- "support_gap_at_8": 1.9486478716135025
 
 
 
 
155
  },
156
  "1919198188": {
 
 
157
  "branch_car_at_8": 0.016854479908943176,
 
 
158
  "outcome_ptr_at_8": 1.0,
159
  "pairwise_causal_calibration_ece": 0.15779226512372985,
 
 
 
 
 
 
 
 
160
  "selector_regret_at_8": 0.016854479908943176,
161
- "support_gap_at_8": 1.9100720435380936
 
 
 
 
162
  },
163
  "1991414747": {
 
 
164
  "branch_car_at_8": 0.0,
 
 
165
  "outcome_ptr_at_8": 0.0,
166
  "pairwise_causal_calibration_ece": 0.05287821364799471,
 
 
 
 
 
 
 
 
167
  "selector_regret_at_8": 0.0,
168
- "support_gap_at_8": 0.0
 
 
 
 
169
  },
170
  "1997090716": {
 
 
171
  "branch_car_at_8": 1.9691698104143143,
 
 
172
  "outcome_ptr_at_8": 1.0,
173
  "pairwise_causal_calibration_ece": 0.4044963838715546,
 
 
 
 
 
 
 
 
174
  "selector_regret_at_8": 1.9691698104143143,
175
- "support_gap_at_8": 0.0
 
 
 
 
176
  },
177
  "2015956957": {
 
 
178
  "branch_car_at_8": 0.01770364921540022,
 
 
179
  "outcome_ptr_at_8": 0.6,
180
  "pairwise_causal_calibration_ece": 0.18664994005668376,
 
 
 
 
 
 
 
 
181
  "selector_regret_at_8": 0.01770364921540022,
182
- "support_gap_at_8": 0.07863824404776096
 
 
 
 
183
  },
184
  "2054899126": {
 
 
185
  "branch_car_at_8": 0.0,
 
 
186
  "outcome_ptr_at_8": 0.0,
187
  "pairwise_causal_calibration_ece": 0.0,
 
 
 
 
 
 
 
 
188
  "selector_regret_at_8": 0.0,
189
- "support_gap_at_8": 0.0
 
 
 
 
190
  },
191
  "211245842": {
 
 
192
  "branch_car_at_8": 0.5908347219228745,
 
 
193
  "outcome_ptr_at_8": 0.0,
194
  "pairwise_causal_calibration_ece": 0.16771630653845432,
 
 
 
 
 
 
 
 
195
  "selector_regret_at_8": 0.5908347219228745,
196
- "support_gap_at_8": 1.1869857907295227
 
 
 
 
197
  },
198
  "2141802816": {
 
 
199
  "branch_car_at_8": 0.07543906569480896,
 
 
200
  "outcome_ptr_at_8": 1.0,
201
  "pairwise_causal_calibration_ece": 0.05242798758366053,
 
 
 
 
 
 
 
 
202
  "selector_regret_at_8": 0.07543906569480896,
203
- "support_gap_at_8": 1.8752704411745071
 
 
 
 
204
  },
205
  "256770874": {
 
 
206
  "branch_car_at_8": 0.10820980370044708,
 
 
207
  "outcome_ptr_at_8": 0.0,
208
  "pairwise_causal_calibration_ece": 0.030709639081801327,
 
 
 
 
 
 
 
 
209
  "selector_regret_at_8": 0.10820980370044708,
210
- "support_gap_at_8": 1.758925274014473
 
 
 
 
211
  },
212
  "278008524": {
 
 
213
  "branch_car_at_8": 0.0016456245793960989,
 
 
214
  "outcome_ptr_at_8": 0.0,
215
  "pairwise_causal_calibration_ece": 0.11925245416072927,
 
 
 
 
 
 
 
 
216
  "selector_regret_at_8": 0.0016456245793960989,
217
- "support_gap_at_8": 1.1023782094707713
 
 
 
 
218
  },
219
  "416183106": {
 
 
220
  "branch_car_at_8": 0.3360496461391449,
 
 
221
  "outcome_ptr_at_8": 1.0,
222
  "pairwise_causal_calibration_ece": 0.10129212810935397,
 
 
 
 
 
 
 
 
223
  "selector_regret_at_8": 0.3360496461391449,
224
- "support_gap_at_8": 1.4221857786178589
 
 
 
 
225
  },
226
  "534717141": {
 
 
227
  "branch_car_at_8": 1.9274959191679955,
 
 
228
  "outcome_ptr_at_8": 0.0,
229
  "pairwise_causal_calibration_ece": 0.6159370955924921,
 
 
 
 
 
 
 
 
230
  "selector_regret_at_8": 1.9274959191679955,
231
- "support_gap_at_8": 0.0
 
 
 
 
232
  },
233
  "551504635": {
 
 
234
  "branch_car_at_8": 0.033253490924835205,
 
 
235
  "outcome_ptr_at_8": 1.0,
236
  "pairwise_causal_calibration_ece": 0.10932818664818449,
 
 
 
 
 
 
 
 
237
  "selector_regret_at_8": 0.033253490924835205,
238
- "support_gap_at_8": 1.2749102115631104
 
 
 
 
239
  },
240
  "565369392": {
 
 
241
  "branch_car_at_8": 0.06443698455890019,
 
 
242
  "outcome_ptr_at_8": 0.0,
243
  "pairwise_causal_calibration_ece": 0.22308845281795123,
 
 
 
 
 
 
 
 
244
  "selector_regret_at_8": 0.06443698455890019,
245
- "support_gap_at_8": 0.6413601487874985
 
 
 
 
246
  },
247
  "565621826": {
 
 
248
  "branch_car_at_8": 0.0,
 
 
249
  "outcome_ptr_at_8": 0.0,
250
  "pairwise_causal_calibration_ece": 0.0,
 
 
 
 
 
 
 
 
251
  "selector_regret_at_8": 0.0,
252
- "support_gap_at_8": 0.0
 
 
 
 
253
  },
254
  "585904130": {
 
 
255
  "branch_car_at_8": 0.043168901621053614,
 
 
256
  "outcome_ptr_at_8": 0.3333333333333333,
257
  "pairwise_causal_calibration_ece": 0.10391809443539522,
 
 
 
 
 
 
 
 
258
  "selector_regret_at_8": 0.043168901621053614,
259
- "support_gap_at_8": 0.1816591378301382
 
 
 
 
260
  },
261
  "626486813": {
 
 
262
  "branch_car_at_8": 0.018570441869087517,
 
 
263
  "outcome_ptr_at_8": 0.75,
264
  "pairwise_causal_calibration_ece": 0.16162412565990822,
 
 
 
 
 
 
 
 
265
  "selector_regret_at_8": 0.018570441869087517,
266
- "support_gap_at_8": 0.0801479306537658
 
 
 
 
267
  },
268
  "636680137": {
 
 
269
  "branch_car_at_8": 0.0,
 
 
270
  "outcome_ptr_at_8": 0.0,
271
  "pairwise_causal_calibration_ece": 0.1699944666253151,
 
 
 
 
 
 
 
 
272
  "selector_regret_at_8": 0.0,
273
- "support_gap_at_8": 1.9875853462144732
 
 
 
 
274
  },
275
  "706479600": {
 
 
276
  "branch_car_at_8": 1.1865743398666382,
 
 
277
  "outcome_ptr_at_8": 1.0,
278
  "pairwise_causal_calibration_ece": 0.0922387871049479,
 
 
 
 
 
 
 
 
279
  "selector_regret_at_8": 1.1865743398666382,
280
- "support_gap_at_8": 0.0
 
 
 
 
281
  },
282
  "750448296": {
 
 
283
  "branch_car_at_8": 0.07732226233929396,
 
 
284
  "outcome_ptr_at_8": 0.5,
285
  "pairwise_causal_calibration_ece": 0.14360585133877676,
 
 
 
 
 
 
 
 
286
  "selector_regret_at_8": 0.07732226233929396,
287
- "support_gap_at_8": 0.12881441414356232
 
 
 
 
288
  },
289
  "75070641": {
 
 
290
  "branch_car_at_8": 0.07482179999351501,
 
 
291
  "outcome_ptr_at_8": 1.0,
292
  "pairwise_causal_calibration_ece": 0.2497251397544613,
 
 
 
 
 
 
 
 
293
  "selector_regret_at_8": 0.07482179999351501,
294
- "support_gap_at_8": 1.7603253424167633
 
 
 
 
295
  },
296
  "767747424": {
 
 
297
  "branch_car_at_8": 0.015960504300892353,
 
 
298
  "outcome_ptr_at_8": 0.25,
299
  "pairwise_causal_calibration_ece": 0.21088094062197144,
 
 
 
 
 
 
 
 
300
  "selector_regret_at_8": 0.015960504300892353,
301
- "support_gap_at_8": 0.05905248410999775
 
 
 
 
302
  },
303
  "77147264": {
 
 
304
  "branch_car_at_8": 1.868594542145729,
 
 
305
  "outcome_ptr_at_8": 0.5,
306
  "pairwise_causal_calibration_ece": 0.06512707818376468,
 
 
 
 
 
 
 
 
307
  "selector_regret_at_8": 1.868594542145729,
308
- "support_gap_at_8": 0.0
 
 
 
 
309
  },
310
  "794181626": {
 
 
311
  "branch_car_at_8": 0.0,
 
 
312
  "outcome_ptr_at_8": 1.0,
313
  "pairwise_causal_calibration_ece": 0.12119924562800825,
 
 
 
 
 
 
 
 
314
  "selector_regret_at_8": 0.0,
315
- "support_gap_at_8": 0.0
 
 
 
 
316
  },
317
  "957101489": {
 
 
318
  "branch_car_at_8": 0.0,
 
 
319
  "outcome_ptr_at_8": 1.0,
320
  "pairwise_causal_calibration_ece": 0.20250855048512742,
 
 
 
 
 
 
 
 
321
  "selector_regret_at_8": 0.0,
322
- "support_gap_at_8": 1.4118301272392273
 
 
 
 
323
  },
324
  "998761162": {
 
 
325
  "branch_car_at_8": 0.07749588787555695,
 
 
326
  "outcome_ptr_at_8": 1.0,
327
  "pairwise_causal_calibration_ece": 0.23606164078349207,
 
 
 
 
 
 
 
 
328
  "selector_regret_at_8": 0.07749588787555695,
329
- "support_gap_at_8": 1.9159605950117111
 
 
 
 
330
  }
331
  }
 
1
  {
2
  "1025740212": {
3
+ "base_success": 0.0,
4
+ "base_utility": 0.528373658657074,
5
  "branch_car_at_8": 0.26188503205776215,
6
+ "hidden_chart_oracle_success_at_8": 1.0,
7
+ "hidden_chart_oracle_utility_at_8": 2.0,
8
  "outcome_ptr_at_8": 0.0,
9
  "pairwise_causal_calibration_ece": 0.08963387331564551,
10
+ "proposal_oracle_success_at_8": 0.0,
11
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
12
+ "proposal_oracle_utility_at_8": 0.4761262834072113,
13
+ "proposal_oracle_utility_gain_over_base_at_8": -0.05224737524986267,
14
+ "selected_success_at_8": 0.0,
15
+ "selected_success_gain_over_base_at_8": 0.0,
16
+ "selected_utility_at_8": 0.21424125134944916,
17
+ "selected_utility_gain_over_base_at_8": -0.3141324073076248,
18
  "selector_regret_at_8": 0.26188503205776215,
19
+ "success_selector_gap_at_8": 0.0,
20
+ "success_support_gap_at_8": 1.0,
21
+ "success_total_car_to_hidden_at_8": 1.0,
22
+ "support_gap_at_8": 1.5238737165927887,
23
+ "total_car_to_hidden_at_8": 1.7857587486505508
24
  },
25
  "1055157354": {
26
+ "base_success": 0.0,
27
+ "base_utility": 0.17649109661579132,
28
  "branch_car_at_8": 1.8516457676887512,
29
+ "hidden_chart_oracle_success_at_8": 1.0,
30
+ "hidden_chart_oracle_utility_at_8": 2.0,
31
  "outcome_ptr_at_8": 1.0,
32
  "pairwise_causal_calibration_ece": 0.2973966626262834,
33
+ "proposal_oracle_success_at_8": 1.0,
34
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
35
+ "proposal_oracle_utility_at_8": 2.0,
36
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8235089033842087,
37
+ "selected_success_at_8": 0.0,
38
+ "selected_success_gain_over_base_at_8": 0.0,
39
+ "selected_utility_at_8": 0.14835423231124878,
40
+ "selected_utility_gain_over_base_at_8": -0.02813686430454254,
41
  "selector_regret_at_8": 1.8516457676887512,
42
+ "success_selector_gap_at_8": 1.0,
43
+ "success_support_gap_at_8": 0.0,
44
+ "success_total_car_to_hidden_at_8": 1.0,
45
+ "support_gap_at_8": 0.0,
46
+ "total_car_to_hidden_at_8": 1.8516457676887512
47
  },
48
  "1055567618": {
49
+ "base_success": 1.0,
50
+ "base_utility": 2.0,
51
  "branch_car_at_8": 1.183156132698059,
52
+ "hidden_chart_oracle_success_at_8": 1.0,
53
+ "hidden_chart_oracle_utility_at_8": 2.0,
54
  "outcome_ptr_at_8": 0.0,
55
  "pairwise_causal_calibration_ece": 0.07486610854656817,
56
+ "proposal_oracle_success_at_8": 1.0,
57
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
58
+ "proposal_oracle_utility_at_8": 2.0,
59
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
60
+ "selected_success_at_8": 0.0,
61
+ "selected_success_gain_over_base_at_8": -1.0,
62
+ "selected_utility_at_8": 0.8168438673019409,
63
+ "selected_utility_gain_over_base_at_8": -1.183156132698059,
64
  "selector_regret_at_8": 1.183156132698059,
65
+ "success_selector_gap_at_8": 1.0,
66
+ "success_support_gap_at_8": 0.0,
67
+ "success_total_car_to_hidden_at_8": 1.0,
68
+ "support_gap_at_8": 0.0,
69
+ "total_car_to_hidden_at_8": 1.183156132698059
70
  },
71
  "1113268940": {
72
+ "base_success": 0.0,
73
+ "base_utility": 0.42231252789497375,
74
  "branch_car_at_8": 0.38470593094825745,
75
+ "hidden_chart_oracle_success_at_8": 1.0,
76
+ "hidden_chart_oracle_utility_at_8": 2.0,
77
  "outcome_ptr_at_8": 1.0,
78
  "pairwise_causal_calibration_ece": 0.1616344352768096,
79
+ "proposal_oracle_success_at_8": 0.0,
80
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
81
+ "proposal_oracle_utility_at_8": 0.6646121144294739,
82
+ "proposal_oracle_utility_gain_over_base_at_8": 0.24229958653450012,
83
+ "selected_success_at_8": 0.0,
84
+ "selected_success_gain_over_base_at_8": 0.0,
85
+ "selected_utility_at_8": 0.27990618348121643,
86
+ "selected_utility_gain_over_base_at_8": -0.14240634441375732,
87
  "selector_regret_at_8": 0.38470593094825745,
88
+ "success_selector_gap_at_8": 0.0,
89
+ "success_support_gap_at_8": 1.0,
90
+ "success_total_car_to_hidden_at_8": 1.0,
91
+ "support_gap_at_8": 1.3353878855705261,
92
+ "total_car_to_hidden_at_8": 1.7200938165187836
93
  },
94
  "1113371261": {
95
+ "base_success": 1.0,
96
+ "base_utility": 2.0,
97
  "branch_car_at_8": 0.0,
98
+ "hidden_chart_oracle_success_at_8": 1.0,
99
+ "hidden_chart_oracle_utility_at_8": 2.0,
100
  "outcome_ptr_at_8": 0.0,
101
  "pairwise_causal_calibration_ece": 0.3179396489681421,
102
+ "proposal_oracle_success_at_8": 1.0,
103
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
104
+ "proposal_oracle_utility_at_8": 2.0,
105
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
106
+ "selected_success_at_8": 1.0,
107
+ "selected_success_gain_over_base_at_8": 0.0,
108
+ "selected_utility_at_8": 2.0,
109
+ "selected_utility_gain_over_base_at_8": 0.0,
110
  "selector_regret_at_8": 0.0,
111
+ "success_selector_gap_at_8": 0.0,
112
+ "success_support_gap_at_8": 0.0,
113
+ "success_total_car_to_hidden_at_8": 0.0,
114
+ "support_gap_at_8": 0.0,
115
+ "total_car_to_hidden_at_8": 0.0
116
  },
117
  "1176943004": {
118
+ "base_success": 0.0,
119
+ "base_utility": 0.05094052851200104,
120
  "branch_car_at_8": 0.0,
121
+ "hidden_chart_oracle_success_at_8": 1.0,
122
+ "hidden_chart_oracle_utility_at_8": 2.0,
123
  "outcome_ptr_at_8": 1.0,
124
  "pairwise_causal_calibration_ece": 0.2176350935010736,
125
+ "proposal_oracle_success_at_8": 1.0,
126
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
127
+ "proposal_oracle_utility_at_8": 2.0,
128
+ "proposal_oracle_utility_gain_over_base_at_8": 1.949059471487999,
129
+ "selected_success_at_8": 1.0,
130
+ "selected_success_gain_over_base_at_8": 1.0,
131
+ "selected_utility_at_8": 2.0,
132
+ "selected_utility_gain_over_base_at_8": 1.949059471487999,
133
  "selector_regret_at_8": 0.0,
134
+ "success_selector_gap_at_8": 0.0,
135
+ "success_support_gap_at_8": 0.0,
136
+ "success_total_car_to_hidden_at_8": 0.0,
137
+ "support_gap_at_8": 0.0,
138
+ "total_car_to_hidden_at_8": 0.0
139
  },
140
  "1189392917": {
141
+ "base_success": 0.0,
142
+ "base_utility": 0.10841864347457886,
143
  "branch_car_at_8": 0.0,
144
+ "hidden_chart_oracle_success_at_8": 1.0,
145
+ "hidden_chart_oracle_utility_at_8": 2.0,
146
  "outcome_ptr_at_8": 1.0,
147
  "pairwise_causal_calibration_ece": 0.1628408162009477,
148
+ "proposal_oracle_success_at_8": 1.0,
149
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
150
+ "proposal_oracle_utility_at_8": 2.0,
151
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211,
152
+ "selected_success_at_8": 1.0,
153
+ "selected_success_gain_over_base_at_8": 1.0,
154
+ "selected_utility_at_8": 2.0,
155
+ "selected_utility_gain_over_base_at_8": 1.8915813565254211,
156
  "selector_regret_at_8": 0.0,
157
+ "success_selector_gap_at_8": 0.0,
158
+ "success_support_gap_at_8": 0.0,
159
+ "success_total_car_to_hidden_at_8": 0.0,
160
+ "support_gap_at_8": 0.0,
161
+ "total_car_to_hidden_at_8": 0.0
162
  },
163
  "1231780997": {
164
+ "base_success": 0.0,
165
+ "base_utility": 0.7270061373710632,
166
  "branch_car_at_8": 0.4897426515817642,
167
+ "hidden_chart_oracle_success_at_8": 1.0,
168
+ "hidden_chart_oracle_utility_at_8": 2.0,
169
  "outcome_ptr_at_8": 0.0,
170
  "pairwise_causal_calibration_ece": 0.18322944379343714,
171
+ "proposal_oracle_success_at_8": 0.0,
172
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
173
+ "proposal_oracle_utility_at_8": 0.7187411189079285,
174
+ "proposal_oracle_utility_gain_over_base_at_8": -0.008265018463134766,
175
+ "selected_success_at_8": 0.0,
176
+ "selected_success_gain_over_base_at_8": 0.0,
177
+ "selected_utility_at_8": 0.22899846732616425,
178
+ "selected_utility_gain_over_base_at_8": -0.498007670044899,
179
  "selector_regret_at_8": 0.4897426515817642,
180
+ "success_selector_gap_at_8": 0.0,
181
+ "success_support_gap_at_8": 1.0,
182
+ "success_total_car_to_hidden_at_8": 1.0,
183
+ "support_gap_at_8": 1.2812588810920715,
184
+ "total_car_to_hidden_at_8": 1.7710015326738358
185
  },
186
  "1271718007": {
187
+ "base_success": 0.5,
188
+ "base_utility": 1.4142604768276215,
189
  "branch_car_at_8": 0.0,
190
+ "hidden_chart_oracle_success_at_8": 1.0,
191
+ "hidden_chart_oracle_utility_at_8": 2.0,
192
  "outcome_ptr_at_8": 0.5,
193
  "pairwise_causal_calibration_ece": 0.07543261691159965,
194
+ "proposal_oracle_success_at_8": 1.0,
195
+ "proposal_oracle_success_gain_over_base_at_8": 0.5,
196
+ "proposal_oracle_utility_at_8": 2.0,
197
+ "proposal_oracle_utility_gain_over_base_at_8": 0.5857395231723785,
198
+ "selected_success_at_8": 1.0,
199
+ "selected_success_gain_over_base_at_8": 0.5,
200
+ "selected_utility_at_8": 2.0,
201
+ "selected_utility_gain_over_base_at_8": 0.5857395231723785,
202
  "selector_regret_at_8": 0.0,
203
+ "success_selector_gap_at_8": 0.0,
204
+ "success_support_gap_at_8": 0.0,
205
+ "success_total_car_to_hidden_at_8": 0.0,
206
+ "support_gap_at_8": 0.0,
207
+ "total_car_to_hidden_at_8": 0.0
208
  },
209
  "1273247072": {
210
+ "base_success": 0.0,
211
+ "base_utility": 0.15342965722084045,
212
  "branch_car_at_8": 0.0,
213
+ "hidden_chart_oracle_success_at_8": 1.0,
214
+ "hidden_chart_oracle_utility_at_8": 2.0,
215
  "outcome_ptr_at_8": 0.0,
216
  "pairwise_causal_calibration_ece": 0.15040783499409596,
217
+ "proposal_oracle_success_at_8": 0.0,
218
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
219
+ "proposal_oracle_utility_at_8": 0.13278456032276154,
220
+ "proposal_oracle_utility_gain_over_base_at_8": -0.02064509689807892,
221
+ "selected_success_at_8": 0.0,
222
+ "selected_success_gain_over_base_at_8": 0.0,
223
+ "selected_utility_at_8": 0.13278456032276154,
224
+ "selected_utility_gain_over_base_at_8": -0.02064509689807892,
225
  "selector_regret_at_8": 0.0,
226
+ "success_selector_gap_at_8": 0.0,
227
+ "success_support_gap_at_8": 1.0,
228
+ "success_total_car_to_hidden_at_8": 1.0,
229
+ "support_gap_at_8": 1.8672154396772385,
230
+ "total_car_to_hidden_at_8": 1.8672154396772385
231
  },
232
  "1358421325": {
233
+ "base_success": 0.0,
234
+ "base_utility": 0.8327022790908813,
235
  "branch_car_at_8": 0.5775325745344162,
236
+ "hidden_chart_oracle_success_at_8": 1.0,
237
+ "hidden_chart_oracle_utility_at_8": 2.0,
238
  "outcome_ptr_at_8": 0.0,
239
  "pairwise_causal_calibration_ece": 0.13502516765102715,
240
+ "proposal_oracle_success_at_8": 0.0,
241
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
242
+ "proposal_oracle_utility_at_8": 0.8142459392547607,
243
+ "proposal_oracle_utility_gain_over_base_at_8": -0.018456339836120605,
244
+ "selected_success_at_8": 0.0,
245
+ "selected_success_gain_over_base_at_8": 0.0,
246
+ "selected_utility_at_8": 0.23671336472034454,
247
+ "selected_utility_gain_over_base_at_8": -0.5959889143705368,
248
  "selector_regret_at_8": 0.5775325745344162,
249
+ "success_selector_gap_at_8": 0.0,
250
+ "success_support_gap_at_8": 1.0,
251
+ "success_total_car_to_hidden_at_8": 1.0,
252
+ "support_gap_at_8": 1.1857540607452393,
253
+ "total_car_to_hidden_at_8": 1.7632866352796555
254
  },
255
  "1375492786": {
256
+ "base_success": 1.0,
257
+ "base_utility": 2.0,
258
  "branch_car_at_8": 0.0002854267368093133,
259
+ "hidden_chart_oracle_success_at_8": 0.0,
260
+ "hidden_chart_oracle_utility_at_8": 0.24345096945762634,
261
  "outcome_ptr_at_8": 0.0,
262
  "pairwise_causal_calibration_ece": 0.19928218350064877,
263
+ "proposal_oracle_success_at_8": 0.0,
264
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
265
+ "proposal_oracle_utility_at_8": 0.0016440550098195672,
266
+ "proposal_oracle_utility_gain_over_base_at_8": -1.9983559449901804,
267
+ "selected_success_at_8": 0.0,
268
+ "selected_success_gain_over_base_at_8": -1.0,
269
+ "selected_utility_at_8": 0.001358628273010254,
270
+ "selected_utility_gain_over_base_at_8": -1.9986413717269897,
271
  "selector_regret_at_8": 0.0002854267368093133,
272
+ "success_selector_gap_at_8": 0.0,
273
+ "success_support_gap_at_8": 0.0,
274
+ "success_total_car_to_hidden_at_8": 0.0,
275
+ "support_gap_at_8": 0.24180691444780678,
276
+ "total_car_to_hidden_at_8": 0.2420923411846161
277
  },
278
  "141984615": {
279
+ "base_success": 0.0,
280
+ "base_utility": 0.09381213908394177,
281
  "branch_car_at_8": 0.03316789741317431,
282
+ "hidden_chart_oracle_success_at_8": 0.0,
283
+ "hidden_chart_oracle_utility_at_8": 0.20044784247875214,
284
  "outcome_ptr_at_8": 0.3333333333333333,
285
  "pairwise_causal_calibration_ece": 0.10926310602682258,
286
+ "proposal_oracle_success_at_8": 0.0,
287
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
288
+ "proposal_oracle_utility_at_8": 0.0797463705142339,
289
+ "proposal_oracle_utility_gain_over_base_at_8": -0.01406576856970787,
290
+ "selected_success_at_8": 0.0,
291
+ "selected_success_gain_over_base_at_8": 0.0,
292
+ "selected_utility_at_8": 0.04657847310105959,
293
+ "selected_utility_gain_over_base_at_8": -0.04723366598288218,
294
  "selector_regret_at_8": 0.03316789741317431,
295
+ "success_selector_gap_at_8": 0.0,
296
+ "success_support_gap_at_8": 0.0,
297
+ "success_total_car_to_hidden_at_8": 0.0,
298
+ "support_gap_at_8": 0.12070147196451823,
299
+ "total_car_to_hidden_at_8": 0.15386936937769255
300
  },
301
  "1475440973": {
302
+ "base_success": 1.0,
303
+ "base_utility": 2.0,
304
  "branch_car_at_8": 0.0,
305
+ "hidden_chart_oracle_success_at_8": 1.0,
306
+ "hidden_chart_oracle_utility_at_8": 2.0,
307
  "outcome_ptr_at_8": 0.0,
308
  "pairwise_causal_calibration_ece": 0.09930428958964055,
309
+ "proposal_oracle_success_at_8": 1.0,
310
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
311
+ "proposal_oracle_utility_at_8": 2.0,
312
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
313
+ "selected_success_at_8": 1.0,
314
+ "selected_success_gain_over_base_at_8": 0.0,
315
+ "selected_utility_at_8": 2.0,
316
+ "selected_utility_gain_over_base_at_8": 0.0,
317
  "selector_regret_at_8": 0.0,
318
+ "success_selector_gap_at_8": 0.0,
319
+ "success_support_gap_at_8": 0.0,
320
+ "success_total_car_to_hidden_at_8": 0.0,
321
+ "support_gap_at_8": 0.0,
322
+ "total_car_to_hidden_at_8": 0.0
323
  },
324
  "1483645336": {
325
+ "base_success": 0.0,
326
+ "base_utility": 0.01641605794429779,
327
  "branch_car_at_8": 0.0,
328
+ "hidden_chart_oracle_success_at_8": 1.0,
329
+ "hidden_chart_oracle_utility_at_8": 2.0,
330
  "outcome_ptr_at_8": 1.0,
331
  "pairwise_causal_calibration_ece": 0.0,
332
+ "proposal_oracle_success_at_8": 1.0,
333
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
334
+ "proposal_oracle_utility_at_8": 2.0,
335
+ "proposal_oracle_utility_gain_over_base_at_8": 1.9835839420557022,
336
+ "selected_success_at_8": 1.0,
337
+ "selected_success_gain_over_base_at_8": 1.0,
338
+ "selected_utility_at_8": 2.0,
339
+ "selected_utility_gain_over_base_at_8": 1.9835839420557022,
340
  "selector_regret_at_8": 0.0,
341
+ "success_selector_gap_at_8": 0.0,
342
+ "success_support_gap_at_8": 0.0,
343
+ "success_total_car_to_hidden_at_8": 0.0,
344
+ "support_gap_at_8": 0.0,
345
+ "total_car_to_hidden_at_8": 0.0
346
  },
347
  "1522272686": {
348
+ "base_success": 0.0,
349
+ "base_utility": 0.19476109743118286,
350
  "branch_car_at_8": 0.009524375200271606,
351
+ "hidden_chart_oracle_success_at_8": 1.0,
352
+ "hidden_chart_oracle_utility_at_8": 2.0,
353
  "outcome_ptr_at_8": 1.0,
354
  "pairwise_causal_calibration_ece": 0.08257819446483214,
355
+ "proposal_oracle_success_at_8": 0.0,
356
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
357
+ "proposal_oracle_utility_at_8": 0.2435554713010788,
358
+ "proposal_oracle_utility_gain_over_base_at_8": 0.048794373869895935,
359
+ "selected_success_at_8": 0.0,
360
+ "selected_success_gain_over_base_at_8": 0.0,
361
+ "selected_utility_at_8": 0.2340310961008072,
362
+ "selected_utility_gain_over_base_at_8": 0.03926999866962433,
363
  "selector_regret_at_8": 0.009524375200271606,
364
+ "success_selector_gap_at_8": 0.0,
365
+ "success_support_gap_at_8": 1.0,
366
+ "success_total_car_to_hidden_at_8": 1.0,
367
+ "support_gap_at_8": 1.7564445286989212,
368
+ "total_car_to_hidden_at_8": 1.7659689038991928
369
  },
370
  "1528740417": {
371
+ "base_success": 1.0,
372
+ "base_utility": 2.0,
373
  "branch_car_at_8": 0.0,
374
+ "hidden_chart_oracle_success_at_8": 1.0,
375
+ "hidden_chart_oracle_utility_at_8": 2.0,
376
  "outcome_ptr_at_8": 0.0,
377
  "pairwise_causal_calibration_ece": 0.015990866806051107,
378
+ "proposal_oracle_success_at_8": 1.0,
379
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
380
+ "proposal_oracle_utility_at_8": 2.0,
381
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
382
+ "selected_success_at_8": 1.0,
383
+ "selected_success_gain_over_base_at_8": 0.0,
384
+ "selected_utility_at_8": 2.0,
385
+ "selected_utility_gain_over_base_at_8": 0.0,
386
  "selector_regret_at_8": 0.0,
387
+ "success_selector_gap_at_8": 0.0,
388
+ "success_support_gap_at_8": 0.0,
389
+ "success_total_car_to_hidden_at_8": 0.0,
390
+ "support_gap_at_8": 0.0,
391
+ "total_car_to_hidden_at_8": 0.0
392
  },
393
  "1549324805": {
394
+ "base_success": 0.6666666666666666,
395
+ "base_utility": 1.3787474483251572,
396
  "branch_car_at_8": 0.0034310482442378998,
397
+ "hidden_chart_oracle_success_at_8": 0.6666666666666666,
398
+ "hidden_chart_oracle_utility_at_8": 1.4027505218982697,
399
  "outcome_ptr_at_8": 0.0,
400
  "pairwise_causal_calibration_ece": 0.19382954770507574,
401
+ "proposal_oracle_success_at_8": 0.6666666666666666,
402
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
403
+ "proposal_oracle_utility_at_8": 1.3427665165315072,
404
+ "proposal_oracle_utility_gain_over_base_at_8": -0.035980931793649994,
405
+ "selected_success_at_8": 0.6666666666666666,
406
+ "selected_success_gain_over_base_at_8": 0.0,
407
+ "selected_utility_at_8": 1.3393354682872693,
408
+ "selected_utility_gain_over_base_at_8": -0.03941198003788789,
409
  "selector_regret_at_8": 0.0034310482442378998,
410
+ "success_selector_gap_at_8": 0.0,
411
+ "success_support_gap_at_8": 0.0,
412
+ "success_total_car_to_hidden_at_8": 0.0,
413
+ "support_gap_at_8": 0.05998400536676248,
414
+ "total_car_to_hidden_at_8": 0.06341505361100037
415
  },
416
  "1607773860": {
417
+ "base_success": 1.0,
418
+ "base_utility": 2.0,
419
  "branch_car_at_8": 1.779732346534729,
420
+ "hidden_chart_oracle_success_at_8": 1.0,
421
+ "hidden_chart_oracle_utility_at_8": 2.0,
422
  "outcome_ptr_at_8": 0.0,
423
  "pairwise_causal_calibration_ece": 0.08588083120555855,
424
+ "proposal_oracle_success_at_8": 1.0,
425
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
426
+ "proposal_oracle_utility_at_8": 2.0,
427
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
428
+ "selected_success_at_8": 0.0,
429
+ "selected_success_gain_over_base_at_8": -1.0,
430
+ "selected_utility_at_8": 0.220267653465271,
431
+ "selected_utility_gain_over_base_at_8": -1.779732346534729,
432
  "selector_regret_at_8": 1.779732346534729,
433
+ "success_selector_gap_at_8": 1.0,
434
+ "success_support_gap_at_8": 0.0,
435
+ "success_total_car_to_hidden_at_8": 1.0,
436
+ "support_gap_at_8": 0.0,
437
+ "total_car_to_hidden_at_8": 1.779732346534729
438
  },
439
  "1665739511": {
440
+ "base_success": 1.0,
441
+ "base_utility": 2.0,
442
  "branch_car_at_8": 0.007551252841949463,
443
+ "hidden_chart_oracle_success_at_8": 1.0,
444
+ "hidden_chart_oracle_utility_at_8": 2.0,
445
  "outcome_ptr_at_8": 0.0,
446
  "pairwise_causal_calibration_ece": 0.050940808914706356,
447
+ "proposal_oracle_success_at_8": 0.0,
448
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
449
+ "proposal_oracle_utility_at_8": 0.8216220736503601,
450
+ "proposal_oracle_utility_gain_over_base_at_8": -1.17837792634964,
451
+ "selected_success_at_8": 0.0,
452
+ "selected_success_gain_over_base_at_8": -1.0,
453
+ "selected_utility_at_8": 0.8140708208084106,
454
+ "selected_utility_gain_over_base_at_8": -1.1859291791915894,
455
  "selector_regret_at_8": 0.007551252841949463,
456
+ "success_selector_gap_at_8": 0.0,
457
+ "success_support_gap_at_8": 1.0,
458
+ "success_total_car_to_hidden_at_8": 1.0,
459
+ "support_gap_at_8": 1.17837792634964,
460
+ "total_car_to_hidden_at_8": 1.1859291791915894
461
  },
462
  "1747178030": {
463
+ "base_success": 0.0,
464
+ "base_utility": 0.1463065892457962,
465
  "branch_car_at_8": 0.0,
466
+ "hidden_chart_oracle_success_at_8": 1.0,
467
+ "hidden_chart_oracle_utility_at_8": 2.0,
468
  "outcome_ptr_at_8": 1.0,
469
  "pairwise_causal_calibration_ece": 0.15825371494561297,
470
+ "proposal_oracle_success_at_8": 1.0,
471
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
472
+ "proposal_oracle_utility_at_8": 2.0,
473
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8536934107542038,
474
+ "selected_success_at_8": 1.0,
475
+ "selected_success_gain_over_base_at_8": 1.0,
476
+ "selected_utility_at_8": 2.0,
477
+ "selected_utility_gain_over_base_at_8": 1.8536934107542038,
478
  "selector_regret_at_8": 0.0,
479
+ "success_selector_gap_at_8": 0.0,
480
+ "success_support_gap_at_8": 0.0,
481
+ "success_total_car_to_hidden_at_8": 0.0,
482
+ "support_gap_at_8": 0.0,
483
+ "total_car_to_hidden_at_8": 0.0
484
  },
485
  "1881223807": {
486
+ "base_success": 0.0,
487
+ "base_utility": 0.0320269912481308,
488
  "branch_car_at_8": 0.020236089825630188,
489
+ "hidden_chart_oracle_success_at_8": 1.0,
490
+ "hidden_chart_oracle_utility_at_8": 2.0,
491
  "outcome_ptr_at_8": 1.0,
492
  "pairwise_causal_calibration_ece": 0.019060791998604134,
493
+ "proposal_oracle_success_at_8": 0.0,
494
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
495
+ "proposal_oracle_utility_at_8": 0.0513521283864975,
496
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0193251371383667,
497
+ "selected_success_at_8": 0.0,
498
+ "selected_success_gain_over_base_at_8": 0.0,
499
+ "selected_utility_at_8": 0.03111603856086731,
500
+ "selected_utility_gain_over_base_at_8": -0.0009109526872634888,
501
  "selector_regret_at_8": 0.020236089825630188,
502
+ "success_selector_gap_at_8": 0.0,
503
+ "success_support_gap_at_8": 1.0,
504
+ "success_total_car_to_hidden_at_8": 1.0,
505
+ "support_gap_at_8": 1.9486478716135025,
506
+ "total_car_to_hidden_at_8": 1.9688839614391327
507
  },
508
  "1919198188": {
509
+ "base_success": 0.0,
510
+ "base_utility": 0.037198856472969055,
511
  "branch_car_at_8": 0.016854479908943176,
512
+ "hidden_chart_oracle_success_at_8": 1.0,
513
+ "hidden_chart_oracle_utility_at_8": 2.0,
514
  "outcome_ptr_at_8": 1.0,
515
  "pairwise_causal_calibration_ece": 0.15779226512372985,
516
+ "proposal_oracle_success_at_8": 0.0,
517
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
518
+ "proposal_oracle_utility_at_8": 0.08992795646190643,
519
+ "proposal_oracle_utility_gain_over_base_at_8": 0.05272909998893738,
520
+ "selected_success_at_8": 0.0,
521
+ "selected_success_gain_over_base_at_8": 0.0,
522
+ "selected_utility_at_8": 0.07307347655296326,
523
+ "selected_utility_gain_over_base_at_8": 0.0358746200799942,
524
  "selector_regret_at_8": 0.016854479908943176,
525
+ "success_selector_gap_at_8": 0.0,
526
+ "success_support_gap_at_8": 1.0,
527
+ "success_total_car_to_hidden_at_8": 1.0,
528
+ "support_gap_at_8": 1.9100720435380936,
529
+ "total_car_to_hidden_at_8": 1.9269265234470367
530
  },
531
  "1991414747": {
532
+ "base_success": 1.0,
533
+ "base_utility": 2.0,
534
  "branch_car_at_8": 0.0,
535
+ "hidden_chart_oracle_success_at_8": 1.0,
536
+ "hidden_chart_oracle_utility_at_8": 2.0,
537
  "outcome_ptr_at_8": 0.0,
538
  "pairwise_causal_calibration_ece": 0.05287821364799471,
539
+ "proposal_oracle_success_at_8": 1.0,
540
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
541
+ "proposal_oracle_utility_at_8": 2.0,
542
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
543
+ "selected_success_at_8": 1.0,
544
+ "selected_success_gain_over_base_at_8": 0.0,
545
+ "selected_utility_at_8": 2.0,
546
+ "selected_utility_gain_over_base_at_8": 0.0,
547
  "selector_regret_at_8": 0.0,
548
+ "success_selector_gap_at_8": 0.0,
549
+ "success_support_gap_at_8": 0.0,
550
+ "success_total_car_to_hidden_at_8": 0.0,
551
+ "support_gap_at_8": 0.0,
552
+ "total_car_to_hidden_at_8": 0.0
553
  },
554
  "1997090716": {
555
+ "base_success": 0.0,
556
+ "base_utility": 0.13927479088306427,
557
  "branch_car_at_8": 1.9691698104143143,
558
+ "hidden_chart_oracle_success_at_8": 1.0,
559
+ "hidden_chart_oracle_utility_at_8": 2.0,
560
  "outcome_ptr_at_8": 1.0,
561
  "pairwise_causal_calibration_ece": 0.4044963838715546,
562
+ "proposal_oracle_success_at_8": 1.0,
563
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
564
+ "proposal_oracle_utility_at_8": 2.0,
565
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357,
566
+ "selected_success_at_8": 0.0,
567
+ "selected_success_gain_over_base_at_8": 0.0,
568
+ "selected_utility_at_8": 0.03083018958568573,
569
+ "selected_utility_gain_over_base_at_8": -0.10844460129737854,
570
  "selector_regret_at_8": 1.9691698104143143,
571
+ "success_selector_gap_at_8": 1.0,
572
+ "success_support_gap_at_8": 0.0,
573
+ "success_total_car_to_hidden_at_8": 1.0,
574
+ "support_gap_at_8": 0.0,
575
+ "total_car_to_hidden_at_8": 1.9691698104143143
576
  },
577
  "2015956957": {
578
+ "base_success": 0.0,
579
+ "base_utility": 0.06711978986859321,
580
  "branch_car_at_8": 0.01770364921540022,
581
+ "hidden_chart_oracle_success_at_8": 0.2,
582
+ "hidden_chart_oracle_utility_at_8": 0.5298813670873642,
583
  "outcome_ptr_at_8": 0.6,
584
  "pairwise_causal_calibration_ece": 0.18664994005668376,
585
+ "proposal_oracle_success_at_8": 0.2,
586
+ "proposal_oracle_success_gain_over_base_at_8": 0.2,
587
+ "proposal_oracle_utility_at_8": 0.45124312303960323,
588
+ "proposal_oracle_utility_gain_over_base_at_8": 0.38412333317101,
589
+ "selected_success_at_8": 0.2,
590
+ "selected_success_gain_over_base_at_8": 0.2,
591
+ "selected_utility_at_8": 0.433539473824203,
592
+ "selected_utility_gain_over_base_at_8": 0.3664196839556098,
593
  "selector_regret_at_8": 0.01770364921540022,
594
+ "success_selector_gap_at_8": 0.0,
595
+ "success_support_gap_at_8": 0.0,
596
+ "success_total_car_to_hidden_at_8": 0.0,
597
+ "support_gap_at_8": 0.07863824404776096,
598
+ "total_car_to_hidden_at_8": 0.09634189326316119
599
  },
600
  "2054899126": {
601
+ "base_success": 1.0,
602
+ "base_utility": 2.0,
603
  "branch_car_at_8": 0.0,
604
+ "hidden_chart_oracle_success_at_8": 1.0,
605
+ "hidden_chart_oracle_utility_at_8": 2.0,
606
  "outcome_ptr_at_8": 0.0,
607
  "pairwise_causal_calibration_ece": 0.0,
608
+ "proposal_oracle_success_at_8": 1.0,
609
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
610
+ "proposal_oracle_utility_at_8": 2.0,
611
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
612
+ "selected_success_at_8": 1.0,
613
+ "selected_success_gain_over_base_at_8": 0.0,
614
+ "selected_utility_at_8": 2.0,
615
+ "selected_utility_gain_over_base_at_8": 0.0,
616
  "selector_regret_at_8": 0.0,
617
+ "success_selector_gap_at_8": 0.0,
618
+ "success_support_gap_at_8": 0.0,
619
+ "success_total_car_to_hidden_at_8": 0.0,
620
+ "support_gap_at_8": 0.0,
621
+ "total_car_to_hidden_at_8": 0.0
622
  },
623
  "211245842": {
624
+ "base_success": 0.0,
625
+ "base_utility": 0.8388344049453735,
626
  "branch_car_at_8": 0.5908347219228745,
627
+ "hidden_chart_oracle_success_at_8": 1.0,
628
+ "hidden_chart_oracle_utility_at_8": 2.0,
629
  "outcome_ptr_at_8": 0.0,
630
  "pairwise_causal_calibration_ece": 0.16771630653845432,
631
+ "proposal_oracle_success_at_8": 0.0,
632
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
633
+ "proposal_oracle_utility_at_8": 0.8130142092704773,
634
+ "proposal_oracle_utility_gain_over_base_at_8": -0.02582019567489624,
635
+ "selected_success_at_8": 0.0,
636
+ "selected_success_gain_over_base_at_8": 0.0,
637
+ "selected_utility_at_8": 0.22217948734760284,
638
+ "selected_utility_gain_over_base_at_8": -0.6166549175977707,
639
  "selector_regret_at_8": 0.5908347219228745,
640
+ "success_selector_gap_at_8": 0.0,
641
+ "success_support_gap_at_8": 1.0,
642
+ "success_total_car_to_hidden_at_8": 1.0,
643
+ "support_gap_at_8": 1.1869857907295227,
644
+ "total_car_to_hidden_at_8": 1.7778205126523972
645
  },
646
  "2141802816": {
647
+ "base_success": 0.0,
648
+ "base_utility": 0.07080724835395813,
649
  "branch_car_at_8": 0.07543906569480896,
650
+ "hidden_chart_oracle_success_at_8": 1.0,
651
+ "hidden_chart_oracle_utility_at_8": 2.0,
652
  "outcome_ptr_at_8": 1.0,
653
  "pairwise_causal_calibration_ece": 0.05242798758366053,
654
+ "proposal_oracle_success_at_8": 0.0,
655
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
656
+ "proposal_oracle_utility_at_8": 0.12472955882549286,
657
+ "proposal_oracle_utility_gain_over_base_at_8": 0.05392231047153473,
658
+ "selected_success_at_8": 0.0,
659
+ "selected_success_gain_over_base_at_8": 0.0,
660
+ "selected_utility_at_8": 0.0492904931306839,
661
+ "selected_utility_gain_over_base_at_8": -0.02151675522327423,
662
  "selector_regret_at_8": 0.07543906569480896,
663
+ "success_selector_gap_at_8": 0.0,
664
+ "success_support_gap_at_8": 1.0,
665
+ "success_total_car_to_hidden_at_8": 1.0,
666
+ "support_gap_at_8": 1.8752704411745071,
667
+ "total_car_to_hidden_at_8": 1.950709506869316
668
  },
669
  "256770874": {
670
+ "base_success": 0.0,
671
+ "base_utility": 0.8557480573654175,
672
  "branch_car_at_8": 0.10820980370044708,
673
+ "hidden_chart_oracle_success_at_8": 1.0,
674
+ "hidden_chart_oracle_utility_at_8": 2.0,
675
  "outcome_ptr_at_8": 0.0,
676
  "pairwise_causal_calibration_ece": 0.030709639081801327,
677
+ "proposal_oracle_success_at_8": 0.0,
678
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
679
+ "proposal_oracle_utility_at_8": 0.24107472598552704,
680
+ "proposal_oracle_utility_gain_over_base_at_8": -0.6146733313798904,
681
+ "selected_success_at_8": 0.0,
682
+ "selected_success_gain_over_base_at_8": 0.0,
683
+ "selected_utility_at_8": 0.13286492228507996,
684
+ "selected_utility_gain_over_base_at_8": -0.7228831350803375,
685
  "selector_regret_at_8": 0.10820980370044708,
686
+ "success_selector_gap_at_8": 0.0,
687
+ "success_support_gap_at_8": 1.0,
688
+ "success_total_car_to_hidden_at_8": 1.0,
689
+ "support_gap_at_8": 1.758925274014473,
690
+ "total_car_to_hidden_at_8": 1.86713507771492
691
  },
692
  "278008524": {
693
+ "base_success": 0.0,
694
+ "base_utility": 0.10747186839580536,
695
  "branch_car_at_8": 0.0016456245793960989,
696
+ "hidden_chart_oracle_success_at_8": 0.5,
697
+ "hidden_chart_oracle_utility_at_8": 1.113811194896698,
698
  "outcome_ptr_at_8": 0.0,
699
  "pairwise_causal_calibration_ece": 0.11925245416072927,
700
+ "proposal_oracle_success_at_8": 0.0,
701
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
702
+ "proposal_oracle_utility_at_8": 0.011432985425926745,
703
+ "proposal_oracle_utility_gain_over_base_at_8": -0.09603888296987861,
704
+ "selected_success_at_8": 0.0,
705
+ "selected_success_gain_over_base_at_8": 0.0,
706
+ "selected_utility_at_8": 0.009787360846530646,
707
+ "selected_utility_gain_over_base_at_8": -0.09768450754927471,
708
  "selector_regret_at_8": 0.0016456245793960989,
709
+ "success_selector_gap_at_8": 0.0,
710
+ "success_support_gap_at_8": 0.5,
711
+ "success_total_car_to_hidden_at_8": 0.5,
712
+ "support_gap_at_8": 1.1023782094707713,
713
+ "total_car_to_hidden_at_8": 1.1040238340501674
714
  },
715
  "416183106": {
716
+ "base_success": 0.0,
717
+ "base_utility": 0.20032641291618347,
718
  "branch_car_at_8": 0.3360496461391449,
719
+ "hidden_chart_oracle_success_at_8": 1.0,
720
+ "hidden_chart_oracle_utility_at_8": 2.0,
721
  "outcome_ptr_at_8": 1.0,
722
  "pairwise_causal_calibration_ece": 0.10129212810935397,
723
+ "proposal_oracle_success_at_8": 0.0,
724
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
725
+ "proposal_oracle_utility_at_8": 0.5778142213821411,
726
+ "proposal_oracle_utility_gain_over_base_at_8": 0.37748780846595764,
727
+ "selected_success_at_8": 0.0,
728
+ "selected_success_gain_over_base_at_8": 0.0,
729
+ "selected_utility_at_8": 0.24176457524299622,
730
+ "selected_utility_gain_over_base_at_8": 0.041438162326812744,
731
  "selector_regret_at_8": 0.3360496461391449,
732
+ "success_selector_gap_at_8": 0.0,
733
+ "success_support_gap_at_8": 1.0,
734
+ "success_total_car_to_hidden_at_8": 1.0,
735
+ "support_gap_at_8": 1.4221857786178589,
736
+ "total_car_to_hidden_at_8": 1.7582354247570038
737
  },
738
  "534717141": {
739
+ "base_success": 1.0,
740
+ "base_utility": 2.0,
741
  "branch_car_at_8": 1.9274959191679955,
742
+ "hidden_chart_oracle_success_at_8": 1.0,
743
+ "hidden_chart_oracle_utility_at_8": 2.0,
744
  "outcome_ptr_at_8": 0.0,
745
  "pairwise_causal_calibration_ece": 0.6159370955924921,
746
+ "proposal_oracle_success_at_8": 1.0,
747
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
748
+ "proposal_oracle_utility_at_8": 2.0,
749
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
750
+ "selected_success_at_8": 0.0,
751
+ "selected_success_gain_over_base_at_8": -1.0,
752
+ "selected_utility_at_8": 0.07250408083200455,
753
+ "selected_utility_gain_over_base_at_8": -1.9274959191679955,
754
  "selector_regret_at_8": 1.9274959191679955,
755
+ "success_selector_gap_at_8": 1.0,
756
+ "success_support_gap_at_8": 0.0,
757
+ "success_total_car_to_hidden_at_8": 1.0,
758
+ "support_gap_at_8": 0.0,
759
+ "total_car_to_hidden_at_8": 1.9274959191679955
760
  },
761
  "551504635": {
762
+ "base_success": 0.0,
763
+ "base_utility": 0.7239725589752197,
764
  "branch_car_at_8": 0.033253490924835205,
765
+ "hidden_chart_oracle_success_at_8": 1.0,
766
+ "hidden_chart_oracle_utility_at_8": 2.0,
767
  "outcome_ptr_at_8": 1.0,
768
  "pairwise_causal_calibration_ece": 0.10932818664818449,
769
+ "proposal_oracle_success_at_8": 0.0,
770
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
771
+ "proposal_oracle_utility_at_8": 0.7250897884368896,
772
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0011172294616699219,
773
+ "selected_success_at_8": 0.0,
774
+ "selected_success_gain_over_base_at_8": 0.0,
775
+ "selected_utility_at_8": 0.6918362975120544,
776
+ "selected_utility_gain_over_base_at_8": -0.03213626146316528,
777
  "selector_regret_at_8": 0.033253490924835205,
778
+ "success_selector_gap_at_8": 0.0,
779
+ "success_support_gap_at_8": 1.0,
780
+ "success_total_car_to_hidden_at_8": 1.0,
781
+ "support_gap_at_8": 1.2749102115631104,
782
+ "total_car_to_hidden_at_8": 1.3081637024879456
783
  },
784
  "565369392": {
785
+ "base_success": 0.3333333333333333,
786
+ "base_utility": 0.7997656464576721,
787
  "branch_car_at_8": 0.06443698455890019,
788
+ "hidden_chart_oracle_success_at_8": 0.6666666666666666,
789
+ "hidden_chart_oracle_utility_at_8": 1.4266315797964733,
790
  "outcome_ptr_at_8": 0.0,
791
  "pairwise_causal_calibration_ece": 0.22308845281795123,
792
+ "proposal_oracle_success_at_8": 0.3333333333333333,
793
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
794
+ "proposal_oracle_utility_at_8": 0.7852714310089747,
795
+ "proposal_oracle_utility_gain_over_base_at_8": -0.014494215448697409,
796
+ "selected_success_at_8": 0.3333333333333333,
797
+ "selected_success_gain_over_base_at_8": 0.0,
798
+ "selected_utility_at_8": 0.7208344464500746,
799
+ "selected_utility_gain_over_base_at_8": -0.07893120000759761,
800
  "selector_regret_at_8": 0.06443698455890019,
801
+ "success_selector_gap_at_8": 0.0,
802
+ "success_support_gap_at_8": 0.3333333333333333,
803
+ "success_total_car_to_hidden_at_8": 0.3333333333333333,
804
+ "support_gap_at_8": 0.6413601487874985,
805
+ "total_car_to_hidden_at_8": 0.7057971333463987
806
  },
807
  "565621826": {
808
+ "base_success": 1.0,
809
+ "base_utility": 2.0,
810
  "branch_car_at_8": 0.0,
811
+ "hidden_chart_oracle_success_at_8": 1.0,
812
+ "hidden_chart_oracle_utility_at_8": 2.0,
813
  "outcome_ptr_at_8": 0.0,
814
  "pairwise_causal_calibration_ece": 0.0,
815
+ "proposal_oracle_success_at_8": 1.0,
816
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
817
+ "proposal_oracle_utility_at_8": 2.0,
818
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
819
+ "selected_success_at_8": 1.0,
820
+ "selected_success_gain_over_base_at_8": 0.0,
821
+ "selected_utility_at_8": 2.0,
822
+ "selected_utility_gain_over_base_at_8": 0.0,
823
  "selector_regret_at_8": 0.0,
824
+ "success_selector_gap_at_8": 0.0,
825
+ "success_support_gap_at_8": 0.0,
826
+ "success_total_car_to_hidden_at_8": 0.0,
827
+ "support_gap_at_8": 0.0,
828
+ "total_car_to_hidden_at_8": 0.0
829
  },
830
  "585904130": {
831
+ "base_success": 0.0,
832
+ "base_utility": 0.07529209802548091,
833
  "branch_car_at_8": 0.043168901621053614,
834
+ "hidden_chart_oracle_success_at_8": 0.0,
835
+ "hidden_chart_oracle_utility_at_8": 0.23764699200789133,
836
  "outcome_ptr_at_8": 0.3333333333333333,
837
  "pairwise_causal_calibration_ece": 0.10391809443539522,
838
+ "proposal_oracle_success_at_8": 0.0,
839
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
840
+ "proposal_oracle_utility_at_8": 0.05598785417775313,
841
+ "proposal_oracle_utility_gain_over_base_at_8": -0.019304243847727776,
842
+ "selected_success_at_8": 0.0,
843
+ "selected_success_gain_over_base_at_8": 0.0,
844
+ "selected_utility_at_8": 0.012818952556699514,
845
+ "selected_utility_gain_over_base_at_8": -0.06247314546878139,
846
  "selector_regret_at_8": 0.043168901621053614,
847
+ "success_selector_gap_at_8": 0.0,
848
+ "success_support_gap_at_8": 0.0,
849
+ "success_total_car_to_hidden_at_8": 0.0,
850
+ "support_gap_at_8": 0.1816591378301382,
851
+ "total_car_to_hidden_at_8": 0.2248280394511918
852
  },
853
  "626486813": {
854
+ "base_success": 0.25,
855
+ "base_utility": 0.5054400166263804,
856
  "branch_car_at_8": 0.018570441869087517,
857
+ "hidden_chart_oracle_success_at_8": 0.25,
858
+ "hidden_chart_oracle_utility_at_8": 0.6124684102833271,
859
  "outcome_ptr_at_8": 0.75,
860
  "pairwise_causal_calibration_ece": 0.16162412565990822,
861
+ "proposal_oracle_success_at_8": 0.25,
862
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
863
+ "proposal_oracle_utility_at_8": 0.5615248780231923,
864
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0560848613968119,
865
+ "selected_success_at_8": 0.25,
866
+ "selected_success_gain_over_base_at_8": 0.0,
867
+ "selected_utility_at_8": 0.5429544361541048,
868
+ "selected_utility_gain_over_base_at_8": 0.037514419527724385,
869
  "selector_regret_at_8": 0.018570441869087517,
870
+ "success_selector_gap_at_8": 0.0,
871
+ "success_support_gap_at_8": 0.0,
872
+ "success_total_car_to_hidden_at_8": 0.0,
873
+ "support_gap_at_8": 0.0801479306537658,
874
+ "total_car_to_hidden_at_8": 0.08356922247912735
875
  },
876
  "636680137": {
877
+ "base_success": 1.0,
878
+ "base_utility": 2.0,
879
  "branch_car_at_8": 0.0,
880
+ "hidden_chart_oracle_success_at_8": 1.0,
881
+ "hidden_chart_oracle_utility_at_8": 2.0,
882
  "outcome_ptr_at_8": 0.0,
883
  "pairwise_causal_calibration_ece": 0.1699944666253151,
884
+ "proposal_oracle_success_at_8": 0.0,
885
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
886
+ "proposal_oracle_utility_at_8": 0.012414653785526752,
887
+ "proposal_oracle_utility_gain_over_base_at_8": -1.9875853462144732,
888
+ "selected_success_at_8": 0.0,
889
+ "selected_success_gain_over_base_at_8": -1.0,
890
+ "selected_utility_at_8": 0.012414653785526752,
891
+ "selected_utility_gain_over_base_at_8": -1.9875853462144732,
892
  "selector_regret_at_8": 0.0,
893
+ "success_selector_gap_at_8": 0.0,
894
+ "success_support_gap_at_8": 1.0,
895
+ "success_total_car_to_hidden_at_8": 1.0,
896
+ "support_gap_at_8": 1.9875853462144732,
897
+ "total_car_to_hidden_at_8": 1.9875853462144732
898
  },
899
  "706479600": {
900
+ "base_success": 0.0,
901
+ "base_utility": 0.8125406503677368,
902
  "branch_car_at_8": 1.1865743398666382,
903
+ "hidden_chart_oracle_success_at_8": 1.0,
904
+ "hidden_chart_oracle_utility_at_8": 2.0,
905
  "outcome_ptr_at_8": 1.0,
906
  "pairwise_causal_calibration_ece": 0.0922387871049479,
907
+ "proposal_oracle_success_at_8": 1.0,
908
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
909
+ "proposal_oracle_utility_at_8": 2.0,
910
+ "proposal_oracle_utility_gain_over_base_at_8": 1.1874593496322632,
911
+ "selected_success_at_8": 0.0,
912
+ "selected_success_gain_over_base_at_8": 0.0,
913
+ "selected_utility_at_8": 0.8134256601333618,
914
+ "selected_utility_gain_over_base_at_8": 0.000885009765625,
915
  "selector_regret_at_8": 1.1865743398666382,
916
+ "success_selector_gap_at_8": 1.0,
917
+ "success_support_gap_at_8": 0.0,
918
+ "success_total_car_to_hidden_at_8": 1.0,
919
+ "support_gap_at_8": 0.0,
920
+ "total_car_to_hidden_at_8": 1.1865743398666382
921
  },
922
  "750448296": {
923
+ "base_success": 0.0,
924
+ "base_utility": 0.11153355985879898,
925
  "branch_car_at_8": 0.07732226233929396,
926
+ "hidden_chart_oracle_success_at_8": 0.0,
927
+ "hidden_chart_oracle_utility_at_8": 0.23580759018659592,
928
  "outcome_ptr_at_8": 0.5,
929
  "pairwise_causal_calibration_ece": 0.14360585133877676,
930
+ "proposal_oracle_success_at_8": 0.0,
931
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
932
+ "proposal_oracle_utility_at_8": 0.1069931760430336,
933
+ "proposal_oracle_utility_gain_over_base_at_8": -0.004540383815765381,
934
+ "selected_success_at_8": 0.0,
935
+ "selected_success_gain_over_base_at_8": 0.0,
936
+ "selected_utility_at_8": 0.029670913703739643,
937
+ "selected_utility_gain_over_base_at_8": -0.08186264615505934,
938
  "selector_regret_at_8": 0.07732226233929396,
939
+ "success_selector_gap_at_8": 0.0,
940
+ "success_support_gap_at_8": 0.0,
941
+ "success_total_car_to_hidden_at_8": 0.0,
942
+ "support_gap_at_8": 0.12881441414356232,
943
+ "total_car_to_hidden_at_8": 0.20613667648285627
944
  },
945
  "75070641": {
946
+ "base_success": 0.0,
947
+ "base_utility": 0.20275163650512695,
948
  "branch_car_at_8": 0.07482179999351501,
949
+ "hidden_chart_oracle_success_at_8": 1.0,
950
+ "hidden_chart_oracle_utility_at_8": 2.0,
951
  "outcome_ptr_at_8": 1.0,
952
  "pairwise_causal_calibration_ece": 0.2497251397544613,
953
+ "proposal_oracle_success_at_8": 0.0,
954
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
955
+ "proposal_oracle_utility_at_8": 0.2396746575832367,
956
+ "proposal_oracle_utility_gain_over_base_at_8": 0.03692302107810974,
957
+ "selected_success_at_8": 0.0,
958
+ "selected_success_gain_over_base_at_8": 0.0,
959
+ "selected_utility_at_8": 0.16485285758972168,
960
+ "selected_utility_gain_over_base_at_8": -0.03789877891540527,
961
  "selector_regret_at_8": 0.07482179999351501,
962
+ "success_selector_gap_at_8": 0.0,
963
+ "success_support_gap_at_8": 1.0,
964
+ "success_total_car_to_hidden_at_8": 1.0,
965
+ "support_gap_at_8": 1.7603253424167633,
966
+ "total_car_to_hidden_at_8": 1.8351471424102783
967
  },
968
  "767747424": {
969
+ "base_success": 0.25,
970
+ "base_utility": 0.5475241877138615,
971
  "branch_car_at_8": 0.015960504300892353,
972
+ "hidden_chart_oracle_success_at_8": 0.0,
973
+ "hidden_chart_oracle_utility_at_8": 0.18863565661013126,
974
  "outcome_ptr_at_8": 0.25,
975
  "pairwise_causal_calibration_ece": 0.21088094062197144,
976
+ "proposal_oracle_success_at_8": 0.25,
977
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
978
+ "proposal_oracle_utility_at_8": 0.5703969299793243,
979
+ "proposal_oracle_utility_gain_over_base_at_8": 0.022872742265462875,
980
+ "selected_success_at_8": 0.25,
981
+ "selected_success_gain_over_base_at_8": 0.0,
982
+ "selected_utility_at_8": 0.554436425678432,
983
+ "selected_utility_gain_over_base_at_8": 0.006912237964570522,
984
  "selector_regret_at_8": 0.015960504300892353,
985
+ "success_selector_gap_at_8": 0.0,
986
+ "success_support_gap_at_8": 0.0,
987
+ "success_total_car_to_hidden_at_8": 0.0,
988
+ "support_gap_at_8": 0.05905248410999775,
989
+ "total_car_to_hidden_at_8": 0.0750129884108901
990
  },
991
  "77147264": {
992
+ "base_success": 0.5,
993
+ "base_utility": 1.4215794801712036,
994
  "branch_car_at_8": 1.868594542145729,
995
+ "hidden_chart_oracle_success_at_8": 1.0,
996
+ "hidden_chart_oracle_utility_at_8": 2.0,
997
  "outcome_ptr_at_8": 0.5,
998
  "pairwise_causal_calibration_ece": 0.06512707818376468,
999
+ "proposal_oracle_success_at_8": 1.0,
1000
+ "proposal_oracle_success_gain_over_base_at_8": 0.5,
1001
+ "proposal_oracle_utility_at_8": 2.0,
1002
+ "proposal_oracle_utility_gain_over_base_at_8": 0.5784205198287964,
1003
+ "selected_success_at_8": 0.0,
1004
+ "selected_success_gain_over_base_at_8": -0.5,
1005
+ "selected_utility_at_8": 0.13140545785427094,
1006
+ "selected_utility_gain_over_base_at_8": -1.2901740223169327,
1007
  "selector_regret_at_8": 1.868594542145729,
1008
+ "success_selector_gap_at_8": 1.0,
1009
+ "success_support_gap_at_8": 0.0,
1010
+ "success_total_car_to_hidden_at_8": 1.0,
1011
+ "support_gap_at_8": 0.0,
1012
+ "total_car_to_hidden_at_8": 1.868594542145729
1013
  },
1014
  "794181626": {
1015
+ "base_success": 0.0,
1016
+ "base_utility": 0.1733371466398239,
1017
  "branch_car_at_8": 0.0,
1018
+ "hidden_chart_oracle_success_at_8": 1.0,
1019
+ "hidden_chart_oracle_utility_at_8": 2.0,
1020
  "outcome_ptr_at_8": 1.0,
1021
  "pairwise_causal_calibration_ece": 0.12119924562800825,
1022
+ "proposal_oracle_success_at_8": 1.0,
1023
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
1024
+ "proposal_oracle_utility_at_8": 2.0,
1025
+ "proposal_oracle_utility_gain_over_base_at_8": 1.826662853360176,
1026
+ "selected_success_at_8": 1.0,
1027
+ "selected_success_gain_over_base_at_8": 1.0,
1028
+ "selected_utility_at_8": 2.0,
1029
+ "selected_utility_gain_over_base_at_8": 1.826662853360176,
1030
  "selector_regret_at_8": 0.0,
1031
+ "success_selector_gap_at_8": 0.0,
1032
+ "success_support_gap_at_8": 0.0,
1033
+ "success_total_car_to_hidden_at_8": 0.0,
1034
+ "support_gap_at_8": 0.0,
1035
+ "total_car_to_hidden_at_8": 0.0
1036
  },
1037
  "957101489": {
1038
+ "base_success": 0.0,
1039
+ "base_utility": 0.2039649486541748,
1040
  "branch_car_at_8": 0.0,
1041
+ "hidden_chart_oracle_success_at_8": 1.0,
1042
+ "hidden_chart_oracle_utility_at_8": 2.0,
1043
  "outcome_ptr_at_8": 1.0,
1044
  "pairwise_causal_calibration_ece": 0.20250855048512742,
1045
+ "proposal_oracle_success_at_8": 0.0,
1046
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
1047
+ "proposal_oracle_utility_at_8": 0.5881698727607727,
1048
+ "proposal_oracle_utility_gain_over_base_at_8": 0.3842049241065979,
1049
+ "selected_success_at_8": 0.0,
1050
+ "selected_success_gain_over_base_at_8": 0.0,
1051
+ "selected_utility_at_8": 0.5881698727607727,
1052
+ "selected_utility_gain_over_base_at_8": 0.3842049241065979,
1053
  "selector_regret_at_8": 0.0,
1054
+ "success_selector_gap_at_8": 0.0,
1055
+ "success_support_gap_at_8": 1.0,
1056
+ "success_total_car_to_hidden_at_8": 1.0,
1057
+ "support_gap_at_8": 1.4118301272392273,
1058
+ "total_car_to_hidden_at_8": 1.4118301272392273
1059
  },
1060
  "998761162": {
1061
+ "base_success": 0.0,
1062
+ "base_utility": 0.0689050555229187,
1063
  "branch_car_at_8": 0.07749588787555695,
1064
+ "hidden_chart_oracle_success_at_8": 1.0,
1065
+ "hidden_chart_oracle_utility_at_8": 2.0,
1066
  "outcome_ptr_at_8": 1.0,
1067
  "pairwise_causal_calibration_ece": 0.23606164078349207,
1068
+ "proposal_oracle_success_at_8": 0.0,
1069
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
1070
+ "proposal_oracle_utility_at_8": 0.08403940498828888,
1071
+ "proposal_oracle_utility_gain_over_base_at_8": 0.015134349465370178,
1072
+ "selected_success_at_8": 0.0,
1073
+ "selected_success_gain_over_base_at_8": 0.0,
1074
+ "selected_utility_at_8": 0.006543517112731934,
1075
+ "selected_utility_gain_over_base_at_8": -0.06236153841018677,
1076
  "selector_regret_at_8": 0.07749588787555695,
1077
+ "success_selector_gap_at_8": 0.0,
1078
+ "success_support_gap_at_8": 1.0,
1079
+ "success_total_car_to_hidden_at_8": 1.0,
1080
+ "support_gap_at_8": 1.9159605950117111,
1081
+ "total_car_to_hidden_at_8": 1.993456482887268
1082
  }
1083
  }
workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics_by_task.json CHANGED
@@ -1,37 +1,117 @@
1
  {
2
  "LiftPegUpright-v1": {
 
 
3
  "branch_car_at_8": 0.19493118425210318,
 
 
4
  "outcome_ptr_at_8": 0.3333333333333333,
5
  "pairwise_causal_calibration_ece": 0.1098535159049614,
 
 
 
 
 
 
 
 
6
  "selector_regret_at_8": 0.19493118425210318,
7
- "support_gap_at_8": 0.9025717824697495
 
 
 
 
8
  },
9
  "PickCube-v1": {
 
 
10
  "branch_car_at_8": 0.0,
 
 
11
  "outcome_ptr_at_8": 0.0,
12
  "pairwise_causal_calibration_ece": 0.15040783499409596,
 
 
 
 
 
 
 
 
13
  "selector_regret_at_8": 0.0,
14
- "support_gap_at_8": 1.8672154396772385
 
 
 
 
15
  },
16
  "PullCube-v1": {
 
 
17
  "branch_car_at_8": 0.08066527944003396,
 
 
18
  "outcome_ptr_at_8": 0.3235294117647059,
19
  "pairwise_causal_calibration_ece": 0.17256325415222457,
 
 
 
 
 
 
 
 
20
  "selector_regret_at_8": 0.08066527944003396,
21
- "support_gap_at_8": 0.1962626721634639
 
 
 
 
22
  },
23
  "PushCube-v1": {
 
 
24
  "branch_car_at_8": 0.31651630768409145,
 
 
25
  "outcome_ptr_at_8": 0.6153846153846154,
26
  "pairwise_causal_calibration_ece": 0.13663407835877941,
 
 
 
 
 
 
 
 
27
  "selector_regret_at_8": 0.31651630768409145,
28
- "support_gap_at_8": 0.593833287174885
 
 
 
 
29
  },
30
  "StackCube-v1": {
 
 
31
  "branch_car_at_8": 0.6326439549525579,
 
 
32
  "outcome_ptr_at_8": 0.6,
33
  "pairwise_causal_calibration_ece": 0.128912518811403,
 
 
 
 
 
 
 
 
34
  "selector_regret_at_8": 0.6326439549525579,
35
- "support_gap_at_8": 0.904726724823316
 
 
 
 
36
  }
37
  }
 
1
  {
2
  "LiftPegUpright-v1": {
3
+ "base_success": 0.3333333333333333,
4
+ "base_utility": 1.0669441471497219,
5
  "branch_car_at_8": 0.19493118425210318,
6
+ "hidden_chart_oracle_success_at_8": 1.0,
7
+ "hidden_chart_oracle_utility_at_8": 2.0,
8
  "outcome_ptr_at_8": 0.3333333333333333,
9
  "pairwise_causal_calibration_ece": 0.1098535159049614,
10
+ "proposal_oracle_success_at_8": 0.3333333333333333,
11
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
12
+ "proposal_oracle_utility_at_8": 1.0974282175302505,
13
+ "proposal_oracle_utility_gain_over_base_at_8": 0.030484070380528767,
14
+ "selected_success_at_8": 0.3333333333333333,
15
+ "selected_success_gain_over_base_at_8": 0.0,
16
+ "selected_utility_at_8": 0.9024970332781473,
17
+ "selected_utility_gain_over_base_at_8": -0.1644471138715744,
18
  "selector_regret_at_8": 0.19493118425210318,
19
+ "success_selector_gap_at_8": 0.0,
20
+ "success_support_gap_at_8": 0.6666666666666666,
21
+ "success_total_car_to_hidden_at_8": 0.6666666666666666,
22
+ "support_gap_at_8": 0.9025717824697495,
23
+ "total_car_to_hidden_at_8": 1.0975029667218525
24
  },
25
  "PickCube-v1": {
26
+ "base_success": 0.0,
27
+ "base_utility": 0.15342965722084045,
28
  "branch_car_at_8": 0.0,
29
+ "hidden_chart_oracle_success_at_8": 1.0,
30
+ "hidden_chart_oracle_utility_at_8": 2.0,
31
  "outcome_ptr_at_8": 0.0,
32
  "pairwise_causal_calibration_ece": 0.15040783499409596,
33
+ "proposal_oracle_success_at_8": 0.0,
34
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
35
+ "proposal_oracle_utility_at_8": 0.13278456032276154,
36
+ "proposal_oracle_utility_gain_over_base_at_8": -0.02064509689807892,
37
+ "selected_success_at_8": 0.0,
38
+ "selected_success_gain_over_base_at_8": 0.0,
39
+ "selected_utility_at_8": 0.13278456032276154,
40
+ "selected_utility_gain_over_base_at_8": -0.02064509689807892,
41
  "selector_regret_at_8": 0.0,
42
+ "success_selector_gap_at_8": 0.0,
43
+ "success_support_gap_at_8": 1.0,
44
+ "success_total_car_to_hidden_at_8": 1.0,
45
+ "support_gap_at_8": 1.8672154396772385,
46
+ "total_car_to_hidden_at_8": 1.8672154396772385
47
  },
48
  "PullCube-v1": {
49
+ "base_success": 0.29411764705882354,
50
+ "base_utility": 0.6512299207833541,
51
  "branch_car_at_8": 0.08066527944003396,
52
+ "hidden_chart_oracle_success_at_8": 0.3235294117647059,
53
+ "hidden_chart_oracle_utility_at_8": 0.7823218367993832,
54
  "outcome_ptr_at_8": 0.3235294117647059,
55
  "pairwise_causal_calibration_ece": 0.17256325415222457,
56
+ "proposal_oracle_success_at_8": 0.29411764705882354,
57
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
58
+ "proposal_oracle_utility_at_8": 0.6413554182680159,
59
+ "proposal_oracle_utility_gain_over_base_at_8": -0.009874502515338142,
60
+ "selected_success_at_8": 0.2647058823529412,
61
+ "selected_success_gain_over_base_at_8": -0.029411764705882353,
62
+ "selected_utility_at_8": 0.560690138827982,
63
+ "selected_utility_gain_over_base_at_8": -0.0905397819553721,
64
  "selector_regret_at_8": 0.08066527944003396,
65
+ "success_selector_gap_at_8": 0.029411764705882353,
66
+ "success_support_gap_at_8": 0.058823529411764705,
67
+ "success_total_car_to_hidden_at_8": 0.08823529411764706,
68
+ "support_gap_at_8": 0.1962626721634639,
69
+ "total_car_to_hidden_at_8": 0.2751456986571772
70
  },
71
  "PushCube-v1": {
72
+ "base_success": 0.3076923076923077,
73
+ "base_utility": 0.6686872587754176,
74
  "branch_car_at_8": 0.31651630768409145,
75
+ "hidden_chart_oracle_success_at_8": 1.0,
76
+ "hidden_chart_oracle_utility_at_8": 2.0,
77
  "outcome_ptr_at_8": 0.6153846153846154,
78
  "pairwise_causal_calibration_ece": 0.13663407835877941,
79
+ "proposal_oracle_success_at_8": 0.6923076923076923,
80
+ "proposal_oracle_success_gain_over_base_at_8": 0.38461538461538464,
81
+ "proposal_oracle_utility_at_8": 1.406166712825115,
82
+ "proposal_oracle_utility_gain_over_base_at_8": 0.7374794540496973,
83
+ "selected_success_at_8": 0.5384615384615384,
84
+ "selected_success_gain_over_base_at_8": 0.23076923076923078,
85
+ "selected_utility_at_8": 1.0896504051410234,
86
+ "selected_utility_gain_over_base_at_8": 0.4209631463656059,
87
  "selector_regret_at_8": 0.31651630768409145,
88
+ "success_selector_gap_at_8": 0.15384615384615385,
89
+ "success_support_gap_at_8": 0.3076923076923077,
90
+ "success_total_car_to_hidden_at_8": 0.46153846153846156,
91
+ "support_gap_at_8": 0.593833287174885,
92
+ "total_car_to_hidden_at_8": 0.9103495948589765
93
  },
94
  "StackCube-v1": {
95
+ "base_success": 0.2,
96
+ "base_utility": 0.8017999569574992,
97
  "branch_car_at_8": 0.6326439549525579,
98
+ "hidden_chart_oracle_success_at_8": 1.0,
99
+ "hidden_chart_oracle_utility_at_8": 2.0,
100
  "outcome_ptr_at_8": 0.6,
101
  "pairwise_causal_calibration_ece": 0.128912518811403,
102
+ "proposal_oracle_success_at_8": 0.4,
103
+ "proposal_oracle_success_gain_over_base_at_8": 0.2,
104
+ "proposal_oracle_utility_at_8": 1.095273275176684,
105
+ "proposal_oracle_utility_gain_over_base_at_8": 0.2934733182191849,
106
+ "selected_success_at_8": 0.06666666666666667,
107
+ "selected_success_gain_over_base_at_8": -0.13333333333333333,
108
+ "selected_utility_at_8": 0.4626293202241262,
109
+ "selected_utility_gain_over_base_at_8": -0.339170636733373,
110
  "selector_regret_at_8": 0.6326439549525579,
111
+ "success_selector_gap_at_8": 0.3333333333333333,
112
+ "success_support_gap_at_8": 0.6,
113
+ "success_total_car_to_hidden_at_8": 0.9333333333333333,
114
+ "support_gap_at_8": 0.904726724823316,
115
+ "total_car_to_hidden_at_8": 1.5373706797758737
116
  }
117
  }
workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/report.md CHANGED
@@ -4,8 +4,24 @@ K: `8`
4
 
5
  | Metric | N | Micro mean | 95% CI | Task macro | Seed macro |
6
  | --- | ---: | ---: | ---: | ---: | ---: |
 
 
7
  | branch_car_at_8 | 69 | 0.2539 | [0.1317, 0.3789] | 0.2450 | 0.3214 |
 
 
8
  | outcome_ptr_at_8 | 69 | 0.4348 | [0.3188, 0.5507] | 0.3744 | 0.4631 |
9
  | pairwise_causal_calibration_ece | 69 | 0.1505 | [0.1261, 0.1764] | 0.1397 | 0.1459 |
 
 
 
 
 
 
 
 
10
  | selector_regret_at_8 | 69 | 0.2539 | [0.1317, 0.3789] | 0.2450 | 0.3214 |
 
 
 
11
  | support_gap_at_8 | 69 | 0.5108 | [0.3299, 0.6784] | 0.8929 | 0.6654 |
 
 
4
 
5
  | Metric | N | Micro mean | 95% CI | Task macro | Seed macro |
6
  | --- | ---: | ---: | ---: | ---: | ---: |
7
+ | base_success | 69 | 0.2754 | [0.1739, 0.3913] | 0.2270 | 0.3085 |
8
+ | base_utility | 69 | 0.7162 | [0.5461, 0.9125] | 0.6684 | 0.8136 |
9
  | branch_car_at_8 | 69 | 0.2539 | [0.1317, 0.3789] | 0.2450 | 0.3214 |
10
+ | hidden_chart_oracle_success_at_8 | 69 | 0.6667 | [0.5507, 0.7826] | 0.8647 | 0.8358 |
11
+ | hidden_chart_oracle_utility_at_8 | 69 | 1.4000 | [1.1855, 1.6062] | 1.7565 | 1.7062 |
12
  | outcome_ptr_at_8 | 69 | 0.4348 | [0.3188, 0.5507] | 0.3744 | 0.4631 |
13
  | pairwise_causal_calibration_ece | 69 | 0.1505 | [0.1261, 0.1764] | 0.1397 | 0.1459 |
14
+ | proposal_oracle_success_at_8 | 69 | 0.3913 | [0.2899, 0.5217] | 0.3440 | 0.4404 |
15
+ | proposal_oracle_success_gain_over_base_at_8 | 69 | 0.1159 | [0.0145, 0.2174] | 0.1169 | 0.1319 |
16
+ | proposal_oracle_utility_at_8 | 69 | 0.9164 | [0.7319, 1.1490] | 0.8746 | 1.0508 |
17
+ | proposal_oracle_utility_gain_over_base_at_8 | 69 | 0.2002 | [0.0165, 0.3763] | 0.2062 | 0.2372 |
18
+ | selected_success_at_8 | 69 | 0.2754 | [0.1884, 0.3768] | 0.2406 | 0.2915 |
19
+ | selected_success_gain_over_base_at_8 | 69 | 0.0000 | [-0.1014, 0.1159] | 0.0136 | -0.0170 |
20
+ | selected_utility_at_8 | 69 | 0.6626 | [0.4843, 0.8693] | 0.6297 | 0.7294 |
21
+ | selected_utility_gain_over_base_at_8 | 69 | -0.0536 | [-0.2450, 0.1422] | -0.0388 | -0.0842 |
22
  | selector_regret_at_8 | 69 | 0.2539 | [0.1317, 0.3789] | 0.2450 | 0.3214 |
23
+ | success_selector_gap_at_8 | 69 | 0.1159 | [0.0435, 0.1884] | 0.1033 | 0.1489 |
24
+ | success_support_gap_at_8 | 69 | 0.2899 | [0.1739, 0.3913] | 0.5266 | 0.4007 |
25
+ | success_total_car_to_hidden_at_8 | 69 | 0.4058 | [0.2754, 0.5217] | 0.6300 | 0.5496 |
26
  | support_gap_at_8 | 69 | 0.5108 | [0.3299, 0.6784] | 0.8929 | 0.6654 |
27
+ | total_car_to_hidden_at_8 | 69 | 0.7638 | [0.5655, 0.9371] | 1.1375 | 0.9865 |
workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/table.tex CHANGED
@@ -3,10 +3,26 @@
3
  \toprule
4
  Metric & N & Micro mean & CI low & CI high \\
5
  \midrule
 
 
6
  branch\_car\_at\_8 & 69 & 0.2539 & 0.1317 & 0.3789 \\
 
 
7
  outcome\_ptr\_at\_8 & 69 & 0.4348 & 0.3188 & 0.5507 \\
8
  pairwise\_causal\_calibration\_ece & 69 & 0.1505 & 0.1261 & 0.1764 \\
 
 
 
 
 
 
 
 
9
  selector\_regret\_at\_8 & 69 & 0.2539 & 0.1317 & 0.3789 \\
 
 
 
10
  support\_gap\_at\_8 & 69 & 0.5108 & 0.3299 & 0.6784 \\
 
11
  \bottomrule
12
  \end{tabular}
 
3
  \toprule
4
  Metric & N & Micro mean & CI low & CI high \\
5
  \midrule
6
+ base\_success & 69 & 0.2754 & 0.1739 & 0.3913 \\
7
+ base\_utility & 69 & 0.7162 & 0.5461 & 0.9125 \\
8
  branch\_car\_at\_8 & 69 & 0.2539 & 0.1317 & 0.3789 \\
9
+ hidden\_chart\_oracle\_success\_at\_8 & 69 & 0.6667 & 0.5507 & 0.7826 \\
10
+ hidden\_chart\_oracle\_utility\_at\_8 & 69 & 1.4000 & 1.1855 & 1.6062 \\
11
  outcome\_ptr\_at\_8 & 69 & 0.4348 & 0.3188 & 0.5507 \\
12
  pairwise\_causal\_calibration\_ece & 69 & 0.1505 & 0.1261 & 0.1764 \\
13
+ proposal\_oracle\_success\_at\_8 & 69 & 0.3913 & 0.2899 & 0.5217 \\
14
+ proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 69 & 0.1159 & 0.0145 & 0.2174 \\
15
+ proposal\_oracle\_utility\_at\_8 & 69 & 0.9164 & 0.7319 & 1.1490 \\
16
+ proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 69 & 0.2002 & 0.0165 & 0.3763 \\
17
+ selected\_success\_at\_8 & 69 & 0.2754 & 0.1884 & 0.3768 \\
18
+ selected\_success\_gain\_over\_base\_at\_8 & 69 & 0.0000 & -0.1014 & 0.1159 \\
19
+ selected\_utility\_at\_8 & 69 & 0.6626 & 0.4843 & 0.8693 \\
20
+ selected\_utility\_gain\_over\_base\_at\_8 & 69 & -0.0536 & -0.2450 & 0.1422 \\
21
  selector\_regret\_at\_8 & 69 & 0.2539 & 0.1317 & 0.3789 \\
22
+ success\_selector\_gap\_at\_8 & 69 & 0.1159 & 0.0435 & 0.1884 \\
23
+ success\_support\_gap\_at\_8 & 69 & 0.2899 & 0.1739 & 0.3913 \\
24
+ success\_total\_car\_to\_hidden\_at\_8 & 69 & 0.4058 & 0.2754 & 0.5217 \\
25
  support\_gap\_at\_8 & 69 & 0.5108 & 0.3299 & 0.6784 \\
26
+ total\_car\_to\_hidden\_at\_8 & 69 & 0.7638 & 0.5655 & 0.9371 \\
27
  \bottomrule
28
  \end{tabular}
workspace/runs/ctt_residual_rollout_val69_seed0/metrics.json ADDED
@@ -0,0 +1,1918 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "runs/ctt_residual_full_seed0/model.pt",
3
+ "k": 8,
4
+ "measured_metric_summary": {
5
+ "base_success": {
6
+ "macro_by_seed": {
7
+ "ci": {
8
+ "confidence": 0.95,
9
+ "high": 0.4397163120567376,
10
+ "low": 0.18794326241134748,
11
+ "mean": 0.3085106382978724,
12
+ "n": 47,
13
+ "num_samples": 500
14
+ },
15
+ "mean": 0.3085106382978724,
16
+ "per_seed": {
17
+ "1025740212": 0.0,
18
+ "1055157354": 0.0,
19
+ "1055567618": 1.0,
20
+ "1113268940": 0.0,
21
+ "1113371261": 1.0,
22
+ "1176943004": 0.0,
23
+ "1189392917": 0.0,
24
+ "1231780997": 0.0,
25
+ "1271718007": 0.5,
26
+ "1273247072": 0.0,
27
+ "1358421325": 0.0,
28
+ "1375492786": 1.0,
29
+ "141984615": 0.0,
30
+ "1475440973": 1.0,
31
+ "1483645336": 0.0,
32
+ "1522272686": 0.0,
33
+ "1528740417": 1.0,
34
+ "1549324805": 0.6666666666666666,
35
+ "1607773860": 1.0,
36
+ "1665739511": 1.0,
37
+ "1747178030": 0.0,
38
+ "1881223807": 0.0,
39
+ "1919198188": 0.0,
40
+ "1991414747": 1.0,
41
+ "1997090716": 0.0,
42
+ "2015956957": 0.0,
43
+ "2054899126": 1.0,
44
+ "211245842": 0.0,
45
+ "2141802816": 0.0,
46
+ "256770874": 0.0,
47
+ "278008524": 0.0,
48
+ "416183106": 0.0,
49
+ "534717141": 1.0,
50
+ "551504635": 0.0,
51
+ "565369392": 0.3333333333333333,
52
+ "565621826": 1.0,
53
+ "585904130": 0.0,
54
+ "626486813": 0.25,
55
+ "636680137": 1.0,
56
+ "706479600": 0.0,
57
+ "750448296": 0.0,
58
+ "75070641": 0.0,
59
+ "767747424": 0.25,
60
+ "77147264": 0.5,
61
+ "794181626": 0.0,
62
+ "957101489": 0.0,
63
+ "998761162": 0.0
64
+ }
65
+ },
66
+ "macro_by_task": {
67
+ "ci": {
68
+ "confidence": 0.95,
69
+ "high": 0.3125188536953243,
70
+ "low": 0.10153846153846154,
71
+ "mean": 0.2270286576168929,
72
+ "n": 5,
73
+ "num_samples": 500
74
+ },
75
+ "mean": 0.2270286576168929,
76
+ "per_task": {
77
+ "LiftPegUpright-v1": 0.3333333333333333,
78
+ "PickCube-v1": 0.0,
79
+ "PullCube-v1": 0.29411764705882354,
80
+ "PushCube-v1": 0.3076923076923077,
81
+ "StackCube-v1": 0.2
82
+ }
83
+ },
84
+ "metric": "base_success",
85
+ "micro": {
86
+ "confidence": 0.95,
87
+ "high": 0.391304347826087,
88
+ "low": 0.17391304347826086,
89
+ "mean": 0.2753623188405797,
90
+ "n": 69,
91
+ "num_samples": 500
92
+ },
93
+ "n": 69
94
+ },
95
+ "base_utility": {
96
+ "macro_by_seed": {
97
+ "ci": {
98
+ "confidence": 0.95,
99
+ "high": 1.0788229198548414,
100
+ "low": 0.582660698415117,
101
+ "mean": 0.8136041222055977,
102
+ "n": 47,
103
+ "num_samples": 500
104
+ },
105
+ "mean": 0.8136041222055977,
106
+ "per_seed": {
107
+ "1025740212": 0.528373658657074,
108
+ "1055157354": 0.17649109661579132,
109
+ "1055567618": 2.0,
110
+ "1113268940": 0.42231252789497375,
111
+ "1113371261": 2.0,
112
+ "1176943004": 0.05094052851200104,
113
+ "1189392917": 0.10841864347457886,
114
+ "1231780997": 0.7270061373710632,
115
+ "1271718007": 1.4142604768276215,
116
+ "1273247072": 0.15342965722084045,
117
+ "1358421325": 0.8327022790908813,
118
+ "1375492786": 2.0,
119
+ "141984615": 0.09381213908394177,
120
+ "1475440973": 2.0,
121
+ "1483645336": 0.01641605794429779,
122
+ "1522272686": 0.19476109743118286,
123
+ "1528740417": 2.0,
124
+ "1549324805": 1.3787474483251572,
125
+ "1607773860": 2.0,
126
+ "1665739511": 2.0,
127
+ "1747178030": 0.1463065892457962,
128
+ "1881223807": 0.0320269912481308,
129
+ "1919198188": 0.037198856472969055,
130
+ "1991414747": 2.0,
131
+ "1997090716": 0.13927479088306427,
132
+ "2015956957": 0.06711978986859321,
133
+ "2054899126": 2.0,
134
+ "211245842": 0.8388344049453735,
135
+ "2141802816": 0.07080724835395813,
136
+ "256770874": 0.8557480573654175,
137
+ "278008524": 0.10747186839580536,
138
+ "416183106": 0.20032641291618347,
139
+ "534717141": 2.0,
140
+ "551504635": 0.7239725589752197,
141
+ "565369392": 0.7997656464576721,
142
+ "565621826": 2.0,
143
+ "585904130": 0.07529209802548091,
144
+ "626486813": 0.5054400166263804,
145
+ "636680137": 2.0,
146
+ "706479600": 0.8125406503677368,
147
+ "750448296": 0.11153355985879898,
148
+ "75070641": 0.20275163650512695,
149
+ "767747424": 0.5475241877138615,
150
+ "77147264": 1.4215794801712036,
151
+ "794181626": 0.1733371466398239,
152
+ "957101489": 0.2039649486541748,
153
+ "998761162": 0.0689050555229187
154
+ }
155
+ },
156
+ "macro_by_task": {
157
+ "ci": {
158
+ "confidence": 0.95,
159
+ "high": 0.9041499242015874,
160
+ "low": 0.38615523747908764,
161
+ "mean": 0.6684181881773666,
162
+ "n": 5,
163
+ "num_samples": 500
164
+ },
165
+ "mean": 0.6684181881773666,
166
+ "per_task": {
167
+ "LiftPegUpright-v1": 1.0669441471497219,
168
+ "PickCube-v1": 0.15342965722084045,
169
+ "PullCube-v1": 0.6512299207833541,
170
+ "PushCube-v1": 0.6686872587754176,
171
+ "StackCube-v1": 0.8017999569574992
172
+ }
173
+ },
174
+ "metric": "base_utility",
175
+ "micro": {
176
+ "confidence": 0.95,
177
+ "high": 0.9124907980254595,
178
+ "low": 0.5461467205060889,
179
+ "mean": 0.7161861676115381,
180
+ "n": 69,
181
+ "num_samples": 500
182
+ },
183
+ "n": 69
184
+ },
185
+ "branch_car_at_8": {
186
+ "macro_by_seed": {
187
+ "ci": {
188
+ "confidence": 0.95,
189
+ "high": 0.5197332343231088,
190
+ "low": 0.16253579634643023,
191
+ "mean": 0.321408476649886,
192
+ "n": 47,
193
+ "num_samples": 500
194
+ },
195
+ "mean": 0.321408476649886,
196
+ "per_seed": {
197
+ "1025740212": 0.26188503205776215,
198
+ "1055157354": 1.8516457676887512,
199
+ "1055567618": 1.183156132698059,
200
+ "1113268940": 0.38470593094825745,
201
+ "1113371261": 0.0,
202
+ "1176943004": 0.0,
203
+ "1189392917": 0.0,
204
+ "1231780997": 0.4897426515817642,
205
+ "1271718007": 0.0,
206
+ "1273247072": 0.0,
207
+ "1358421325": 0.5775325745344162,
208
+ "1375492786": 0.0002854267368093133,
209
+ "141984615": 0.03316789741317431,
210
+ "1475440973": 0.0,
211
+ "1483645336": 0.0,
212
+ "1522272686": 0.009524375200271606,
213
+ "1528740417": 0.0,
214
+ "1549324805": 0.0034310482442378998,
215
+ "1607773860": 1.779732346534729,
216
+ "1665739511": 0.007551252841949463,
217
+ "1747178030": 0.0,
218
+ "1881223807": 0.020236089825630188,
219
+ "1919198188": 0.016854479908943176,
220
+ "1991414747": 0.0,
221
+ "1997090716": 1.9691698104143143,
222
+ "2015956957": 0.01770364921540022,
223
+ "2054899126": 0.0,
224
+ "211245842": 0.5908347219228745,
225
+ "2141802816": 0.07543906569480896,
226
+ "256770874": 0.10820980370044708,
227
+ "278008524": 0.0016456245793960989,
228
+ "416183106": 0.3360496461391449,
229
+ "534717141": 1.9274959191679955,
230
+ "551504635": 0.033253490924835205,
231
+ "565369392": 0.06443698455890019,
232
+ "565621826": 0.0,
233
+ "585904130": 0.043168901621053614,
234
+ "626486813": 0.018570441869087517,
235
+ "636680137": 0.0,
236
+ "706479600": 1.1865743398666382,
237
+ "750448296": 0.07732226233929396,
238
+ "75070641": 0.07482179999351501,
239
+ "767747424": 0.015960504300892353,
240
+ "77147264": 1.868594542145729,
241
+ "794181626": 0.0,
242
+ "957101489": 0.0,
243
+ "998761162": 0.07749588787555695
244
+ }
245
+ },
246
+ "macro_by_task": {
247
+ "ci": {
248
+ "confidence": 0.95,
249
+ "high": 0.5061151639620463,
250
+ "low": 0.07125234862643422,
251
+ "mean": 0.24495134526575732,
252
+ "n": 5,
253
+ "num_samples": 500
254
+ },
255
+ "mean": 0.24495134526575732,
256
+ "per_task": {
257
+ "LiftPegUpright-v1": 0.19493118425210318,
258
+ "PickCube-v1": 0.0,
259
+ "PullCube-v1": 0.08066527944003396,
260
+ "PushCube-v1": 0.31651630768409145,
261
+ "StackCube-v1": 0.6326439549525579
262
+ }
263
+ },
264
+ "metric": "branch_car_at_8",
265
+ "micro": {
266
+ "confidence": 0.95,
267
+ "high": 0.378890726060701,
268
+ "low": 0.13165242864372398,
269
+ "mean": 0.2538634482703671,
270
+ "n": 69,
271
+ "num_samples": 500
272
+ },
273
+ "n": 69
274
+ },
275
+ "hidden_chart_oracle_success_at_8": {
276
+ "macro_by_seed": {
277
+ "ci": {
278
+ "confidence": 0.95,
279
+ "high": 0.922695035460993,
280
+ "low": 0.7351063829787235,
281
+ "mean": 0.8358156028368794,
282
+ "n": 47,
283
+ "num_samples": 500
284
+ },
285
+ "mean": 0.8358156028368794,
286
+ "per_seed": {
287
+ "1025740212": 1.0,
288
+ "1055157354": 1.0,
289
+ "1055567618": 1.0,
290
+ "1113268940": 1.0,
291
+ "1113371261": 1.0,
292
+ "1176943004": 1.0,
293
+ "1189392917": 1.0,
294
+ "1231780997": 1.0,
295
+ "1271718007": 1.0,
296
+ "1273247072": 1.0,
297
+ "1358421325": 1.0,
298
+ "1375492786": 0.0,
299
+ "141984615": 0.0,
300
+ "1475440973": 1.0,
301
+ "1483645336": 1.0,
302
+ "1522272686": 1.0,
303
+ "1528740417": 1.0,
304
+ "1549324805": 0.6666666666666666,
305
+ "1607773860": 1.0,
306
+ "1665739511": 1.0,
307
+ "1747178030": 1.0,
308
+ "1881223807": 1.0,
309
+ "1919198188": 1.0,
310
+ "1991414747": 1.0,
311
+ "1997090716": 1.0,
312
+ "2015956957": 0.2,
313
+ "2054899126": 1.0,
314
+ "211245842": 1.0,
315
+ "2141802816": 1.0,
316
+ "256770874": 1.0,
317
+ "278008524": 0.5,
318
+ "416183106": 1.0,
319
+ "534717141": 1.0,
320
+ "551504635": 1.0,
321
+ "565369392": 0.6666666666666666,
322
+ "565621826": 1.0,
323
+ "585904130": 0.0,
324
+ "626486813": 0.25,
325
+ "636680137": 1.0,
326
+ "706479600": 1.0,
327
+ "750448296": 0.0,
328
+ "75070641": 1.0,
329
+ "767747424": 0.0,
330
+ "77147264": 1.0,
331
+ "794181626": 1.0,
332
+ "957101489": 1.0,
333
+ "998761162": 1.0
334
+ }
335
+ },
336
+ "macro_by_task": {
337
+ "ci": {
338
+ "confidence": 0.95,
339
+ "high": 1.0,
340
+ "low": 0.5941176470588235,
341
+ "mean": 0.8647058823529413,
342
+ "n": 5,
343
+ "num_samples": 500
344
+ },
345
+ "mean": 0.8647058823529413,
346
+ "per_task": {
347
+ "LiftPegUpright-v1": 1.0,
348
+ "PickCube-v1": 1.0,
349
+ "PullCube-v1": 0.3235294117647059,
350
+ "PushCube-v1": 1.0,
351
+ "StackCube-v1": 1.0
352
+ }
353
+ },
354
+ "metric": "hidden_chart_oracle_success_at_8",
355
+ "micro": {
356
+ "confidence": 0.95,
357
+ "high": 0.782608695652174,
358
+ "low": 0.5507246376811594,
359
+ "mean": 0.6666666666666666,
360
+ "n": 69,
361
+ "num_samples": 500
362
+ },
363
+ "n": 69
364
+ },
365
+ "hidden_chart_oracle_utility_at_8": {
366
+ "macro_by_seed": {
367
+ "ci": {
368
+ "confidence": 0.95,
369
+ "high": 1.8615907955016442,
370
+ "low": 1.529367521088174,
371
+ "mean": 1.7062028111638965,
372
+ "n": 47,
373
+ "num_samples": 500
374
+ },
375
+ "mean": 1.7062028111638965,
376
+ "per_seed": {
377
+ "1025740212": 2.0,
378
+ "1055157354": 2.0,
379
+ "1055567618": 2.0,
380
+ "1113268940": 2.0,
381
+ "1113371261": 2.0,
382
+ "1176943004": 2.0,
383
+ "1189392917": 2.0,
384
+ "1231780997": 2.0,
385
+ "1271718007": 2.0,
386
+ "1273247072": 2.0,
387
+ "1358421325": 2.0,
388
+ "1375492786": 0.24345096945762634,
389
+ "141984615": 0.20044784247875214,
390
+ "1475440973": 2.0,
391
+ "1483645336": 2.0,
392
+ "1522272686": 2.0,
393
+ "1528740417": 2.0,
394
+ "1549324805": 1.4027505218982697,
395
+ "1607773860": 2.0,
396
+ "1665739511": 2.0,
397
+ "1747178030": 2.0,
398
+ "1881223807": 2.0,
399
+ "1919198188": 2.0,
400
+ "1991414747": 2.0,
401
+ "1997090716": 2.0,
402
+ "2015956957": 0.5298813670873642,
403
+ "2054899126": 2.0,
404
+ "211245842": 2.0,
405
+ "2141802816": 2.0,
406
+ "256770874": 2.0,
407
+ "278008524": 1.113811194896698,
408
+ "416183106": 2.0,
409
+ "534717141": 2.0,
410
+ "551504635": 2.0,
411
+ "565369392": 1.4266315797964733,
412
+ "565621826": 2.0,
413
+ "585904130": 0.23764699200789133,
414
+ "626486813": 0.6124684102833271,
415
+ "636680137": 2.0,
416
+ "706479600": 2.0,
417
+ "750448296": 0.23580759018659592,
418
+ "75070641": 2.0,
419
+ "767747424": 0.18863565661013126,
420
+ "77147264": 2.0,
421
+ "794181626": 2.0,
422
+ "957101489": 2.0,
423
+ "998761162": 2.0
424
+ }
425
+ },
426
+ "macro_by_task": {
427
+ "ci": {
428
+ "confidence": 0.95,
429
+ "high": 2.0,
430
+ "low": 1.2693931020796299,
431
+ "mean": 1.7564643673598765,
432
+ "n": 5,
433
+ "num_samples": 500
434
+ },
435
+ "mean": 1.7564643673598765,
436
+ "per_task": {
437
+ "LiftPegUpright-v1": 2.0,
438
+ "PickCube-v1": 2.0,
439
+ "PullCube-v1": 0.7823218367993832,
440
+ "PushCube-v1": 2.0,
441
+ "StackCube-v1": 2.0
442
+ }
443
+ },
444
+ "metric": "hidden_chart_oracle_utility_at_8",
445
+ "micro": {
446
+ "confidence": 0.95,
447
+ "high": 1.6062410234109215,
448
+ "low": 1.1855228276572365,
449
+ "mean": 1.3999846732054932,
450
+ "n": 69,
451
+ "num_samples": 500
452
+ },
453
+ "n": 69
454
+ },
455
+ "outcome_ptr_at_8": {
456
+ "macro_by_seed": {
457
+ "ci": {
458
+ "confidence": 0.95,
459
+ "high": 0.5868794326241134,
460
+ "low": 0.3124113475177305,
461
+ "mean": 0.4631205673758865,
462
+ "n": 47,
463
+ "num_samples": 500
464
+ },
465
+ "mean": 0.4631205673758865,
466
+ "per_seed": {
467
+ "1025740212": 0.0,
468
+ "1055157354": 1.0,
469
+ "1055567618": 0.0,
470
+ "1113268940": 1.0,
471
+ "1113371261": 0.0,
472
+ "1176943004": 1.0,
473
+ "1189392917": 1.0,
474
+ "1231780997": 0.0,
475
+ "1271718007": 0.5,
476
+ "1273247072": 0.0,
477
+ "1358421325": 0.0,
478
+ "1375492786": 0.0,
479
+ "141984615": 0.3333333333333333,
480
+ "1475440973": 0.0,
481
+ "1483645336": 1.0,
482
+ "1522272686": 1.0,
483
+ "1528740417": 0.0,
484
+ "1549324805": 0.0,
485
+ "1607773860": 0.0,
486
+ "1665739511": 0.0,
487
+ "1747178030": 1.0,
488
+ "1881223807": 1.0,
489
+ "1919198188": 1.0,
490
+ "1991414747": 0.0,
491
+ "1997090716": 1.0,
492
+ "2015956957": 0.6,
493
+ "2054899126": 0.0,
494
+ "211245842": 0.0,
495
+ "2141802816": 1.0,
496
+ "256770874": 0.0,
497
+ "278008524": 0.0,
498
+ "416183106": 1.0,
499
+ "534717141": 0.0,
500
+ "551504635": 1.0,
501
+ "565369392": 0.0,
502
+ "565621826": 0.0,
503
+ "585904130": 0.3333333333333333,
504
+ "626486813": 0.75,
505
+ "636680137": 0.0,
506
+ "706479600": 1.0,
507
+ "750448296": 0.5,
508
+ "75070641": 1.0,
509
+ "767747424": 0.25,
510
+ "77147264": 0.5,
511
+ "794181626": 1.0,
512
+ "957101489": 1.0,
513
+ "998761162": 1.0
514
+ }
515
+ },
516
+ "macro_by_task": {
517
+ "ci": {
518
+ "confidence": 0.95,
519
+ "high": 0.555897435897436,
520
+ "low": 0.18666666666666668,
521
+ "mean": 0.3744494720965309,
522
+ "n": 5,
523
+ "num_samples": 500
524
+ },
525
+ "mean": 0.3744494720965309,
526
+ "per_task": {
527
+ "LiftPegUpright-v1": 0.3333333333333333,
528
+ "PickCube-v1": 0.0,
529
+ "PullCube-v1": 0.3235294117647059,
530
+ "PushCube-v1": 0.6153846153846154,
531
+ "StackCube-v1": 0.6
532
+ }
533
+ },
534
+ "metric": "outcome_ptr_at_8",
535
+ "micro": {
536
+ "confidence": 0.95,
537
+ "high": 0.5507246376811594,
538
+ "low": 0.3188405797101449,
539
+ "mean": 0.43478260869565216,
540
+ "n": 69,
541
+ "num_samples": 500
542
+ },
543
+ "n": 69
544
+ },
545
+ "pairwise_causal_calibration_ece": {
546
+ "macro_by_seed": {
547
+ "ci": {
548
+ "confidence": 0.95,
549
+ "high": 0.1773022288327493,
550
+ "low": 0.11479435257646622,
551
+ "mean": 0.14593359597440297,
552
+ "n": 47,
553
+ "num_samples": 500
554
+ },
555
+ "mean": 0.14593359597440297,
556
+ "per_seed": {
557
+ "1025740212": 0.08963387331564551,
558
+ "1055157354": 0.2973966626262834,
559
+ "1055567618": 0.07486610854656817,
560
+ "1113268940": 0.1616344352768096,
561
+ "1113371261": 0.3179396489681421,
562
+ "1176943004": 0.2176350935010736,
563
+ "1189392917": 0.1628408162009477,
564
+ "1231780997": 0.18322944379343714,
565
+ "1271718007": 0.07543261691159965,
566
+ "1273247072": 0.15040783499409596,
567
+ "1358421325": 0.13502516765102715,
568
+ "1375492786": 0.19928218350064877,
569
+ "141984615": 0.10926310602682258,
570
+ "1475440973": 0.09930428958964055,
571
+ "1483645336": 0.0,
572
+ "1522272686": 0.08257819446483214,
573
+ "1528740417": 0.015990866806051107,
574
+ "1549324805": 0.19382954770507574,
575
+ "1607773860": 0.08588083120555855,
576
+ "1665739511": 0.050940808914706356,
577
+ "1747178030": 0.15825371494561297,
578
+ "1881223807": 0.019060791998604134,
579
+ "1919198188": 0.15779226512372985,
580
+ "1991414747": 0.05287821364799471,
581
+ "1997090716": 0.4044963838715546,
582
+ "2015956957": 0.18664994005668376,
583
+ "2054899126": 0.0,
584
+ "211245842": 0.16771630653845432,
585
+ "2141802816": 0.05242798758366053,
586
+ "256770874": 0.030709639081801327,
587
+ "278008524": 0.11925245416072927,
588
+ "416183106": 0.10129212810935397,
589
+ "534717141": 0.6159370955924921,
590
+ "551504635": 0.10932818664818449,
591
+ "565369392": 0.22308845281795123,
592
+ "565621826": 0.0,
593
+ "585904130": 0.10391809443539522,
594
+ "626486813": 0.16162412565990822,
595
+ "636680137": 0.1699944666253151,
596
+ "706479600": 0.0922387871049479,
597
+ "750448296": 0.14360585133877676,
598
+ "75070641": 0.2497251397544613,
599
+ "767747424": 0.21088094062197144,
600
+ "77147264": 0.06512707818376468,
601
+ "794181626": 0.12119924562800825,
602
+ "957101489": 0.20250855048512742,
603
+ "998761162": 0.23606164078349207
604
+ }
605
+ },
606
+ "macro_by_task": {
607
+ "ci": {
608
+ "confidence": 0.95,
609
+ "high": 0.15940202325243455,
610
+ "low": 0.1233204922135519,
611
+ "mean": 0.13967424044429286,
612
+ "n": 5,
613
+ "num_samples": 500
614
+ },
615
+ "mean": 0.13967424044429286,
616
+ "per_task": {
617
+ "LiftPegUpright-v1": 0.1098535159049614,
618
+ "PickCube-v1": 0.15040783499409596,
619
+ "PullCube-v1": 0.17256325415222457,
620
+ "PushCube-v1": 0.13663407835877941,
621
+ "StackCube-v1": 0.128912518811403
622
+ }
623
+ },
624
+ "metric": "pairwise_causal_calibration_ece",
625
+ "micro": {
626
+ "confidence": 0.95,
627
+ "high": 0.17644750878364412,
628
+ "low": 0.12609473303549118,
629
+ "mean": 0.15053058510774892,
630
+ "n": 69,
631
+ "num_samples": 500
632
+ },
633
+ "n": 69
634
+ },
635
+ "proposal_oracle_success_at_8": {
636
+ "macro_by_seed": {
637
+ "ci": {
638
+ "confidence": 0.95,
639
+ "high": 0.5698581560283689,
640
+ "low": 0.29964539007092195,
641
+ "mean": 0.4404255319148936,
642
+ "n": 47,
643
+ "num_samples": 500
644
+ },
645
+ "mean": 0.4404255319148936,
646
+ "per_seed": {
647
+ "1025740212": 0.0,
648
+ "1055157354": 1.0,
649
+ "1055567618": 1.0,
650
+ "1113268940": 0.0,
651
+ "1113371261": 1.0,
652
+ "1176943004": 1.0,
653
+ "1189392917": 1.0,
654
+ "1231780997": 0.0,
655
+ "1271718007": 1.0,
656
+ "1273247072": 0.0,
657
+ "1358421325": 0.0,
658
+ "1375492786": 0.0,
659
+ "141984615": 0.0,
660
+ "1475440973": 1.0,
661
+ "1483645336": 1.0,
662
+ "1522272686": 0.0,
663
+ "1528740417": 1.0,
664
+ "1549324805": 0.6666666666666666,
665
+ "1607773860": 1.0,
666
+ "1665739511": 0.0,
667
+ "1747178030": 1.0,
668
+ "1881223807": 0.0,
669
+ "1919198188": 0.0,
670
+ "1991414747": 1.0,
671
+ "1997090716": 1.0,
672
+ "2015956957": 0.2,
673
+ "2054899126": 1.0,
674
+ "211245842": 0.0,
675
+ "2141802816": 0.0,
676
+ "256770874": 0.0,
677
+ "278008524": 0.0,
678
+ "416183106": 0.0,
679
+ "534717141": 1.0,
680
+ "551504635": 0.0,
681
+ "565369392": 0.3333333333333333,
682
+ "565621826": 1.0,
683
+ "585904130": 0.0,
684
+ "626486813": 0.25,
685
+ "636680137": 0.0,
686
+ "706479600": 1.0,
687
+ "750448296": 0.0,
688
+ "75070641": 0.0,
689
+ "767747424": 0.25,
690
+ "77147264": 1.0,
691
+ "794181626": 1.0,
692
+ "957101489": 0.0,
693
+ "998761162": 0.0
694
+ }
695
+ },
696
+ "macro_by_task": {
697
+ "ci": {
698
+ "confidence": 0.95,
699
+ "high": 0.5408748114630468,
700
+ "low": 0.14666666666666667,
701
+ "mean": 0.34395173453996986,
702
+ "n": 5,
703
+ "num_samples": 500
704
+ },
705
+ "mean": 0.34395173453996986,
706
+ "per_task": {
707
+ "LiftPegUpright-v1": 0.3333333333333333,
708
+ "PickCube-v1": 0.0,
709
+ "PullCube-v1": 0.29411764705882354,
710
+ "PushCube-v1": 0.6923076923076923,
711
+ "StackCube-v1": 0.4
712
+ }
713
+ },
714
+ "metric": "proposal_oracle_success_at_8",
715
+ "micro": {
716
+ "confidence": 0.95,
717
+ "high": 0.5217391304347826,
718
+ "low": 0.2898550724637681,
719
+ "mean": 0.391304347826087,
720
+ "n": 69,
721
+ "num_samples": 500
722
+ },
723
+ "n": 69
724
+ },
725
+ "proposal_oracle_success_gain_over_base_at_8": {
726
+ "macro_by_seed": {
727
+ "ci": {
728
+ "confidence": 0.95,
729
+ "high": 0.27446808510638293,
730
+ "low": -0.010638297872340425,
731
+ "mean": 0.13191489361702127,
732
+ "n": 47,
733
+ "num_samples": 500
734
+ },
735
+ "mean": 0.13191489361702127,
736
+ "per_seed": {
737
+ "1025740212": 0.0,
738
+ "1055157354": 1.0,
739
+ "1055567618": 0.0,
740
+ "1113268940": 0.0,
741
+ "1113371261": 0.0,
742
+ "1176943004": 1.0,
743
+ "1189392917": 1.0,
744
+ "1231780997": 0.0,
745
+ "1271718007": 0.5,
746
+ "1273247072": 0.0,
747
+ "1358421325": 0.0,
748
+ "1375492786": -1.0,
749
+ "141984615": 0.0,
750
+ "1475440973": 0.0,
751
+ "1483645336": 1.0,
752
+ "1522272686": 0.0,
753
+ "1528740417": 0.0,
754
+ "1549324805": 0.0,
755
+ "1607773860": 0.0,
756
+ "1665739511": -1.0,
757
+ "1747178030": 1.0,
758
+ "1881223807": 0.0,
759
+ "1919198188": 0.0,
760
+ "1991414747": 0.0,
761
+ "1997090716": 1.0,
762
+ "2015956957": 0.2,
763
+ "2054899126": 0.0,
764
+ "211245842": 0.0,
765
+ "2141802816": 0.0,
766
+ "256770874": 0.0,
767
+ "278008524": 0.0,
768
+ "416183106": 0.0,
769
+ "534717141": 0.0,
770
+ "551504635": 0.0,
771
+ "565369392": 0.0,
772
+ "565621826": 0.0,
773
+ "585904130": 0.0,
774
+ "626486813": 0.0,
775
+ "636680137": -1.0,
776
+ "706479600": 1.0,
777
+ "750448296": 0.0,
778
+ "75070641": 0.0,
779
+ "767747424": 0.0,
780
+ "77147264": 0.5,
781
+ "794181626": 1.0,
782
+ "957101489": 0.0,
783
+ "998761162": 0.0
784
+ }
785
+ },
786
+ "macro_by_task": {
787
+ "ci": {
788
+ "confidence": 0.95,
789
+ "high": 0.27076923076923076,
790
+ "low": 0.0,
791
+ "mean": 0.11692307692307694,
792
+ "n": 5,
793
+ "num_samples": 500
794
+ },
795
+ "mean": 0.11692307692307694,
796
+ "per_task": {
797
+ "LiftPegUpright-v1": 0.0,
798
+ "PickCube-v1": 0.0,
799
+ "PullCube-v1": 0.0,
800
+ "PushCube-v1": 0.38461538461538464,
801
+ "StackCube-v1": 0.2
802
+ }
803
+ },
804
+ "metric": "proposal_oracle_success_gain_over_base_at_8",
805
+ "micro": {
806
+ "confidence": 0.95,
807
+ "high": 0.21739130434782608,
808
+ "low": 0.014492753623188406,
809
+ "mean": 0.11594202898550725,
810
+ "n": 69,
811
+ "num_samples": 500
812
+ },
813
+ "n": 69
814
+ },
815
+ "proposal_oracle_utility_at_8": {
816
+ "macro_by_seed": {
817
+ "ci": {
818
+ "confidence": 0.95,
819
+ "high": 1.2912854919257963,
820
+ "low": 0.8075891729536738,
821
+ "mean": 1.0507658735934828,
822
+ "n": 47,
823
+ "num_samples": 500
824
+ },
825
+ "mean": 1.0507658735934828,
826
+ "per_seed": {
827
+ "1025740212": 0.4761262834072113,
828
+ "1055157354": 2.0,
829
+ "1055567618": 2.0,
830
+ "1113268940": 0.6646121144294739,
831
+ "1113371261": 2.0,
832
+ "1176943004": 2.0,
833
+ "1189392917": 2.0,
834
+ "1231780997": 0.7187411189079285,
835
+ "1271718007": 2.0,
836
+ "1273247072": 0.13278456032276154,
837
+ "1358421325": 0.8142459392547607,
838
+ "1375492786": 0.0016440550098195672,
839
+ "141984615": 0.0797463705142339,
840
+ "1475440973": 2.0,
841
+ "1483645336": 2.0,
842
+ "1522272686": 0.2435554713010788,
843
+ "1528740417": 2.0,
844
+ "1549324805": 1.3427665165315072,
845
+ "1607773860": 2.0,
846
+ "1665739511": 0.8216220736503601,
847
+ "1747178030": 2.0,
848
+ "1881223807": 0.0513521283864975,
849
+ "1919198188": 0.08992795646190643,
850
+ "1991414747": 2.0,
851
+ "1997090716": 2.0,
852
+ "2015956957": 0.45124312303960323,
853
+ "2054899126": 2.0,
854
+ "211245842": 0.8130142092704773,
855
+ "2141802816": 0.12472955882549286,
856
+ "256770874": 0.24107472598552704,
857
+ "278008524": 0.011432985425926745,
858
+ "416183106": 0.5778142213821411,
859
+ "534717141": 2.0,
860
+ "551504635": 0.7250897884368896,
861
+ "565369392": 0.7852714310089747,
862
+ "565621826": 2.0,
863
+ "585904130": 0.05598785417775313,
864
+ "626486813": 0.5615248780231923,
865
+ "636680137": 0.012414653785526752,
866
+ "706479600": 2.0,
867
+ "750448296": 0.1069931760430336,
868
+ "75070641": 0.2396746575832367,
869
+ "767747424": 0.5703969299793243,
870
+ "77147264": 2.0,
871
+ "794181626": 2.0,
872
+ "957101489": 0.5881698727607727,
873
+ "998761162": 0.08403940498828888
874
+ }
875
+ },
876
+ "macro_by_task": {
877
+ "ci": {
878
+ "confidence": 0.95,
879
+ "high": 1.2200616387067698,
880
+ "low": 0.4891751624122831,
881
+ "mean": 0.8746016368245654,
882
+ "n": 5,
883
+ "num_samples": 500
884
+ },
885
+ "mean": 0.8746016368245654,
886
+ "per_task": {
887
+ "LiftPegUpright-v1": 1.0974282175302505,
888
+ "PickCube-v1": 0.13278456032276154,
889
+ "PullCube-v1": 0.6413554182680159,
890
+ "PushCube-v1": 1.406166712825115,
891
+ "StackCube-v1": 1.095273275176684
892
+ }
893
+ },
894
+ "metric": "proposal_oracle_utility_at_8",
895
+ "micro": {
896
+ "confidence": 0.95,
897
+ "high": 1.149046370209586,
898
+ "low": 0.73185485545917,
899
+ "mean": 0.9164160069709212,
900
+ "n": 69,
901
+ "num_samples": 500
902
+ },
903
+ "n": 69
904
+ },
905
+ "proposal_oracle_utility_gain_over_base_at_8": {
906
+ "macro_by_seed": {
907
+ "ci": {
908
+ "confidence": 0.95,
909
+ "high": 0.5003876065769306,
910
+ "low": -0.0012803047515636864,
911
+ "mean": 0.23716175138788523,
912
+ "n": 47,
913
+ "num_samples": 500
914
+ },
915
+ "mean": 0.23716175138788523,
916
+ "per_seed": {
917
+ "1025740212": -0.05224737524986267,
918
+ "1055157354": 1.8235089033842087,
919
+ "1055567618": 0.0,
920
+ "1113268940": 0.24229958653450012,
921
+ "1113371261": 0.0,
922
+ "1176943004": 1.949059471487999,
923
+ "1189392917": 1.8915813565254211,
924
+ "1231780997": -0.008265018463134766,
925
+ "1271718007": 0.5857395231723785,
926
+ "1273247072": -0.02064509689807892,
927
+ "1358421325": -0.018456339836120605,
928
+ "1375492786": -1.9983559449901804,
929
+ "141984615": -0.01406576856970787,
930
+ "1475440973": 0.0,
931
+ "1483645336": 1.9835839420557022,
932
+ "1522272686": 0.048794373869895935,
933
+ "1528740417": 0.0,
934
+ "1549324805": -0.035980931793649994,
935
+ "1607773860": 0.0,
936
+ "1665739511": -1.17837792634964,
937
+ "1747178030": 1.8536934107542038,
938
+ "1881223807": 0.0193251371383667,
939
+ "1919198188": 0.05272909998893738,
940
+ "1991414747": 0.0,
941
+ "1997090716": 1.8607252091169357,
942
+ "2015956957": 0.38412333317101,
943
+ "2054899126": 0.0,
944
+ "211245842": -0.02582019567489624,
945
+ "2141802816": 0.05392231047153473,
946
+ "256770874": -0.6146733313798904,
947
+ "278008524": -0.09603888296987861,
948
+ "416183106": 0.37748780846595764,
949
+ "534717141": 0.0,
950
+ "551504635": 0.0011172294616699219,
951
+ "565369392": -0.014494215448697409,
952
+ "565621826": 0.0,
953
+ "585904130": -0.019304243847727776,
954
+ "626486813": 0.0560848613968119,
955
+ "636680137": -1.9875853462144732,
956
+ "706479600": 1.1874593496322632,
957
+ "750448296": -0.004540383815765381,
958
+ "75070641": 0.03692302107810974,
959
+ "767747424": 0.022872742265462875,
960
+ "77147264": 0.5784205198287964,
961
+ "794181626": 1.826662853360176,
962
+ "957101489": 0.3842049241065979,
963
+ "998761162": 0.015134349465370178
964
+ }
965
+ },
966
+ "macro_by_task": {
967
+ "ci": {
968
+ "confidence": 0.95,
969
+ "high": 0.4710757725513899,
970
+ "low": -0.00611102568926107,
971
+ "mean": 0.20618344864719879,
972
+ "n": 5,
973
+ "num_samples": 500
974
+ },
975
+ "mean": 0.20618344864719879,
976
+ "per_task": {
977
+ "LiftPegUpright-v1": 0.030484070380528767,
978
+ "PickCube-v1": -0.02064509689807892,
979
+ "PullCube-v1": -0.009874502515338142,
980
+ "PushCube-v1": 0.7374794540496973,
981
+ "StackCube-v1": 0.2934733182191849
982
+ }
983
+ },
984
+ "metric": "proposal_oracle_utility_gain_over_base_at_8",
985
+ "micro": {
986
+ "confidence": 0.95,
987
+ "high": 0.37626325682588463,
988
+ "low": 0.0164921339938714,
989
+ "mean": 0.2002298393593831,
990
+ "n": 69,
991
+ "num_samples": 500
992
+ },
993
+ "n": 69
994
+ },
995
+ "selected_success_at_8": {
996
+ "macro_by_seed": {
997
+ "ci": {
998
+ "confidence": 0.95,
999
+ "high": 0.4219858156028369,
1000
+ "low": 0.17978723404255317,
1001
+ "mean": 0.2914893617021277,
1002
+ "n": 47,
1003
+ "num_samples": 500
1004
+ },
1005
+ "mean": 0.2914893617021277,
1006
+ "per_seed": {
1007
+ "1025740212": 0.0,
1008
+ "1055157354": 0.0,
1009
+ "1055567618": 0.0,
1010
+ "1113268940": 0.0,
1011
+ "1113371261": 1.0,
1012
+ "1176943004": 1.0,
1013
+ "1189392917": 1.0,
1014
+ "1231780997": 0.0,
1015
+ "1271718007": 1.0,
1016
+ "1273247072": 0.0,
1017
+ "1358421325": 0.0,
1018
+ "1375492786": 0.0,
1019
+ "141984615": 0.0,
1020
+ "1475440973": 1.0,
1021
+ "1483645336": 1.0,
1022
+ "1522272686": 0.0,
1023
+ "1528740417": 1.0,
1024
+ "1549324805": 0.6666666666666666,
1025
+ "1607773860": 0.0,
1026
+ "1665739511": 0.0,
1027
+ "1747178030": 1.0,
1028
+ "1881223807": 0.0,
1029
+ "1919198188": 0.0,
1030
+ "1991414747": 1.0,
1031
+ "1997090716": 0.0,
1032
+ "2015956957": 0.2,
1033
+ "2054899126": 1.0,
1034
+ "211245842": 0.0,
1035
+ "2141802816": 0.0,
1036
+ "256770874": 0.0,
1037
+ "278008524": 0.0,
1038
+ "416183106": 0.0,
1039
+ "534717141": 0.0,
1040
+ "551504635": 0.0,
1041
+ "565369392": 0.3333333333333333,
1042
+ "565621826": 1.0,
1043
+ "585904130": 0.0,
1044
+ "626486813": 0.25,
1045
+ "636680137": 0.0,
1046
+ "706479600": 0.0,
1047
+ "750448296": 0.0,
1048
+ "75070641": 0.0,
1049
+ "767747424": 0.25,
1050
+ "77147264": 0.0,
1051
+ "794181626": 1.0,
1052
+ "957101489": 0.0,
1053
+ "998761162": 0.0
1054
+ }
1055
+ },
1056
+ "macro_by_task": {
1057
+ "ci": {
1058
+ "confidence": 0.95,
1059
+ "high": 0.40307692307692305,
1060
+ "low": 0.0796078431372549,
1061
+ "mean": 0.24063348416289593,
1062
+ "n": 5,
1063
+ "num_samples": 500
1064
+ },
1065
+ "mean": 0.24063348416289593,
1066
+ "per_task": {
1067
+ "LiftPegUpright-v1": 0.3333333333333333,
1068
+ "PickCube-v1": 0.0,
1069
+ "PullCube-v1": 0.2647058823529412,
1070
+ "PushCube-v1": 0.5384615384615384,
1071
+ "StackCube-v1": 0.06666666666666667
1072
+ }
1073
+ },
1074
+ "metric": "selected_success_at_8",
1075
+ "micro": {
1076
+ "confidence": 0.95,
1077
+ "high": 0.37681159420289856,
1078
+ "low": 0.18840579710144928,
1079
+ "mean": 0.2753623188405797,
1080
+ "n": 69,
1081
+ "num_samples": 500
1082
+ },
1083
+ "n": 69
1084
+ },
1085
+ "selected_success_gain_over_base_at_8": {
1086
+ "macro_by_seed": {
1087
+ "ci": {
1088
+ "confidence": 0.95,
1089
+ "high": 0.12127659574468086,
1090
+ "low": -0.16595744680851063,
1091
+ "mean": -0.01702127659574468,
1092
+ "n": 47,
1093
+ "num_samples": 500
1094
+ },
1095
+ "mean": -0.01702127659574468,
1096
+ "per_seed": {
1097
+ "1025740212": 0.0,
1098
+ "1055157354": 0.0,
1099
+ "1055567618": -1.0,
1100
+ "1113268940": 0.0,
1101
+ "1113371261": 0.0,
1102
+ "1176943004": 1.0,
1103
+ "1189392917": 1.0,
1104
+ "1231780997": 0.0,
1105
+ "1271718007": 0.5,
1106
+ "1273247072": 0.0,
1107
+ "1358421325": 0.0,
1108
+ "1375492786": -1.0,
1109
+ "141984615": 0.0,
1110
+ "1475440973": 0.0,
1111
+ "1483645336": 1.0,
1112
+ "1522272686": 0.0,
1113
+ "1528740417": 0.0,
1114
+ "1549324805": 0.0,
1115
+ "1607773860": -1.0,
1116
+ "1665739511": -1.0,
1117
+ "1747178030": 1.0,
1118
+ "1881223807": 0.0,
1119
+ "1919198188": 0.0,
1120
+ "1991414747": 0.0,
1121
+ "1997090716": 0.0,
1122
+ "2015956957": 0.2,
1123
+ "2054899126": 0.0,
1124
+ "211245842": 0.0,
1125
+ "2141802816": 0.0,
1126
+ "256770874": 0.0,
1127
+ "278008524": 0.0,
1128
+ "416183106": 0.0,
1129
+ "534717141": -1.0,
1130
+ "551504635": 0.0,
1131
+ "565369392": 0.0,
1132
+ "565621826": 0.0,
1133
+ "585904130": 0.0,
1134
+ "626486813": 0.0,
1135
+ "636680137": -1.0,
1136
+ "706479600": 0.0,
1137
+ "750448296": 0.0,
1138
+ "75070641": 0.0,
1139
+ "767747424": 0.0,
1140
+ "77147264": -0.5,
1141
+ "794181626": 1.0,
1142
+ "957101489": 0.0,
1143
+ "998761162": 0.0
1144
+ }
1145
+ },
1146
+ "macro_by_task": {
1147
+ "ci": {
1148
+ "confidence": 0.95,
1149
+ "high": 0.12669683257918551,
1150
+ "low": -0.08588235294117648,
1151
+ "mean": 0.013604826546003018,
1152
+ "n": 5,
1153
+ "num_samples": 500
1154
+ },
1155
+ "mean": 0.013604826546003018,
1156
+ "per_task": {
1157
+ "LiftPegUpright-v1": 0.0,
1158
+ "PickCube-v1": 0.0,
1159
+ "PullCube-v1": -0.029411764705882353,
1160
+ "PushCube-v1": 0.23076923076923078,
1161
+ "StackCube-v1": -0.13333333333333333
1162
+ }
1163
+ },
1164
+ "metric": "selected_success_gain_over_base_at_8",
1165
+ "micro": {
1166
+ "confidence": 0.95,
1167
+ "high": 0.11594202898550725,
1168
+ "low": -0.10144927536231885,
1169
+ "mean": 0.0,
1170
+ "n": 69,
1171
+ "num_samples": 500
1172
+ },
1173
+ "n": 69
1174
+ },
1175
+ "selected_utility_at_8": {
1176
+ "macro_by_seed": {
1177
+ "ci": {
1178
+ "confidence": 0.95,
1179
+ "high": 0.9708039672328622,
1180
+ "low": 0.5131759628765814,
1181
+ "mean": 0.729357396943597,
1182
+ "n": 47,
1183
+ "num_samples": 500
1184
+ },
1185
+ "mean": 0.729357396943597,
1186
+ "per_seed": {
1187
+ "1025740212": 0.21424125134944916,
1188
+ "1055157354": 0.14835423231124878,
1189
+ "1055567618": 0.8168438673019409,
1190
+ "1113268940": 0.27990618348121643,
1191
+ "1113371261": 2.0,
1192
+ "1176943004": 2.0,
1193
+ "1189392917": 2.0,
1194
+ "1231780997": 0.22899846732616425,
1195
+ "1271718007": 2.0,
1196
+ "1273247072": 0.13278456032276154,
1197
+ "1358421325": 0.23671336472034454,
1198
+ "1375492786": 0.001358628273010254,
1199
+ "141984615": 0.04657847310105959,
1200
+ "1475440973": 2.0,
1201
+ "1483645336": 2.0,
1202
+ "1522272686": 0.2340310961008072,
1203
+ "1528740417": 2.0,
1204
+ "1549324805": 1.3393354682872693,
1205
+ "1607773860": 0.220267653465271,
1206
+ "1665739511": 0.8140708208084106,
1207
+ "1747178030": 2.0,
1208
+ "1881223807": 0.03111603856086731,
1209
+ "1919198188": 0.07307347655296326,
1210
+ "1991414747": 2.0,
1211
+ "1997090716": 0.03083018958568573,
1212
+ "2015956957": 0.433539473824203,
1213
+ "2054899126": 2.0,
1214
+ "211245842": 0.22217948734760284,
1215
+ "2141802816": 0.0492904931306839,
1216
+ "256770874": 0.13286492228507996,
1217
+ "278008524": 0.009787360846530646,
1218
+ "416183106": 0.24176457524299622,
1219
+ "534717141": 0.07250408083200455,
1220
+ "551504635": 0.6918362975120544,
1221
+ "565369392": 0.7208344464500746,
1222
+ "565621826": 2.0,
1223
+ "585904130": 0.012818952556699514,
1224
+ "626486813": 0.5429544361541048,
1225
+ "636680137": 0.012414653785526752,
1226
+ "706479600": 0.8134256601333618,
1227
+ "750448296": 0.029670913703739643,
1228
+ "75070641": 0.16485285758972168,
1229
+ "767747424": 0.554436425678432,
1230
+ "77147264": 0.13140545785427094,
1231
+ "794181626": 2.0,
1232
+ "957101489": 0.5881698727607727,
1233
+ "998761162": 0.006543517112731934
1234
+ }
1235
+ },
1236
+ "macro_by_task": {
1237
+ "ci": {
1238
+ "confidence": 0.95,
1239
+ "high": 0.9089970031332648,
1240
+ "low": 0.35030357998435147,
1241
+ "mean": 0.629650291558808,
1242
+ "n": 5,
1243
+ "num_samples": 500
1244
+ },
1245
+ "mean": 0.629650291558808,
1246
+ "per_task": {
1247
+ "LiftPegUpright-v1": 0.9024970332781473,
1248
+ "PickCube-v1": 0.13278456032276154,
1249
+ "PullCube-v1": 0.560690138827982,
1250
+ "PushCube-v1": 1.0896504051410234,
1251
+ "StackCube-v1": 0.4626293202241262
1252
+ }
1253
+ },
1254
+ "metric": "selected_utility_at_8",
1255
+ "micro": {
1256
+ "confidence": 0.95,
1257
+ "high": 0.8693176263022115,
1258
+ "low": 0.48426623860259366,
1259
+ "mean": 0.6625525587005541,
1260
+ "n": 69,
1261
+ "num_samples": 500
1262
+ },
1263
+ "n": 69
1264
+ },
1265
+ "selected_utility_gain_over_base_at_8": {
1266
+ "macro_by_seed": {
1267
+ "ci": {
1268
+ "confidence": 0.95,
1269
+ "high": 0.19248003160666863,
1270
+ "low": -0.359282114742413,
1271
+ "mean": -0.0842467252620007,
1272
+ "n": 47,
1273
+ "num_samples": 500
1274
+ },
1275
+ "mean": -0.0842467252620007,
1276
+ "per_seed": {
1277
+ "1025740212": -0.3141324073076248,
1278
+ "1055157354": -0.02813686430454254,
1279
+ "1055567618": -1.183156132698059,
1280
+ "1113268940": -0.14240634441375732,
1281
+ "1113371261": 0.0,
1282
+ "1176943004": 1.949059471487999,
1283
+ "1189392917": 1.8915813565254211,
1284
+ "1231780997": -0.498007670044899,
1285
+ "1271718007": 0.5857395231723785,
1286
+ "1273247072": -0.02064509689807892,
1287
+ "1358421325": -0.5959889143705368,
1288
+ "1375492786": -1.9986413717269897,
1289
+ "141984615": -0.04723366598288218,
1290
+ "1475440973": 0.0,
1291
+ "1483645336": 1.9835839420557022,
1292
+ "1522272686": 0.03926999866962433,
1293
+ "1528740417": 0.0,
1294
+ "1549324805": -0.03941198003788789,
1295
+ "1607773860": -1.779732346534729,
1296
+ "1665739511": -1.1859291791915894,
1297
+ "1747178030": 1.8536934107542038,
1298
+ "1881223807": -0.0009109526872634888,
1299
+ "1919198188": 0.0358746200799942,
1300
+ "1991414747": 0.0,
1301
+ "1997090716": -0.10844460129737854,
1302
+ "2015956957": 0.3664196839556098,
1303
+ "2054899126": 0.0,
1304
+ "211245842": -0.6166549175977707,
1305
+ "2141802816": -0.02151675522327423,
1306
+ "256770874": -0.7228831350803375,
1307
+ "278008524": -0.09768450754927471,
1308
+ "416183106": 0.041438162326812744,
1309
+ "534717141": -1.9274959191679955,
1310
+ "551504635": -0.03213626146316528,
1311
+ "565369392": -0.07893120000759761,
1312
+ "565621826": 0.0,
1313
+ "585904130": -0.06247314546878139,
1314
+ "626486813": 0.037514419527724385,
1315
+ "636680137": -1.9875853462144732,
1316
+ "706479600": 0.000885009765625,
1317
+ "750448296": -0.08186264615505934,
1318
+ "75070641": -0.03789877891540527,
1319
+ "767747424": 0.006912237964570522,
1320
+ "77147264": -1.2901740223169327,
1321
+ "794181626": 1.826662853360176,
1322
+ "957101489": 0.3842049241065979,
1323
+ "998761162": -0.06236153841018677
1324
+ }
1325
+ },
1326
+ "macro_by_task": {
1327
+ "ci": {
1328
+ "confidence": 0.95,
1329
+ "high": 0.20158050865397423,
1330
+ "low": -0.25449976120541307,
1331
+ "mean": -0.038767896618558506,
1332
+ "n": 5,
1333
+ "num_samples": 500
1334
+ },
1335
+ "mean": -0.038767896618558506,
1336
+ "per_task": {
1337
+ "LiftPegUpright-v1": -0.1644471138715744,
1338
+ "PickCube-v1": -0.02064509689807892,
1339
+ "PullCube-v1": -0.0905397819553721,
1340
+ "PushCube-v1": 0.4209631463656059,
1341
+ "StackCube-v1": -0.339170636733373
1342
+ }
1343
+ },
1344
+ "metric": "selected_utility_gain_over_base_at_8",
1345
+ "micro": {
1346
+ "confidence": 0.95,
1347
+ "high": 0.14216782323901367,
1348
+ "low": -0.2450303274233812,
1349
+ "mean": -0.053633608910983996,
1350
+ "n": 69,
1351
+ "num_samples": 500
1352
+ },
1353
+ "n": 69
1354
+ },
1355
+ "selector_regret_at_8": {
1356
+ "macro_by_seed": {
1357
+ "ci": {
1358
+ "confidence": 0.95,
1359
+ "high": 0.5197332343231088,
1360
+ "low": 0.16253579634643023,
1361
+ "mean": 0.321408476649886,
1362
+ "n": 47,
1363
+ "num_samples": 500
1364
+ },
1365
+ "mean": 0.321408476649886,
1366
+ "per_seed": {
1367
+ "1025740212": 0.26188503205776215,
1368
+ "1055157354": 1.8516457676887512,
1369
+ "1055567618": 1.183156132698059,
1370
+ "1113268940": 0.38470593094825745,
1371
+ "1113371261": 0.0,
1372
+ "1176943004": 0.0,
1373
+ "1189392917": 0.0,
1374
+ "1231780997": 0.4897426515817642,
1375
+ "1271718007": 0.0,
1376
+ "1273247072": 0.0,
1377
+ "1358421325": 0.5775325745344162,
1378
+ "1375492786": 0.0002854267368093133,
1379
+ "141984615": 0.03316789741317431,
1380
+ "1475440973": 0.0,
1381
+ "1483645336": 0.0,
1382
+ "1522272686": 0.009524375200271606,
1383
+ "1528740417": 0.0,
1384
+ "1549324805": 0.0034310482442378998,
1385
+ "1607773860": 1.779732346534729,
1386
+ "1665739511": 0.007551252841949463,
1387
+ "1747178030": 0.0,
1388
+ "1881223807": 0.020236089825630188,
1389
+ "1919198188": 0.016854479908943176,
1390
+ "1991414747": 0.0,
1391
+ "1997090716": 1.9691698104143143,
1392
+ "2015956957": 0.01770364921540022,
1393
+ "2054899126": 0.0,
1394
+ "211245842": 0.5908347219228745,
1395
+ "2141802816": 0.07543906569480896,
1396
+ "256770874": 0.10820980370044708,
1397
+ "278008524": 0.0016456245793960989,
1398
+ "416183106": 0.3360496461391449,
1399
+ "534717141": 1.9274959191679955,
1400
+ "551504635": 0.033253490924835205,
1401
+ "565369392": 0.06443698455890019,
1402
+ "565621826": 0.0,
1403
+ "585904130": 0.043168901621053614,
1404
+ "626486813": 0.018570441869087517,
1405
+ "636680137": 0.0,
1406
+ "706479600": 1.1865743398666382,
1407
+ "750448296": 0.07732226233929396,
1408
+ "75070641": 0.07482179999351501,
1409
+ "767747424": 0.015960504300892353,
1410
+ "77147264": 1.868594542145729,
1411
+ "794181626": 0.0,
1412
+ "957101489": 0.0,
1413
+ "998761162": 0.07749588787555695
1414
+ }
1415
+ },
1416
+ "macro_by_task": {
1417
+ "ci": {
1418
+ "confidence": 0.95,
1419
+ "high": 0.5061151639620463,
1420
+ "low": 0.07125234862643422,
1421
+ "mean": 0.24495134526575732,
1422
+ "n": 5,
1423
+ "num_samples": 500
1424
+ },
1425
+ "mean": 0.24495134526575732,
1426
+ "per_task": {
1427
+ "LiftPegUpright-v1": 0.19493118425210318,
1428
+ "PickCube-v1": 0.0,
1429
+ "PullCube-v1": 0.08066527944003396,
1430
+ "PushCube-v1": 0.31651630768409145,
1431
+ "StackCube-v1": 0.6326439549525579
1432
+ }
1433
+ },
1434
+ "metric": "selector_regret_at_8",
1435
+ "micro": {
1436
+ "confidence": 0.95,
1437
+ "high": 0.378890726060701,
1438
+ "low": 0.13165242864372398,
1439
+ "mean": 0.2538634482703671,
1440
+ "n": 69,
1441
+ "num_samples": 500
1442
+ },
1443
+ "n": 69
1444
+ },
1445
+ "success_selector_gap_at_8": {
1446
+ "macro_by_seed": {
1447
+ "ci": {
1448
+ "confidence": 0.95,
1449
+ "high": 0.2553191489361702,
1450
+ "low": 0.0425531914893617,
1451
+ "mean": 0.14893617021276595,
1452
+ "n": 47,
1453
+ "num_samples": 500
1454
+ },
1455
+ "mean": 0.14893617021276595,
1456
+ "per_seed": {
1457
+ "1025740212": 0.0,
1458
+ "1055157354": 1.0,
1459
+ "1055567618": 1.0,
1460
+ "1113268940": 0.0,
1461
+ "1113371261": 0.0,
1462
+ "1176943004": 0.0,
1463
+ "1189392917": 0.0,
1464
+ "1231780997": 0.0,
1465
+ "1271718007": 0.0,
1466
+ "1273247072": 0.0,
1467
+ "1358421325": 0.0,
1468
+ "1375492786": 0.0,
1469
+ "141984615": 0.0,
1470
+ "1475440973": 0.0,
1471
+ "1483645336": 0.0,
1472
+ "1522272686": 0.0,
1473
+ "1528740417": 0.0,
1474
+ "1549324805": 0.0,
1475
+ "1607773860": 1.0,
1476
+ "1665739511": 0.0,
1477
+ "1747178030": 0.0,
1478
+ "1881223807": 0.0,
1479
+ "1919198188": 0.0,
1480
+ "1991414747": 0.0,
1481
+ "1997090716": 1.0,
1482
+ "2015956957": 0.0,
1483
+ "2054899126": 0.0,
1484
+ "211245842": 0.0,
1485
+ "2141802816": 0.0,
1486
+ "256770874": 0.0,
1487
+ "278008524": 0.0,
1488
+ "416183106": 0.0,
1489
+ "534717141": 1.0,
1490
+ "551504635": 0.0,
1491
+ "565369392": 0.0,
1492
+ "565621826": 0.0,
1493
+ "585904130": 0.0,
1494
+ "626486813": 0.0,
1495
+ "636680137": 0.0,
1496
+ "706479600": 1.0,
1497
+ "750448296": 0.0,
1498
+ "75070641": 0.0,
1499
+ "767747424": 0.0,
1500
+ "77147264": 1.0,
1501
+ "794181626": 0.0,
1502
+ "957101489": 0.0,
1503
+ "998761162": 0.0
1504
+ }
1505
+ },
1506
+ "macro_by_task": {
1507
+ "ci": {
1508
+ "confidence": 0.95,
1509
+ "high": 0.2615384615384615,
1510
+ "low": 0.011764705882352941,
1511
+ "mean": 0.1033182503770739,
1512
+ "n": 5,
1513
+ "num_samples": 500
1514
+ },
1515
+ "mean": 0.1033182503770739,
1516
+ "per_task": {
1517
+ "LiftPegUpright-v1": 0.0,
1518
+ "PickCube-v1": 0.0,
1519
+ "PullCube-v1": 0.029411764705882353,
1520
+ "PushCube-v1": 0.15384615384615385,
1521
+ "StackCube-v1": 0.3333333333333333
1522
+ }
1523
+ },
1524
+ "metric": "success_selector_gap_at_8",
1525
+ "micro": {
1526
+ "confidence": 0.95,
1527
+ "high": 0.18840579710144928,
1528
+ "low": 0.043478260869565216,
1529
+ "mean": 0.11594202898550725,
1530
+ "n": 69,
1531
+ "num_samples": 500
1532
+ },
1533
+ "n": 69
1534
+ },
1535
+ "success_support_gap_at_8": {
1536
+ "macro_by_seed": {
1537
+ "ci": {
1538
+ "confidence": 0.95,
1539
+ "high": 0.5531914893617021,
1540
+ "low": 0.2695035460992908,
1541
+ "mean": 0.40070921985815605,
1542
+ "n": 47,
1543
+ "num_samples": 500
1544
+ },
1545
+ "mean": 0.40070921985815605,
1546
+ "per_seed": {
1547
+ "1025740212": 1.0,
1548
+ "1055157354": 0.0,
1549
+ "1055567618": 0.0,
1550
+ "1113268940": 1.0,
1551
+ "1113371261": 0.0,
1552
+ "1176943004": 0.0,
1553
+ "1189392917": 0.0,
1554
+ "1231780997": 1.0,
1555
+ "1271718007": 0.0,
1556
+ "1273247072": 1.0,
1557
+ "1358421325": 1.0,
1558
+ "1375492786": 0.0,
1559
+ "141984615": 0.0,
1560
+ "1475440973": 0.0,
1561
+ "1483645336": 0.0,
1562
+ "1522272686": 1.0,
1563
+ "1528740417": 0.0,
1564
+ "1549324805": 0.0,
1565
+ "1607773860": 0.0,
1566
+ "1665739511": 1.0,
1567
+ "1747178030": 0.0,
1568
+ "1881223807": 1.0,
1569
+ "1919198188": 1.0,
1570
+ "1991414747": 0.0,
1571
+ "1997090716": 0.0,
1572
+ "2015956957": 0.0,
1573
+ "2054899126": 0.0,
1574
+ "211245842": 1.0,
1575
+ "2141802816": 1.0,
1576
+ "256770874": 1.0,
1577
+ "278008524": 0.5,
1578
+ "416183106": 1.0,
1579
+ "534717141": 0.0,
1580
+ "551504635": 1.0,
1581
+ "565369392": 0.3333333333333333,
1582
+ "565621826": 0.0,
1583
+ "585904130": 0.0,
1584
+ "626486813": 0.0,
1585
+ "636680137": 1.0,
1586
+ "706479600": 0.0,
1587
+ "750448296": 0.0,
1588
+ "75070641": 1.0,
1589
+ "767747424": 0.0,
1590
+ "77147264": 0.0,
1591
+ "794181626": 0.0,
1592
+ "957101489": 1.0,
1593
+ "998761162": 1.0
1594
+ }
1595
+ },
1596
+ "macro_by_task": {
1597
+ "ci": {
1598
+ "confidence": 0.95,
1599
+ "high": 0.7948717948717949,
1600
+ "low": 0.21683257918552035,
1601
+ "mean": 0.5266365007541478,
1602
+ "n": 5,
1603
+ "num_samples": 500
1604
+ },
1605
+ "mean": 0.5266365007541478,
1606
+ "per_task": {
1607
+ "LiftPegUpright-v1": 0.6666666666666666,
1608
+ "PickCube-v1": 1.0,
1609
+ "PullCube-v1": 0.058823529411764705,
1610
+ "PushCube-v1": 0.3076923076923077,
1611
+ "StackCube-v1": 0.6
1612
+ }
1613
+ },
1614
+ "metric": "success_support_gap_at_8",
1615
+ "micro": {
1616
+ "confidence": 0.95,
1617
+ "high": 0.391304347826087,
1618
+ "low": 0.17391304347826086,
1619
+ "mean": 0.2898550724637681,
1620
+ "n": 69,
1621
+ "num_samples": 500
1622
+ },
1623
+ "n": 69
1624
+ },
1625
+ "success_total_car_to_hidden_at_8": {
1626
+ "macro_by_seed": {
1627
+ "ci": {
1628
+ "confidence": 0.95,
1629
+ "high": 0.6950354609929079,
1630
+ "low": 0.4113475177304965,
1631
+ "mean": 0.549645390070922,
1632
+ "n": 47,
1633
+ "num_samples": 500
1634
+ },
1635
+ "mean": 0.549645390070922,
1636
+ "per_seed": {
1637
+ "1025740212": 1.0,
1638
+ "1055157354": 1.0,
1639
+ "1055567618": 1.0,
1640
+ "1113268940": 1.0,
1641
+ "1113371261": 0.0,
1642
+ "1176943004": 0.0,
1643
+ "1189392917": 0.0,
1644
+ "1231780997": 1.0,
1645
+ "1271718007": 0.0,
1646
+ "1273247072": 1.0,
1647
+ "1358421325": 1.0,
1648
+ "1375492786": 0.0,
1649
+ "141984615": 0.0,
1650
+ "1475440973": 0.0,
1651
+ "1483645336": 0.0,
1652
+ "1522272686": 1.0,
1653
+ "1528740417": 0.0,
1654
+ "1549324805": 0.0,
1655
+ "1607773860": 1.0,
1656
+ "1665739511": 1.0,
1657
+ "1747178030": 0.0,
1658
+ "1881223807": 1.0,
1659
+ "1919198188": 1.0,
1660
+ "1991414747": 0.0,
1661
+ "1997090716": 1.0,
1662
+ "2015956957": 0.0,
1663
+ "2054899126": 0.0,
1664
+ "211245842": 1.0,
1665
+ "2141802816": 1.0,
1666
+ "256770874": 1.0,
1667
+ "278008524": 0.5,
1668
+ "416183106": 1.0,
1669
+ "534717141": 1.0,
1670
+ "551504635": 1.0,
1671
+ "565369392": 0.3333333333333333,
1672
+ "565621826": 0.0,
1673
+ "585904130": 0.0,
1674
+ "626486813": 0.0,
1675
+ "636680137": 1.0,
1676
+ "706479600": 1.0,
1677
+ "750448296": 0.0,
1678
+ "75070641": 1.0,
1679
+ "767747424": 0.0,
1680
+ "77147264": 1.0,
1681
+ "794181626": 0.0,
1682
+ "957101489": 1.0,
1683
+ "998761162": 1.0
1684
+ }
1685
+ },
1686
+ "macro_by_task": {
1687
+ "ci": {
1688
+ "confidence": 0.95,
1689
+ "high": 0.9066666666666666,
1690
+ "low": 0.3122171945701358,
1691
+ "mean": 0.6299547511312216,
1692
+ "n": 5,
1693
+ "num_samples": 500
1694
+ },
1695
+ "mean": 0.6299547511312216,
1696
+ "per_task": {
1697
+ "LiftPegUpright-v1": 0.6666666666666666,
1698
+ "PickCube-v1": 1.0,
1699
+ "PullCube-v1": 0.08823529411764706,
1700
+ "PushCube-v1": 0.46153846153846156,
1701
+ "StackCube-v1": 0.9333333333333333
1702
+ }
1703
+ },
1704
+ "metric": "success_total_car_to_hidden_at_8",
1705
+ "micro": {
1706
+ "confidence": 0.95,
1707
+ "high": 0.5217391304347826,
1708
+ "low": 0.2753623188405797,
1709
+ "mean": 0.4057971014492754,
1710
+ "n": 69,
1711
+ "num_samples": 500
1712
+ },
1713
+ "n": 69
1714
+ },
1715
+ "support_gap_at_8": {
1716
+ "macro_by_seed": {
1717
+ "ci": {
1718
+ "confidence": 0.95,
1719
+ "high": 0.8930026109046337,
1720
+ "low": 0.45526805628827577,
1721
+ "mean": 0.6654373238655799,
1722
+ "n": 47,
1723
+ "num_samples": 500
1724
+ },
1725
+ "mean": 0.6654373238655799,
1726
+ "per_seed": {
1727
+ "1025740212": 1.5238737165927887,
1728
+ "1055157354": 0.0,
1729
+ "1055567618": 0.0,
1730
+ "1113268940": 1.3353878855705261,
1731
+ "1113371261": 0.0,
1732
+ "1176943004": 0.0,
1733
+ "1189392917": 0.0,
1734
+ "1231780997": 1.2812588810920715,
1735
+ "1271718007": 0.0,
1736
+ "1273247072": 1.8672154396772385,
1737
+ "1358421325": 1.1857540607452393,
1738
+ "1375492786": 0.24180691444780678,
1739
+ "141984615": 0.12070147196451823,
1740
+ "1475440973": 0.0,
1741
+ "1483645336": 0.0,
1742
+ "1522272686": 1.7564445286989212,
1743
+ "1528740417": 0.0,
1744
+ "1549324805": 0.05998400536676248,
1745
+ "1607773860": 0.0,
1746
+ "1665739511": 1.17837792634964,
1747
+ "1747178030": 0.0,
1748
+ "1881223807": 1.9486478716135025,
1749
+ "1919198188": 1.9100720435380936,
1750
+ "1991414747": 0.0,
1751
+ "1997090716": 0.0,
1752
+ "2015956957": 0.07863824404776096,
1753
+ "2054899126": 0.0,
1754
+ "211245842": 1.1869857907295227,
1755
+ "2141802816": 1.8752704411745071,
1756
+ "256770874": 1.758925274014473,
1757
+ "278008524": 1.1023782094707713,
1758
+ "416183106": 1.4221857786178589,
1759
+ "534717141": 0.0,
1760
+ "551504635": 1.2749102115631104,
1761
+ "565369392": 0.6413601487874985,
1762
+ "565621826": 0.0,
1763
+ "585904130": 0.1816591378301382,
1764
+ "626486813": 0.0801479306537658,
1765
+ "636680137": 1.9875853462144732,
1766
+ "706479600": 0.0,
1767
+ "750448296": 0.12881441414356232,
1768
+ "75070641": 1.7603253424167633,
1769
+ "767747424": 0.05905248410999775,
1770
+ "77147264": 0.0,
1771
+ "794181626": 0.0,
1772
+ "957101489": 1.4118301272392273,
1773
+ "998761162": 1.9159605950117111
1774
+ }
1775
+ },
1776
+ "macro_by_task": {
1777
+ "ci": {
1778
+ "confidence": 0.95,
1779
+ "high": 1.4200412662059834,
1780
+ "low": 0.41746960569771857,
1781
+ "mean": 0.8929219812617306,
1782
+ "n": 5,
1783
+ "num_samples": 500
1784
+ },
1785
+ "mean": 0.8929219812617306,
1786
+ "per_task": {
1787
+ "LiftPegUpright-v1": 0.9025717824697495,
1788
+ "PickCube-v1": 1.8672154396772385,
1789
+ "PullCube-v1": 0.1962626721634639,
1790
+ "PushCube-v1": 0.593833287174885,
1791
+ "StackCube-v1": 0.904726724823316
1792
+ }
1793
+ },
1794
+ "metric": "support_gap_at_8",
1795
+ "micro": {
1796
+ "confidence": 0.95,
1797
+ "high": 0.6783581883714035,
1798
+ "low": 0.3298995522124207,
1799
+ "mean": 0.5108160955605326,
1800
+ "n": 69,
1801
+ "num_samples": 500
1802
+ },
1803
+ "n": 69
1804
+ },
1805
+ "total_car_to_hidden_at_8": {
1806
+ "macro_by_seed": {
1807
+ "ci": {
1808
+ "confidence": 0.95,
1809
+ "high": 1.235173605110724,
1810
+ "low": 0.7539350982827253,
1811
+ "mean": 0.9865234781741099,
1812
+ "n": 47,
1813
+ "num_samples": 500
1814
+ },
1815
+ "mean": 0.9865234781741099,
1816
+ "per_seed": {
1817
+ "1025740212": 1.7857587486505508,
1818
+ "1055157354": 1.8516457676887512,
1819
+ "1055567618": 1.183156132698059,
1820
+ "1113268940": 1.7200938165187836,
1821
+ "1113371261": 0.0,
1822
+ "1176943004": 0.0,
1823
+ "1189392917": 0.0,
1824
+ "1231780997": 1.7710015326738358,
1825
+ "1271718007": 0.0,
1826
+ "1273247072": 1.8672154396772385,
1827
+ "1358421325": 1.7632866352796555,
1828
+ "1375492786": 0.2420923411846161,
1829
+ "141984615": 0.15386936937769255,
1830
+ "1475440973": 0.0,
1831
+ "1483645336": 0.0,
1832
+ "1522272686": 1.7659689038991928,
1833
+ "1528740417": 0.0,
1834
+ "1549324805": 0.06341505361100037,
1835
+ "1607773860": 1.779732346534729,
1836
+ "1665739511": 1.1859291791915894,
1837
+ "1747178030": 0.0,
1838
+ "1881223807": 1.9688839614391327,
1839
+ "1919198188": 1.9269265234470367,
1840
+ "1991414747": 0.0,
1841
+ "1997090716": 1.9691698104143143,
1842
+ "2015956957": 0.09634189326316119,
1843
+ "2054899126": 0.0,
1844
+ "211245842": 1.7778205126523972,
1845
+ "2141802816": 1.950709506869316,
1846
+ "256770874": 1.86713507771492,
1847
+ "278008524": 1.1040238340501674,
1848
+ "416183106": 1.7582354247570038,
1849
+ "534717141": 1.9274959191679955,
1850
+ "551504635": 1.3081637024879456,
1851
+ "565369392": 0.7057971333463987,
1852
+ "565621826": 0.0,
1853
+ "585904130": 0.2248280394511918,
1854
+ "626486813": 0.08356922247912735,
1855
+ "636680137": 1.9875853462144732,
1856
+ "706479600": 1.1865743398666382,
1857
+ "750448296": 0.20613667648285627,
1858
+ "75070641": 1.8351471424102783,
1859
+ "767747424": 0.0750129884108901,
1860
+ "77147264": 1.868594542145729,
1861
+ "794181626": 0.0,
1862
+ "957101489": 1.4118301272392273,
1863
+ "998761162": 1.993456482887268
1864
+ }
1865
+ },
1866
+ "macro_by_task": {
1867
+ "ci": {
1868
+ "confidence": 0.95,
1869
+ "high": 1.6098733187333132,
1870
+ "low": 0.6546314741212763,
1871
+ "mean": 1.1375168759382237,
1872
+ "n": 5,
1873
+ "num_samples": 500
1874
+ },
1875
+ "mean": 1.1375168759382237,
1876
+ "per_task": {
1877
+ "LiftPegUpright-v1": 1.0975029667218525,
1878
+ "PickCube-v1": 1.8672154396772385,
1879
+ "PullCube-v1": 0.2751456986571772,
1880
+ "PushCube-v1": 0.9103495948589765,
1881
+ "StackCube-v1": 1.5373706797758737
1882
+ }
1883
+ },
1884
+ "metric": "total_car_to_hidden_at_8",
1885
+ "micro": {
1886
+ "confidence": 0.95,
1887
+ "high": 0.9371206429449545,
1888
+ "low": 0.5655091602922134,
1889
+ "mean": 0.763801332234162,
1890
+ "n": 69,
1891
+ "num_samples": 500
1892
+ },
1893
+ "n": 69
1894
+ }
1895
+ },
1896
+ "num_rows": 69,
1897
+ "report_type": "ctt_generated_measured_rollout",
1898
+ "schema_version": 1,
1899
+ "source_content_hash": "90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957",
1900
+ "source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372",
1901
+ "success_summary": {
1902
+ "base_success_rate": 0.2753623188405797,
1903
+ "base_utility_mean": 0.7161861676115381,
1904
+ "hidden_chart_oracle_success_rate": 0.6666666666666666,
1905
+ "hidden_chart_oracle_utility_mean": 1.3999846732054932,
1906
+ "max_restore_error": 1.4901161193847656e-08,
1907
+ "proposal_oracle_success_gain_over_base": 0.11594202898550725,
1908
+ "proposal_oracle_success_rate": 0.391304347826087,
1909
+ "proposal_oracle_utility_mean": 0.9164160069709212,
1910
+ "selected_success_gain_over_base": 0.0,
1911
+ "selected_success_rate": 0.2753623188405797,
1912
+ "selected_utility_mean": 0.6625525587005541,
1913
+ "success_selector_gap": 0.11594202898550725,
1914
+ "success_support_gap": 0.2898550724637681
1915
+ },
1916
+ "target_content_hash": "fa0fe0e3881ee70ce91492445871614d01b3a47fc7c576659c53ff17ddee4e42",
1917
+ "target_split_hash": "9d1dc1c10868f76fa080895df0f2730c173f12b6db14932599d55c2b60859381"
1918
+ }
workspace/runs/ctt_residual_rollout_val69_seed0/metrics_by_seed.json ADDED
@@ -0,0 +1,1083 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "1025740212": {
3
+ "base_success": 0.0,
4
+ "base_utility": 0.528373658657074,
5
+ "branch_car_at_8": 0.26188503205776215,
6
+ "hidden_chart_oracle_success_at_8": 1.0,
7
+ "hidden_chart_oracle_utility_at_8": 2.0,
8
+ "outcome_ptr_at_8": 0.0,
9
+ "pairwise_causal_calibration_ece": 0.08963387331564551,
10
+ "proposal_oracle_success_at_8": 0.0,
11
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
12
+ "proposal_oracle_utility_at_8": 0.4761262834072113,
13
+ "proposal_oracle_utility_gain_over_base_at_8": -0.05224737524986267,
14
+ "selected_success_at_8": 0.0,
15
+ "selected_success_gain_over_base_at_8": 0.0,
16
+ "selected_utility_at_8": 0.21424125134944916,
17
+ "selected_utility_gain_over_base_at_8": -0.3141324073076248,
18
+ "selector_regret_at_8": 0.26188503205776215,
19
+ "success_selector_gap_at_8": 0.0,
20
+ "success_support_gap_at_8": 1.0,
21
+ "success_total_car_to_hidden_at_8": 1.0,
22
+ "support_gap_at_8": 1.5238737165927887,
23
+ "total_car_to_hidden_at_8": 1.7857587486505508
24
+ },
25
+ "1055157354": {
26
+ "base_success": 0.0,
27
+ "base_utility": 0.17649109661579132,
28
+ "branch_car_at_8": 1.8516457676887512,
29
+ "hidden_chart_oracle_success_at_8": 1.0,
30
+ "hidden_chart_oracle_utility_at_8": 2.0,
31
+ "outcome_ptr_at_8": 1.0,
32
+ "pairwise_causal_calibration_ece": 0.2973966626262834,
33
+ "proposal_oracle_success_at_8": 1.0,
34
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
35
+ "proposal_oracle_utility_at_8": 2.0,
36
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8235089033842087,
37
+ "selected_success_at_8": 0.0,
38
+ "selected_success_gain_over_base_at_8": 0.0,
39
+ "selected_utility_at_8": 0.14835423231124878,
40
+ "selected_utility_gain_over_base_at_8": -0.02813686430454254,
41
+ "selector_regret_at_8": 1.8516457676887512,
42
+ "success_selector_gap_at_8": 1.0,
43
+ "success_support_gap_at_8": 0.0,
44
+ "success_total_car_to_hidden_at_8": 1.0,
45
+ "support_gap_at_8": 0.0,
46
+ "total_car_to_hidden_at_8": 1.8516457676887512
47
+ },
48
+ "1055567618": {
49
+ "base_success": 1.0,
50
+ "base_utility": 2.0,
51
+ "branch_car_at_8": 1.183156132698059,
52
+ "hidden_chart_oracle_success_at_8": 1.0,
53
+ "hidden_chart_oracle_utility_at_8": 2.0,
54
+ "outcome_ptr_at_8": 0.0,
55
+ "pairwise_causal_calibration_ece": 0.07486610854656817,
56
+ "proposal_oracle_success_at_8": 1.0,
57
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
58
+ "proposal_oracle_utility_at_8": 2.0,
59
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
60
+ "selected_success_at_8": 0.0,
61
+ "selected_success_gain_over_base_at_8": -1.0,
62
+ "selected_utility_at_8": 0.8168438673019409,
63
+ "selected_utility_gain_over_base_at_8": -1.183156132698059,
64
+ "selector_regret_at_8": 1.183156132698059,
65
+ "success_selector_gap_at_8": 1.0,
66
+ "success_support_gap_at_8": 0.0,
67
+ "success_total_car_to_hidden_at_8": 1.0,
68
+ "support_gap_at_8": 0.0,
69
+ "total_car_to_hidden_at_8": 1.183156132698059
70
+ },
71
+ "1113268940": {
72
+ "base_success": 0.0,
73
+ "base_utility": 0.42231252789497375,
74
+ "branch_car_at_8": 0.38470593094825745,
75
+ "hidden_chart_oracle_success_at_8": 1.0,
76
+ "hidden_chart_oracle_utility_at_8": 2.0,
77
+ "outcome_ptr_at_8": 1.0,
78
+ "pairwise_causal_calibration_ece": 0.1616344352768096,
79
+ "proposal_oracle_success_at_8": 0.0,
80
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
81
+ "proposal_oracle_utility_at_8": 0.6646121144294739,
82
+ "proposal_oracle_utility_gain_over_base_at_8": 0.24229958653450012,
83
+ "selected_success_at_8": 0.0,
84
+ "selected_success_gain_over_base_at_8": 0.0,
85
+ "selected_utility_at_8": 0.27990618348121643,
86
+ "selected_utility_gain_over_base_at_8": -0.14240634441375732,
87
+ "selector_regret_at_8": 0.38470593094825745,
88
+ "success_selector_gap_at_8": 0.0,
89
+ "success_support_gap_at_8": 1.0,
90
+ "success_total_car_to_hidden_at_8": 1.0,
91
+ "support_gap_at_8": 1.3353878855705261,
92
+ "total_car_to_hidden_at_8": 1.7200938165187836
93
+ },
94
+ "1113371261": {
95
+ "base_success": 1.0,
96
+ "base_utility": 2.0,
97
+ "branch_car_at_8": 0.0,
98
+ "hidden_chart_oracle_success_at_8": 1.0,
99
+ "hidden_chart_oracle_utility_at_8": 2.0,
100
+ "outcome_ptr_at_8": 0.0,
101
+ "pairwise_causal_calibration_ece": 0.3179396489681421,
102
+ "proposal_oracle_success_at_8": 1.0,
103
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
104
+ "proposal_oracle_utility_at_8": 2.0,
105
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
106
+ "selected_success_at_8": 1.0,
107
+ "selected_success_gain_over_base_at_8": 0.0,
108
+ "selected_utility_at_8": 2.0,
109
+ "selected_utility_gain_over_base_at_8": 0.0,
110
+ "selector_regret_at_8": 0.0,
111
+ "success_selector_gap_at_8": 0.0,
112
+ "success_support_gap_at_8": 0.0,
113
+ "success_total_car_to_hidden_at_8": 0.0,
114
+ "support_gap_at_8": 0.0,
115
+ "total_car_to_hidden_at_8": 0.0
116
+ },
117
+ "1176943004": {
118
+ "base_success": 0.0,
119
+ "base_utility": 0.05094052851200104,
120
+ "branch_car_at_8": 0.0,
121
+ "hidden_chart_oracle_success_at_8": 1.0,
122
+ "hidden_chart_oracle_utility_at_8": 2.0,
123
+ "outcome_ptr_at_8": 1.0,
124
+ "pairwise_causal_calibration_ece": 0.2176350935010736,
125
+ "proposal_oracle_success_at_8": 1.0,
126
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
127
+ "proposal_oracle_utility_at_8": 2.0,
128
+ "proposal_oracle_utility_gain_over_base_at_8": 1.949059471487999,
129
+ "selected_success_at_8": 1.0,
130
+ "selected_success_gain_over_base_at_8": 1.0,
131
+ "selected_utility_at_8": 2.0,
132
+ "selected_utility_gain_over_base_at_8": 1.949059471487999,
133
+ "selector_regret_at_8": 0.0,
134
+ "success_selector_gap_at_8": 0.0,
135
+ "success_support_gap_at_8": 0.0,
136
+ "success_total_car_to_hidden_at_8": 0.0,
137
+ "support_gap_at_8": 0.0,
138
+ "total_car_to_hidden_at_8": 0.0
139
+ },
140
+ "1189392917": {
141
+ "base_success": 0.0,
142
+ "base_utility": 0.10841864347457886,
143
+ "branch_car_at_8": 0.0,
144
+ "hidden_chart_oracle_success_at_8": 1.0,
145
+ "hidden_chart_oracle_utility_at_8": 2.0,
146
+ "outcome_ptr_at_8": 1.0,
147
+ "pairwise_causal_calibration_ece": 0.1628408162009477,
148
+ "proposal_oracle_success_at_8": 1.0,
149
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
150
+ "proposal_oracle_utility_at_8": 2.0,
151
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211,
152
+ "selected_success_at_8": 1.0,
153
+ "selected_success_gain_over_base_at_8": 1.0,
154
+ "selected_utility_at_8": 2.0,
155
+ "selected_utility_gain_over_base_at_8": 1.8915813565254211,
156
+ "selector_regret_at_8": 0.0,
157
+ "success_selector_gap_at_8": 0.0,
158
+ "success_support_gap_at_8": 0.0,
159
+ "success_total_car_to_hidden_at_8": 0.0,
160
+ "support_gap_at_8": 0.0,
161
+ "total_car_to_hidden_at_8": 0.0
162
+ },
163
+ "1231780997": {
164
+ "base_success": 0.0,
165
+ "base_utility": 0.7270061373710632,
166
+ "branch_car_at_8": 0.4897426515817642,
167
+ "hidden_chart_oracle_success_at_8": 1.0,
168
+ "hidden_chart_oracle_utility_at_8": 2.0,
169
+ "outcome_ptr_at_8": 0.0,
170
+ "pairwise_causal_calibration_ece": 0.18322944379343714,
171
+ "proposal_oracle_success_at_8": 0.0,
172
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
173
+ "proposal_oracle_utility_at_8": 0.7187411189079285,
174
+ "proposal_oracle_utility_gain_over_base_at_8": -0.008265018463134766,
175
+ "selected_success_at_8": 0.0,
176
+ "selected_success_gain_over_base_at_8": 0.0,
177
+ "selected_utility_at_8": 0.22899846732616425,
178
+ "selected_utility_gain_over_base_at_8": -0.498007670044899,
179
+ "selector_regret_at_8": 0.4897426515817642,
180
+ "success_selector_gap_at_8": 0.0,
181
+ "success_support_gap_at_8": 1.0,
182
+ "success_total_car_to_hidden_at_8": 1.0,
183
+ "support_gap_at_8": 1.2812588810920715,
184
+ "total_car_to_hidden_at_8": 1.7710015326738358
185
+ },
186
+ "1271718007": {
187
+ "base_success": 0.5,
188
+ "base_utility": 1.4142604768276215,
189
+ "branch_car_at_8": 0.0,
190
+ "hidden_chart_oracle_success_at_8": 1.0,
191
+ "hidden_chart_oracle_utility_at_8": 2.0,
192
+ "outcome_ptr_at_8": 0.5,
193
+ "pairwise_causal_calibration_ece": 0.07543261691159965,
194
+ "proposal_oracle_success_at_8": 1.0,
195
+ "proposal_oracle_success_gain_over_base_at_8": 0.5,
196
+ "proposal_oracle_utility_at_8": 2.0,
197
+ "proposal_oracle_utility_gain_over_base_at_8": 0.5857395231723785,
198
+ "selected_success_at_8": 1.0,
199
+ "selected_success_gain_over_base_at_8": 0.5,
200
+ "selected_utility_at_8": 2.0,
201
+ "selected_utility_gain_over_base_at_8": 0.5857395231723785,
202
+ "selector_regret_at_8": 0.0,
203
+ "success_selector_gap_at_8": 0.0,
204
+ "success_support_gap_at_8": 0.0,
205
+ "success_total_car_to_hidden_at_8": 0.0,
206
+ "support_gap_at_8": 0.0,
207
+ "total_car_to_hidden_at_8": 0.0
208
+ },
209
+ "1273247072": {
210
+ "base_success": 0.0,
211
+ "base_utility": 0.15342965722084045,
212
+ "branch_car_at_8": 0.0,
213
+ "hidden_chart_oracle_success_at_8": 1.0,
214
+ "hidden_chart_oracle_utility_at_8": 2.0,
215
+ "outcome_ptr_at_8": 0.0,
216
+ "pairwise_causal_calibration_ece": 0.15040783499409596,
217
+ "proposal_oracle_success_at_8": 0.0,
218
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
219
+ "proposal_oracle_utility_at_8": 0.13278456032276154,
220
+ "proposal_oracle_utility_gain_over_base_at_8": -0.02064509689807892,
221
+ "selected_success_at_8": 0.0,
222
+ "selected_success_gain_over_base_at_8": 0.0,
223
+ "selected_utility_at_8": 0.13278456032276154,
224
+ "selected_utility_gain_over_base_at_8": -0.02064509689807892,
225
+ "selector_regret_at_8": 0.0,
226
+ "success_selector_gap_at_8": 0.0,
227
+ "success_support_gap_at_8": 1.0,
228
+ "success_total_car_to_hidden_at_8": 1.0,
229
+ "support_gap_at_8": 1.8672154396772385,
230
+ "total_car_to_hidden_at_8": 1.8672154396772385
231
+ },
232
+ "1358421325": {
233
+ "base_success": 0.0,
234
+ "base_utility": 0.8327022790908813,
235
+ "branch_car_at_8": 0.5775325745344162,
236
+ "hidden_chart_oracle_success_at_8": 1.0,
237
+ "hidden_chart_oracle_utility_at_8": 2.0,
238
+ "outcome_ptr_at_8": 0.0,
239
+ "pairwise_causal_calibration_ece": 0.13502516765102715,
240
+ "proposal_oracle_success_at_8": 0.0,
241
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
242
+ "proposal_oracle_utility_at_8": 0.8142459392547607,
243
+ "proposal_oracle_utility_gain_over_base_at_8": -0.018456339836120605,
244
+ "selected_success_at_8": 0.0,
245
+ "selected_success_gain_over_base_at_8": 0.0,
246
+ "selected_utility_at_8": 0.23671336472034454,
247
+ "selected_utility_gain_over_base_at_8": -0.5959889143705368,
248
+ "selector_regret_at_8": 0.5775325745344162,
249
+ "success_selector_gap_at_8": 0.0,
250
+ "success_support_gap_at_8": 1.0,
251
+ "success_total_car_to_hidden_at_8": 1.0,
252
+ "support_gap_at_8": 1.1857540607452393,
253
+ "total_car_to_hidden_at_8": 1.7632866352796555
254
+ },
255
+ "1375492786": {
256
+ "base_success": 1.0,
257
+ "base_utility": 2.0,
258
+ "branch_car_at_8": 0.0002854267368093133,
259
+ "hidden_chart_oracle_success_at_8": 0.0,
260
+ "hidden_chart_oracle_utility_at_8": 0.24345096945762634,
261
+ "outcome_ptr_at_8": 0.0,
262
+ "pairwise_causal_calibration_ece": 0.19928218350064877,
263
+ "proposal_oracle_success_at_8": 0.0,
264
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
265
+ "proposal_oracle_utility_at_8": 0.0016440550098195672,
266
+ "proposal_oracle_utility_gain_over_base_at_8": -1.9983559449901804,
267
+ "selected_success_at_8": 0.0,
268
+ "selected_success_gain_over_base_at_8": -1.0,
269
+ "selected_utility_at_8": 0.001358628273010254,
270
+ "selected_utility_gain_over_base_at_8": -1.9986413717269897,
271
+ "selector_regret_at_8": 0.0002854267368093133,
272
+ "success_selector_gap_at_8": 0.0,
273
+ "success_support_gap_at_8": 0.0,
274
+ "success_total_car_to_hidden_at_8": 0.0,
275
+ "support_gap_at_8": 0.24180691444780678,
276
+ "total_car_to_hidden_at_8": 0.2420923411846161
277
+ },
278
+ "141984615": {
279
+ "base_success": 0.0,
280
+ "base_utility": 0.09381213908394177,
281
+ "branch_car_at_8": 0.03316789741317431,
282
+ "hidden_chart_oracle_success_at_8": 0.0,
283
+ "hidden_chart_oracle_utility_at_8": 0.20044784247875214,
284
+ "outcome_ptr_at_8": 0.3333333333333333,
285
+ "pairwise_causal_calibration_ece": 0.10926310602682258,
286
+ "proposal_oracle_success_at_8": 0.0,
287
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
288
+ "proposal_oracle_utility_at_8": 0.0797463705142339,
289
+ "proposal_oracle_utility_gain_over_base_at_8": -0.01406576856970787,
290
+ "selected_success_at_8": 0.0,
291
+ "selected_success_gain_over_base_at_8": 0.0,
292
+ "selected_utility_at_8": 0.04657847310105959,
293
+ "selected_utility_gain_over_base_at_8": -0.04723366598288218,
294
+ "selector_regret_at_8": 0.03316789741317431,
295
+ "success_selector_gap_at_8": 0.0,
296
+ "success_support_gap_at_8": 0.0,
297
+ "success_total_car_to_hidden_at_8": 0.0,
298
+ "support_gap_at_8": 0.12070147196451823,
299
+ "total_car_to_hidden_at_8": 0.15386936937769255
300
+ },
301
+ "1475440973": {
302
+ "base_success": 1.0,
303
+ "base_utility": 2.0,
304
+ "branch_car_at_8": 0.0,
305
+ "hidden_chart_oracle_success_at_8": 1.0,
306
+ "hidden_chart_oracle_utility_at_8": 2.0,
307
+ "outcome_ptr_at_8": 0.0,
308
+ "pairwise_causal_calibration_ece": 0.09930428958964055,
309
+ "proposal_oracle_success_at_8": 1.0,
310
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
311
+ "proposal_oracle_utility_at_8": 2.0,
312
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
313
+ "selected_success_at_8": 1.0,
314
+ "selected_success_gain_over_base_at_8": 0.0,
315
+ "selected_utility_at_8": 2.0,
316
+ "selected_utility_gain_over_base_at_8": 0.0,
317
+ "selector_regret_at_8": 0.0,
318
+ "success_selector_gap_at_8": 0.0,
319
+ "success_support_gap_at_8": 0.0,
320
+ "success_total_car_to_hidden_at_8": 0.0,
321
+ "support_gap_at_8": 0.0,
322
+ "total_car_to_hidden_at_8": 0.0
323
+ },
324
+ "1483645336": {
325
+ "base_success": 0.0,
326
+ "base_utility": 0.01641605794429779,
327
+ "branch_car_at_8": 0.0,
328
+ "hidden_chart_oracle_success_at_8": 1.0,
329
+ "hidden_chart_oracle_utility_at_8": 2.0,
330
+ "outcome_ptr_at_8": 1.0,
331
+ "pairwise_causal_calibration_ece": 0.0,
332
+ "proposal_oracle_success_at_8": 1.0,
333
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
334
+ "proposal_oracle_utility_at_8": 2.0,
335
+ "proposal_oracle_utility_gain_over_base_at_8": 1.9835839420557022,
336
+ "selected_success_at_8": 1.0,
337
+ "selected_success_gain_over_base_at_8": 1.0,
338
+ "selected_utility_at_8": 2.0,
339
+ "selected_utility_gain_over_base_at_8": 1.9835839420557022,
340
+ "selector_regret_at_8": 0.0,
341
+ "success_selector_gap_at_8": 0.0,
342
+ "success_support_gap_at_8": 0.0,
343
+ "success_total_car_to_hidden_at_8": 0.0,
344
+ "support_gap_at_8": 0.0,
345
+ "total_car_to_hidden_at_8": 0.0
346
+ },
347
+ "1522272686": {
348
+ "base_success": 0.0,
349
+ "base_utility": 0.19476109743118286,
350
+ "branch_car_at_8": 0.009524375200271606,
351
+ "hidden_chart_oracle_success_at_8": 1.0,
352
+ "hidden_chart_oracle_utility_at_8": 2.0,
353
+ "outcome_ptr_at_8": 1.0,
354
+ "pairwise_causal_calibration_ece": 0.08257819446483214,
355
+ "proposal_oracle_success_at_8": 0.0,
356
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
357
+ "proposal_oracle_utility_at_8": 0.2435554713010788,
358
+ "proposal_oracle_utility_gain_over_base_at_8": 0.048794373869895935,
359
+ "selected_success_at_8": 0.0,
360
+ "selected_success_gain_over_base_at_8": 0.0,
361
+ "selected_utility_at_8": 0.2340310961008072,
362
+ "selected_utility_gain_over_base_at_8": 0.03926999866962433,
363
+ "selector_regret_at_8": 0.009524375200271606,
364
+ "success_selector_gap_at_8": 0.0,
365
+ "success_support_gap_at_8": 1.0,
366
+ "success_total_car_to_hidden_at_8": 1.0,
367
+ "support_gap_at_8": 1.7564445286989212,
368
+ "total_car_to_hidden_at_8": 1.7659689038991928
369
+ },
370
+ "1528740417": {
371
+ "base_success": 1.0,
372
+ "base_utility": 2.0,
373
+ "branch_car_at_8": 0.0,
374
+ "hidden_chart_oracle_success_at_8": 1.0,
375
+ "hidden_chart_oracle_utility_at_8": 2.0,
376
+ "outcome_ptr_at_8": 0.0,
377
+ "pairwise_causal_calibration_ece": 0.015990866806051107,
378
+ "proposal_oracle_success_at_8": 1.0,
379
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
380
+ "proposal_oracle_utility_at_8": 2.0,
381
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
382
+ "selected_success_at_8": 1.0,
383
+ "selected_success_gain_over_base_at_8": 0.0,
384
+ "selected_utility_at_8": 2.0,
385
+ "selected_utility_gain_over_base_at_8": 0.0,
386
+ "selector_regret_at_8": 0.0,
387
+ "success_selector_gap_at_8": 0.0,
388
+ "success_support_gap_at_8": 0.0,
389
+ "success_total_car_to_hidden_at_8": 0.0,
390
+ "support_gap_at_8": 0.0,
391
+ "total_car_to_hidden_at_8": 0.0
392
+ },
393
+ "1549324805": {
394
+ "base_success": 0.6666666666666666,
395
+ "base_utility": 1.3787474483251572,
396
+ "branch_car_at_8": 0.0034310482442378998,
397
+ "hidden_chart_oracle_success_at_8": 0.6666666666666666,
398
+ "hidden_chart_oracle_utility_at_8": 1.4027505218982697,
399
+ "outcome_ptr_at_8": 0.0,
400
+ "pairwise_causal_calibration_ece": 0.19382954770507574,
401
+ "proposal_oracle_success_at_8": 0.6666666666666666,
402
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
403
+ "proposal_oracle_utility_at_8": 1.3427665165315072,
404
+ "proposal_oracle_utility_gain_over_base_at_8": -0.035980931793649994,
405
+ "selected_success_at_8": 0.6666666666666666,
406
+ "selected_success_gain_over_base_at_8": 0.0,
407
+ "selected_utility_at_8": 1.3393354682872693,
408
+ "selected_utility_gain_over_base_at_8": -0.03941198003788789,
409
+ "selector_regret_at_8": 0.0034310482442378998,
410
+ "success_selector_gap_at_8": 0.0,
411
+ "success_support_gap_at_8": 0.0,
412
+ "success_total_car_to_hidden_at_8": 0.0,
413
+ "support_gap_at_8": 0.05998400536676248,
414
+ "total_car_to_hidden_at_8": 0.06341505361100037
415
+ },
416
+ "1607773860": {
417
+ "base_success": 1.0,
418
+ "base_utility": 2.0,
419
+ "branch_car_at_8": 1.779732346534729,
420
+ "hidden_chart_oracle_success_at_8": 1.0,
421
+ "hidden_chart_oracle_utility_at_8": 2.0,
422
+ "outcome_ptr_at_8": 0.0,
423
+ "pairwise_causal_calibration_ece": 0.08588083120555855,
424
+ "proposal_oracle_success_at_8": 1.0,
425
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
426
+ "proposal_oracle_utility_at_8": 2.0,
427
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
428
+ "selected_success_at_8": 0.0,
429
+ "selected_success_gain_over_base_at_8": -1.0,
430
+ "selected_utility_at_8": 0.220267653465271,
431
+ "selected_utility_gain_over_base_at_8": -1.779732346534729,
432
+ "selector_regret_at_8": 1.779732346534729,
433
+ "success_selector_gap_at_8": 1.0,
434
+ "success_support_gap_at_8": 0.0,
435
+ "success_total_car_to_hidden_at_8": 1.0,
436
+ "support_gap_at_8": 0.0,
437
+ "total_car_to_hidden_at_8": 1.779732346534729
438
+ },
439
+ "1665739511": {
440
+ "base_success": 1.0,
441
+ "base_utility": 2.0,
442
+ "branch_car_at_8": 0.007551252841949463,
443
+ "hidden_chart_oracle_success_at_8": 1.0,
444
+ "hidden_chart_oracle_utility_at_8": 2.0,
445
+ "outcome_ptr_at_8": 0.0,
446
+ "pairwise_causal_calibration_ece": 0.050940808914706356,
447
+ "proposal_oracle_success_at_8": 0.0,
448
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
449
+ "proposal_oracle_utility_at_8": 0.8216220736503601,
450
+ "proposal_oracle_utility_gain_over_base_at_8": -1.17837792634964,
451
+ "selected_success_at_8": 0.0,
452
+ "selected_success_gain_over_base_at_8": -1.0,
453
+ "selected_utility_at_8": 0.8140708208084106,
454
+ "selected_utility_gain_over_base_at_8": -1.1859291791915894,
455
+ "selector_regret_at_8": 0.007551252841949463,
456
+ "success_selector_gap_at_8": 0.0,
457
+ "success_support_gap_at_8": 1.0,
458
+ "success_total_car_to_hidden_at_8": 1.0,
459
+ "support_gap_at_8": 1.17837792634964,
460
+ "total_car_to_hidden_at_8": 1.1859291791915894
461
+ },
462
+ "1747178030": {
463
+ "base_success": 0.0,
464
+ "base_utility": 0.1463065892457962,
465
+ "branch_car_at_8": 0.0,
466
+ "hidden_chart_oracle_success_at_8": 1.0,
467
+ "hidden_chart_oracle_utility_at_8": 2.0,
468
+ "outcome_ptr_at_8": 1.0,
469
+ "pairwise_causal_calibration_ece": 0.15825371494561297,
470
+ "proposal_oracle_success_at_8": 1.0,
471
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
472
+ "proposal_oracle_utility_at_8": 2.0,
473
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8536934107542038,
474
+ "selected_success_at_8": 1.0,
475
+ "selected_success_gain_over_base_at_8": 1.0,
476
+ "selected_utility_at_8": 2.0,
477
+ "selected_utility_gain_over_base_at_8": 1.8536934107542038,
478
+ "selector_regret_at_8": 0.0,
479
+ "success_selector_gap_at_8": 0.0,
480
+ "success_support_gap_at_8": 0.0,
481
+ "success_total_car_to_hidden_at_8": 0.0,
482
+ "support_gap_at_8": 0.0,
483
+ "total_car_to_hidden_at_8": 0.0
484
+ },
485
+ "1881223807": {
486
+ "base_success": 0.0,
487
+ "base_utility": 0.0320269912481308,
488
+ "branch_car_at_8": 0.020236089825630188,
489
+ "hidden_chart_oracle_success_at_8": 1.0,
490
+ "hidden_chart_oracle_utility_at_8": 2.0,
491
+ "outcome_ptr_at_8": 1.0,
492
+ "pairwise_causal_calibration_ece": 0.019060791998604134,
493
+ "proposal_oracle_success_at_8": 0.0,
494
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
495
+ "proposal_oracle_utility_at_8": 0.0513521283864975,
496
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0193251371383667,
497
+ "selected_success_at_8": 0.0,
498
+ "selected_success_gain_over_base_at_8": 0.0,
499
+ "selected_utility_at_8": 0.03111603856086731,
500
+ "selected_utility_gain_over_base_at_8": -0.0009109526872634888,
501
+ "selector_regret_at_8": 0.020236089825630188,
502
+ "success_selector_gap_at_8": 0.0,
503
+ "success_support_gap_at_8": 1.0,
504
+ "success_total_car_to_hidden_at_8": 1.0,
505
+ "support_gap_at_8": 1.9486478716135025,
506
+ "total_car_to_hidden_at_8": 1.9688839614391327
507
+ },
508
+ "1919198188": {
509
+ "base_success": 0.0,
510
+ "base_utility": 0.037198856472969055,
511
+ "branch_car_at_8": 0.016854479908943176,
512
+ "hidden_chart_oracle_success_at_8": 1.0,
513
+ "hidden_chart_oracle_utility_at_8": 2.0,
514
+ "outcome_ptr_at_8": 1.0,
515
+ "pairwise_causal_calibration_ece": 0.15779226512372985,
516
+ "proposal_oracle_success_at_8": 0.0,
517
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
518
+ "proposal_oracle_utility_at_8": 0.08992795646190643,
519
+ "proposal_oracle_utility_gain_over_base_at_8": 0.05272909998893738,
520
+ "selected_success_at_8": 0.0,
521
+ "selected_success_gain_over_base_at_8": 0.0,
522
+ "selected_utility_at_8": 0.07307347655296326,
523
+ "selected_utility_gain_over_base_at_8": 0.0358746200799942,
524
+ "selector_regret_at_8": 0.016854479908943176,
525
+ "success_selector_gap_at_8": 0.0,
526
+ "success_support_gap_at_8": 1.0,
527
+ "success_total_car_to_hidden_at_8": 1.0,
528
+ "support_gap_at_8": 1.9100720435380936,
529
+ "total_car_to_hidden_at_8": 1.9269265234470367
530
+ },
531
+ "1991414747": {
532
+ "base_success": 1.0,
533
+ "base_utility": 2.0,
534
+ "branch_car_at_8": 0.0,
535
+ "hidden_chart_oracle_success_at_8": 1.0,
536
+ "hidden_chart_oracle_utility_at_8": 2.0,
537
+ "outcome_ptr_at_8": 0.0,
538
+ "pairwise_causal_calibration_ece": 0.05287821364799471,
539
+ "proposal_oracle_success_at_8": 1.0,
540
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
541
+ "proposal_oracle_utility_at_8": 2.0,
542
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
543
+ "selected_success_at_8": 1.0,
544
+ "selected_success_gain_over_base_at_8": 0.0,
545
+ "selected_utility_at_8": 2.0,
546
+ "selected_utility_gain_over_base_at_8": 0.0,
547
+ "selector_regret_at_8": 0.0,
548
+ "success_selector_gap_at_8": 0.0,
549
+ "success_support_gap_at_8": 0.0,
550
+ "success_total_car_to_hidden_at_8": 0.0,
551
+ "support_gap_at_8": 0.0,
552
+ "total_car_to_hidden_at_8": 0.0
553
+ },
554
+ "1997090716": {
555
+ "base_success": 0.0,
556
+ "base_utility": 0.13927479088306427,
557
+ "branch_car_at_8": 1.9691698104143143,
558
+ "hidden_chart_oracle_success_at_8": 1.0,
559
+ "hidden_chart_oracle_utility_at_8": 2.0,
560
+ "outcome_ptr_at_8": 1.0,
561
+ "pairwise_causal_calibration_ece": 0.4044963838715546,
562
+ "proposal_oracle_success_at_8": 1.0,
563
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
564
+ "proposal_oracle_utility_at_8": 2.0,
565
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357,
566
+ "selected_success_at_8": 0.0,
567
+ "selected_success_gain_over_base_at_8": 0.0,
568
+ "selected_utility_at_8": 0.03083018958568573,
569
+ "selected_utility_gain_over_base_at_8": -0.10844460129737854,
570
+ "selector_regret_at_8": 1.9691698104143143,
571
+ "success_selector_gap_at_8": 1.0,
572
+ "success_support_gap_at_8": 0.0,
573
+ "success_total_car_to_hidden_at_8": 1.0,
574
+ "support_gap_at_8": 0.0,
575
+ "total_car_to_hidden_at_8": 1.9691698104143143
576
+ },
577
+ "2015956957": {
578
+ "base_success": 0.0,
579
+ "base_utility": 0.06711978986859321,
580
+ "branch_car_at_8": 0.01770364921540022,
581
+ "hidden_chart_oracle_success_at_8": 0.2,
582
+ "hidden_chart_oracle_utility_at_8": 0.5298813670873642,
583
+ "outcome_ptr_at_8": 0.6,
584
+ "pairwise_causal_calibration_ece": 0.18664994005668376,
585
+ "proposal_oracle_success_at_8": 0.2,
586
+ "proposal_oracle_success_gain_over_base_at_8": 0.2,
587
+ "proposal_oracle_utility_at_8": 0.45124312303960323,
588
+ "proposal_oracle_utility_gain_over_base_at_8": 0.38412333317101,
589
+ "selected_success_at_8": 0.2,
590
+ "selected_success_gain_over_base_at_8": 0.2,
591
+ "selected_utility_at_8": 0.433539473824203,
592
+ "selected_utility_gain_over_base_at_8": 0.3664196839556098,
593
+ "selector_regret_at_8": 0.01770364921540022,
594
+ "success_selector_gap_at_8": 0.0,
595
+ "success_support_gap_at_8": 0.0,
596
+ "success_total_car_to_hidden_at_8": 0.0,
597
+ "support_gap_at_8": 0.07863824404776096,
598
+ "total_car_to_hidden_at_8": 0.09634189326316119
599
+ },
600
+ "2054899126": {
601
+ "base_success": 1.0,
602
+ "base_utility": 2.0,
603
+ "branch_car_at_8": 0.0,
604
+ "hidden_chart_oracle_success_at_8": 1.0,
605
+ "hidden_chart_oracle_utility_at_8": 2.0,
606
+ "outcome_ptr_at_8": 0.0,
607
+ "pairwise_causal_calibration_ece": 0.0,
608
+ "proposal_oracle_success_at_8": 1.0,
609
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
610
+ "proposal_oracle_utility_at_8": 2.0,
611
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
612
+ "selected_success_at_8": 1.0,
613
+ "selected_success_gain_over_base_at_8": 0.0,
614
+ "selected_utility_at_8": 2.0,
615
+ "selected_utility_gain_over_base_at_8": 0.0,
616
+ "selector_regret_at_8": 0.0,
617
+ "success_selector_gap_at_8": 0.0,
618
+ "success_support_gap_at_8": 0.0,
619
+ "success_total_car_to_hidden_at_8": 0.0,
620
+ "support_gap_at_8": 0.0,
621
+ "total_car_to_hidden_at_8": 0.0
622
+ },
623
+ "211245842": {
624
+ "base_success": 0.0,
625
+ "base_utility": 0.8388344049453735,
626
+ "branch_car_at_8": 0.5908347219228745,
627
+ "hidden_chart_oracle_success_at_8": 1.0,
628
+ "hidden_chart_oracle_utility_at_8": 2.0,
629
+ "outcome_ptr_at_8": 0.0,
630
+ "pairwise_causal_calibration_ece": 0.16771630653845432,
631
+ "proposal_oracle_success_at_8": 0.0,
632
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
633
+ "proposal_oracle_utility_at_8": 0.8130142092704773,
634
+ "proposal_oracle_utility_gain_over_base_at_8": -0.02582019567489624,
635
+ "selected_success_at_8": 0.0,
636
+ "selected_success_gain_over_base_at_8": 0.0,
637
+ "selected_utility_at_8": 0.22217948734760284,
638
+ "selected_utility_gain_over_base_at_8": -0.6166549175977707,
639
+ "selector_regret_at_8": 0.5908347219228745,
640
+ "success_selector_gap_at_8": 0.0,
641
+ "success_support_gap_at_8": 1.0,
642
+ "success_total_car_to_hidden_at_8": 1.0,
643
+ "support_gap_at_8": 1.1869857907295227,
644
+ "total_car_to_hidden_at_8": 1.7778205126523972
645
+ },
646
+ "2141802816": {
647
+ "base_success": 0.0,
648
+ "base_utility": 0.07080724835395813,
649
+ "branch_car_at_8": 0.07543906569480896,
650
+ "hidden_chart_oracle_success_at_8": 1.0,
651
+ "hidden_chart_oracle_utility_at_8": 2.0,
652
+ "outcome_ptr_at_8": 1.0,
653
+ "pairwise_causal_calibration_ece": 0.05242798758366053,
654
+ "proposal_oracle_success_at_8": 0.0,
655
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
656
+ "proposal_oracle_utility_at_8": 0.12472955882549286,
657
+ "proposal_oracle_utility_gain_over_base_at_8": 0.05392231047153473,
658
+ "selected_success_at_8": 0.0,
659
+ "selected_success_gain_over_base_at_8": 0.0,
660
+ "selected_utility_at_8": 0.0492904931306839,
661
+ "selected_utility_gain_over_base_at_8": -0.02151675522327423,
662
+ "selector_regret_at_8": 0.07543906569480896,
663
+ "success_selector_gap_at_8": 0.0,
664
+ "success_support_gap_at_8": 1.0,
665
+ "success_total_car_to_hidden_at_8": 1.0,
666
+ "support_gap_at_8": 1.8752704411745071,
667
+ "total_car_to_hidden_at_8": 1.950709506869316
668
+ },
669
+ "256770874": {
670
+ "base_success": 0.0,
671
+ "base_utility": 0.8557480573654175,
672
+ "branch_car_at_8": 0.10820980370044708,
673
+ "hidden_chart_oracle_success_at_8": 1.0,
674
+ "hidden_chart_oracle_utility_at_8": 2.0,
675
+ "outcome_ptr_at_8": 0.0,
676
+ "pairwise_causal_calibration_ece": 0.030709639081801327,
677
+ "proposal_oracle_success_at_8": 0.0,
678
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
679
+ "proposal_oracle_utility_at_8": 0.24107472598552704,
680
+ "proposal_oracle_utility_gain_over_base_at_8": -0.6146733313798904,
681
+ "selected_success_at_8": 0.0,
682
+ "selected_success_gain_over_base_at_8": 0.0,
683
+ "selected_utility_at_8": 0.13286492228507996,
684
+ "selected_utility_gain_over_base_at_8": -0.7228831350803375,
685
+ "selector_regret_at_8": 0.10820980370044708,
686
+ "success_selector_gap_at_8": 0.0,
687
+ "success_support_gap_at_8": 1.0,
688
+ "success_total_car_to_hidden_at_8": 1.0,
689
+ "support_gap_at_8": 1.758925274014473,
690
+ "total_car_to_hidden_at_8": 1.86713507771492
691
+ },
692
+ "278008524": {
693
+ "base_success": 0.0,
694
+ "base_utility": 0.10747186839580536,
695
+ "branch_car_at_8": 0.0016456245793960989,
696
+ "hidden_chart_oracle_success_at_8": 0.5,
697
+ "hidden_chart_oracle_utility_at_8": 1.113811194896698,
698
+ "outcome_ptr_at_8": 0.0,
699
+ "pairwise_causal_calibration_ece": 0.11925245416072927,
700
+ "proposal_oracle_success_at_8": 0.0,
701
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
702
+ "proposal_oracle_utility_at_8": 0.011432985425926745,
703
+ "proposal_oracle_utility_gain_over_base_at_8": -0.09603888296987861,
704
+ "selected_success_at_8": 0.0,
705
+ "selected_success_gain_over_base_at_8": 0.0,
706
+ "selected_utility_at_8": 0.009787360846530646,
707
+ "selected_utility_gain_over_base_at_8": -0.09768450754927471,
708
+ "selector_regret_at_8": 0.0016456245793960989,
709
+ "success_selector_gap_at_8": 0.0,
710
+ "success_support_gap_at_8": 0.5,
711
+ "success_total_car_to_hidden_at_8": 0.5,
712
+ "support_gap_at_8": 1.1023782094707713,
713
+ "total_car_to_hidden_at_8": 1.1040238340501674
714
+ },
715
+ "416183106": {
716
+ "base_success": 0.0,
717
+ "base_utility": 0.20032641291618347,
718
+ "branch_car_at_8": 0.3360496461391449,
719
+ "hidden_chart_oracle_success_at_8": 1.0,
720
+ "hidden_chart_oracle_utility_at_8": 2.0,
721
+ "outcome_ptr_at_8": 1.0,
722
+ "pairwise_causal_calibration_ece": 0.10129212810935397,
723
+ "proposal_oracle_success_at_8": 0.0,
724
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
725
+ "proposal_oracle_utility_at_8": 0.5778142213821411,
726
+ "proposal_oracle_utility_gain_over_base_at_8": 0.37748780846595764,
727
+ "selected_success_at_8": 0.0,
728
+ "selected_success_gain_over_base_at_8": 0.0,
729
+ "selected_utility_at_8": 0.24176457524299622,
730
+ "selected_utility_gain_over_base_at_8": 0.041438162326812744,
731
+ "selector_regret_at_8": 0.3360496461391449,
732
+ "success_selector_gap_at_8": 0.0,
733
+ "success_support_gap_at_8": 1.0,
734
+ "success_total_car_to_hidden_at_8": 1.0,
735
+ "support_gap_at_8": 1.4221857786178589,
736
+ "total_car_to_hidden_at_8": 1.7582354247570038
737
+ },
738
+ "534717141": {
739
+ "base_success": 1.0,
740
+ "base_utility": 2.0,
741
+ "branch_car_at_8": 1.9274959191679955,
742
+ "hidden_chart_oracle_success_at_8": 1.0,
743
+ "hidden_chart_oracle_utility_at_8": 2.0,
744
+ "outcome_ptr_at_8": 0.0,
745
+ "pairwise_causal_calibration_ece": 0.6159370955924921,
746
+ "proposal_oracle_success_at_8": 1.0,
747
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
748
+ "proposal_oracle_utility_at_8": 2.0,
749
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
750
+ "selected_success_at_8": 0.0,
751
+ "selected_success_gain_over_base_at_8": -1.0,
752
+ "selected_utility_at_8": 0.07250408083200455,
753
+ "selected_utility_gain_over_base_at_8": -1.9274959191679955,
754
+ "selector_regret_at_8": 1.9274959191679955,
755
+ "success_selector_gap_at_8": 1.0,
756
+ "success_support_gap_at_8": 0.0,
757
+ "success_total_car_to_hidden_at_8": 1.0,
758
+ "support_gap_at_8": 0.0,
759
+ "total_car_to_hidden_at_8": 1.9274959191679955
760
+ },
761
+ "551504635": {
762
+ "base_success": 0.0,
763
+ "base_utility": 0.7239725589752197,
764
+ "branch_car_at_8": 0.033253490924835205,
765
+ "hidden_chart_oracle_success_at_8": 1.0,
766
+ "hidden_chart_oracle_utility_at_8": 2.0,
767
+ "outcome_ptr_at_8": 1.0,
768
+ "pairwise_causal_calibration_ece": 0.10932818664818449,
769
+ "proposal_oracle_success_at_8": 0.0,
770
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
771
+ "proposal_oracle_utility_at_8": 0.7250897884368896,
772
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0011172294616699219,
773
+ "selected_success_at_8": 0.0,
774
+ "selected_success_gain_over_base_at_8": 0.0,
775
+ "selected_utility_at_8": 0.6918362975120544,
776
+ "selected_utility_gain_over_base_at_8": -0.03213626146316528,
777
+ "selector_regret_at_8": 0.033253490924835205,
778
+ "success_selector_gap_at_8": 0.0,
779
+ "success_support_gap_at_8": 1.0,
780
+ "success_total_car_to_hidden_at_8": 1.0,
781
+ "support_gap_at_8": 1.2749102115631104,
782
+ "total_car_to_hidden_at_8": 1.3081637024879456
783
+ },
784
+ "565369392": {
785
+ "base_success": 0.3333333333333333,
786
+ "base_utility": 0.7997656464576721,
787
+ "branch_car_at_8": 0.06443698455890019,
788
+ "hidden_chart_oracle_success_at_8": 0.6666666666666666,
789
+ "hidden_chart_oracle_utility_at_8": 1.4266315797964733,
790
+ "outcome_ptr_at_8": 0.0,
791
+ "pairwise_causal_calibration_ece": 0.22308845281795123,
792
+ "proposal_oracle_success_at_8": 0.3333333333333333,
793
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
794
+ "proposal_oracle_utility_at_8": 0.7852714310089747,
795
+ "proposal_oracle_utility_gain_over_base_at_8": -0.014494215448697409,
796
+ "selected_success_at_8": 0.3333333333333333,
797
+ "selected_success_gain_over_base_at_8": 0.0,
798
+ "selected_utility_at_8": 0.7208344464500746,
799
+ "selected_utility_gain_over_base_at_8": -0.07893120000759761,
800
+ "selector_regret_at_8": 0.06443698455890019,
801
+ "success_selector_gap_at_8": 0.0,
802
+ "success_support_gap_at_8": 0.3333333333333333,
803
+ "success_total_car_to_hidden_at_8": 0.3333333333333333,
804
+ "support_gap_at_8": 0.6413601487874985,
805
+ "total_car_to_hidden_at_8": 0.7057971333463987
806
+ },
807
+ "565621826": {
808
+ "base_success": 1.0,
809
+ "base_utility": 2.0,
810
+ "branch_car_at_8": 0.0,
811
+ "hidden_chart_oracle_success_at_8": 1.0,
812
+ "hidden_chart_oracle_utility_at_8": 2.0,
813
+ "outcome_ptr_at_8": 0.0,
814
+ "pairwise_causal_calibration_ece": 0.0,
815
+ "proposal_oracle_success_at_8": 1.0,
816
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
817
+ "proposal_oracle_utility_at_8": 2.0,
818
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
819
+ "selected_success_at_8": 1.0,
820
+ "selected_success_gain_over_base_at_8": 0.0,
821
+ "selected_utility_at_8": 2.0,
822
+ "selected_utility_gain_over_base_at_8": 0.0,
823
+ "selector_regret_at_8": 0.0,
824
+ "success_selector_gap_at_8": 0.0,
825
+ "success_support_gap_at_8": 0.0,
826
+ "success_total_car_to_hidden_at_8": 0.0,
827
+ "support_gap_at_8": 0.0,
828
+ "total_car_to_hidden_at_8": 0.0
829
+ },
830
+ "585904130": {
831
+ "base_success": 0.0,
832
+ "base_utility": 0.07529209802548091,
833
+ "branch_car_at_8": 0.043168901621053614,
834
+ "hidden_chart_oracle_success_at_8": 0.0,
835
+ "hidden_chart_oracle_utility_at_8": 0.23764699200789133,
836
+ "outcome_ptr_at_8": 0.3333333333333333,
837
+ "pairwise_causal_calibration_ece": 0.10391809443539522,
838
+ "proposal_oracle_success_at_8": 0.0,
839
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
840
+ "proposal_oracle_utility_at_8": 0.05598785417775313,
841
+ "proposal_oracle_utility_gain_over_base_at_8": -0.019304243847727776,
842
+ "selected_success_at_8": 0.0,
843
+ "selected_success_gain_over_base_at_8": 0.0,
844
+ "selected_utility_at_8": 0.012818952556699514,
845
+ "selected_utility_gain_over_base_at_8": -0.06247314546878139,
846
+ "selector_regret_at_8": 0.043168901621053614,
847
+ "success_selector_gap_at_8": 0.0,
848
+ "success_support_gap_at_8": 0.0,
849
+ "success_total_car_to_hidden_at_8": 0.0,
850
+ "support_gap_at_8": 0.1816591378301382,
851
+ "total_car_to_hidden_at_8": 0.2248280394511918
852
+ },
853
+ "626486813": {
854
+ "base_success": 0.25,
855
+ "base_utility": 0.5054400166263804,
856
+ "branch_car_at_8": 0.018570441869087517,
857
+ "hidden_chart_oracle_success_at_8": 0.25,
858
+ "hidden_chart_oracle_utility_at_8": 0.6124684102833271,
859
+ "outcome_ptr_at_8": 0.75,
860
+ "pairwise_causal_calibration_ece": 0.16162412565990822,
861
+ "proposal_oracle_success_at_8": 0.25,
862
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
863
+ "proposal_oracle_utility_at_8": 0.5615248780231923,
864
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0560848613968119,
865
+ "selected_success_at_8": 0.25,
866
+ "selected_success_gain_over_base_at_8": 0.0,
867
+ "selected_utility_at_8": 0.5429544361541048,
868
+ "selected_utility_gain_over_base_at_8": 0.037514419527724385,
869
+ "selector_regret_at_8": 0.018570441869087517,
870
+ "success_selector_gap_at_8": 0.0,
871
+ "success_support_gap_at_8": 0.0,
872
+ "success_total_car_to_hidden_at_8": 0.0,
873
+ "support_gap_at_8": 0.0801479306537658,
874
+ "total_car_to_hidden_at_8": 0.08356922247912735
875
+ },
876
+ "636680137": {
877
+ "base_success": 1.0,
878
+ "base_utility": 2.0,
879
+ "branch_car_at_8": 0.0,
880
+ "hidden_chart_oracle_success_at_8": 1.0,
881
+ "hidden_chart_oracle_utility_at_8": 2.0,
882
+ "outcome_ptr_at_8": 0.0,
883
+ "pairwise_causal_calibration_ece": 0.1699944666253151,
884
+ "proposal_oracle_success_at_8": 0.0,
885
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
886
+ "proposal_oracle_utility_at_8": 0.012414653785526752,
887
+ "proposal_oracle_utility_gain_over_base_at_8": -1.9875853462144732,
888
+ "selected_success_at_8": 0.0,
889
+ "selected_success_gain_over_base_at_8": -1.0,
890
+ "selected_utility_at_8": 0.012414653785526752,
891
+ "selected_utility_gain_over_base_at_8": -1.9875853462144732,
892
+ "selector_regret_at_8": 0.0,
893
+ "success_selector_gap_at_8": 0.0,
894
+ "success_support_gap_at_8": 1.0,
895
+ "success_total_car_to_hidden_at_8": 1.0,
896
+ "support_gap_at_8": 1.9875853462144732,
897
+ "total_car_to_hidden_at_8": 1.9875853462144732
898
+ },
899
+ "706479600": {
900
+ "base_success": 0.0,
901
+ "base_utility": 0.8125406503677368,
902
+ "branch_car_at_8": 1.1865743398666382,
903
+ "hidden_chart_oracle_success_at_8": 1.0,
904
+ "hidden_chart_oracle_utility_at_8": 2.0,
905
+ "outcome_ptr_at_8": 1.0,
906
+ "pairwise_causal_calibration_ece": 0.0922387871049479,
907
+ "proposal_oracle_success_at_8": 1.0,
908
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
909
+ "proposal_oracle_utility_at_8": 2.0,
910
+ "proposal_oracle_utility_gain_over_base_at_8": 1.1874593496322632,
911
+ "selected_success_at_8": 0.0,
912
+ "selected_success_gain_over_base_at_8": 0.0,
913
+ "selected_utility_at_8": 0.8134256601333618,
914
+ "selected_utility_gain_over_base_at_8": 0.000885009765625,
915
+ "selector_regret_at_8": 1.1865743398666382,
916
+ "success_selector_gap_at_8": 1.0,
917
+ "success_support_gap_at_8": 0.0,
918
+ "success_total_car_to_hidden_at_8": 1.0,
919
+ "support_gap_at_8": 0.0,
920
+ "total_car_to_hidden_at_8": 1.1865743398666382
921
+ },
922
+ "750448296": {
923
+ "base_success": 0.0,
924
+ "base_utility": 0.11153355985879898,
925
+ "branch_car_at_8": 0.07732226233929396,
926
+ "hidden_chart_oracle_success_at_8": 0.0,
927
+ "hidden_chart_oracle_utility_at_8": 0.23580759018659592,
928
+ "outcome_ptr_at_8": 0.5,
929
+ "pairwise_causal_calibration_ece": 0.14360585133877676,
930
+ "proposal_oracle_success_at_8": 0.0,
931
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
932
+ "proposal_oracle_utility_at_8": 0.1069931760430336,
933
+ "proposal_oracle_utility_gain_over_base_at_8": -0.004540383815765381,
934
+ "selected_success_at_8": 0.0,
935
+ "selected_success_gain_over_base_at_8": 0.0,
936
+ "selected_utility_at_8": 0.029670913703739643,
937
+ "selected_utility_gain_over_base_at_8": -0.08186264615505934,
938
+ "selector_regret_at_8": 0.07732226233929396,
939
+ "success_selector_gap_at_8": 0.0,
940
+ "success_support_gap_at_8": 0.0,
941
+ "success_total_car_to_hidden_at_8": 0.0,
942
+ "support_gap_at_8": 0.12881441414356232,
943
+ "total_car_to_hidden_at_8": 0.20613667648285627
944
+ },
945
+ "75070641": {
946
+ "base_success": 0.0,
947
+ "base_utility": 0.20275163650512695,
948
+ "branch_car_at_8": 0.07482179999351501,
949
+ "hidden_chart_oracle_success_at_8": 1.0,
950
+ "hidden_chart_oracle_utility_at_8": 2.0,
951
+ "outcome_ptr_at_8": 1.0,
952
+ "pairwise_causal_calibration_ece": 0.2497251397544613,
953
+ "proposal_oracle_success_at_8": 0.0,
954
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
955
+ "proposal_oracle_utility_at_8": 0.2396746575832367,
956
+ "proposal_oracle_utility_gain_over_base_at_8": 0.03692302107810974,
957
+ "selected_success_at_8": 0.0,
958
+ "selected_success_gain_over_base_at_8": 0.0,
959
+ "selected_utility_at_8": 0.16485285758972168,
960
+ "selected_utility_gain_over_base_at_8": -0.03789877891540527,
961
+ "selector_regret_at_8": 0.07482179999351501,
962
+ "success_selector_gap_at_8": 0.0,
963
+ "success_support_gap_at_8": 1.0,
964
+ "success_total_car_to_hidden_at_8": 1.0,
965
+ "support_gap_at_8": 1.7603253424167633,
966
+ "total_car_to_hidden_at_8": 1.8351471424102783
967
+ },
968
+ "767747424": {
969
+ "base_success": 0.25,
970
+ "base_utility": 0.5475241877138615,
971
+ "branch_car_at_8": 0.015960504300892353,
972
+ "hidden_chart_oracle_success_at_8": 0.0,
973
+ "hidden_chart_oracle_utility_at_8": 0.18863565661013126,
974
+ "outcome_ptr_at_8": 0.25,
975
+ "pairwise_causal_calibration_ece": 0.21088094062197144,
976
+ "proposal_oracle_success_at_8": 0.25,
977
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
978
+ "proposal_oracle_utility_at_8": 0.5703969299793243,
979
+ "proposal_oracle_utility_gain_over_base_at_8": 0.022872742265462875,
980
+ "selected_success_at_8": 0.25,
981
+ "selected_success_gain_over_base_at_8": 0.0,
982
+ "selected_utility_at_8": 0.554436425678432,
983
+ "selected_utility_gain_over_base_at_8": 0.006912237964570522,
984
+ "selector_regret_at_8": 0.015960504300892353,
985
+ "success_selector_gap_at_8": 0.0,
986
+ "success_support_gap_at_8": 0.0,
987
+ "success_total_car_to_hidden_at_8": 0.0,
988
+ "support_gap_at_8": 0.05905248410999775,
989
+ "total_car_to_hidden_at_8": 0.0750129884108901
990
+ },
991
+ "77147264": {
992
+ "base_success": 0.5,
993
+ "base_utility": 1.4215794801712036,
994
+ "branch_car_at_8": 1.868594542145729,
995
+ "hidden_chart_oracle_success_at_8": 1.0,
996
+ "hidden_chart_oracle_utility_at_8": 2.0,
997
+ "outcome_ptr_at_8": 0.5,
998
+ "pairwise_causal_calibration_ece": 0.06512707818376468,
999
+ "proposal_oracle_success_at_8": 1.0,
1000
+ "proposal_oracle_success_gain_over_base_at_8": 0.5,
1001
+ "proposal_oracle_utility_at_8": 2.0,
1002
+ "proposal_oracle_utility_gain_over_base_at_8": 0.5784205198287964,
1003
+ "selected_success_at_8": 0.0,
1004
+ "selected_success_gain_over_base_at_8": -0.5,
1005
+ "selected_utility_at_8": 0.13140545785427094,
1006
+ "selected_utility_gain_over_base_at_8": -1.2901740223169327,
1007
+ "selector_regret_at_8": 1.868594542145729,
1008
+ "success_selector_gap_at_8": 1.0,
1009
+ "success_support_gap_at_8": 0.0,
1010
+ "success_total_car_to_hidden_at_8": 1.0,
1011
+ "support_gap_at_8": 0.0,
1012
+ "total_car_to_hidden_at_8": 1.868594542145729
1013
+ },
1014
+ "794181626": {
1015
+ "base_success": 0.0,
1016
+ "base_utility": 0.1733371466398239,
1017
+ "branch_car_at_8": 0.0,
1018
+ "hidden_chart_oracle_success_at_8": 1.0,
1019
+ "hidden_chart_oracle_utility_at_8": 2.0,
1020
+ "outcome_ptr_at_8": 1.0,
1021
+ "pairwise_causal_calibration_ece": 0.12119924562800825,
1022
+ "proposal_oracle_success_at_8": 1.0,
1023
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
1024
+ "proposal_oracle_utility_at_8": 2.0,
1025
+ "proposal_oracle_utility_gain_over_base_at_8": 1.826662853360176,
1026
+ "selected_success_at_8": 1.0,
1027
+ "selected_success_gain_over_base_at_8": 1.0,
1028
+ "selected_utility_at_8": 2.0,
1029
+ "selected_utility_gain_over_base_at_8": 1.826662853360176,
1030
+ "selector_regret_at_8": 0.0,
1031
+ "success_selector_gap_at_8": 0.0,
1032
+ "success_support_gap_at_8": 0.0,
1033
+ "success_total_car_to_hidden_at_8": 0.0,
1034
+ "support_gap_at_8": 0.0,
1035
+ "total_car_to_hidden_at_8": 0.0
1036
+ },
1037
+ "957101489": {
1038
+ "base_success": 0.0,
1039
+ "base_utility": 0.2039649486541748,
1040
+ "branch_car_at_8": 0.0,
1041
+ "hidden_chart_oracle_success_at_8": 1.0,
1042
+ "hidden_chart_oracle_utility_at_8": 2.0,
1043
+ "outcome_ptr_at_8": 1.0,
1044
+ "pairwise_causal_calibration_ece": 0.20250855048512742,
1045
+ "proposal_oracle_success_at_8": 0.0,
1046
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
1047
+ "proposal_oracle_utility_at_8": 0.5881698727607727,
1048
+ "proposal_oracle_utility_gain_over_base_at_8": 0.3842049241065979,
1049
+ "selected_success_at_8": 0.0,
1050
+ "selected_success_gain_over_base_at_8": 0.0,
1051
+ "selected_utility_at_8": 0.5881698727607727,
1052
+ "selected_utility_gain_over_base_at_8": 0.3842049241065979,
1053
+ "selector_regret_at_8": 0.0,
1054
+ "success_selector_gap_at_8": 0.0,
1055
+ "success_support_gap_at_8": 1.0,
1056
+ "success_total_car_to_hidden_at_8": 1.0,
1057
+ "support_gap_at_8": 1.4118301272392273,
1058
+ "total_car_to_hidden_at_8": 1.4118301272392273
1059
+ },
1060
+ "998761162": {
1061
+ "base_success": 0.0,
1062
+ "base_utility": 0.0689050555229187,
1063
+ "branch_car_at_8": 0.07749588787555695,
1064
+ "hidden_chart_oracle_success_at_8": 1.0,
1065
+ "hidden_chart_oracle_utility_at_8": 2.0,
1066
+ "outcome_ptr_at_8": 1.0,
1067
+ "pairwise_causal_calibration_ece": 0.23606164078349207,
1068
+ "proposal_oracle_success_at_8": 0.0,
1069
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
1070
+ "proposal_oracle_utility_at_8": 0.08403940498828888,
1071
+ "proposal_oracle_utility_gain_over_base_at_8": 0.015134349465370178,
1072
+ "selected_success_at_8": 0.0,
1073
+ "selected_success_gain_over_base_at_8": 0.0,
1074
+ "selected_utility_at_8": 0.006543517112731934,
1075
+ "selected_utility_gain_over_base_at_8": -0.06236153841018677,
1076
+ "selector_regret_at_8": 0.07749588787555695,
1077
+ "success_selector_gap_at_8": 0.0,
1078
+ "success_support_gap_at_8": 1.0,
1079
+ "success_total_car_to_hidden_at_8": 1.0,
1080
+ "support_gap_at_8": 1.9159605950117111,
1081
+ "total_car_to_hidden_at_8": 1.993456482887268
1082
+ }
1083
+ }
workspace/runs/ctt_residual_rollout_val69_seed0/metrics_by_task.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "LiftPegUpright-v1": {
3
+ "base_success": 0.3333333333333333,
4
+ "base_utility": 1.0669441471497219,
5
+ "branch_car_at_8": 0.19493118425210318,
6
+ "hidden_chart_oracle_success_at_8": 1.0,
7
+ "hidden_chart_oracle_utility_at_8": 2.0,
8
+ "outcome_ptr_at_8": 0.3333333333333333,
9
+ "pairwise_causal_calibration_ece": 0.1098535159049614,
10
+ "proposal_oracle_success_at_8": 0.3333333333333333,
11
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
12
+ "proposal_oracle_utility_at_8": 1.0974282175302505,
13
+ "proposal_oracle_utility_gain_over_base_at_8": 0.030484070380528767,
14
+ "selected_success_at_8": 0.3333333333333333,
15
+ "selected_success_gain_over_base_at_8": 0.0,
16
+ "selected_utility_at_8": 0.9024970332781473,
17
+ "selected_utility_gain_over_base_at_8": -0.1644471138715744,
18
+ "selector_regret_at_8": 0.19493118425210318,
19
+ "success_selector_gap_at_8": 0.0,
20
+ "success_support_gap_at_8": 0.6666666666666666,
21
+ "success_total_car_to_hidden_at_8": 0.6666666666666666,
22
+ "support_gap_at_8": 0.9025717824697495,
23
+ "total_car_to_hidden_at_8": 1.0975029667218525
24
+ },
25
+ "PickCube-v1": {
26
+ "base_success": 0.0,
27
+ "base_utility": 0.15342965722084045,
28
+ "branch_car_at_8": 0.0,
29
+ "hidden_chart_oracle_success_at_8": 1.0,
30
+ "hidden_chart_oracle_utility_at_8": 2.0,
31
+ "outcome_ptr_at_8": 0.0,
32
+ "pairwise_causal_calibration_ece": 0.15040783499409596,
33
+ "proposal_oracle_success_at_8": 0.0,
34
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
35
+ "proposal_oracle_utility_at_8": 0.13278456032276154,
36
+ "proposal_oracle_utility_gain_over_base_at_8": -0.02064509689807892,
37
+ "selected_success_at_8": 0.0,
38
+ "selected_success_gain_over_base_at_8": 0.0,
39
+ "selected_utility_at_8": 0.13278456032276154,
40
+ "selected_utility_gain_over_base_at_8": -0.02064509689807892,
41
+ "selector_regret_at_8": 0.0,
42
+ "success_selector_gap_at_8": 0.0,
43
+ "success_support_gap_at_8": 1.0,
44
+ "success_total_car_to_hidden_at_8": 1.0,
45
+ "support_gap_at_8": 1.8672154396772385,
46
+ "total_car_to_hidden_at_8": 1.8672154396772385
47
+ },
48
+ "PullCube-v1": {
49
+ "base_success": 0.29411764705882354,
50
+ "base_utility": 0.6512299207833541,
51
+ "branch_car_at_8": 0.08066527944003396,
52
+ "hidden_chart_oracle_success_at_8": 0.3235294117647059,
53
+ "hidden_chart_oracle_utility_at_8": 0.7823218367993832,
54
+ "outcome_ptr_at_8": 0.3235294117647059,
55
+ "pairwise_causal_calibration_ece": 0.17256325415222457,
56
+ "proposal_oracle_success_at_8": 0.29411764705882354,
57
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
58
+ "proposal_oracle_utility_at_8": 0.6413554182680159,
59
+ "proposal_oracle_utility_gain_over_base_at_8": -0.009874502515338142,
60
+ "selected_success_at_8": 0.2647058823529412,
61
+ "selected_success_gain_over_base_at_8": -0.029411764705882353,
62
+ "selected_utility_at_8": 0.560690138827982,
63
+ "selected_utility_gain_over_base_at_8": -0.0905397819553721,
64
+ "selector_regret_at_8": 0.08066527944003396,
65
+ "success_selector_gap_at_8": 0.029411764705882353,
66
+ "success_support_gap_at_8": 0.058823529411764705,
67
+ "success_total_car_to_hidden_at_8": 0.08823529411764706,
68
+ "support_gap_at_8": 0.1962626721634639,
69
+ "total_car_to_hidden_at_8": 0.2751456986571772
70
+ },
71
+ "PushCube-v1": {
72
+ "base_success": 0.3076923076923077,
73
+ "base_utility": 0.6686872587754176,
74
+ "branch_car_at_8": 0.31651630768409145,
75
+ "hidden_chart_oracle_success_at_8": 1.0,
76
+ "hidden_chart_oracle_utility_at_8": 2.0,
77
+ "outcome_ptr_at_8": 0.6153846153846154,
78
+ "pairwise_causal_calibration_ece": 0.13663407835877941,
79
+ "proposal_oracle_success_at_8": 0.6923076923076923,
80
+ "proposal_oracle_success_gain_over_base_at_8": 0.38461538461538464,
81
+ "proposal_oracle_utility_at_8": 1.406166712825115,
82
+ "proposal_oracle_utility_gain_over_base_at_8": 0.7374794540496973,
83
+ "selected_success_at_8": 0.5384615384615384,
84
+ "selected_success_gain_over_base_at_8": 0.23076923076923078,
85
+ "selected_utility_at_8": 1.0896504051410234,
86
+ "selected_utility_gain_over_base_at_8": 0.4209631463656059,
87
+ "selector_regret_at_8": 0.31651630768409145,
88
+ "success_selector_gap_at_8": 0.15384615384615385,
89
+ "success_support_gap_at_8": 0.3076923076923077,
90
+ "success_total_car_to_hidden_at_8": 0.46153846153846156,
91
+ "support_gap_at_8": 0.593833287174885,
92
+ "total_car_to_hidden_at_8": 0.9103495948589765
93
+ },
94
+ "StackCube-v1": {
95
+ "base_success": 0.2,
96
+ "base_utility": 0.8017999569574992,
97
+ "branch_car_at_8": 0.6326439549525579,
98
+ "hidden_chart_oracle_success_at_8": 1.0,
99
+ "hidden_chart_oracle_utility_at_8": 2.0,
100
+ "outcome_ptr_at_8": 0.6,
101
+ "pairwise_causal_calibration_ece": 0.128912518811403,
102
+ "proposal_oracle_success_at_8": 0.4,
103
+ "proposal_oracle_success_gain_over_base_at_8": 0.2,
104
+ "proposal_oracle_utility_at_8": 1.095273275176684,
105
+ "proposal_oracle_utility_gain_over_base_at_8": 0.2934733182191849,
106
+ "selected_success_at_8": 0.06666666666666667,
107
+ "selected_success_gain_over_base_at_8": -0.13333333333333333,
108
+ "selected_utility_at_8": 0.4626293202241262,
109
+ "selected_utility_gain_over_base_at_8": -0.339170636733373,
110
+ "selector_regret_at_8": 0.6326439549525579,
111
+ "success_selector_gap_at_8": 0.3333333333333333,
112
+ "success_support_gap_at_8": 0.6,
113
+ "success_total_car_to_hidden_at_8": 0.9333333333333333,
114
+ "support_gap_at_8": 0.904726724823316,
115
+ "total_car_to_hidden_at_8": 1.5373706797758737
116
+ }
117
+ }
workspace/runs/ctt_residual_rollout_val69_seed0/source_data_hash.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957
workspace/runs/ctt_residual_rollout_val69_seed0/source_split_hash.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372
workspace/runs/ctt_residual_rollout_val69_seed0/split_hash.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 9d1dc1c10868f76fa080895df0f2730c173f12b6db14932599d55c2b60859381
workspace/runs/ctt_residual_rollout_val69_seed0/table.tex ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ % Auto-generated by scripts/eval_metrics.py
2
+ \begin{tabular}{lrrrr}
3
+ \toprule
4
+ Metric & N & Micro mean & CI low & CI high \\
5
+ \midrule
6
+ base\_success & 69 & 0.2754 & 0.1739 & 0.3913 \\
7
+ base\_utility & 69 & 0.7162 & 0.5461 & 0.9125 \\
8
+ branch\_car\_at\_8 & 69 & 0.2539 & 0.1317 & 0.3789 \\
9
+ hidden\_chart\_oracle\_success\_at\_8 & 69 & 0.6667 & 0.5507 & 0.7826 \\
10
+ hidden\_chart\_oracle\_utility\_at\_8 & 69 & 1.4000 & 1.1855 & 1.6062 \\
11
+ outcome\_ptr\_at\_8 & 69 & 0.4348 & 0.3188 & 0.5507 \\
12
+ pairwise\_causal\_calibration\_ece & 69 & 0.1505 & 0.1261 & 0.1764 \\
13
+ proposal\_oracle\_success\_at\_8 & 69 & 0.3913 & 0.2899 & 0.5217 \\
14
+ proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 69 & 0.1159 & 0.0145 & 0.2174 \\
15
+ proposal\_oracle\_utility\_at\_8 & 69 & 0.9164 & 0.7319 & 1.1490 \\
16
+ proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 69 & 0.2002 & 0.0165 & 0.3763 \\
17
+ selected\_success\_at\_8 & 69 & 0.2754 & 0.1884 & 0.3768 \\
18
+ selected\_success\_gain\_over\_base\_at\_8 & 69 & 0.0000 & -0.1014 & 0.1159 \\
19
+ selected\_utility\_at\_8 & 69 & 0.6626 & 0.4843 & 0.8693 \\
20
+ selected\_utility\_gain\_over\_base\_at\_8 & 69 & -0.0536 & -0.2450 & 0.1422 \\
21
+ selector\_regret\_at\_8 & 69 & 0.2539 & 0.1317 & 0.3789 \\
22
+ success\_selector\_gap\_at\_8 & 69 & 0.1159 & 0.0435 & 0.1884 \\
23
+ success\_support\_gap\_at\_8 & 69 & 0.2899 & 0.1739 & 0.3913 \\
24
+ success\_total\_car\_to\_hidden\_at\_8 & 69 & 0.4058 & 0.2754 & 0.5217 \\
25
+ support\_gap\_at\_8 & 69 & 0.5108 & 0.3299 & 0.6784 \\
26
+ total\_car\_to\_hidden\_at\_8 & 69 & 0.7638 & 0.5655 & 0.9371 \\
27
+ \bottomrule
28
+ \end{tabular}
workspace/runs/ctt_residual_rollout_val69_seed0/train.log ADDED
@@ -0,0 +1 @@
 
 
1
+ CTT checkpoint trained separately; rollout used checkpoint=runs/ctt_residual_full_seed0/model.pt
workspace/runs/ctt_residual_rollout_val69_seed1/data_hash.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ fa0fe0e3881ee70ce91492445871614d01b3a47fc7c576659c53ff17ddee4e42
workspace/runs/ctt_residual_rollout_val69_seed1/eval.log ADDED
@@ -0,0 +1,668 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ start
2
+ importing gymnasium/mani_skill
3
+ imported gymnasium/mani_skill
4
+ loaded checkpoint=runs/ctt_residual_full_seed1/model.pt
5
+ loaded charts source=301 target=69
6
+ generated proposals rows=69 total=552
7
+ env init sequential task=LiftPegUpright-v1 batch=4 sim=physx_cpu render=cpu
8
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=0 shape=(1, 1, 16, 7)
9
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=1 shape=(1, 1, 16, 7)
10
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=2 shape=(1, 1, 16, 7)
11
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=3 shape=(1, 1, 16, 7)
12
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=4 shape=(1, 1, 16, 7)
13
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=5 shape=(1, 1, 16, 7)
14
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=6 shape=(1, 1, 16, 7)
15
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=7 shape=(1, 1, 16, 7)
16
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=8 shape=(1, 1, 16, 7)
17
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=0 shape=(1, 1, 16, 7)
18
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=1 shape=(1, 1, 16, 7)
19
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=2 shape=(1, 1, 16, 7)
20
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=3 shape=(1, 1, 16, 7)
21
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=4 shape=(1, 1, 16, 7)
22
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=5 shape=(1, 1, 16, 7)
23
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=6 shape=(1, 1, 16, 7)
24
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=7 shape=(1, 1, 16, 7)
25
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=8 shape=(1, 1, 16, 7)
26
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=0 shape=(1, 1, 16, 7)
27
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=1 shape=(1, 1, 16, 7)
28
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=2 shape=(1, 1, 16, 7)
29
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=3 shape=(1, 1, 16, 7)
30
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=4 shape=(1, 1, 16, 7)
31
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=5 shape=(1, 1, 16, 7)
32
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=6 shape=(1, 1, 16, 7)
33
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=7 shape=(1, 1, 16, 7)
34
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=8 shape=(1, 1, 16, 7)
35
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=0 shape=(1, 1, 16, 7)
36
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=1 shape=(1, 1, 16, 7)
37
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=2 shape=(1, 1, 16, 7)
38
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=3 shape=(1, 1, 16, 7)
39
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=4 shape=(1, 1, 16, 7)
40
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=5 shape=(1, 1, 16, 7)
41
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=6 shape=(1, 1, 16, 7)
42
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=7 shape=(1, 1, 16, 7)
43
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=8 shape=(1, 1, 16, 7)
44
+ batch done sequential task=LiftPegUpright-v1
45
+ env init sequential task=LiftPegUpright-v1 batch=2 sim=physx_cpu render=cpu
46
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=0 shape=(1, 1, 16, 7)
47
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=1 shape=(1, 1, 16, 7)
48
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=2 shape=(1, 1, 16, 7)
49
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=3 shape=(1, 1, 16, 7)
50
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=4 shape=(1, 1, 16, 7)
51
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=5 shape=(1, 1, 16, 7)
52
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=6 shape=(1, 1, 16, 7)
53
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=7 shape=(1, 1, 16, 7)
54
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=8 shape=(1, 1, 16, 7)
55
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=0 shape=(1, 1, 16, 7)
56
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=1 shape=(1, 1, 16, 7)
57
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=2 shape=(1, 1, 16, 7)
58
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=3 shape=(1, 1, 16, 7)
59
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=4 shape=(1, 1, 16, 7)
60
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=5 shape=(1, 1, 16, 7)
61
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=6 shape=(1, 1, 16, 7)
62
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=7 shape=(1, 1, 16, 7)
63
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=8 shape=(1, 1, 16, 7)
64
+ batch done sequential task=LiftPegUpright-v1
65
+ env init sequential task=PickCube-v1 batch=1 sim=physx_cpu render=cpu
66
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=0 shape=(1, 1, 16, 7)
67
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=1 shape=(1, 1, 16, 7)
68
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=2 shape=(1, 1, 16, 7)
69
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=3 shape=(1, 1, 16, 7)
70
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=4 shape=(1, 1, 16, 7)
71
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=5 shape=(1, 1, 16, 7)
72
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=6 shape=(1, 1, 16, 7)
73
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=7 shape=(1, 1, 16, 7)
74
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=8 shape=(1, 1, 16, 7)
75
+ batch done sequential task=PickCube-v1
76
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
77
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=0 shape=(1, 1, 16, 7)
78
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=1 shape=(1, 1, 16, 7)
79
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=2 shape=(1, 1, 16, 7)
80
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=3 shape=(1, 1, 16, 7)
81
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=4 shape=(1, 1, 16, 7)
82
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=5 shape=(1, 1, 16, 7)
83
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=6 shape=(1, 1, 16, 7)
84
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=7 shape=(1, 1, 16, 7)
85
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=8 shape=(1, 1, 16, 7)
86
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=0 shape=(1, 1, 16, 7)
87
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=1 shape=(1, 1, 16, 7)
88
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=2 shape=(1, 1, 16, 7)
89
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=3 shape=(1, 1, 16, 7)
90
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=4 shape=(1, 1, 16, 7)
91
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=5 shape=(1, 1, 16, 7)
92
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=6 shape=(1, 1, 16, 7)
93
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=7 shape=(1, 1, 16, 7)
94
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=8 shape=(1, 1, 16, 7)
95
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=0 shape=(1, 1, 16, 7)
96
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=1 shape=(1, 1, 16, 7)
97
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=2 shape=(1, 1, 16, 7)
98
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=3 shape=(1, 1, 16, 7)
99
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=4 shape=(1, 1, 16, 7)
100
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=5 shape=(1, 1, 16, 7)
101
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=6 shape=(1, 1, 16, 7)
102
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=7 shape=(1, 1, 16, 7)
103
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=8 shape=(1, 1, 16, 7)
104
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=0 shape=(1, 1, 16, 7)
105
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=1 shape=(1, 1, 16, 7)
106
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=2 shape=(1, 1, 16, 7)
107
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=3 shape=(1, 1, 16, 7)
108
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=4 shape=(1, 1, 16, 7)
109
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=5 shape=(1, 1, 16, 7)
110
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=6 shape=(1, 1, 16, 7)
111
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=7 shape=(1, 1, 16, 7)
112
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=8 shape=(1, 1, 16, 7)
113
+ batch done sequential task=PullCube-v1
114
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
115
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=0 shape=(1, 1, 16, 7)
116
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=1 shape=(1, 1, 16, 7)
117
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=2 shape=(1, 1, 16, 7)
118
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=3 shape=(1, 1, 16, 7)
119
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=4 shape=(1, 1, 16, 7)
120
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=5 shape=(1, 1, 16, 7)
121
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=6 shape=(1, 1, 16, 7)
122
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=7 shape=(1, 1, 16, 7)
123
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=8 shape=(1, 1, 16, 7)
124
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=0 shape=(1, 1, 16, 7)
125
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=1 shape=(1, 1, 16, 7)
126
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=2 shape=(1, 1, 16, 7)
127
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=3 shape=(1, 1, 16, 7)
128
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=4 shape=(1, 1, 16, 7)
129
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=5 shape=(1, 1, 16, 7)
130
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=6 shape=(1, 1, 16, 7)
131
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=7 shape=(1, 1, 16, 7)
132
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=8 shape=(1, 1, 16, 7)
133
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=0 shape=(1, 1, 16, 7)
134
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=1 shape=(1, 1, 16, 7)
135
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=2 shape=(1, 1, 16, 7)
136
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=3 shape=(1, 1, 16, 7)
137
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=4 shape=(1, 1, 16, 7)
138
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=5 shape=(1, 1, 16, 7)
139
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=6 shape=(1, 1, 16, 7)
140
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=7 shape=(1, 1, 16, 7)
141
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=8 shape=(1, 1, 16, 7)
142
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=0 shape=(1, 1, 16, 7)
143
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=1 shape=(1, 1, 16, 7)
144
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=2 shape=(1, 1, 16, 7)
145
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=3 shape=(1, 1, 16, 7)
146
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=4 shape=(1, 1, 16, 7)
147
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=5 shape=(1, 1, 16, 7)
148
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=6 shape=(1, 1, 16, 7)
149
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=7 shape=(1, 1, 16, 7)
150
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=8 shape=(1, 1, 16, 7)
151
+ batch done sequential task=PullCube-v1
152
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
153
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=0 shape=(1, 1, 16, 7)
154
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=1 shape=(1, 1, 16, 7)
155
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=2 shape=(1, 1, 16, 7)
156
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=3 shape=(1, 1, 16, 7)
157
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=4 shape=(1, 1, 16, 7)
158
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=5 shape=(1, 1, 16, 7)
159
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=6 shape=(1, 1, 16, 7)
160
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=7 shape=(1, 1, 16, 7)
161
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=8 shape=(1, 1, 16, 7)
162
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=0 shape=(1, 1, 16, 7)
163
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=1 shape=(1, 1, 16, 7)
164
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=2 shape=(1, 1, 16, 7)
165
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=3 shape=(1, 1, 16, 7)
166
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=4 shape=(1, 1, 16, 7)
167
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=5 shape=(1, 1, 16, 7)
168
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=6 shape=(1, 1, 16, 7)
169
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=7 shape=(1, 1, 16, 7)
170
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=8 shape=(1, 1, 16, 7)
171
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=0 shape=(1, 1, 16, 7)
172
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=1 shape=(1, 1, 16, 7)
173
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=2 shape=(1, 1, 16, 7)
174
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=3 shape=(1, 1, 16, 7)
175
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=4 shape=(1, 1, 16, 7)
176
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=5 shape=(1, 1, 16, 7)
177
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=6 shape=(1, 1, 16, 7)
178
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=7 shape=(1, 1, 16, 7)
179
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=8 shape=(1, 1, 16, 7)
180
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=0 shape=(1, 1, 16, 7)
181
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=1 shape=(1, 1, 16, 7)
182
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=2 shape=(1, 1, 16, 7)
183
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=3 shape=(1, 1, 16, 7)
184
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=4 shape=(1, 1, 16, 7)
185
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=5 shape=(1, 1, 16, 7)
186
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=6 shape=(1, 1, 16, 7)
187
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=7 shape=(1, 1, 16, 7)
188
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=8 shape=(1, 1, 16, 7)
189
+ batch done sequential task=PullCube-v1
190
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
191
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=0 shape=(1, 1, 16, 7)
192
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=1 shape=(1, 1, 16, 7)
193
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=2 shape=(1, 1, 16, 7)
194
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=3 shape=(1, 1, 16, 7)
195
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=4 shape=(1, 1, 16, 7)
196
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=5 shape=(1, 1, 16, 7)
197
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=6 shape=(1, 1, 16, 7)
198
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=7 shape=(1, 1, 16, 7)
199
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=8 shape=(1, 1, 16, 7)
200
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=0 shape=(1, 1, 16, 7)
201
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=1 shape=(1, 1, 16, 7)
202
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=2 shape=(1, 1, 16, 7)
203
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=3 shape=(1, 1, 16, 7)
204
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=4 shape=(1, 1, 16, 7)
205
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=5 shape=(1, 1, 16, 7)
206
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=6 shape=(1, 1, 16, 7)
207
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=7 shape=(1, 1, 16, 7)
208
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=8 shape=(1, 1, 16, 7)
209
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=0 shape=(1, 1, 16, 7)
210
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=1 shape=(1, 1, 16, 7)
211
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=2 shape=(1, 1, 16, 7)
212
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=3 shape=(1, 1, 16, 7)
213
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=4 shape=(1, 1, 16, 7)
214
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=5 shape=(1, 1, 16, 7)
215
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=6 shape=(1, 1, 16, 7)
216
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=7 shape=(1, 1, 16, 7)
217
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=8 shape=(1, 1, 16, 7)
218
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=0 shape=(1, 1, 16, 7)
219
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=1 shape=(1, 1, 16, 7)
220
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=2 shape=(1, 1, 16, 7)
221
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=3 shape=(1, 1, 16, 7)
222
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=4 shape=(1, 1, 16, 7)
223
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=5 shape=(1, 1, 16, 7)
224
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=6 shape=(1, 1, 16, 7)
225
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=7 shape=(1, 1, 16, 7)
226
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=8 shape=(1, 1, 16, 7)
227
+ batch done sequential task=PullCube-v1
228
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
229
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=0 shape=(1, 1, 16, 7)
230
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=1 shape=(1, 1, 16, 7)
231
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=2 shape=(1, 1, 16, 7)
232
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=3 shape=(1, 1, 16, 7)
233
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=4 shape=(1, 1, 16, 7)
234
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=5 shape=(1, 1, 16, 7)
235
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=6 shape=(1, 1, 16, 7)
236
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=7 shape=(1, 1, 16, 7)
237
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=8 shape=(1, 1, 16, 7)
238
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=0 shape=(1, 1, 16, 7)
239
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=1 shape=(1, 1, 16, 7)
240
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=2 shape=(1, 1, 16, 7)
241
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=3 shape=(1, 1, 16, 7)
242
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=4 shape=(1, 1, 16, 7)
243
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=5 shape=(1, 1, 16, 7)
244
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=6 shape=(1, 1, 16, 7)
245
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=7 shape=(1, 1, 16, 7)
246
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=8 shape=(1, 1, 16, 7)
247
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=0 shape=(1, 1, 16, 7)
248
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=1 shape=(1, 1, 16, 7)
249
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=2 shape=(1, 1, 16, 7)
250
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=3 shape=(1, 1, 16, 7)
251
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=4 shape=(1, 1, 16, 7)
252
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=5 shape=(1, 1, 16, 7)
253
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=6 shape=(1, 1, 16, 7)
254
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=7 shape=(1, 1, 16, 7)
255
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=8 shape=(1, 1, 16, 7)
256
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=0 shape=(1, 1, 16, 7)
257
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=1 shape=(1, 1, 16, 7)
258
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=2 shape=(1, 1, 16, 7)
259
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=3 shape=(1, 1, 16, 7)
260
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=4 shape=(1, 1, 16, 7)
261
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=5 shape=(1, 1, 16, 7)
262
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=6 shape=(1, 1, 16, 7)
263
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=7 shape=(1, 1, 16, 7)
264
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=8 shape=(1, 1, 16, 7)
265
+ batch done sequential task=PullCube-v1
266
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
267
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=0 shape=(1, 1, 16, 7)
268
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=1 shape=(1, 1, 16, 7)
269
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=2 shape=(1, 1, 16, 7)
270
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=3 shape=(1, 1, 16, 7)
271
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=4 shape=(1, 1, 16, 7)
272
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=5 shape=(1, 1, 16, 7)
273
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=6 shape=(1, 1, 16, 7)
274
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=7 shape=(1, 1, 16, 7)
275
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=8 shape=(1, 1, 16, 7)
276
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=0 shape=(1, 1, 16, 7)
277
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=1 shape=(1, 1, 16, 7)
278
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=2 shape=(1, 1, 16, 7)
279
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=3 shape=(1, 1, 16, 7)
280
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=4 shape=(1, 1, 16, 7)
281
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=5 shape=(1, 1, 16, 7)
282
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=6 shape=(1, 1, 16, 7)
283
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=7 shape=(1, 1, 16, 7)
284
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=8 shape=(1, 1, 16, 7)
285
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=0 shape=(1, 1, 16, 7)
286
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=1 shape=(1, 1, 16, 7)
287
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=2 shape=(1, 1, 16, 7)
288
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=3 shape=(1, 1, 16, 7)
289
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=4 shape=(1, 1, 16, 7)
290
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=5 shape=(1, 1, 16, 7)
291
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=6 shape=(1, 1, 16, 7)
292
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=7 shape=(1, 1, 16, 7)
293
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=8 shape=(1, 1, 16, 7)
294
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=0 shape=(1, 1, 16, 7)
295
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=1 shape=(1, 1, 16, 7)
296
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=2 shape=(1, 1, 16, 7)
297
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=3 shape=(1, 1, 16, 7)
298
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=4 shape=(1, 1, 16, 7)
299
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=5 shape=(1, 1, 16, 7)
300
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=6 shape=(1, 1, 16, 7)
301
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=7 shape=(1, 1, 16, 7)
302
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=8 shape=(1, 1, 16, 7)
303
+ batch done sequential task=PullCube-v1
304
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
305
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=0 shape=(1, 1, 16, 7)
306
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=1 shape=(1, 1, 16, 7)
307
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=2 shape=(1, 1, 16, 7)
308
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=3 shape=(1, 1, 16, 7)
309
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=4 shape=(1, 1, 16, 7)
310
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=5 shape=(1, 1, 16, 7)
311
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=6 shape=(1, 1, 16, 7)
312
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=7 shape=(1, 1, 16, 7)
313
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=8 shape=(1, 1, 16, 7)
314
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=0 shape=(1, 1, 16, 7)
315
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=1 shape=(1, 1, 16, 7)
316
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=2 shape=(1, 1, 16, 7)
317
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=3 shape=(1, 1, 16, 7)
318
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=4 shape=(1, 1, 16, 7)
319
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=5 shape=(1, 1, 16, 7)
320
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=6 shape=(1, 1, 16, 7)
321
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=7 shape=(1, 1, 16, 7)
322
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=8 shape=(1, 1, 16, 7)
323
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=0 shape=(1, 1, 16, 7)
324
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=1 shape=(1, 1, 16, 7)
325
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=2 shape=(1, 1, 16, 7)
326
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=3 shape=(1, 1, 16, 7)
327
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=4 shape=(1, 1, 16, 7)
328
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=5 shape=(1, 1, 16, 7)
329
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=6 shape=(1, 1, 16, 7)
330
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=7 shape=(1, 1, 16, 7)
331
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=8 shape=(1, 1, 16, 7)
332
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=0 shape=(1, 1, 16, 7)
333
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=1 shape=(1, 1, 16, 7)
334
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=2 shape=(1, 1, 16, 7)
335
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=3 shape=(1, 1, 16, 7)
336
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=4 shape=(1, 1, 16, 7)
337
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=5 shape=(1, 1, 16, 7)
338
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=6 shape=(1, 1, 16, 7)
339
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=7 shape=(1, 1, 16, 7)
340
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=8 shape=(1, 1, 16, 7)
341
+ batch done sequential task=PullCube-v1
342
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
343
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=0 shape=(1, 1, 16, 7)
344
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=1 shape=(1, 1, 16, 7)
345
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=2 shape=(1, 1, 16, 7)
346
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=3 shape=(1, 1, 16, 7)
347
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=4 shape=(1, 1, 16, 7)
348
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=5 shape=(1, 1, 16, 7)
349
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=6 shape=(1, 1, 16, 7)
350
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=7 shape=(1, 1, 16, 7)
351
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=8 shape=(1, 1, 16, 7)
352
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=0 shape=(1, 1, 16, 7)
353
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=1 shape=(1, 1, 16, 7)
354
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=2 shape=(1, 1, 16, 7)
355
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=3 shape=(1, 1, 16, 7)
356
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=4 shape=(1, 1, 16, 7)
357
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=5 shape=(1, 1, 16, 7)
358
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=6 shape=(1, 1, 16, 7)
359
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=7 shape=(1, 1, 16, 7)
360
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=8 shape=(1, 1, 16, 7)
361
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=0 shape=(1, 1, 16, 7)
362
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=1 shape=(1, 1, 16, 7)
363
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=2 shape=(1, 1, 16, 7)
364
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=3 shape=(1, 1, 16, 7)
365
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=4 shape=(1, 1, 16, 7)
366
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=5 shape=(1, 1, 16, 7)
367
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=6 shape=(1, 1, 16, 7)
368
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=7 shape=(1, 1, 16, 7)
369
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=8 shape=(1, 1, 16, 7)
370
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=0 shape=(1, 1, 16, 7)
371
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=1 shape=(1, 1, 16, 7)
372
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=2 shape=(1, 1, 16, 7)
373
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=3 shape=(1, 1, 16, 7)
374
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=4 shape=(1, 1, 16, 7)
375
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=5 shape=(1, 1, 16, 7)
376
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=6 shape=(1, 1, 16, 7)
377
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=7 shape=(1, 1, 16, 7)
378
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=8 shape=(1, 1, 16, 7)
379
+ batch done sequential task=PullCube-v1
380
+ env init sequential task=PullCube-v1 batch=2 sim=physx_cpu render=cpu
381
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=0 shape=(1, 1, 16, 7)
382
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=1 shape=(1, 1, 16, 7)
383
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=2 shape=(1, 1, 16, 7)
384
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=3 shape=(1, 1, 16, 7)
385
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=4 shape=(1, 1, 16, 7)
386
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=5 shape=(1, 1, 16, 7)
387
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=6 shape=(1, 1, 16, 7)
388
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=7 shape=(1, 1, 16, 7)
389
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=8 shape=(1, 1, 16, 7)
390
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=0 shape=(1, 1, 16, 7)
391
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=1 shape=(1, 1, 16, 7)
392
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=2 shape=(1, 1, 16, 7)
393
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=3 shape=(1, 1, 16, 7)
394
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=4 shape=(1, 1, 16, 7)
395
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=5 shape=(1, 1, 16, 7)
396
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=6 shape=(1, 1, 16, 7)
397
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=7 shape=(1, 1, 16, 7)
398
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=8 shape=(1, 1, 16, 7)
399
+ batch done sequential task=PullCube-v1
400
+ env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu
401
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=0 shape=(1, 1, 16, 7)
402
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=1 shape=(1, 1, 16, 7)
403
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=2 shape=(1, 1, 16, 7)
404
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=3 shape=(1, 1, 16, 7)
405
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=4 shape=(1, 1, 16, 7)
406
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=5 shape=(1, 1, 16, 7)
407
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=6 shape=(1, 1, 16, 7)
408
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=7 shape=(1, 1, 16, 7)
409
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=8 shape=(1, 1, 16, 7)
410
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=0 shape=(1, 1, 16, 7)
411
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=1 shape=(1, 1, 16, 7)
412
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=2 shape=(1, 1, 16, 7)
413
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=3 shape=(1, 1, 16, 7)
414
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=4 shape=(1, 1, 16, 7)
415
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=5 shape=(1, 1, 16, 7)
416
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=6 shape=(1, 1, 16, 7)
417
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=7 shape=(1, 1, 16, 7)
418
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=8 shape=(1, 1, 16, 7)
419
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=0 shape=(1, 1, 16, 7)
420
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=1 shape=(1, 1, 16, 7)
421
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=2 shape=(1, 1, 16, 7)
422
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=3 shape=(1, 1, 16, 7)
423
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=4 shape=(1, 1, 16, 7)
424
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=5 shape=(1, 1, 16, 7)
425
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=6 shape=(1, 1, 16, 7)
426
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=7 shape=(1, 1, 16, 7)
427
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=8 shape=(1, 1, 16, 7)
428
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=0 shape=(1, 1, 16, 7)
429
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=1 shape=(1, 1, 16, 7)
430
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=2 shape=(1, 1, 16, 7)
431
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=3 shape=(1, 1, 16, 7)
432
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=4 shape=(1, 1, 16, 7)
433
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=5 shape=(1, 1, 16, 7)
434
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=6 shape=(1, 1, 16, 7)
435
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=7 shape=(1, 1, 16, 7)
436
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=8 shape=(1, 1, 16, 7)
437
+ batch done sequential task=PushCube-v1
438
+ env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu
439
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=0 shape=(1, 1, 16, 7)
440
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=1 shape=(1, 1, 16, 7)
441
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=2 shape=(1, 1, 16, 7)
442
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=3 shape=(1, 1, 16, 7)
443
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=4 shape=(1, 1, 16, 7)
444
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=5 shape=(1, 1, 16, 7)
445
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=6 shape=(1, 1, 16, 7)
446
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=7 shape=(1, 1, 16, 7)
447
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=8 shape=(1, 1, 16, 7)
448
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=0 shape=(1, 1, 16, 7)
449
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=1 shape=(1, 1, 16, 7)
450
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=2 shape=(1, 1, 16, 7)
451
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=3 shape=(1, 1, 16, 7)
452
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=4 shape=(1, 1, 16, 7)
453
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=5 shape=(1, 1, 16, 7)
454
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=6 shape=(1, 1, 16, 7)
455
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=7 shape=(1, 1, 16, 7)
456
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=8 shape=(1, 1, 16, 7)
457
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=0 shape=(1, 1, 16, 7)
458
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=1 shape=(1, 1, 16, 7)
459
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=2 shape=(1, 1, 16, 7)
460
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=3 shape=(1, 1, 16, 7)
461
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=4 shape=(1, 1, 16, 7)
462
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=5 shape=(1, 1, 16, 7)
463
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=6 shape=(1, 1, 16, 7)
464
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=7 shape=(1, 1, 16, 7)
465
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=8 shape=(1, 1, 16, 7)
466
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=0 shape=(1, 1, 16, 7)
467
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=1 shape=(1, 1, 16, 7)
468
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=2 shape=(1, 1, 16, 7)
469
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=3 shape=(1, 1, 16, 7)
470
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=4 shape=(1, 1, 16, 7)
471
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=5 shape=(1, 1, 16, 7)
472
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=6 shape=(1, 1, 16, 7)
473
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=7 shape=(1, 1, 16, 7)
474
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=8 shape=(1, 1, 16, 7)
475
+ batch done sequential task=PushCube-v1
476
+ env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu
477
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=0 shape=(1, 1, 16, 7)
478
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=1 shape=(1, 1, 16, 7)
479
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=2 shape=(1, 1, 16, 7)
480
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=3 shape=(1, 1, 16, 7)
481
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=4 shape=(1, 1, 16, 7)
482
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=5 shape=(1, 1, 16, 7)
483
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=6 shape=(1, 1, 16, 7)
484
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=7 shape=(1, 1, 16, 7)
485
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=8 shape=(1, 1, 16, 7)
486
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=0 shape=(1, 1, 16, 7)
487
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=1 shape=(1, 1, 16, 7)
488
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=2 shape=(1, 1, 16, 7)
489
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=3 shape=(1, 1, 16, 7)
490
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=4 shape=(1, 1, 16, 7)
491
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=5 shape=(1, 1, 16, 7)
492
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=6 shape=(1, 1, 16, 7)
493
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=7 shape=(1, 1, 16, 7)
494
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=8 shape=(1, 1, 16, 7)
495
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=0 shape=(1, 1, 16, 7)
496
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=1 shape=(1, 1, 16, 7)
497
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=2 shape=(1, 1, 16, 7)
498
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=3 shape=(1, 1, 16, 7)
499
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=4 shape=(1, 1, 16, 7)
500
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=5 shape=(1, 1, 16, 7)
501
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=6 shape=(1, 1, 16, 7)
502
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=7 shape=(1, 1, 16, 7)
503
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=8 shape=(1, 1, 16, 7)
504
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=0 shape=(1, 1, 16, 7)
505
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=1 shape=(1, 1, 16, 7)
506
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=2 shape=(1, 1, 16, 7)
507
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=3 shape=(1, 1, 16, 7)
508
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=4 shape=(1, 1, 16, 7)
509
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=5 shape=(1, 1, 16, 7)
510
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=6 shape=(1, 1, 16, 7)
511
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=7 shape=(1, 1, 16, 7)
512
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=8 shape=(1, 1, 16, 7)
513
+ batch done sequential task=PushCube-v1
514
+ env init sequential task=PushCube-v1 batch=1 sim=physx_cpu render=cpu
515
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=0 shape=(1, 1, 16, 7)
516
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=1 shape=(1, 1, 16, 7)
517
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=2 shape=(1, 1, 16, 7)
518
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=3 shape=(1, 1, 16, 7)
519
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=4 shape=(1, 1, 16, 7)
520
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=5 shape=(1, 1, 16, 7)
521
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=6 shape=(1, 1, 16, 7)
522
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=7 shape=(1, 1, 16, 7)
523
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=8 shape=(1, 1, 16, 7)
524
+ batch done sequential task=PushCube-v1
525
+ env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu
526
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=0 shape=(1, 1, 16, 7)
527
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=1 shape=(1, 1, 16, 7)
528
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=2 shape=(1, 1, 16, 7)
529
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=3 shape=(1, 1, 16, 7)
530
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=4 shape=(1, 1, 16, 7)
531
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=5 shape=(1, 1, 16, 7)
532
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=6 shape=(1, 1, 16, 7)
533
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=7 shape=(1, 1, 16, 7)
534
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=8 shape=(1, 1, 16, 7)
535
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=0 shape=(1, 1, 16, 7)
536
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=1 shape=(1, 1, 16, 7)
537
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=2 shape=(1, 1, 16, 7)
538
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=3 shape=(1, 1, 16, 7)
539
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=4 shape=(1, 1, 16, 7)
540
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=5 shape=(1, 1, 16, 7)
541
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=6 shape=(1, 1, 16, 7)
542
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=7 shape=(1, 1, 16, 7)
543
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=8 shape=(1, 1, 16, 7)
544
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=0 shape=(1, 1, 16, 7)
545
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=1 shape=(1, 1, 16, 7)
546
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=2 shape=(1, 1, 16, 7)
547
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=3 shape=(1, 1, 16, 7)
548
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=4 shape=(1, 1, 16, 7)
549
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=5 shape=(1, 1, 16, 7)
550
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=6 shape=(1, 1, 16, 7)
551
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=7 shape=(1, 1, 16, 7)
552
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=8 shape=(1, 1, 16, 7)
553
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=0 shape=(1, 1, 16, 7)
554
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=1 shape=(1, 1, 16, 7)
555
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=2 shape=(1, 1, 16, 7)
556
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=3 shape=(1, 1, 16, 7)
557
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=4 shape=(1, 1, 16, 7)
558
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=5 shape=(1, 1, 16, 7)
559
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=6 shape=(1, 1, 16, 7)
560
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=7 shape=(1, 1, 16, 7)
561
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=8 shape=(1, 1, 16, 7)
562
+ batch done sequential task=StackCube-v1
563
+ env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu
564
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=0 shape=(1, 1, 16, 7)
565
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=1 shape=(1, 1, 16, 7)
566
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=2 shape=(1, 1, 16, 7)
567
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=3 shape=(1, 1, 16, 7)
568
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=4 shape=(1, 1, 16, 7)
569
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=5 shape=(1, 1, 16, 7)
570
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=6 shape=(1, 1, 16, 7)
571
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=7 shape=(1, 1, 16, 7)
572
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=8 shape=(1, 1, 16, 7)
573
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=0 shape=(1, 1, 16, 7)
574
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=1 shape=(1, 1, 16, 7)
575
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=2 shape=(1, 1, 16, 7)
576
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=3 shape=(1, 1, 16, 7)
577
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=4 shape=(1, 1, 16, 7)
578
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=5 shape=(1, 1, 16, 7)
579
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=6 shape=(1, 1, 16, 7)
580
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=7 shape=(1, 1, 16, 7)
581
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=8 shape=(1, 1, 16, 7)
582
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=0 shape=(1, 1, 16, 7)
583
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=1 shape=(1, 1, 16, 7)
584
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=2 shape=(1, 1, 16, 7)
585
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=3 shape=(1, 1, 16, 7)
586
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=4 shape=(1, 1, 16, 7)
587
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=5 shape=(1, 1, 16, 7)
588
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=6 shape=(1, 1, 16, 7)
589
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=7 shape=(1, 1, 16, 7)
590
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=8 shape=(1, 1, 16, 7)
591
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=0 shape=(1, 1, 16, 7)
592
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=1 shape=(1, 1, 16, 7)
593
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=2 shape=(1, 1, 16, 7)
594
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=3 shape=(1, 1, 16, 7)
595
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=4 shape=(1, 1, 16, 7)
596
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=5 shape=(1, 1, 16, 7)
597
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=6 shape=(1, 1, 16, 7)
598
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=7 shape=(1, 1, 16, 7)
599
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=8 shape=(1, 1, 16, 7)
600
+ batch done sequential task=StackCube-v1
601
+ env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu
602
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=0 shape=(1, 1, 16, 7)
603
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=1 shape=(1, 1, 16, 7)
604
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=2 shape=(1, 1, 16, 7)
605
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=3 shape=(1, 1, 16, 7)
606
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=4 shape=(1, 1, 16, 7)
607
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=5 shape=(1, 1, 16, 7)
608
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=6 shape=(1, 1, 16, 7)
609
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=7 shape=(1, 1, 16, 7)
610
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=8 shape=(1, 1, 16, 7)
611
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=0 shape=(1, 1, 16, 7)
612
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=1 shape=(1, 1, 16, 7)
613
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=2 shape=(1, 1, 16, 7)
614
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=3 shape=(1, 1, 16, 7)
615
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=4 shape=(1, 1, 16, 7)
616
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=5 shape=(1, 1, 16, 7)
617
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=6 shape=(1, 1, 16, 7)
618
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=7 shape=(1, 1, 16, 7)
619
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=8 shape=(1, 1, 16, 7)
620
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=0 shape=(1, 1, 16, 7)
621
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=1 shape=(1, 1, 16, 7)
622
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=2 shape=(1, 1, 16, 7)
623
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=3 shape=(1, 1, 16, 7)
624
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=4 shape=(1, 1, 16, 7)
625
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=5 shape=(1, 1, 16, 7)
626
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=6 shape=(1, 1, 16, 7)
627
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=7 shape=(1, 1, 16, 7)
628
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=8 shape=(1, 1, 16, 7)
629
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=0 shape=(1, 1, 16, 7)
630
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=1 shape=(1, 1, 16, 7)
631
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=2 shape=(1, 1, 16, 7)
632
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=3 shape=(1, 1, 16, 7)
633
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=4 shape=(1, 1, 16, 7)
634
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=5 shape=(1, 1, 16, 7)
635
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=6 shape=(1, 1, 16, 7)
636
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=7 shape=(1, 1, 16, 7)
637
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=8 shape=(1, 1, 16, 7)
638
+ batch done sequential task=StackCube-v1
639
+ env init sequential task=StackCube-v1 batch=3 sim=physx_cpu render=cpu
640
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=0 shape=(1, 1, 16, 7)
641
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=1 shape=(1, 1, 16, 7)
642
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=2 shape=(1, 1, 16, 7)
643
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=3 shape=(1, 1, 16, 7)
644
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=4 shape=(1, 1, 16, 7)
645
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=5 shape=(1, 1, 16, 7)
646
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=6 shape=(1, 1, 16, 7)
647
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=7 shape=(1, 1, 16, 7)
648
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=8 shape=(1, 1, 16, 7)
649
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=0 shape=(1, 1, 16, 7)
650
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=1 shape=(1, 1, 16, 7)
651
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=2 shape=(1, 1, 16, 7)
652
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=3 shape=(1, 1, 16, 7)
653
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=4 shape=(1, 1, 16, 7)
654
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=5 shape=(1, 1, 16, 7)
655
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=6 shape=(1, 1, 16, 7)
656
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=7 shape=(1, 1, 16, 7)
657
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=8 shape=(1, 1, 16, 7)
658
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=0 shape=(1, 1, 16, 7)
659
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=1 shape=(1, 1, 16, 7)
660
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=2 shape=(1, 1, 16, 7)
661
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=3 shape=(1, 1, 16, 7)
662
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=4 shape=(1, 1, 16, 7)
663
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=5 shape=(1, 1, 16, 7)
664
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=6 shape=(1, 1, 16, 7)
665
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=7 shape=(1, 1, 16, 7)
666
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=8 shape=(1, 1, 16, 7)
667
+ batch done sequential task=StackCube-v1
668
+ rollout complete rows=69
workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics.json CHANGED
The diff for this file is too large to render. See raw diff
 
workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics_by_seed.json CHANGED
@@ -1,331 +1,1083 @@
1
  {
2
  "1025740212": {
 
 
3
  "branch_car_at_8": 0.47739143669605255,
 
 
4
  "outcome_ptr_at_8": 1.0,
5
  "pairwise_causal_calibration_ece": 0.12992986623548633,
 
 
 
 
 
 
 
 
6
  "selector_regret_at_8": 0.47739143669605255,
7
- "support_gap_at_8": 1.3016598224639893
 
 
 
 
8
  },
9
  "1055157354": {
 
 
10
  "branch_car_at_8": 0.6370042562484741,
 
 
11
  "outcome_ptr_at_8": 1.0,
12
  "pairwise_causal_calibration_ece": 0.07450750352404123,
 
 
 
 
 
 
 
 
13
  "selector_regret_at_8": 0.6370042562484741,
14
- "support_gap_at_8": 1.1767837405204773
 
 
 
 
15
  },
16
  "1055567618": {
 
 
17
  "branch_car_at_8": 1.8253512382507324,
 
 
18
  "outcome_ptr_at_8": 0.0,
19
  "pairwise_causal_calibration_ece": 0.318234334809661,
 
 
 
 
 
 
 
 
20
  "selector_regret_at_8": 1.8253512382507324,
21
- "support_gap_at_8": 0.0
 
 
 
 
22
  },
23
  "1113268940": {
 
 
24
  "branch_car_at_8": 0.18306860327720642,
 
 
25
  "outcome_ptr_at_8": 1.0,
26
  "pairwise_causal_calibration_ece": 0.10725283874677279,
 
 
 
 
 
 
 
 
27
  "selector_regret_at_8": 0.18306860327720642,
28
- "support_gap_at_8": 1.33825021982193
 
 
 
 
29
  },
30
  "1113371261": {
 
 
31
  "branch_car_at_8": 0.0,
 
 
32
  "outcome_ptr_at_8": 0.0,
33
  "pairwise_causal_calibration_ece": 0.04977883974023356,
 
 
 
 
 
 
 
 
34
  "selector_regret_at_8": 0.0,
35
- "support_gap_at_8": 0.0
 
 
 
 
36
  },
37
  "1176943004": {
 
 
38
  "branch_car_at_8": 1.974869817495346,
 
 
39
  "outcome_ptr_at_8": 1.0,
40
  "pairwise_causal_calibration_ece": 0.1815411965846294,
 
 
 
 
 
 
 
 
41
  "selector_regret_at_8": 1.974869817495346,
42
- "support_gap_at_8": 0.0
 
 
 
 
43
  },
44
  "1189392917": {
 
 
45
  "branch_car_at_8": 0.0,
 
 
46
  "outcome_ptr_at_8": 1.0,
47
  "pairwise_causal_calibration_ece": 0.2180235973132697,
 
 
 
 
 
 
 
 
48
  "selector_regret_at_8": 0.0,
49
- "support_gap_at_8": 0.0
 
 
 
 
50
  },
51
  "1231780997": {
 
 
52
  "branch_car_at_8": 0.0,
 
 
53
  "outcome_ptr_at_8": 1.0,
54
  "pairwise_causal_calibration_ece": 0.28322114525972475,
 
 
 
 
 
 
 
 
55
  "selector_regret_at_8": 0.0,
56
- "support_gap_at_8": 0.0
 
 
 
 
57
  },
58
  "1271718007": {
 
 
59
  "branch_car_at_8": 0.00353144109249115,
 
 
60
  "outcome_ptr_at_8": 0.0,
61
  "pairwise_causal_calibration_ece": 0.07023931523066618,
 
 
 
 
 
 
 
 
62
  "selector_regret_at_8": 0.00353144109249115,
63
- "support_gap_at_8": 0.8889523223042488
 
 
 
 
64
  },
65
  "1273247072": {
 
 
66
  "branch_car_at_8": 0.45726464316248894,
 
 
67
  "outcome_ptr_at_8": 1.0,
68
  "pairwise_causal_calibration_ece": 0.096994407338536,
 
 
 
 
 
 
 
 
69
  "selector_regret_at_8": 0.45726464316248894,
70
- "support_gap_at_8": 1.500250667333603
 
 
 
 
71
  },
72
  "1358421325": {
 
 
73
  "branch_car_at_8": 0.6467066705226898,
 
 
74
  "outcome_ptr_at_8": 0.0,
75
  "pairwise_causal_calibration_ece": 0.046694794435454434,
 
 
 
 
 
 
 
 
76
  "selector_regret_at_8": 0.6467066705226898,
77
- "support_gap_at_8": 1.1724565625190735
 
 
 
 
78
  },
79
  "1375492786": {
 
 
80
  "branch_car_at_8": 0.00047226750757545233,
 
 
81
  "outcome_ptr_at_8": 0.0,
82
  "pairwise_causal_calibration_ece": 0.14027669602920534,
 
 
 
 
 
 
 
 
83
  "selector_regret_at_8": 0.00047226750757545233,
84
- "support_gap_at_8": 0.2416609227657318
 
 
 
 
85
  },
86
  "141984615": {
 
 
87
  "branch_car_at_8": 0.05730179324746132,
 
 
88
  "outcome_ptr_at_8": 0.6666666666666666,
89
  "pairwise_causal_calibration_ece": 0.08533231968443573,
 
 
 
 
 
 
 
 
90
  "selector_regret_at_8": 0.05730179324746132,
91
- "support_gap_at_8": 0.11134201536575954
 
 
 
 
92
  },
93
  "1475440973": {
 
 
94
  "branch_car_at_8": 1.289921224117279,
 
 
95
  "outcome_ptr_at_8": 0.0,
96
  "pairwise_causal_calibration_ece": 0.2443552809335613,
 
 
 
 
 
 
 
 
97
  "selector_regret_at_8": 1.289921224117279,
98
- "support_gap_at_8": 0.0
 
 
 
 
99
  },
100
  "1483645336": {
 
 
101
  "branch_car_at_8": 1.9307887554168701,
 
 
102
  "outcome_ptr_at_8": 0.0,
103
  "pairwise_causal_calibration_ece": 0.13752192684190806,
 
 
 
 
 
 
 
 
104
  "selector_regret_at_8": 1.9307887554168701,
105
- "support_gap_at_8": 0.0
 
 
 
 
106
  },
107
  "1522272686": {
 
 
108
  "branch_car_at_8": 0.03616611659526825,
 
 
109
  "outcome_ptr_at_8": 1.0,
110
  "pairwise_causal_calibration_ece": 0.28555947838938045,
 
 
 
 
 
 
 
 
111
  "selector_regret_at_8": 0.03616611659526825,
112
- "support_gap_at_8": 1.7675106227397919
 
 
 
 
113
  },
114
  "1528740417": {
 
 
115
  "branch_car_at_8": 0.0,
 
 
116
  "outcome_ptr_at_8": 0.0,
117
  "pairwise_causal_calibration_ece": 0.2757671830981365,
 
 
 
 
 
 
 
 
118
  "selector_regret_at_8": 0.0,
119
- "support_gap_at_8": 0.0
 
 
 
 
120
  },
121
  "1549324805": {
 
 
122
  "branch_car_at_8": 0.0015795694974561532,
 
 
123
  "outcome_ptr_at_8": 0.0,
124
  "pairwise_causal_calibration_ece": 0.28625156462411294,
 
 
 
 
 
 
 
 
125
  "selector_regret_at_8": 0.0015795694974561532,
126
- "support_gap_at_8": 0.7242437700430552
 
 
 
 
127
  },
128
  "1607773860": {
 
 
129
  "branch_car_at_8": 0.6036040186882019,
 
 
130
  "outcome_ptr_at_8": 0.0,
131
  "pairwise_causal_calibration_ece": 0.18975917078230178,
 
 
 
 
 
 
 
 
132
  "selector_regret_at_8": 0.6036040186882019,
133
- "support_gap_at_8": 1.173292636871338
 
 
 
 
134
  },
135
  "1665739511": {
 
 
136
  "branch_car_at_8": 0.0,
 
 
137
  "outcome_ptr_at_8": 0.0,
138
  "pairwise_causal_calibration_ece": 0.0,
 
 
 
 
 
 
 
 
139
  "selector_regret_at_8": 0.0,
140
- "support_gap_at_8": 0.0
 
 
 
 
141
  },
142
  "1747178030": {
 
 
143
  "branch_car_at_8": 0.027157030999660492,
 
 
144
  "outcome_ptr_at_8": 0.0,
145
  "pairwise_causal_calibration_ece": 0.3089551740932353,
 
 
 
 
 
 
 
 
146
  "selector_regret_at_8": 0.027157030999660492,
147
- "support_gap_at_8": 1.8886577263474464
 
 
 
 
148
  },
149
  "1881223807": {
 
 
150
  "branch_car_at_8": 0.009028241038322449,
 
 
151
  "outcome_ptr_at_8": 1.0,
152
  "pairwise_causal_calibration_ece": 0.13674585276395504,
 
 
 
 
 
 
 
 
153
  "selector_regret_at_8": 0.009028241038322449,
154
- "support_gap_at_8": 1.9651138186454773
 
 
 
 
155
  },
156
  "1919198188": {
 
 
157
  "branch_car_at_8": 0.0,
 
 
158
  "outcome_ptr_at_8": 1.0,
159
  "pairwise_causal_calibration_ece": 0.13114443223224567,
 
 
 
 
 
 
 
 
160
  "selector_regret_at_8": 0.0,
161
- "support_gap_at_8": 1.9537556618452072
 
 
 
 
162
  },
163
  "1991414747": {
 
 
164
  "branch_car_at_8": 1.9863876849412918,
 
 
165
  "outcome_ptr_at_8": 0.0,
166
  "pairwise_causal_calibration_ece": 0.1306768270010748,
 
 
 
 
 
 
 
 
167
  "selector_regret_at_8": 1.9863876849412918,
168
- "support_gap_at_8": 0.0
 
 
 
 
169
  },
170
  "1997090716": {
 
 
171
  "branch_car_at_8": 0.0,
 
 
172
  "outcome_ptr_at_8": 1.0,
173
  "pairwise_causal_calibration_ece": 0.2554442550173647,
 
 
 
 
 
 
 
 
174
  "selector_regret_at_8": 0.0,
175
- "support_gap_at_8": 0.0
 
 
 
 
176
  },
177
  "2015956957": {
 
 
178
  "branch_car_at_8": 0.022195855714380743,
 
 
179
  "outcome_ptr_at_8": 0.6,
180
  "pairwise_causal_calibration_ece": 0.16317219295797403,
 
 
 
 
 
 
 
 
181
  "selector_regret_at_8": 0.022195855714380743,
182
- "support_gap_at_8": 0.4735892705619335
 
 
 
 
183
  },
184
  "2054899126": {
 
 
185
  "branch_car_at_8": 0.0,
 
 
186
  "outcome_ptr_at_8": 0.0,
187
  "pairwise_causal_calibration_ece": 0.0,
 
 
 
 
 
 
 
 
188
  "selector_regret_at_8": 0.0,
189
- "support_gap_at_8": 0.0
 
 
 
 
190
  },
191
  "211245842": {
 
 
192
  "branch_car_at_8": 0.6792904287576675,
 
 
193
  "outcome_ptr_at_8": 0.0,
194
  "pairwise_causal_calibration_ece": 0.07548939908426067,
 
 
 
 
 
 
 
 
195
  "selector_regret_at_8": 0.6792904287576675,
196
- "support_gap_at_8": 1.1641557812690735
 
 
 
 
197
  },
198
  "2141802816": {
 
 
199
  "branch_car_at_8": 0.019766509532928467,
 
 
200
  "outcome_ptr_at_8": 1.0,
201
  "pairwise_causal_calibration_ece": 0.07786951284417709,
 
 
 
 
 
 
 
 
202
  "selector_regret_at_8": 0.019766509532928467,
203
- "support_gap_at_8": 1.904106765985489
 
 
 
 
204
  },
205
  "256770874": {
 
 
206
  "branch_car_at_8": 0.7044661641120911,
 
 
207
  "outcome_ptr_at_8": 0.0,
208
  "pairwise_causal_calibration_ece": 0.03135306008748828,
 
 
 
 
 
 
 
 
209
  "selector_regret_at_8": 0.7044661641120911,
210
- "support_gap_at_8": 1.1875
 
 
 
 
211
  },
212
  "278008524": {
 
 
213
  "branch_car_at_8": 0.011409573198761791,
 
 
214
  "outcome_ptr_at_8": 0.5,
215
  "pairwise_causal_calibration_ece": 0.24605586147071373,
 
 
 
 
 
 
 
 
216
  "selector_regret_at_8": 0.011409573198761791,
217
- "support_gap_at_8": 1.082339386222884
 
 
 
 
218
  },
219
  "416183106": {
 
 
220
  "branch_car_at_8": 1.7668711841106415,
 
 
221
  "outcome_ptr_at_8": 1.0,
222
  "pairwise_causal_calibration_ece": 0.07978206783428046,
 
 
 
 
 
 
 
 
223
  "selector_regret_at_8": 1.7668711841106415,
224
- "support_gap_at_8": 0.0
 
 
 
 
225
  },
226
  "534717141": {
 
 
227
  "branch_car_at_8": 1.9558932781219482,
 
 
228
  "outcome_ptr_at_8": 0.0,
229
  "pairwise_causal_calibration_ece": 0.14369576234023818,
 
 
 
 
 
 
 
 
230
  "selector_regret_at_8": 1.9558932781219482,
231
- "support_gap_at_8": 0.0
 
 
 
 
232
  },
233
  "551504635": {
 
 
234
  "branch_car_at_8": 0.0,
 
 
235
  "outcome_ptr_at_8": 0.0,
236
  "pairwise_causal_calibration_ece": 0.2299325778712667,
 
 
 
 
 
 
 
 
237
  "selector_regret_at_8": 0.0,
238
- "support_gap_at_8": 1.2775209546089172
 
 
 
 
239
  },
240
  "565369392": {
 
 
241
  "branch_car_at_8": 0.046260833740234375,
 
 
242
  "outcome_ptr_at_8": 0.0,
243
  "pairwise_causal_calibration_ece": 0.13991651305345584,
 
 
 
 
 
 
 
 
244
  "selector_regret_at_8": 0.046260833740234375,
245
- "support_gap_at_8": 1.2529667715231578
 
 
 
 
246
  },
247
  "565621826": {
 
 
248
  "branch_car_at_8": 0.0,
 
 
249
  "outcome_ptr_at_8": 0.0,
250
  "pairwise_causal_calibration_ece": 0.0,
 
 
 
 
 
 
 
 
251
  "selector_regret_at_8": 0.0,
252
- "support_gap_at_8": 0.0
 
 
 
 
253
  },
254
  "585904130": {
 
 
255
  "branch_car_at_8": 0.012068324722349644,
 
 
256
  "outcome_ptr_at_8": 0.0,
257
  "pairwise_causal_calibration_ece": 0.07276320034061332,
 
 
 
 
 
 
 
 
258
  "selector_regret_at_8": 0.012068324722349644,
259
- "support_gap_at_8": 0.20472841958204904
 
 
 
 
260
  },
261
  "626486813": {
 
 
262
  "branch_car_at_8": 0.02500777377281338,
 
 
263
  "outcome_ptr_at_8": 0.5,
264
  "pairwise_causal_calibration_ece": 0.2420589895144555,
 
 
 
 
 
 
 
 
265
  "selector_regret_at_8": 0.02500777377281338,
266
- "support_gap_at_8": 0.5414478182792664
 
 
 
 
267
  },
268
  "636680137": {
 
 
269
  "branch_car_at_8": 0.0,
 
 
270
  "outcome_ptr_at_8": 0.0,
271
  "pairwise_causal_calibration_ece": 0.07628525991805817,
 
 
 
 
 
 
 
 
272
  "selector_regret_at_8": 0.0,
273
- "support_gap_at_8": 1.986071527004242
 
 
 
 
274
  },
275
  "706479600": {
 
 
276
  "branch_car_at_8": 0.0,
 
 
277
  "outcome_ptr_at_8": 1.0,
278
  "pairwise_causal_calibration_ece": 0.16443847051324711,
 
 
 
 
 
 
 
 
279
  "selector_regret_at_8": 0.0,
280
- "support_gap_at_8": 0.0
 
 
 
 
281
  },
282
  "750448296": {
 
 
283
  "branch_car_at_8": 0.02701035887002945,
 
 
284
  "outcome_ptr_at_8": 0.0,
285
  "pairwise_causal_calibration_ece": 0.2344413298276417,
 
 
 
 
 
 
 
 
286
  "selector_regret_at_8": 0.02701035887002945,
287
- "support_gap_at_8": 0.16238094866275787
 
 
 
 
288
  },
289
  "75070641": {
 
 
290
  "branch_car_at_8": 0.5863077491521835,
 
 
291
  "outcome_ptr_at_8": 1.0,
292
  "pairwise_causal_calibration_ece": 0.18559462278331035,
 
 
 
 
 
 
 
 
293
  "selector_regret_at_8": 0.5863077491521835,
294
- "support_gap_at_8": 1.1871125102043152
 
 
 
 
295
  },
296
  "767747424": {
 
 
297
  "branch_car_at_8": 0.04561196081340313,
 
 
298
  "outcome_ptr_at_8": 0.75,
299
  "pairwise_causal_calibration_ece": 0.07531396139335009,
 
 
 
 
 
 
 
 
300
  "selector_regret_at_8": 0.04561196081340313,
301
- "support_gap_at_8": 0.08526064571924508
 
 
 
 
302
  },
303
  "77147264": {
 
 
304
  "branch_car_at_8": 0.0,
 
 
305
  "outcome_ptr_at_8": 0.5,
306
  "pairwise_causal_calibration_ece": 0.2630803856693186,
 
 
 
 
 
 
 
 
307
  "selector_regret_at_8": 0.0,
308
- "support_gap_at_8": 0.573363870382309
 
 
 
 
309
  },
310
  "794181626": {
 
 
311
  "branch_car_at_8": 0.0,
 
 
312
  "outcome_ptr_at_8": 1.0,
313
  "pairwise_causal_calibration_ece": 0.28339362113630795,
 
 
 
 
 
 
 
 
314
  "selector_regret_at_8": 0.0,
315
- "support_gap_at_8": 0.0
 
 
 
 
316
  },
317
  "957101489": {
 
 
318
  "branch_car_at_8": 1.7757539749145508,
 
 
319
  "outcome_ptr_at_8": 1.0,
320
  "pairwise_causal_calibration_ece": 0.07643021229779011,
 
 
 
 
 
 
 
 
321
  "selector_regret_at_8": 1.7757539749145508,
322
- "support_gap_at_8": 0.0
 
 
 
 
323
  },
324
  "998761162": {
 
 
325
  "branch_car_at_8": 1.9512543678283691,
 
 
326
  "outcome_ptr_at_8": 1.0,
327
  "pairwise_causal_calibration_ece": 0.28991947553571784,
 
 
 
 
 
 
 
 
328
  "selector_regret_at_8": 1.9512543678283691,
329
- "support_gap_at_8": 0.0
 
 
 
 
330
  }
331
  }
 
1
  {
2
  "1025740212": {
3
+ "base_success": 0.0,
4
+ "base_utility": 0.528373658657074,
5
  "branch_car_at_8": 0.47739143669605255,
6
+ "hidden_chart_oracle_success_at_8": 1.0,
7
+ "hidden_chart_oracle_utility_at_8": 2.0,
8
  "outcome_ptr_at_8": 1.0,
9
  "pairwise_causal_calibration_ece": 0.12992986623548633,
10
+ "proposal_oracle_success_at_8": 0.0,
11
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
12
+ "proposal_oracle_utility_at_8": 0.6983401775360107,
13
+ "proposal_oracle_utility_gain_over_base_at_8": 0.16996651887893677,
14
+ "selected_success_at_8": 0.0,
15
+ "selected_success_gain_over_base_at_8": 0.0,
16
+ "selected_utility_at_8": 0.2209487408399582,
17
+ "selected_utility_gain_over_base_at_8": -0.3074249178171158,
18
  "selector_regret_at_8": 0.47739143669605255,
19
+ "success_selector_gap_at_8": 0.0,
20
+ "success_support_gap_at_8": 1.0,
21
+ "success_total_car_to_hidden_at_8": 1.0,
22
+ "support_gap_at_8": 1.3016598224639893,
23
+ "total_car_to_hidden_at_8": 1.7790512591600418
24
  },
25
  "1055157354": {
26
+ "base_success": 0.0,
27
+ "base_utility": 0.17649109661579132,
28
  "branch_car_at_8": 0.6370042562484741,
29
+ "hidden_chart_oracle_success_at_8": 1.0,
30
+ "hidden_chart_oracle_utility_at_8": 2.0,
31
  "outcome_ptr_at_8": 1.0,
32
  "pairwise_causal_calibration_ece": 0.07450750352404123,
33
+ "proposal_oracle_success_at_8": 0.0,
34
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
35
+ "proposal_oracle_utility_at_8": 0.8232162594795227,
36
+ "proposal_oracle_utility_gain_over_base_at_8": 0.6467251628637314,
37
+ "selected_success_at_8": 0.0,
38
+ "selected_success_gain_over_base_at_8": 0.0,
39
+ "selected_utility_at_8": 0.18621200323104858,
40
+ "selected_utility_gain_over_base_at_8": 0.009720906615257263,
41
  "selector_regret_at_8": 0.6370042562484741,
42
+ "success_selector_gap_at_8": 0.0,
43
+ "success_support_gap_at_8": 1.0,
44
+ "success_total_car_to_hidden_at_8": 1.0,
45
+ "support_gap_at_8": 1.1767837405204773,
46
+ "total_car_to_hidden_at_8": 1.8137879967689514
47
  },
48
  "1055567618": {
49
+ "base_success": 1.0,
50
+ "base_utility": 2.0,
51
  "branch_car_at_8": 1.8253512382507324,
52
+ "hidden_chart_oracle_success_at_8": 1.0,
53
+ "hidden_chart_oracle_utility_at_8": 2.0,
54
  "outcome_ptr_at_8": 0.0,
55
  "pairwise_causal_calibration_ece": 0.318234334809661,
56
+ "proposal_oracle_success_at_8": 1.0,
57
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
58
+ "proposal_oracle_utility_at_8": 2.0,
59
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
60
+ "selected_success_at_8": 0.0,
61
+ "selected_success_gain_over_base_at_8": -1.0,
62
+ "selected_utility_at_8": 0.17464876174926758,
63
+ "selected_utility_gain_over_base_at_8": -1.8253512382507324,
64
  "selector_regret_at_8": 1.8253512382507324,
65
+ "success_selector_gap_at_8": 1.0,
66
+ "success_support_gap_at_8": 0.0,
67
+ "success_total_car_to_hidden_at_8": 1.0,
68
+ "support_gap_at_8": 0.0,
69
+ "total_car_to_hidden_at_8": 1.8253512382507324
70
  },
71
  "1113268940": {
72
+ "base_success": 0.0,
73
+ "base_utility": 0.42231252789497375,
74
  "branch_car_at_8": 0.18306860327720642,
75
+ "hidden_chart_oracle_success_at_8": 1.0,
76
+ "hidden_chart_oracle_utility_at_8": 2.0,
77
  "outcome_ptr_at_8": 1.0,
78
  "pairwise_causal_calibration_ece": 0.10725283874677279,
79
+ "proposal_oracle_success_at_8": 0.0,
80
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
81
+ "proposal_oracle_utility_at_8": 0.6617497801780701,
82
+ "proposal_oracle_utility_gain_over_base_at_8": 0.2394372522830963,
83
+ "selected_success_at_8": 0.0,
84
+ "selected_success_gain_over_base_at_8": 0.0,
85
+ "selected_utility_at_8": 0.47868117690086365,
86
+ "selected_utility_gain_over_base_at_8": 0.05636864900588989,
87
  "selector_regret_at_8": 0.18306860327720642,
88
+ "success_selector_gap_at_8": 0.0,
89
+ "success_support_gap_at_8": 1.0,
90
+ "success_total_car_to_hidden_at_8": 1.0,
91
+ "support_gap_at_8": 1.33825021982193,
92
+ "total_car_to_hidden_at_8": 1.5213188230991364
93
  },
94
  "1113371261": {
95
+ "base_success": 1.0,
96
+ "base_utility": 2.0,
97
  "branch_car_at_8": 0.0,
98
+ "hidden_chart_oracle_success_at_8": 1.0,
99
+ "hidden_chart_oracle_utility_at_8": 2.0,
100
  "outcome_ptr_at_8": 0.0,
101
  "pairwise_causal_calibration_ece": 0.04977883974023356,
102
+ "proposal_oracle_success_at_8": 1.0,
103
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
104
+ "proposal_oracle_utility_at_8": 2.0,
105
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
106
+ "selected_success_at_8": 1.0,
107
+ "selected_success_gain_over_base_at_8": 0.0,
108
+ "selected_utility_at_8": 2.0,
109
+ "selected_utility_gain_over_base_at_8": 0.0,
110
  "selector_regret_at_8": 0.0,
111
+ "success_selector_gap_at_8": 0.0,
112
+ "success_support_gap_at_8": 0.0,
113
+ "success_total_car_to_hidden_at_8": 0.0,
114
+ "support_gap_at_8": 0.0,
115
+ "total_car_to_hidden_at_8": 0.0
116
  },
117
  "1176943004": {
118
+ "base_success": 0.0,
119
+ "base_utility": 0.058033183217048645,
120
  "branch_car_at_8": 1.974869817495346,
121
+ "hidden_chart_oracle_success_at_8": 1.0,
122
+ "hidden_chart_oracle_utility_at_8": 2.0,
123
  "outcome_ptr_at_8": 1.0,
124
  "pairwise_causal_calibration_ece": 0.1815411965846294,
125
+ "proposal_oracle_success_at_8": 1.0,
126
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
127
+ "proposal_oracle_utility_at_8": 2.0,
128
+ "proposal_oracle_utility_gain_over_base_at_8": 1.9419668167829514,
129
+ "selected_success_at_8": 0.0,
130
+ "selected_success_gain_over_base_at_8": 0.0,
131
+ "selected_utility_at_8": 0.02513018250465393,
132
+ "selected_utility_gain_over_base_at_8": -0.032903000712394714,
133
  "selector_regret_at_8": 1.974869817495346,
134
+ "success_selector_gap_at_8": 1.0,
135
+ "success_support_gap_at_8": 0.0,
136
+ "success_total_car_to_hidden_at_8": 1.0,
137
+ "support_gap_at_8": 0.0,
138
+ "total_car_to_hidden_at_8": 1.974869817495346
139
  },
140
  "1189392917": {
141
+ "base_success": 0.0,
142
+ "base_utility": 0.10841864347457886,
143
  "branch_car_at_8": 0.0,
144
+ "hidden_chart_oracle_success_at_8": 1.0,
145
+ "hidden_chart_oracle_utility_at_8": 2.0,
146
  "outcome_ptr_at_8": 1.0,
147
  "pairwise_causal_calibration_ece": 0.2180235973132697,
148
+ "proposal_oracle_success_at_8": 1.0,
149
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
150
+ "proposal_oracle_utility_at_8": 2.0,
151
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211,
152
+ "selected_success_at_8": 1.0,
153
+ "selected_success_gain_over_base_at_8": 1.0,
154
+ "selected_utility_at_8": 2.0,
155
+ "selected_utility_gain_over_base_at_8": 1.8915813565254211,
156
  "selector_regret_at_8": 0.0,
157
+ "success_selector_gap_at_8": 0.0,
158
+ "success_support_gap_at_8": 0.0,
159
+ "success_total_car_to_hidden_at_8": 0.0,
160
+ "support_gap_at_8": 0.0,
161
+ "total_car_to_hidden_at_8": 0.0
162
  },
163
  "1231780997": {
164
+ "base_success": 0.0,
165
+ "base_utility": 0.7270061373710632,
166
  "branch_car_at_8": 0.0,
167
+ "hidden_chart_oracle_success_at_8": 1.0,
168
+ "hidden_chart_oracle_utility_at_8": 2.0,
169
  "outcome_ptr_at_8": 1.0,
170
  "pairwise_causal_calibration_ece": 0.28322114525972475,
171
+ "proposal_oracle_success_at_8": 1.0,
172
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
173
+ "proposal_oracle_utility_at_8": 2.0,
174
+ "proposal_oracle_utility_gain_over_base_at_8": 1.2729938626289368,
175
+ "selected_success_at_8": 1.0,
176
+ "selected_success_gain_over_base_at_8": 1.0,
177
+ "selected_utility_at_8": 2.0,
178
+ "selected_utility_gain_over_base_at_8": 1.2729938626289368,
179
  "selector_regret_at_8": 0.0,
180
+ "success_selector_gap_at_8": 0.0,
181
+ "success_support_gap_at_8": 0.0,
182
+ "success_total_car_to_hidden_at_8": 0.0,
183
+ "support_gap_at_8": 0.0,
184
+ "total_car_to_hidden_at_8": 0.0
185
  },
186
  "1271718007": {
187
+ "base_success": 0.5,
188
+ "base_utility": 1.415381133556366,
189
  "branch_car_at_8": 0.00353144109249115,
190
+ "hidden_chart_oracle_success_at_8": 1.0,
191
+ "hidden_chart_oracle_utility_at_8": 2.0,
192
  "outcome_ptr_at_8": 0.0,
193
  "pairwise_causal_calibration_ece": 0.07023931523066618,
194
+ "proposal_oracle_success_at_8": 0.5,
195
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
196
+ "proposal_oracle_utility_at_8": 1.1110476776957512,
197
+ "proposal_oracle_utility_gain_over_base_at_8": -0.3043334558606148,
198
+ "selected_success_at_8": 0.5,
199
+ "selected_success_gain_over_base_at_8": 0.0,
200
+ "selected_utility_at_8": 1.10751623660326,
201
+ "selected_utility_gain_over_base_at_8": -0.3078648969531059,
202
  "selector_regret_at_8": 0.00353144109249115,
203
+ "success_selector_gap_at_8": 0.0,
204
+ "success_support_gap_at_8": 0.5,
205
+ "success_total_car_to_hidden_at_8": 0.5,
206
+ "support_gap_at_8": 0.8889523223042488,
207
+ "total_car_to_hidden_at_8": 0.89248376339674
208
  },
209
  "1273247072": {
210
+ "base_success": 0.0,
211
+ "base_utility": 0.15342965722084045,
212
  "branch_car_at_8": 0.45726464316248894,
213
+ "hidden_chart_oracle_success_at_8": 1.0,
214
+ "hidden_chart_oracle_utility_at_8": 2.0,
215
  "outcome_ptr_at_8": 1.0,
216
  "pairwise_causal_calibration_ece": 0.096994407338536,
217
+ "proposal_oracle_success_at_8": 0.0,
218
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
219
+ "proposal_oracle_utility_at_8": 0.4997493326663971,
220
+ "proposal_oracle_utility_gain_over_base_at_8": 0.34631967544555664,
221
+ "selected_success_at_8": 0.0,
222
+ "selected_success_gain_over_base_at_8": 0.0,
223
+ "selected_utility_at_8": 0.04248468950390816,
224
+ "selected_utility_gain_over_base_at_8": -0.1109449677169323,
225
  "selector_regret_at_8": 0.45726464316248894,
226
+ "success_selector_gap_at_8": 0.0,
227
+ "success_support_gap_at_8": 1.0,
228
+ "success_total_car_to_hidden_at_8": 1.0,
229
+ "support_gap_at_8": 1.500250667333603,
230
+ "total_car_to_hidden_at_8": 1.9575153104960918
231
  },
232
  "1358421325": {
233
+ "base_success": 0.0,
234
+ "base_utility": 0.8324764966964722,
235
  "branch_car_at_8": 0.6467066705226898,
236
+ "hidden_chart_oracle_success_at_8": 1.0,
237
+ "hidden_chart_oracle_utility_at_8": 2.0,
238
  "outcome_ptr_at_8": 0.0,
239
  "pairwise_causal_calibration_ece": 0.046694794435454434,
240
+ "proposal_oracle_success_at_8": 0.0,
241
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
242
+ "proposal_oracle_utility_at_8": 0.8275434374809265,
243
+ "proposal_oracle_utility_gain_over_base_at_8": -0.004933059215545654,
244
+ "selected_success_at_8": 0.0,
245
+ "selected_success_gain_over_base_at_8": 0.0,
246
+ "selected_utility_at_8": 0.1808367669582367,
247
+ "selected_utility_gain_over_base_at_8": -0.6516397297382355,
248
  "selector_regret_at_8": 0.6467066705226898,
249
+ "success_selector_gap_at_8": 0.0,
250
+ "success_support_gap_at_8": 1.0,
251
+ "success_total_car_to_hidden_at_8": 1.0,
252
+ "support_gap_at_8": 1.1724565625190735,
253
+ "total_car_to_hidden_at_8": 1.8191632330417633
254
  },
255
  "1375492786": {
256
+ "base_success": 1.0,
257
+ "base_utility": 2.0,
258
  "branch_car_at_8": 0.00047226750757545233,
259
+ "hidden_chart_oracle_success_at_8": 0.0,
260
+ "hidden_chart_oracle_utility_at_8": 0.24345096945762634,
261
  "outcome_ptr_at_8": 0.0,
262
  "pairwise_causal_calibration_ece": 0.14027669602920534,
263
+ "proposal_oracle_success_at_8": 0.0,
264
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
265
+ "proposal_oracle_utility_at_8": 0.0017900466918945312,
266
+ "proposal_oracle_utility_gain_over_base_at_8": -1.9982099533081055,
267
+ "selected_success_at_8": 0.0,
268
+ "selected_success_gain_over_base_at_8": -1.0,
269
+ "selected_utility_at_8": 0.001317779184319079,
270
+ "selected_utility_gain_over_base_at_8": -1.998682220815681,
271
  "selector_regret_at_8": 0.00047226750757545233,
272
+ "success_selector_gap_at_8": 0.0,
273
+ "success_support_gap_at_8": 0.0,
274
+ "success_total_car_to_hidden_at_8": 0.0,
275
+ "support_gap_at_8": 0.2416609227657318,
276
+ "total_car_to_hidden_at_8": 0.24213319027330726
277
  },
278
  "141984615": {
279
+ "base_success": 0.0,
280
+ "base_utility": 0.09381213908394177,
281
  "branch_car_at_8": 0.05730179324746132,
282
+ "hidden_chart_oracle_success_at_8": 0.0,
283
+ "hidden_chart_oracle_utility_at_8": 0.20044784247875214,
284
  "outcome_ptr_at_8": 0.6666666666666666,
285
  "pairwise_causal_calibration_ece": 0.08533231968443573,
286
+ "proposal_oracle_success_at_8": 0.0,
287
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
288
+ "proposal_oracle_utility_at_8": 0.09240151196718216,
289
+ "proposal_oracle_utility_gain_over_base_at_8": -0.001410627116759618,
290
+ "selected_success_at_8": 0.0,
291
+ "selected_success_gain_over_base_at_8": 0.0,
292
+ "selected_utility_at_8": 0.03509971871972084,
293
+ "selected_utility_gain_over_base_at_8": -0.05871242036422094,
294
  "selector_regret_at_8": 0.05730179324746132,
295
+ "success_selector_gap_at_8": 0.0,
296
+ "success_support_gap_at_8": 0.0,
297
+ "success_total_car_to_hidden_at_8": 0.0,
298
+ "support_gap_at_8": 0.11134201536575954,
299
+ "total_car_to_hidden_at_8": 0.1653481237590313
300
  },
301
  "1475440973": {
302
+ "base_success": 1.0,
303
+ "base_utility": 2.0,
304
  "branch_car_at_8": 1.289921224117279,
305
+ "hidden_chart_oracle_success_at_8": 1.0,
306
+ "hidden_chart_oracle_utility_at_8": 2.0,
307
  "outcome_ptr_at_8": 0.0,
308
  "pairwise_causal_calibration_ece": 0.2443552809335613,
309
+ "proposal_oracle_success_at_8": 1.0,
310
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
311
+ "proposal_oracle_utility_at_8": 2.0,
312
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
313
+ "selected_success_at_8": 0.0,
314
+ "selected_success_gain_over_base_at_8": -1.0,
315
+ "selected_utility_at_8": 0.710078775882721,
316
+ "selected_utility_gain_over_base_at_8": -1.289921224117279,
317
  "selector_regret_at_8": 1.289921224117279,
318
+ "success_selector_gap_at_8": 1.0,
319
+ "success_support_gap_at_8": 0.0,
320
+ "success_total_car_to_hidden_at_8": 1.0,
321
+ "support_gap_at_8": 0.0,
322
+ "total_car_to_hidden_at_8": 1.289921224117279
323
  },
324
  "1483645336": {
325
+ "base_success": 1.0,
326
+ "base_utility": 2.0,
327
  "branch_car_at_8": 1.9307887554168701,
328
+ "hidden_chart_oracle_success_at_8": 1.0,
329
+ "hidden_chart_oracle_utility_at_8": 2.0,
330
  "outcome_ptr_at_8": 0.0,
331
  "pairwise_causal_calibration_ece": 0.13752192684190806,
332
+ "proposal_oracle_success_at_8": 1.0,
333
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
334
+ "proposal_oracle_utility_at_8": 2.0,
335
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
336
+ "selected_success_at_8": 0.0,
337
+ "selected_success_gain_over_base_at_8": -1.0,
338
+ "selected_utility_at_8": 0.06921124458312988,
339
+ "selected_utility_gain_over_base_at_8": -1.9307887554168701,
340
  "selector_regret_at_8": 1.9307887554168701,
341
+ "success_selector_gap_at_8": 1.0,
342
+ "success_support_gap_at_8": 0.0,
343
+ "success_total_car_to_hidden_at_8": 1.0,
344
+ "support_gap_at_8": 0.0,
345
+ "total_car_to_hidden_at_8": 1.9307887554168701
346
  },
347
  "1522272686": {
348
+ "base_success": 0.0,
349
+ "base_utility": 0.19844311475753784,
350
  "branch_car_at_8": 0.03616611659526825,
351
+ "hidden_chart_oracle_success_at_8": 1.0,
352
+ "hidden_chart_oracle_utility_at_8": 2.0,
353
  "outcome_ptr_at_8": 1.0,
354
  "pairwise_causal_calibration_ece": 0.28555947838938045,
355
+ "proposal_oracle_success_at_8": 0.0,
356
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
357
+ "proposal_oracle_utility_at_8": 0.23248937726020813,
358
+ "proposal_oracle_utility_gain_over_base_at_8": 0.03404626250267029,
359
+ "selected_success_at_8": 0.0,
360
+ "selected_success_gain_over_base_at_8": 0.0,
361
+ "selected_utility_at_8": 0.19632326066493988,
362
+ "selected_utility_gain_over_base_at_8": -0.0021198540925979614,
363
  "selector_regret_at_8": 0.03616611659526825,
364
+ "success_selector_gap_at_8": 0.0,
365
+ "success_support_gap_at_8": 1.0,
366
+ "success_total_car_to_hidden_at_8": 1.0,
367
+ "support_gap_at_8": 1.7675106227397919,
368
+ "total_car_to_hidden_at_8": 1.8036767393350601
369
  },
370
  "1528740417": {
371
+ "base_success": 1.0,
372
+ "base_utility": 2.0,
373
  "branch_car_at_8": 0.0,
374
+ "hidden_chart_oracle_success_at_8": 1.0,
375
+ "hidden_chart_oracle_utility_at_8": 2.0,
376
  "outcome_ptr_at_8": 0.0,
377
  "pairwise_causal_calibration_ece": 0.2757671830981365,
378
+ "proposal_oracle_success_at_8": 1.0,
379
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
380
+ "proposal_oracle_utility_at_8": 2.0,
381
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
382
+ "selected_success_at_8": 1.0,
383
+ "selected_success_gain_over_base_at_8": 0.0,
384
+ "selected_utility_at_8": 2.0,
385
+ "selected_utility_gain_over_base_at_8": 0.0,
386
  "selector_regret_at_8": 0.0,
387
+ "success_selector_gap_at_8": 0.0,
388
+ "success_support_gap_at_8": 0.0,
389
+ "success_total_car_to_hidden_at_8": 0.0,
390
+ "support_gap_at_8": 0.0,
391
+ "total_car_to_hidden_at_8": 0.0
392
  },
393
  "1549324805": {
394
+ "base_success": 0.6666666666666666,
395
+ "base_utility": 1.3787474483251572,
396
  "branch_car_at_8": 0.0015795694974561532,
397
+ "hidden_chart_oracle_success_at_8": 0.6666666666666666,
398
+ "hidden_chart_oracle_utility_at_8": 1.4027505218982697,
399
  "outcome_ptr_at_8": 0.0,
400
  "pairwise_causal_calibration_ece": 0.28625156462411294,
401
+ "proposal_oracle_success_at_8": 0.3333333333333333,
402
+ "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333,
403
+ "proposal_oracle_utility_at_8": 0.6785067518552145,
404
+ "proposal_oracle_utility_gain_over_base_at_8": -0.7002406964699427,
405
+ "selected_success_at_8": 0.3333333333333333,
406
+ "selected_success_gain_over_base_at_8": -0.3333333333333333,
407
+ "selected_utility_at_8": 0.6769271823577583,
408
+ "selected_utility_gain_over_base_at_8": -0.7018202659673989,
409
  "selector_regret_at_8": 0.0015795694974561532,
410
+ "success_selector_gap_at_8": 0.0,
411
+ "success_support_gap_at_8": 0.3333333333333333,
412
+ "success_total_car_to_hidden_at_8": 0.3333333333333333,
413
+ "support_gap_at_8": 0.7242437700430552,
414
+ "total_car_to_hidden_at_8": 0.7258233395405114
415
  },
416
  "1607773860": {
417
+ "base_success": 1.0,
418
+ "base_utility": 2.0,
419
  "branch_car_at_8": 0.6036040186882019,
420
+ "hidden_chart_oracle_success_at_8": 1.0,
421
+ "hidden_chart_oracle_utility_at_8": 2.0,
422
  "outcome_ptr_at_8": 0.0,
423
  "pairwise_causal_calibration_ece": 0.18975917078230178,
424
+ "proposal_oracle_success_at_8": 0.0,
425
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
426
+ "proposal_oracle_utility_at_8": 0.8267073631286621,
427
+ "proposal_oracle_utility_gain_over_base_at_8": -1.173292636871338,
428
+ "selected_success_at_8": 0.0,
429
+ "selected_success_gain_over_base_at_8": -1.0,
430
+ "selected_utility_at_8": 0.2231033444404602,
431
+ "selected_utility_gain_over_base_at_8": -1.7768966555595398,
432
  "selector_regret_at_8": 0.6036040186882019,
433
+ "success_selector_gap_at_8": 0.0,
434
+ "success_support_gap_at_8": 1.0,
435
+ "success_total_car_to_hidden_at_8": 1.0,
436
+ "support_gap_at_8": 1.173292636871338,
437
+ "total_car_to_hidden_at_8": 1.7768966555595398
438
  },
439
  "1665739511": {
440
+ "base_success": 1.0,
441
+ "base_utility": 2.0,
442
  "branch_car_at_8": 0.0,
443
+ "hidden_chart_oracle_success_at_8": 1.0,
444
+ "hidden_chart_oracle_utility_at_8": 2.0,
445
  "outcome_ptr_at_8": 0.0,
446
  "pairwise_causal_calibration_ece": 0.0,
447
+ "proposal_oracle_success_at_8": 1.0,
448
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
449
+ "proposal_oracle_utility_at_8": 2.0,
450
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
451
+ "selected_success_at_8": 1.0,
452
+ "selected_success_gain_over_base_at_8": 0.0,
453
+ "selected_utility_at_8": 2.0,
454
+ "selected_utility_gain_over_base_at_8": 0.0,
455
  "selector_regret_at_8": 0.0,
456
+ "success_selector_gap_at_8": 0.0,
457
+ "success_support_gap_at_8": 0.0,
458
+ "success_total_car_to_hidden_at_8": 0.0,
459
+ "support_gap_at_8": 0.0,
460
+ "total_car_to_hidden_at_8": 0.0
461
  },
462
  "1747178030": {
463
+ "base_success": 0.0,
464
+ "base_utility": 0.1463065892457962,
465
  "branch_car_at_8": 0.027157030999660492,
466
+ "hidden_chart_oracle_success_at_8": 1.0,
467
+ "hidden_chart_oracle_utility_at_8": 2.0,
468
  "outcome_ptr_at_8": 0.0,
469
  "pairwise_causal_calibration_ece": 0.3089551740932353,
470
+ "proposal_oracle_success_at_8": 0.0,
471
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
472
+ "proposal_oracle_utility_at_8": 0.11134227365255356,
473
+ "proposal_oracle_utility_gain_over_base_at_8": -0.034964315593242645,
474
+ "selected_success_at_8": 0.0,
475
+ "selected_success_gain_over_base_at_8": 0.0,
476
+ "selected_utility_at_8": 0.08418524265289307,
477
+ "selected_utility_gain_over_base_at_8": -0.06212134659290314,
478
  "selector_regret_at_8": 0.027157030999660492,
479
+ "success_selector_gap_at_8": 0.0,
480
+ "success_support_gap_at_8": 1.0,
481
+ "success_total_car_to_hidden_at_8": 1.0,
482
+ "support_gap_at_8": 1.8886577263474464,
483
+ "total_car_to_hidden_at_8": 1.915814757347107
484
  },
485
  "1881223807": {
486
+ "base_success": 0.0,
487
+ "base_utility": 0.0320269912481308,
488
  "branch_car_at_8": 0.009028241038322449,
489
+ "hidden_chart_oracle_success_at_8": 1.0,
490
+ "hidden_chart_oracle_utility_at_8": 2.0,
491
  "outcome_ptr_at_8": 1.0,
492
  "pairwise_causal_calibration_ece": 0.13674585276395504,
493
+ "proposal_oracle_success_at_8": 0.0,
494
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
495
+ "proposal_oracle_utility_at_8": 0.034886181354522705,
496
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0028591901063919067,
497
+ "selected_success_at_8": 0.0,
498
+ "selected_success_gain_over_base_at_8": 0.0,
499
+ "selected_utility_at_8": 0.025857940316200256,
500
+ "selected_utility_gain_over_base_at_8": -0.006169050931930542,
501
  "selector_regret_at_8": 0.009028241038322449,
502
+ "success_selector_gap_at_8": 0.0,
503
+ "success_support_gap_at_8": 1.0,
504
+ "success_total_car_to_hidden_at_8": 1.0,
505
+ "support_gap_at_8": 1.9651138186454773,
506
+ "total_car_to_hidden_at_8": 1.9741420596837997
507
  },
508
  "1919198188": {
509
+ "base_success": 0.0,
510
+ "base_utility": 0.03719887137413025,
511
  "branch_car_at_8": 0.0,
512
+ "hidden_chart_oracle_success_at_8": 1.0,
513
+ "hidden_chart_oracle_utility_at_8": 2.0,
514
  "outcome_ptr_at_8": 1.0,
515
  "pairwise_causal_calibration_ece": 0.13114443223224567,
516
+ "proposal_oracle_success_at_8": 0.0,
517
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
518
+ "proposal_oracle_utility_at_8": 0.046244338154792786,
519
+ "proposal_oracle_utility_gain_over_base_at_8": 0.009045466780662537,
520
+ "selected_success_at_8": 0.0,
521
+ "selected_success_gain_over_base_at_8": 0.0,
522
+ "selected_utility_at_8": 0.046244338154792786,
523
+ "selected_utility_gain_over_base_at_8": 0.009045466780662537,
524
  "selector_regret_at_8": 0.0,
525
+ "success_selector_gap_at_8": 0.0,
526
+ "success_support_gap_at_8": 1.0,
527
+ "success_total_car_to_hidden_at_8": 1.0,
528
+ "support_gap_at_8": 1.9537556618452072,
529
+ "total_car_to_hidden_at_8": 1.9537556618452072
530
  },
531
  "1991414747": {
532
+ "base_success": 1.0,
533
+ "base_utility": 2.0,
534
  "branch_car_at_8": 1.9863876849412918,
535
+ "hidden_chart_oracle_success_at_8": 1.0,
536
+ "hidden_chart_oracle_utility_at_8": 2.0,
537
  "outcome_ptr_at_8": 0.0,
538
  "pairwise_causal_calibration_ece": 0.1306768270010748,
539
+ "proposal_oracle_success_at_8": 1.0,
540
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
541
+ "proposal_oracle_utility_at_8": 2.0,
542
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
543
+ "selected_success_at_8": 0.0,
544
+ "selected_success_gain_over_base_at_8": -1.0,
545
+ "selected_utility_at_8": 0.013612315058708191,
546
+ "selected_utility_gain_over_base_at_8": -1.9863876849412918,
547
  "selector_regret_at_8": 1.9863876849412918,
548
+ "success_selector_gap_at_8": 1.0,
549
+ "success_support_gap_at_8": 0.0,
550
+ "success_total_car_to_hidden_at_8": 1.0,
551
+ "support_gap_at_8": 0.0,
552
+ "total_car_to_hidden_at_8": 1.9863876849412918
553
  },
554
  "1997090716": {
555
+ "base_success": 0.0,
556
+ "base_utility": 0.13927479088306427,
557
  "branch_car_at_8": 0.0,
558
+ "hidden_chart_oracle_success_at_8": 1.0,
559
+ "hidden_chart_oracle_utility_at_8": 2.0,
560
  "outcome_ptr_at_8": 1.0,
561
  "pairwise_causal_calibration_ece": 0.2554442550173647,
562
+ "proposal_oracle_success_at_8": 1.0,
563
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
564
+ "proposal_oracle_utility_at_8": 2.0,
565
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357,
566
+ "selected_success_at_8": 1.0,
567
+ "selected_success_gain_over_base_at_8": 1.0,
568
+ "selected_utility_at_8": 2.0,
569
+ "selected_utility_gain_over_base_at_8": 1.8607252091169357,
570
  "selector_regret_at_8": 0.0,
571
+ "success_selector_gap_at_8": 0.0,
572
+ "success_support_gap_at_8": 0.0,
573
+ "success_total_car_to_hidden_at_8": 0.0,
574
+ "support_gap_at_8": 0.0,
575
+ "total_car_to_hidden_at_8": 0.0
576
  },
577
  "2015956957": {
578
+ "base_success": 0.0,
579
+ "base_utility": 0.06711978986859321,
580
  "branch_car_at_8": 0.022195855714380743,
581
+ "hidden_chart_oracle_success_at_8": 0.2,
582
+ "hidden_chart_oracle_utility_at_8": 0.5298813670873642,
583
  "outcome_ptr_at_8": 0.6,
584
  "pairwise_causal_calibration_ece": 0.16317219295797403,
585
+ "proposal_oracle_success_at_8": 0.0,
586
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
587
+ "proposal_oracle_utility_at_8": 0.05629209652543068,
588
+ "proposal_oracle_utility_gain_over_base_at_8": -0.010827693343162536,
589
+ "selected_success_at_8": 0.0,
590
+ "selected_success_gain_over_base_at_8": 0.0,
591
+ "selected_utility_at_8": 0.034096240811049935,
592
+ "selected_utility_gain_over_base_at_8": -0.033023549057543275,
593
  "selector_regret_at_8": 0.022195855714380743,
594
+ "success_selector_gap_at_8": 0.0,
595
+ "success_support_gap_at_8": 0.2,
596
+ "success_total_car_to_hidden_at_8": 0.2,
597
+ "support_gap_at_8": 0.4735892705619335,
598
+ "total_car_to_hidden_at_8": 0.49578512627631427
599
  },
600
  "2054899126": {
601
+ "base_success": 1.0,
602
+ "base_utility": 2.0,
603
  "branch_car_at_8": 0.0,
604
+ "hidden_chart_oracle_success_at_8": 1.0,
605
+ "hidden_chart_oracle_utility_at_8": 2.0,
606
  "outcome_ptr_at_8": 0.0,
607
  "pairwise_causal_calibration_ece": 0.0,
608
+ "proposal_oracle_success_at_8": 1.0,
609
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
610
+ "proposal_oracle_utility_at_8": 2.0,
611
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
612
+ "selected_success_at_8": 1.0,
613
+ "selected_success_gain_over_base_at_8": 0.0,
614
+ "selected_utility_at_8": 2.0,
615
+ "selected_utility_gain_over_base_at_8": 0.0,
616
  "selector_regret_at_8": 0.0,
617
+ "success_selector_gap_at_8": 0.0,
618
+ "success_support_gap_at_8": 0.0,
619
+ "success_total_car_to_hidden_at_8": 0.0,
620
+ "support_gap_at_8": 0.0,
621
+ "total_car_to_hidden_at_8": 0.0
622
  },
623
  "211245842": {
624
+ "base_success": 0.0,
625
+ "base_utility": 0.8388344049453735,
626
  "branch_car_at_8": 0.6792904287576675,
627
+ "hidden_chart_oracle_success_at_8": 1.0,
628
+ "hidden_chart_oracle_utility_at_8": 2.0,
629
  "outcome_ptr_at_8": 0.0,
630
  "pairwise_causal_calibration_ece": 0.07548939908426067,
631
+ "proposal_oracle_success_at_8": 0.0,
632
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
633
+ "proposal_oracle_utility_at_8": 0.8358442187309265,
634
+ "proposal_oracle_utility_gain_over_base_at_8": -0.0029901862144470215,
635
+ "selected_success_at_8": 0.0,
636
+ "selected_success_gain_over_base_at_8": 0.0,
637
+ "selected_utility_at_8": 0.15655378997325897,
638
+ "selected_utility_gain_over_base_at_8": -0.6822806149721146,
639
  "selector_regret_at_8": 0.6792904287576675,
640
+ "success_selector_gap_at_8": 0.0,
641
+ "success_support_gap_at_8": 1.0,
642
+ "success_total_car_to_hidden_at_8": 1.0,
643
+ "support_gap_at_8": 1.1641557812690735,
644
+ "total_car_to_hidden_at_8": 1.843446210026741
645
  },
646
  "2141802816": {
647
+ "base_success": 0.0,
648
+ "base_utility": 0.07080724835395813,
649
  "branch_car_at_8": 0.019766509532928467,
650
+ "hidden_chart_oracle_success_at_8": 1.0,
651
+ "hidden_chart_oracle_utility_at_8": 2.0,
652
  "outcome_ptr_at_8": 1.0,
653
  "pairwise_causal_calibration_ece": 0.07786951284417709,
654
+ "proposal_oracle_success_at_8": 0.0,
655
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
656
+ "proposal_oracle_utility_at_8": 0.09589323401451111,
657
+ "proposal_oracle_utility_gain_over_base_at_8": 0.02508598566055298,
658
+ "selected_success_at_8": 0.0,
659
+ "selected_success_gain_over_base_at_8": 0.0,
660
+ "selected_utility_at_8": 0.07612672448158264,
661
+ "selected_utility_gain_over_base_at_8": 0.005319476127624512,
662
  "selector_regret_at_8": 0.019766509532928467,
663
+ "success_selector_gap_at_8": 0.0,
664
+ "success_support_gap_at_8": 1.0,
665
+ "success_total_car_to_hidden_at_8": 1.0,
666
+ "support_gap_at_8": 1.904106765985489,
667
+ "total_car_to_hidden_at_8": 1.9238732755184174
668
  },
669
  "256770874": {
670
+ "base_success": 0.0,
671
+ "base_utility": 0.8557480573654175,
672
  "branch_car_at_8": 0.7044661641120911,
673
+ "hidden_chart_oracle_success_at_8": 1.0,
674
+ "hidden_chart_oracle_utility_at_8": 2.0,
675
  "outcome_ptr_at_8": 0.0,
676
  "pairwise_causal_calibration_ece": 0.03135306008748828,
677
+ "proposal_oracle_success_at_8": 0.0,
678
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
679
+ "proposal_oracle_utility_at_8": 0.8125,
680
+ "proposal_oracle_utility_gain_over_base_at_8": -0.04324805736541748,
681
+ "selected_success_at_8": 0.0,
682
+ "selected_success_gain_over_base_at_8": 0.0,
683
+ "selected_utility_at_8": 0.10803383588790894,
684
+ "selected_utility_gain_over_base_at_8": -0.7477142214775085,
685
  "selector_regret_at_8": 0.7044661641120911,
686
+ "success_selector_gap_at_8": 0.0,
687
+ "success_support_gap_at_8": 1.0,
688
+ "success_total_car_to_hidden_at_8": 1.0,
689
+ "support_gap_at_8": 1.1875,
690
+ "total_car_to_hidden_at_8": 1.891966164112091
691
  },
692
  "278008524": {
693
+ "base_success": 0.0,
694
+ "base_utility": 0.10747186839580536,
695
  "branch_car_at_8": 0.011409573198761791,
696
+ "hidden_chart_oracle_success_at_8": 0.5,
697
+ "hidden_chart_oracle_utility_at_8": 1.113811194896698,
698
  "outcome_ptr_at_8": 0.5,
699
  "pairwise_causal_calibration_ece": 0.24605586147071373,
700
+ "proposal_oracle_success_at_8": 0.0,
701
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
702
+ "proposal_oracle_utility_at_8": 0.03147180867381394,
703
+ "proposal_oracle_utility_gain_over_base_at_8": -0.07600005972199142,
704
+ "selected_success_at_8": 0.0,
705
+ "selected_success_gain_over_base_at_8": 0.0,
706
+ "selected_utility_at_8": 0.020062235475052148,
707
+ "selected_utility_gain_over_base_at_8": -0.08740963292075321,
708
  "selector_regret_at_8": 0.011409573198761791,
709
+ "success_selector_gap_at_8": 0.0,
710
+ "success_support_gap_at_8": 0.5,
711
+ "success_total_car_to_hidden_at_8": 0.5,
712
+ "support_gap_at_8": 1.082339386222884,
713
+ "total_car_to_hidden_at_8": 1.0937489594216458
714
  },
715
  "416183106": {
716
+ "base_success": 0.0,
717
+ "base_utility": 0.20032641291618347,
718
  "branch_car_at_8": 1.7668711841106415,
719
+ "hidden_chart_oracle_success_at_8": 1.0,
720
+ "hidden_chart_oracle_utility_at_8": 2.0,
721
  "outcome_ptr_at_8": 1.0,
722
  "pairwise_causal_calibration_ece": 0.07978206783428046,
723
+ "proposal_oracle_success_at_8": 1.0,
724
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
725
+ "proposal_oracle_utility_at_8": 2.0,
726
+ "proposal_oracle_utility_gain_over_base_at_8": 1.7996735870838165,
727
+ "selected_success_at_8": 0.0,
728
+ "selected_success_gain_over_base_at_8": 0.0,
729
+ "selected_utility_at_8": 0.23312881588935852,
730
+ "selected_utility_gain_over_base_at_8": 0.03280240297317505,
731
  "selector_regret_at_8": 1.7668711841106415,
732
+ "success_selector_gap_at_8": 1.0,
733
+ "success_support_gap_at_8": 0.0,
734
+ "success_total_car_to_hidden_at_8": 1.0,
735
+ "support_gap_at_8": 0.0,
736
+ "total_car_to_hidden_at_8": 1.7668711841106415
737
  },
738
  "534717141": {
739
+ "base_success": 1.0,
740
+ "base_utility": 2.0,
741
  "branch_car_at_8": 1.9558932781219482,
742
+ "hidden_chart_oracle_success_at_8": 1.0,
743
+ "hidden_chart_oracle_utility_at_8": 2.0,
744
  "outcome_ptr_at_8": 0.0,
745
  "pairwise_causal_calibration_ece": 0.14369576234023818,
746
+ "proposal_oracle_success_at_8": 1.0,
747
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
748
+ "proposal_oracle_utility_at_8": 2.0,
749
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
750
+ "selected_success_at_8": 0.0,
751
+ "selected_success_gain_over_base_at_8": -1.0,
752
+ "selected_utility_at_8": 0.04410672187805176,
753
+ "selected_utility_gain_over_base_at_8": -1.9558932781219482,
754
  "selector_regret_at_8": 1.9558932781219482,
755
+ "success_selector_gap_at_8": 1.0,
756
+ "success_support_gap_at_8": 0.0,
757
+ "success_total_car_to_hidden_at_8": 1.0,
758
+ "support_gap_at_8": 0.0,
759
+ "total_car_to_hidden_at_8": 1.9558932781219482
760
  },
761
  "551504635": {
762
+ "base_success": 0.0,
763
+ "base_utility": 0.7247228622436523,
764
  "branch_car_at_8": 0.0,
765
+ "hidden_chart_oracle_success_at_8": 1.0,
766
+ "hidden_chart_oracle_utility_at_8": 2.0,
767
  "outcome_ptr_at_8": 0.0,
768
  "pairwise_causal_calibration_ece": 0.2299325778712667,
769
+ "proposal_oracle_success_at_8": 0.0,
770
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
771
+ "proposal_oracle_utility_at_8": 0.7224790453910828,
772
+ "proposal_oracle_utility_gain_over_base_at_8": -0.00224381685256958,
773
+ "selected_success_at_8": 0.0,
774
+ "selected_success_gain_over_base_at_8": 0.0,
775
+ "selected_utility_at_8": 0.7224790453910828,
776
+ "selected_utility_gain_over_base_at_8": -0.00224381685256958,
777
  "selector_regret_at_8": 0.0,
778
+ "success_selector_gap_at_8": 0.0,
779
+ "success_support_gap_at_8": 1.0,
780
+ "success_total_car_to_hidden_at_8": 1.0,
781
+ "support_gap_at_8": 1.2775209546089172,
782
+ "total_car_to_hidden_at_8": 1.2775209546089172
783
  },
784
  "565369392": {
785
+ "base_success": 0.3333333333333333,
786
+ "base_utility": 0.7997656464576721,
787
  "branch_car_at_8": 0.046260833740234375,
788
+ "hidden_chart_oracle_success_at_8": 0.6666666666666666,
789
+ "hidden_chart_oracle_utility_at_8": 1.4266315797964733,
790
  "outcome_ptr_at_8": 0.0,
791
  "pairwise_causal_calibration_ece": 0.13991651305345584,
792
+ "proposal_oracle_success_at_8": 0.0,
793
+ "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333,
794
+ "proposal_oracle_utility_at_8": 0.17366480827331543,
795
+ "proposal_oracle_utility_gain_over_base_at_8": -0.6261008381843567,
796
+ "selected_success_at_8": 0.0,
797
+ "selected_success_gain_over_base_at_8": -0.3333333333333333,
798
+ "selected_utility_at_8": 0.12740397453308105,
799
+ "selected_utility_gain_over_base_at_8": -0.6723616719245911,
800
  "selector_regret_at_8": 0.046260833740234375,
801
+ "success_selector_gap_at_8": 0.0,
802
+ "success_support_gap_at_8": 0.6666666666666666,
803
+ "success_total_car_to_hidden_at_8": 0.6666666666666666,
804
+ "support_gap_at_8": 1.2529667715231578,
805
+ "total_car_to_hidden_at_8": 1.2992276052633922
806
  },
807
  "565621826": {
808
+ "base_success": 1.0,
809
+ "base_utility": 2.0,
810
  "branch_car_at_8": 0.0,
811
+ "hidden_chart_oracle_success_at_8": 1.0,
812
+ "hidden_chart_oracle_utility_at_8": 2.0,
813
  "outcome_ptr_at_8": 0.0,
814
  "pairwise_causal_calibration_ece": 0.0,
815
+ "proposal_oracle_success_at_8": 1.0,
816
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
817
+ "proposal_oracle_utility_at_8": 2.0,
818
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
819
+ "selected_success_at_8": 1.0,
820
+ "selected_success_gain_over_base_at_8": 0.0,
821
+ "selected_utility_at_8": 2.0,
822
+ "selected_utility_gain_over_base_at_8": 0.0,
823
  "selector_regret_at_8": 0.0,
824
+ "success_selector_gap_at_8": 0.0,
825
+ "success_support_gap_at_8": 0.0,
826
+ "success_total_car_to_hidden_at_8": 0.0,
827
+ "support_gap_at_8": 0.0,
828
+ "total_car_to_hidden_at_8": 0.0
829
  },
830
  "585904130": {
831
+ "base_success": 0.0,
832
+ "base_utility": 0.07529209802548091,
833
  "branch_car_at_8": 0.012068324722349644,
834
+ "hidden_chart_oracle_success_at_8": 0.0,
835
+ "hidden_chart_oracle_utility_at_8": 0.23764699200789133,
836
  "outcome_ptr_at_8": 0.0,
837
  "pairwise_causal_calibration_ece": 0.07276320034061332,
838
+ "proposal_oracle_success_at_8": 0.0,
839
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
840
+ "proposal_oracle_utility_at_8": 0.032918572425842285,
841
+ "proposal_oracle_utility_gain_over_base_at_8": -0.04237352559963862,
842
+ "selected_success_at_8": 0.0,
843
+ "selected_success_gain_over_base_at_8": 0.0,
844
+ "selected_utility_at_8": 0.02085024770349264,
845
+ "selected_utility_gain_over_base_at_8": -0.05444185032198826,
846
  "selector_regret_at_8": 0.012068324722349644,
847
+ "success_selector_gap_at_8": 0.0,
848
+ "success_support_gap_at_8": 0.0,
849
+ "success_total_car_to_hidden_at_8": 0.0,
850
+ "support_gap_at_8": 0.20472841958204904,
851
+ "total_car_to_hidden_at_8": 0.2167967443043987
852
  },
853
  "626486813": {
854
+ "base_success": 0.25,
855
+ "base_utility": 0.5054400166263804,
856
  "branch_car_at_8": 0.02500777377281338,
857
+ "hidden_chart_oracle_success_at_8": 0.25,
858
+ "hidden_chart_oracle_utility_at_8": 0.6124684102833271,
859
  "outcome_ptr_at_8": 0.5,
860
  "pairwise_causal_calibration_ece": 0.2420589895144555,
861
+ "proposal_oracle_success_at_8": 0.0,
862
+ "proposal_oracle_success_gain_over_base_at_8": -0.25,
863
+ "proposal_oracle_utility_at_8": 0.07365635968744755,
864
+ "proposal_oracle_utility_gain_over_base_at_8": -0.43178365693893284,
865
+ "selected_success_at_8": 0.0,
866
+ "selected_success_gain_over_base_at_8": -0.25,
867
+ "selected_utility_at_8": 0.04864858591463417,
868
+ "selected_utility_gain_over_base_at_8": -0.4567914307117462,
869
  "selector_regret_at_8": 0.02500777377281338,
870
+ "success_selector_gap_at_8": 0.0,
871
+ "success_support_gap_at_8": 0.25,
872
+ "success_total_car_to_hidden_at_8": 0.25,
873
+ "support_gap_at_8": 0.5414478182792664,
874
+ "total_car_to_hidden_at_8": 0.5638198243686929
875
  },
876
  "636680137": {
877
+ "base_success": 1.0,
878
+ "base_utility": 2.0,
879
  "branch_car_at_8": 0.0,
880
+ "hidden_chart_oracle_success_at_8": 1.0,
881
+ "hidden_chart_oracle_utility_at_8": 2.0,
882
  "outcome_ptr_at_8": 0.0,
883
  "pairwise_causal_calibration_ece": 0.07628525991805817,
884
+ "proposal_oracle_success_at_8": 0.0,
885
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
886
+ "proposal_oracle_utility_at_8": 0.013928472995758057,
887
+ "proposal_oracle_utility_gain_over_base_at_8": -1.986071527004242,
888
+ "selected_success_at_8": 0.0,
889
+ "selected_success_gain_over_base_at_8": -1.0,
890
+ "selected_utility_at_8": 0.013928472995758057,
891
+ "selected_utility_gain_over_base_at_8": -1.986071527004242,
892
  "selector_regret_at_8": 0.0,
893
+ "success_selector_gap_at_8": 0.0,
894
+ "success_support_gap_at_8": 1.0,
895
+ "success_total_car_to_hidden_at_8": 1.0,
896
+ "support_gap_at_8": 1.986071527004242,
897
+ "total_car_to_hidden_at_8": 1.986071527004242
898
  },
899
  "706479600": {
900
+ "base_success": 0.0,
901
+ "base_utility": 0.8350887894630432,
902
  "branch_car_at_8": 0.0,
903
+ "hidden_chart_oracle_success_at_8": 1.0,
904
+ "hidden_chart_oracle_utility_at_8": 2.0,
905
  "outcome_ptr_at_8": 1.0,
906
  "pairwise_causal_calibration_ece": 0.16443847051324711,
907
+ "proposal_oracle_success_at_8": 1.0,
908
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
909
+ "proposal_oracle_utility_at_8": 2.0,
910
+ "proposal_oracle_utility_gain_over_base_at_8": 1.1649112105369568,
911
+ "selected_success_at_8": 1.0,
912
+ "selected_success_gain_over_base_at_8": 1.0,
913
+ "selected_utility_at_8": 2.0,
914
+ "selected_utility_gain_over_base_at_8": 1.1649112105369568,
915
  "selector_regret_at_8": 0.0,
916
+ "success_selector_gap_at_8": 0.0,
917
+ "success_support_gap_at_8": 0.0,
918
+ "success_total_car_to_hidden_at_8": 0.0,
919
+ "support_gap_at_8": 0.0,
920
+ "total_car_to_hidden_at_8": 0.0
921
  },
922
  "750448296": {
923
+ "base_success": 0.0,
924
+ "base_utility": 0.11153355985879898,
925
  "branch_car_at_8": 0.02701035887002945,
926
+ "hidden_chart_oracle_success_at_8": 0.0,
927
+ "hidden_chart_oracle_utility_at_8": 0.23580759018659592,
928
  "outcome_ptr_at_8": 0.0,
929
  "pairwise_causal_calibration_ece": 0.2344413298276417,
930
+ "proposal_oracle_success_at_8": 0.0,
931
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
932
+ "proposal_oracle_utility_at_8": 0.07342664152383804,
933
+ "proposal_oracle_utility_gain_over_base_at_8": -0.03810691833496094,
934
+ "selected_success_at_8": 0.0,
935
+ "selected_success_gain_over_base_at_8": 0.0,
936
+ "selected_utility_at_8": 0.046416282653808594,
937
+ "selected_utility_gain_over_base_at_8": -0.06511727720499039,
938
  "selector_regret_at_8": 0.02701035887002945,
939
+ "success_selector_gap_at_8": 0.0,
940
+ "success_support_gap_at_8": 0.0,
941
+ "success_total_car_to_hidden_at_8": 0.0,
942
+ "support_gap_at_8": 0.16238094866275787,
943
+ "total_car_to_hidden_at_8": 0.18939130753278732
944
  },
945
  "75070641": {
946
+ "base_success": 0.0,
947
+ "base_utility": 0.20275163650512695,
948
  "branch_car_at_8": 0.5863077491521835,
949
+ "hidden_chart_oracle_success_at_8": 1.0,
950
+ "hidden_chart_oracle_utility_at_8": 2.0,
951
  "outcome_ptr_at_8": 1.0,
952
  "pairwise_causal_calibration_ece": 0.18559462278331035,
953
+ "proposal_oracle_success_at_8": 0.0,
954
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
955
+ "proposal_oracle_utility_at_8": 0.8128874897956848,
956
+ "proposal_oracle_utility_gain_over_base_at_8": 0.6101358532905579,
957
+ "selected_success_at_8": 0.0,
958
+ "selected_success_gain_over_base_at_8": 0.0,
959
+ "selected_utility_at_8": 0.22657974064350128,
960
+ "selected_utility_gain_over_base_at_8": 0.02382810413837433,
961
  "selector_regret_at_8": 0.5863077491521835,
962
+ "success_selector_gap_at_8": 0.0,
963
+ "success_support_gap_at_8": 1.0,
964
+ "success_total_car_to_hidden_at_8": 1.0,
965
+ "support_gap_at_8": 1.1871125102043152,
966
+ "total_car_to_hidden_at_8": 1.7734202593564987
967
  },
968
  "767747424": {
969
+ "base_success": 0.25,
970
+ "base_utility": 0.5475241877138615,
971
  "branch_car_at_8": 0.04561196081340313,
972
+ "hidden_chart_oracle_success_at_8": 0.0,
973
+ "hidden_chart_oracle_utility_at_8": 0.18863565661013126,
974
  "outcome_ptr_at_8": 0.75,
975
  "pairwise_causal_calibration_ece": 0.07531396139335009,
976
+ "proposal_oracle_success_at_8": 0.0,
977
+ "proposal_oracle_success_gain_over_base_at_8": -0.25,
978
+ "proposal_oracle_utility_at_8": 0.11342880013398826,
979
+ "proposal_oracle_utility_gain_over_base_at_8": -0.4340953875798732,
980
+ "selected_success_at_8": 0.0,
981
+ "selected_success_gain_over_base_at_8": -0.25,
982
+ "selected_utility_at_8": 0.06781683932058513,
983
+ "selected_utility_gain_over_base_at_8": -0.47970734839327633,
984
  "selector_regret_at_8": 0.04561196081340313,
985
+ "success_selector_gap_at_8": 0.0,
986
+ "success_support_gap_at_8": 0.0,
987
+ "success_total_car_to_hidden_at_8": 0.0,
988
+ "support_gap_at_8": 0.08526064571924508,
989
+ "total_car_to_hidden_at_8": 0.12081881728954613
990
  },
991
  "77147264": {
992
+ "base_success": 0.5,
993
+ "base_utility": 1.4215794801712036,
994
  "branch_car_at_8": 0.0,
995
+ "hidden_chart_oracle_success_at_8": 1.0,
996
+ "hidden_chart_oracle_utility_at_8": 2.0,
997
  "outcome_ptr_at_8": 0.5,
998
  "pairwise_causal_calibration_ece": 0.2630803856693186,
999
+ "proposal_oracle_success_at_8": 0.5,
1000
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
1001
+ "proposal_oracle_utility_at_8": 1.426636129617691,
1002
+ "proposal_oracle_utility_gain_over_base_at_8": 0.005056649446487427,
1003
+ "selected_success_at_8": 0.5,
1004
+ "selected_success_gain_over_base_at_8": 0.0,
1005
+ "selected_utility_at_8": 1.426636129617691,
1006
+ "selected_utility_gain_over_base_at_8": 0.005056649446487427,
1007
  "selector_regret_at_8": 0.0,
1008
+ "success_selector_gap_at_8": 0.0,
1009
+ "success_support_gap_at_8": 0.5,
1010
+ "success_total_car_to_hidden_at_8": 0.5,
1011
+ "support_gap_at_8": 0.573363870382309,
1012
+ "total_car_to_hidden_at_8": 0.573363870382309
1013
  },
1014
  "794181626": {
1015
+ "base_success": 0.0,
1016
+ "base_utility": 0.18095605075359344,
1017
  "branch_car_at_8": 0.0,
1018
+ "hidden_chart_oracle_success_at_8": 1.0,
1019
+ "hidden_chart_oracle_utility_at_8": 2.0,
1020
  "outcome_ptr_at_8": 1.0,
1021
  "pairwise_causal_calibration_ece": 0.28339362113630795,
1022
+ "proposal_oracle_success_at_8": 1.0,
1023
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
1024
+ "proposal_oracle_utility_at_8": 2.0,
1025
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8190439492464066,
1026
+ "selected_success_at_8": 1.0,
1027
+ "selected_success_gain_over_base_at_8": 1.0,
1028
+ "selected_utility_at_8": 2.0,
1029
+ "selected_utility_gain_over_base_at_8": 1.8190439492464066,
1030
  "selector_regret_at_8": 0.0,
1031
+ "success_selector_gap_at_8": 0.0,
1032
+ "success_support_gap_at_8": 0.0,
1033
+ "success_total_car_to_hidden_at_8": 0.0,
1034
+ "support_gap_at_8": 0.0,
1035
+ "total_car_to_hidden_at_8": 0.0
1036
  },
1037
  "957101489": {
1038
+ "base_success": 0.0,
1039
+ "base_utility": 0.20372186601161957,
1040
  "branch_car_at_8": 1.7757539749145508,
1041
+ "hidden_chart_oracle_success_at_8": 1.0,
1042
+ "hidden_chart_oracle_utility_at_8": 2.0,
1043
  "outcome_ptr_at_8": 1.0,
1044
  "pairwise_causal_calibration_ece": 0.07643021229779011,
1045
+ "proposal_oracle_success_at_8": 1.0,
1046
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
1047
+ "proposal_oracle_utility_at_8": 2.0,
1048
+ "proposal_oracle_utility_gain_over_base_at_8": 1.7962781339883804,
1049
+ "selected_success_at_8": 0.0,
1050
+ "selected_success_gain_over_base_at_8": 0.0,
1051
+ "selected_utility_at_8": 0.22424602508544922,
1052
+ "selected_utility_gain_over_base_at_8": 0.02052415907382965,
1053
  "selector_regret_at_8": 1.7757539749145508,
1054
+ "success_selector_gap_at_8": 1.0,
1055
+ "success_support_gap_at_8": 0.0,
1056
+ "success_total_car_to_hidden_at_8": 1.0,
1057
+ "support_gap_at_8": 0.0,
1058
+ "total_car_to_hidden_at_8": 1.7757539749145508
1059
  },
1060
  "998761162": {
1061
+ "base_success": 0.0,
1062
+ "base_utility": 0.06781657040119171,
1063
  "branch_car_at_8": 1.9512543678283691,
1064
+ "hidden_chart_oracle_success_at_8": 1.0,
1065
+ "hidden_chart_oracle_utility_at_8": 2.0,
1066
  "outcome_ptr_at_8": 1.0,
1067
  "pairwise_causal_calibration_ece": 0.28991947553571784,
1068
+ "proposal_oracle_success_at_8": 1.0,
1069
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
1070
+ "proposal_oracle_utility_at_8": 2.0,
1071
+ "proposal_oracle_utility_gain_over_base_at_8": 1.9321834295988083,
1072
+ "selected_success_at_8": 0.0,
1073
+ "selected_success_gain_over_base_at_8": 0.0,
1074
+ "selected_utility_at_8": 0.04874563217163086,
1075
+ "selected_utility_gain_over_base_at_8": -0.019070938229560852,
1076
  "selector_regret_at_8": 1.9512543678283691,
1077
+ "success_selector_gap_at_8": 1.0,
1078
+ "success_support_gap_at_8": 0.0,
1079
+ "success_total_car_to_hidden_at_8": 1.0,
1080
+ "support_gap_at_8": 0.0,
1081
+ "total_car_to_hidden_at_8": 1.9512543678283691
1082
  }
1083
  }
workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics_by_task.json CHANGED
@@ -1,37 +1,117 @@
1
  {
2
  "LiftPegUpright-v1": {
 
 
3
  "branch_car_at_8": 0.3250635440150897,
 
 
4
  "outcome_ptr_at_8": 0.5,
5
  "pairwise_causal_calibration_ece": 0.21174314869082475,
 
 
 
 
 
 
 
 
6
  "selector_regret_at_8": 0.3250635440150897,
7
- "support_gap_at_8": 0.6529051661491394
 
 
 
 
8
  },
9
  "PickCube-v1": {
 
 
10
  "branch_car_at_8": 0.45726464316248894,
 
 
11
  "outcome_ptr_at_8": 1.0,
12
  "pairwise_causal_calibration_ece": 0.096994407338536,
 
 
 
 
 
 
 
 
13
  "selector_regret_at_8": 0.45726464316248894,
14
- "support_gap_at_8": 1.500250667333603
 
 
 
 
15
  },
16
  "PullCube-v1": {
 
 
17
  "branch_car_at_8": 0.08196123232654132,
 
 
18
  "outcome_ptr_at_8": 0.29411764705882354,
19
  "pairwise_causal_calibration_ece": 0.14892514417161398,
 
 
 
 
 
 
 
 
20
  "selector_regret_at_8": 0.08196123232654132,
21
- "support_gap_at_8": 0.4828804947901517
 
 
 
 
22
  },
23
  "PushCube-v1": {
 
 
24
  "branch_car_at_8": 0.6069896691120588,
 
 
25
  "outcome_ptr_at_8": 0.6153846153846154,
26
  "pairwise_causal_calibration_ece": 0.19653272589828236,
 
 
 
 
 
 
 
 
27
  "selector_regret_at_8": 0.6069896691120588,
28
- "support_gap_at_8": 0.44705674969233
 
 
 
 
29
  },
30
  "StackCube-v1": {
 
 
31
  "branch_car_at_8": 0.6179056455691655,
 
 
32
  "outcome_ptr_at_8": 0.5333333333333333,
33
  "pairwise_causal_calibration_ece": 0.12910083675639442,
 
 
 
 
 
 
 
 
34
  "selector_regret_at_8": 0.6179056455691655,
35
- "support_gap_at_8": 0.9138133267561594
 
 
 
 
36
  }
37
  }
 
1
  {
2
  "LiftPegUpright-v1": {
3
+ "base_success": 0.3333333333333333,
4
+ "base_utility": 1.0670691976944606,
5
  "branch_car_at_8": 0.3250635440150897,
6
+ "hidden_chart_oracle_success_at_8": 1.0,
7
+ "hidden_chart_oracle_utility_at_8": 2.0,
8
  "outcome_ptr_at_8": 0.5,
9
  "pairwise_causal_calibration_ece": 0.21174314869082475,
10
+ "proposal_oracle_success_at_8": 0.5,
11
+ "proposal_oracle_success_gain_over_base_at_8": 0.16666666666666666,
12
+ "proposal_oracle_utility_at_8": 1.3470948338508606,
13
+ "proposal_oracle_utility_gain_over_base_at_8": 0.2800256361564,
14
+ "selected_success_at_8": 0.3333333333333333,
15
+ "selected_success_gain_over_base_at_8": 0.0,
16
+ "selected_utility_at_8": 1.0220312898357709,
17
+ "selected_utility_gain_over_base_at_8": -0.04503790785868963,
18
  "selector_regret_at_8": 0.3250635440150897,
19
+ "success_selector_gap_at_8": 0.16666666666666666,
20
+ "success_support_gap_at_8": 0.5,
21
+ "success_total_car_to_hidden_at_8": 0.6666666666666666,
22
+ "support_gap_at_8": 0.6529051661491394,
23
+ "total_car_to_hidden_at_8": 0.977968710164229
24
  },
25
  "PickCube-v1": {
26
+ "base_success": 0.0,
27
+ "base_utility": 0.15342965722084045,
28
  "branch_car_at_8": 0.45726464316248894,
29
+ "hidden_chart_oracle_success_at_8": 1.0,
30
+ "hidden_chart_oracle_utility_at_8": 2.0,
31
  "outcome_ptr_at_8": 1.0,
32
  "pairwise_causal_calibration_ece": 0.096994407338536,
33
+ "proposal_oracle_success_at_8": 0.0,
34
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
35
+ "proposal_oracle_utility_at_8": 0.4997493326663971,
36
+ "proposal_oracle_utility_gain_over_base_at_8": 0.34631967544555664,
37
+ "selected_success_at_8": 0.0,
38
+ "selected_success_gain_over_base_at_8": 0.0,
39
+ "selected_utility_at_8": 0.04248468950390816,
40
+ "selected_utility_gain_over_base_at_8": -0.1109449677169323,
41
  "selector_regret_at_8": 0.45726464316248894,
42
+ "success_selector_gap_at_8": 0.0,
43
+ "success_support_gap_at_8": 1.0,
44
+ "success_total_car_to_hidden_at_8": 1.0,
45
+ "support_gap_at_8": 1.500250667333603,
46
+ "total_car_to_hidden_at_8": 1.9575153104960918
47
  },
48
  "PullCube-v1": {
49
+ "base_success": 0.29411764705882354,
50
+ "base_utility": 0.6512299207833541,
51
  "branch_car_at_8": 0.08196123232654132,
52
+ "hidden_chart_oracle_success_at_8": 0.3235294117647059,
53
+ "hidden_chart_oracle_utility_at_8": 0.7823218367993832,
54
  "outcome_ptr_at_8": 0.29411764705882354,
55
  "pairwise_causal_calibration_ece": 0.14892514417161398,
56
+ "proposal_oracle_success_at_8": 0.11764705882352941,
57
+ "proposal_oracle_success_gain_over_base_at_8": -0.17647058823529413,
58
+ "proposal_oracle_utility_at_8": 0.3012250267818351,
59
+ "proposal_oracle_utility_gain_over_base_at_8": -0.35000489400151896,
60
+ "selected_success_at_8": 0.08823529411764706,
61
+ "selected_success_gain_over_base_at_8": -0.20588235294117646,
62
+ "selected_utility_at_8": 0.2192637944552938,
63
+ "selected_utility_gain_over_base_at_8": -0.4319661263280603,
64
  "selector_regret_at_8": 0.08196123232654132,
65
+ "success_selector_gap_at_8": 0.029411764705882353,
66
+ "success_support_gap_at_8": 0.20588235294117646,
67
+ "success_total_car_to_hidden_at_8": 0.23529411764705882,
68
+ "support_gap_at_8": 0.4828804947901517,
69
+ "total_car_to_hidden_at_8": 0.5630580423440894
70
  },
71
  "PushCube-v1": {
72
+ "base_success": 0.38461538461538464,
73
+ "base_utility": 0.8223185676794785,
74
  "branch_car_at_8": 0.6069896691120588,
75
+ "hidden_chart_oracle_success_at_8": 1.0,
76
+ "hidden_chart_oracle_utility_at_8": 2.0,
77
  "outcome_ptr_at_8": 0.6153846153846154,
78
  "pairwise_causal_calibration_ece": 0.19653272589828236,
79
+ "proposal_oracle_success_at_8": 0.7692307692307693,
80
+ "proposal_oracle_success_gain_over_base_at_8": 0.38461538461538464,
81
+ "proposal_oracle_utility_at_8": 1.55294325030767,
82
+ "proposal_oracle_utility_gain_over_base_at_8": 0.7306246826281915,
83
+ "selected_success_at_8": 0.46153846153846156,
84
+ "selected_success_gain_over_base_at_8": 0.07692307692307693,
85
+ "selected_utility_at_8": 0.9459535811956112,
86
+ "selected_utility_gain_over_base_at_8": 0.12363501351613265,
87
  "selector_regret_at_8": 0.6069896691120588,
88
+ "success_selector_gap_at_8": 0.3076923076923077,
89
+ "success_support_gap_at_8": 0.23076923076923078,
90
+ "success_total_car_to_hidden_at_8": 0.5384615384615384,
91
+ "support_gap_at_8": 0.44705674969233,
92
+ "total_car_to_hidden_at_8": 1.0540464188043888
93
  },
94
  "StackCube-v1": {
95
+ "base_success": 0.2,
96
+ "base_utility": 0.8036667982737223,
97
  "branch_car_at_8": 0.6179056455691655,
98
+ "hidden_chart_oracle_success_at_8": 1.0,
99
+ "hidden_chart_oracle_utility_at_8": 2.0,
100
  "outcome_ptr_at_8": 0.5333333333333333,
101
  "pairwise_causal_calibration_ece": 0.12910083675639442,
102
+ "proposal_oracle_success_at_8": 0.3333333333333333,
103
+ "proposal_oracle_success_gain_over_base_at_8": 0.13333333333333333,
104
+ "proposal_oracle_utility_at_8": 1.0861866732438406,
105
+ "proposal_oracle_utility_gain_over_base_at_8": 0.2825198749701182,
106
+ "selected_success_at_8": 0.13333333333333333,
107
+ "selected_success_gain_over_base_at_8": -0.06666666666666667,
108
+ "selected_utility_at_8": 0.468281027674675,
109
+ "selected_utility_gain_over_base_at_8": -0.33538577059904734,
110
  "selector_regret_at_8": 0.6179056455691655,
111
+ "success_selector_gap_at_8": 0.2,
112
+ "success_support_gap_at_8": 0.6666666666666666,
113
+ "success_total_car_to_hidden_at_8": 0.8666666666666667,
114
+ "support_gap_at_8": 0.9138133267561594,
115
+ "total_car_to_hidden_at_8": 1.531718972325325
116
  }
117
  }
workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/report.md CHANGED
@@ -4,8 +4,24 @@ K: `8`
4
 
5
  | Metric | N | Micro mean | 95% CI | Task macro | Seed macro |
6
  | --- | ---: | ---: | ---: | ---: | ---: |
 
 
7
  | branch_car_at_8 | 69 | 0.3240 | [0.1877, 0.4805] | 0.4178 | 0.4633 |
 
 
8
  | outcome_ptr_at_8 | 69 | 0.4348 | [0.3043, 0.5362] | 0.5886 | 0.4578 |
9
  | pairwise_causal_calibration_ece | 69 | 0.1583 | [0.1336, 0.1862] | 0.1567 | 0.1561 |
 
 
 
 
 
 
 
 
10
  | selector_regret_at_8 | 69 | 0.3240 | [0.1877, 0.4805] | 0.4178 | 0.4633 |
 
 
 
11
  | support_gap_at_8 | 69 | 0.5993 | [0.4135, 0.7763] | 0.7994 | 0.6444 |
 
 
4
 
5
  | Metric | N | Micro mean | 95% CI | Task macro | Seed macro |
6
  | --- | ---: | ---: | ---: | ---: | ---: |
7
+ | base_success | 69 | 0.2899 | [0.1884, 0.4203] | 0.2424 | 0.3298 |
8
+ | base_utility | 69 | 0.7455 | [0.5693, 0.9621] | 0.6995 | 0.8567 |
9
  | branch_car_at_8 | 69 | 0.3240 | [0.1877, 0.4805] | 0.4178 | 0.4633 |
10
+ | hidden_chart_oracle_success_at_8 | 69 | 0.6667 | [0.5507, 0.7826] | 0.8647 | 0.8358 |
11
+ | hidden_chart_oracle_utility_at_8 | 69 | 1.4000 | [1.1855, 1.6062] | 1.7565 | 1.7062 |
12
  | outcome_ptr_at_8 | 69 | 0.4348 | [0.3043, 0.5362] | 0.5886 | 0.4578 |
13
  | pairwise_causal_calibration_ece | 69 | 0.1583 | [0.1336, 0.1862] | 0.1567 | 0.1561 |
14
+ | proposal_oracle_success_at_8 | 69 | 0.3188 | [0.2174, 0.4348] | 0.3440 | 0.4326 |
15
+ | proposal_oracle_success_gain_over_base_at_8 | 69 | 0.0290 | [-0.0870, 0.1449] | 0.1016 | 0.1028 |
16
+ | proposal_oracle_utility_at_8 | 69 | 0.8015 | [0.6154, 1.0082] | 0.9574 | 1.0621 |
17
+ | proposal_oracle_utility_gain_over_base_at_8 | 69 | 0.0560 | [-0.1555, 0.2702] | 0.2579 | 0.2055 |
18
+ | selected_success_at_8 | 69 | 0.1884 | [0.1159, 0.2899] | 0.2033 | 0.2411 |
19
+ | selected_success_gain_over_base_at_8 | 69 | -0.1014 | [-0.2174, 0.0000] | -0.0391 | -0.0887 |
20
+ | selected_utility_at_8 | 69 | 0.4776 | [0.3230, 0.6844] | 0.5396 | 0.5988 |
21
+ | selected_utility_gain_over_base_at_8 | 69 | -0.2680 | [-0.4864, -0.0695] | -0.1599 | -0.2579 |
22
  | selector_regret_at_8 | 69 | 0.3240 | [0.1877, 0.4805] | 0.4178 | 0.4633 |
23
+ | success_selector_gap_at_8 | 69 | 0.1304 | [0.0580, 0.2174] | 0.1408 | 0.1915 |
24
+ | success_support_gap_at_8 | 69 | 0.3478 | [0.2319, 0.4638] | 0.5207 | 0.4032 |
25
+ | success_total_car_to_hidden_at_8 | 69 | 0.4783 | [0.3623, 0.5942] | 0.6614 | 0.5947 |
26
  | support_gap_at_8 | 69 | 0.5993 | [0.4135, 0.7763] | 0.7994 | 0.6444 |
27
+ | total_car_to_hidden_at_8 | 69 | 0.9224 | [0.7123, 1.1263] | 1.2169 | 1.1074 |
workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/table.tex CHANGED
@@ -3,10 +3,26 @@
3
  \toprule
4
  Metric & N & Micro mean & CI low & CI high \\
5
  \midrule
 
 
6
  branch\_car\_at\_8 & 69 & 0.3240 & 0.1877 & 0.4805 \\
 
 
7
  outcome\_ptr\_at\_8 & 69 & 0.4348 & 0.3043 & 0.5362 \\
8
  pairwise\_causal\_calibration\_ece & 69 & 0.1583 & 0.1336 & 0.1862 \\
 
 
 
 
 
 
 
 
9
  selector\_regret\_at\_8 & 69 & 0.3240 & 0.1877 & 0.4805 \\
 
 
 
10
  support\_gap\_at\_8 & 69 & 0.5993 & 0.4135 & 0.7763 \\
 
11
  \bottomrule
12
  \end{tabular}
 
3
  \toprule
4
  Metric & N & Micro mean & CI low & CI high \\
5
  \midrule
6
+ base\_success & 69 & 0.2899 & 0.1884 & 0.4203 \\
7
+ base\_utility & 69 & 0.7455 & 0.5693 & 0.9621 \\
8
  branch\_car\_at\_8 & 69 & 0.3240 & 0.1877 & 0.4805 \\
9
+ hidden\_chart\_oracle\_success\_at\_8 & 69 & 0.6667 & 0.5507 & 0.7826 \\
10
+ hidden\_chart\_oracle\_utility\_at\_8 & 69 & 1.4000 & 1.1855 & 1.6062 \\
11
  outcome\_ptr\_at\_8 & 69 & 0.4348 & 0.3043 & 0.5362 \\
12
  pairwise\_causal\_calibration\_ece & 69 & 0.1583 & 0.1336 & 0.1862 \\
13
+ proposal\_oracle\_success\_at\_8 & 69 & 0.3188 & 0.2174 & 0.4348 \\
14
+ proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 69 & 0.0290 & -0.0870 & 0.1449 \\
15
+ proposal\_oracle\_utility\_at\_8 & 69 & 0.8015 & 0.6154 & 1.0082 \\
16
+ proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 69 & 0.0560 & -0.1555 & 0.2702 \\
17
+ selected\_success\_at\_8 & 69 & 0.1884 & 0.1159 & 0.2899 \\
18
+ selected\_success\_gain\_over\_base\_at\_8 & 69 & -0.1014 & -0.2174 & 0.0000 \\
19
+ selected\_utility\_at\_8 & 69 & 0.4776 & 0.3230 & 0.6844 \\
20
+ selected\_utility\_gain\_over\_base\_at\_8 & 69 & -0.2680 & -0.4864 & -0.0695 \\
21
  selector\_regret\_at\_8 & 69 & 0.3240 & 0.1877 & 0.4805 \\
22
+ success\_selector\_gap\_at\_8 & 69 & 0.1304 & 0.0580 & 0.2174 \\
23
+ success\_support\_gap\_at\_8 & 69 & 0.3478 & 0.2319 & 0.4638 \\
24
+ success\_total\_car\_to\_hidden\_at\_8 & 69 & 0.4783 & 0.3623 & 0.5942 \\
25
  support\_gap\_at\_8 & 69 & 0.5993 & 0.4135 & 0.7763 \\
26
+ total\_car\_to\_hidden\_at\_8 & 69 & 0.9224 & 0.7123 & 1.1263 \\
27
  \bottomrule
28
  \end{tabular}
workspace/runs/ctt_residual_rollout_val69_seed1/metrics.json ADDED
@@ -0,0 +1,1918 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "runs/ctt_residual_full_seed1/model.pt",
3
+ "k": 8,
4
+ "measured_metric_summary": {
5
+ "base_success": {
6
+ "macro_by_seed": {
7
+ "ci": {
8
+ "confidence": 0.95,
9
+ "high": 0.4521276595744681,
10
+ "low": 0.20921985815602834,
11
+ "mean": 0.3297872340425532,
12
+ "n": 47,
13
+ "num_samples": 500
14
+ },
15
+ "mean": 0.3297872340425532,
16
+ "per_seed": {
17
+ "1025740212": 0.0,
18
+ "1055157354": 0.0,
19
+ "1055567618": 1.0,
20
+ "1113268940": 0.0,
21
+ "1113371261": 1.0,
22
+ "1176943004": 0.0,
23
+ "1189392917": 0.0,
24
+ "1231780997": 0.0,
25
+ "1271718007": 0.5,
26
+ "1273247072": 0.0,
27
+ "1358421325": 0.0,
28
+ "1375492786": 1.0,
29
+ "141984615": 0.0,
30
+ "1475440973": 1.0,
31
+ "1483645336": 1.0,
32
+ "1522272686": 0.0,
33
+ "1528740417": 1.0,
34
+ "1549324805": 0.6666666666666666,
35
+ "1607773860": 1.0,
36
+ "1665739511": 1.0,
37
+ "1747178030": 0.0,
38
+ "1881223807": 0.0,
39
+ "1919198188": 0.0,
40
+ "1991414747": 1.0,
41
+ "1997090716": 0.0,
42
+ "2015956957": 0.0,
43
+ "2054899126": 1.0,
44
+ "211245842": 0.0,
45
+ "2141802816": 0.0,
46
+ "256770874": 0.0,
47
+ "278008524": 0.0,
48
+ "416183106": 0.0,
49
+ "534717141": 1.0,
50
+ "551504635": 0.0,
51
+ "565369392": 0.3333333333333333,
52
+ "565621826": 1.0,
53
+ "585904130": 0.0,
54
+ "626486813": 0.25,
55
+ "636680137": 1.0,
56
+ "706479600": 0.0,
57
+ "750448296": 0.0,
58
+ "75070641": 0.0,
59
+ "767747424": 0.25,
60
+ "77147264": 0.5,
61
+ "794181626": 0.0,
62
+ "957101489": 0.0,
63
+ "998761162": 0.0
64
+ }
65
+ },
66
+ "macro_by_task": {
67
+ "ci": {
68
+ "confidence": 0.95,
69
+ "high": 0.33815987933634994,
70
+ "low": 0.10666666666666666,
71
+ "mean": 0.24241327300150828,
72
+ "n": 5,
73
+ "num_samples": 500
74
+ },
75
+ "mean": 0.24241327300150828,
76
+ "per_task": {
77
+ "LiftPegUpright-v1": 0.3333333333333333,
78
+ "PickCube-v1": 0.0,
79
+ "PullCube-v1": 0.29411764705882354,
80
+ "PushCube-v1": 0.38461538461538464,
81
+ "StackCube-v1": 0.2
82
+ }
83
+ },
84
+ "metric": "base_success",
85
+ "micro": {
86
+ "confidence": 0.95,
87
+ "high": 0.42028985507246375,
88
+ "low": 0.18840579710144928,
89
+ "mean": 0.2898550724637681,
90
+ "n": 69,
91
+ "num_samples": 500
92
+ },
93
+ "n": 69
94
+ },
95
+ "base_utility": {
96
+ "macro_by_seed": {
97
+ "ci": {
98
+ "confidence": 0.95,
99
+ "high": 1.0927389474422176,
100
+ "low": 0.6324926499567979,
101
+ "mean": 0.8566858090574239,
102
+ "n": 47,
103
+ "num_samples": 500
104
+ },
105
+ "mean": 0.8566858090574239,
106
+ "per_seed": {
107
+ "1025740212": 0.528373658657074,
108
+ "1055157354": 0.17649109661579132,
109
+ "1055567618": 2.0,
110
+ "1113268940": 0.42231252789497375,
111
+ "1113371261": 2.0,
112
+ "1176943004": 0.058033183217048645,
113
+ "1189392917": 0.10841864347457886,
114
+ "1231780997": 0.7270061373710632,
115
+ "1271718007": 1.415381133556366,
116
+ "1273247072": 0.15342965722084045,
117
+ "1358421325": 0.8324764966964722,
118
+ "1375492786": 2.0,
119
+ "141984615": 0.09381213908394177,
120
+ "1475440973": 2.0,
121
+ "1483645336": 2.0,
122
+ "1522272686": 0.19844311475753784,
123
+ "1528740417": 2.0,
124
+ "1549324805": 1.3787474483251572,
125
+ "1607773860": 2.0,
126
+ "1665739511": 2.0,
127
+ "1747178030": 0.1463065892457962,
128
+ "1881223807": 0.0320269912481308,
129
+ "1919198188": 0.03719887137413025,
130
+ "1991414747": 2.0,
131
+ "1997090716": 0.13927479088306427,
132
+ "2015956957": 0.06711978986859321,
133
+ "2054899126": 2.0,
134
+ "211245842": 0.8388344049453735,
135
+ "2141802816": 0.07080724835395813,
136
+ "256770874": 0.8557480573654175,
137
+ "278008524": 0.10747186839580536,
138
+ "416183106": 0.20032641291618347,
139
+ "534717141": 2.0,
140
+ "551504635": 0.7247228622436523,
141
+ "565369392": 0.7997656464576721,
142
+ "565621826": 2.0,
143
+ "585904130": 0.07529209802548091,
144
+ "626486813": 0.5054400166263804,
145
+ "636680137": 2.0,
146
+ "706479600": 0.8350887894630432,
147
+ "750448296": 0.11153355985879898,
148
+ "75070641": 0.20275163650512695,
149
+ "767747424": 0.5475241877138615,
150
+ "77147264": 1.4215794801712036,
151
+ "794181626": 0.18095605075359344,
152
+ "957101489": 0.20372186601161957,
153
+ "998761162": 0.06781657040119171
154
+ }
155
+ },
156
+ "macro_by_task": {
157
+ "ci": {
158
+ "confidence": 0.95,
159
+ "high": 0.9312208624280917,
160
+ "low": 0.4135245136419933,
161
+ "mean": 0.6995428283303713,
162
+ "n": 5,
163
+ "num_samples": 500
164
+ },
165
+ "mean": 0.6995428283303713,
166
+ "per_task": {
167
+ "LiftPegUpright-v1": 1.0670691976944606,
168
+ "PickCube-v1": 0.15342965722084045,
169
+ "PullCube-v1": 0.6512299207833541,
170
+ "PushCube-v1": 0.8223185676794785,
171
+ "StackCube-v1": 0.8036667982737223
172
+ }
173
+ },
174
+ "metric": "base_utility",
175
+ "micro": {
176
+ "confidence": 0.95,
177
+ "high": 0.9620560415399572,
178
+ "low": 0.5693137269615587,
179
+ "mean": 0.7455479058545029,
180
+ "n": 69,
181
+ "num_samples": 500
182
+ },
183
+ "n": 69
184
+ },
185
+ "branch_car_at_8": {
186
+ "macro_by_seed": {
187
+ "ci": {
188
+ "confidence": 0.95,
189
+ "high": 0.6905135366465427,
190
+ "low": 0.2873830998237971,
191
+ "mean": 0.46333538608845154,
192
+ "n": 47,
193
+ "num_samples": 500
194
+ },
195
+ "mean": 0.46333538608845154,
196
+ "per_seed": {
197
+ "1025740212": 0.47739143669605255,
198
+ "1055157354": 0.6370042562484741,
199
+ "1055567618": 1.8253512382507324,
200
+ "1113268940": 0.18306860327720642,
201
+ "1113371261": 0.0,
202
+ "1176943004": 1.974869817495346,
203
+ "1189392917": 0.0,
204
+ "1231780997": 0.0,
205
+ "1271718007": 0.00353144109249115,
206
+ "1273247072": 0.45726464316248894,
207
+ "1358421325": 0.6467066705226898,
208
+ "1375492786": 0.00047226750757545233,
209
+ "141984615": 0.05730179324746132,
210
+ "1475440973": 1.289921224117279,
211
+ "1483645336": 1.9307887554168701,
212
+ "1522272686": 0.03616611659526825,
213
+ "1528740417": 0.0,
214
+ "1549324805": 0.0015795694974561532,
215
+ "1607773860": 0.6036040186882019,
216
+ "1665739511": 0.0,
217
+ "1747178030": 0.027157030999660492,
218
+ "1881223807": 0.009028241038322449,
219
+ "1919198188": 0.0,
220
+ "1991414747": 1.9863876849412918,
221
+ "1997090716": 0.0,
222
+ "2015956957": 0.022195855714380743,
223
+ "2054899126": 0.0,
224
+ "211245842": 0.6792904287576675,
225
+ "2141802816": 0.019766509532928467,
226
+ "256770874": 0.7044661641120911,
227
+ "278008524": 0.011409573198761791,
228
+ "416183106": 1.7668711841106415,
229
+ "534717141": 1.9558932781219482,
230
+ "551504635": 0.0,
231
+ "565369392": 0.046260833740234375,
232
+ "565621826": 0.0,
233
+ "585904130": 0.012068324722349644,
234
+ "626486813": 0.02500777377281338,
235
+ "636680137": 0.0,
236
+ "706479600": 0.0,
237
+ "750448296": 0.02701035887002945,
238
+ "75070641": 0.5863077491521835,
239
+ "767747424": 0.04561196081340313,
240
+ "77147264": 0.0,
241
+ "794181626": 0.0,
242
+ "957101489": 1.7757539749145508,
243
+ "998761162": 1.9512543678283691
244
+ }
245
+ },
246
+ "macro_by_task": {
247
+ "ci": {
248
+ "confidence": 0.95,
249
+ "high": 0.5835942497964088,
250
+ "low": 0.22782261933967035,
251
+ "mean": 0.4178369468370689,
252
+ "n": 5,
253
+ "num_samples": 500
254
+ },
255
+ "mean": 0.4178369468370689,
256
+ "per_task": {
257
+ "LiftPegUpright-v1": 0.3250635440150897,
258
+ "PickCube-v1": 0.45726464316248894,
259
+ "PullCube-v1": 0.08196123232654132,
260
+ "PushCube-v1": 0.6069896691120588,
261
+ "StackCube-v1": 0.6179056455691655
262
+ }
263
+ },
264
+ "metric": "branch_car_at_8",
265
+ "micro": {
266
+ "confidence": 0.95,
267
+ "high": 0.48046939117341314,
268
+ "low": 0.18767418079685583,
269
+ "mean": 0.32396779983115476,
270
+ "n": 69,
271
+ "num_samples": 500
272
+ },
273
+ "n": 69
274
+ },
275
+ "hidden_chart_oracle_success_at_8": {
276
+ "macro_by_seed": {
277
+ "ci": {
278
+ "confidence": 0.95,
279
+ "high": 0.922695035460993,
280
+ "low": 0.7351063829787235,
281
+ "mean": 0.8358156028368794,
282
+ "n": 47,
283
+ "num_samples": 500
284
+ },
285
+ "mean": 0.8358156028368794,
286
+ "per_seed": {
287
+ "1025740212": 1.0,
288
+ "1055157354": 1.0,
289
+ "1055567618": 1.0,
290
+ "1113268940": 1.0,
291
+ "1113371261": 1.0,
292
+ "1176943004": 1.0,
293
+ "1189392917": 1.0,
294
+ "1231780997": 1.0,
295
+ "1271718007": 1.0,
296
+ "1273247072": 1.0,
297
+ "1358421325": 1.0,
298
+ "1375492786": 0.0,
299
+ "141984615": 0.0,
300
+ "1475440973": 1.0,
301
+ "1483645336": 1.0,
302
+ "1522272686": 1.0,
303
+ "1528740417": 1.0,
304
+ "1549324805": 0.6666666666666666,
305
+ "1607773860": 1.0,
306
+ "1665739511": 1.0,
307
+ "1747178030": 1.0,
308
+ "1881223807": 1.0,
309
+ "1919198188": 1.0,
310
+ "1991414747": 1.0,
311
+ "1997090716": 1.0,
312
+ "2015956957": 0.2,
313
+ "2054899126": 1.0,
314
+ "211245842": 1.0,
315
+ "2141802816": 1.0,
316
+ "256770874": 1.0,
317
+ "278008524": 0.5,
318
+ "416183106": 1.0,
319
+ "534717141": 1.0,
320
+ "551504635": 1.0,
321
+ "565369392": 0.6666666666666666,
322
+ "565621826": 1.0,
323
+ "585904130": 0.0,
324
+ "626486813": 0.25,
325
+ "636680137": 1.0,
326
+ "706479600": 1.0,
327
+ "750448296": 0.0,
328
+ "75070641": 1.0,
329
+ "767747424": 0.0,
330
+ "77147264": 1.0,
331
+ "794181626": 1.0,
332
+ "957101489": 1.0,
333
+ "998761162": 1.0
334
+ }
335
+ },
336
+ "macro_by_task": {
337
+ "ci": {
338
+ "confidence": 0.95,
339
+ "high": 1.0,
340
+ "low": 0.5941176470588235,
341
+ "mean": 0.8647058823529413,
342
+ "n": 5,
343
+ "num_samples": 500
344
+ },
345
+ "mean": 0.8647058823529413,
346
+ "per_task": {
347
+ "LiftPegUpright-v1": 1.0,
348
+ "PickCube-v1": 1.0,
349
+ "PullCube-v1": 0.3235294117647059,
350
+ "PushCube-v1": 1.0,
351
+ "StackCube-v1": 1.0
352
+ }
353
+ },
354
+ "metric": "hidden_chart_oracle_success_at_8",
355
+ "micro": {
356
+ "confidence": 0.95,
357
+ "high": 0.782608695652174,
358
+ "low": 0.5507246376811594,
359
+ "mean": 0.6666666666666666,
360
+ "n": 69,
361
+ "num_samples": 500
362
+ },
363
+ "n": 69
364
+ },
365
+ "hidden_chart_oracle_utility_at_8": {
366
+ "macro_by_seed": {
367
+ "ci": {
368
+ "confidence": 0.95,
369
+ "high": 1.8615907955016442,
370
+ "low": 1.529367521088174,
371
+ "mean": 1.7062028111638965,
372
+ "n": 47,
373
+ "num_samples": 500
374
+ },
375
+ "mean": 1.7062028111638965,
376
+ "per_seed": {
377
+ "1025740212": 2.0,
378
+ "1055157354": 2.0,
379
+ "1055567618": 2.0,
380
+ "1113268940": 2.0,
381
+ "1113371261": 2.0,
382
+ "1176943004": 2.0,
383
+ "1189392917": 2.0,
384
+ "1231780997": 2.0,
385
+ "1271718007": 2.0,
386
+ "1273247072": 2.0,
387
+ "1358421325": 2.0,
388
+ "1375492786": 0.24345096945762634,
389
+ "141984615": 0.20044784247875214,
390
+ "1475440973": 2.0,
391
+ "1483645336": 2.0,
392
+ "1522272686": 2.0,
393
+ "1528740417": 2.0,
394
+ "1549324805": 1.4027505218982697,
395
+ "1607773860": 2.0,
396
+ "1665739511": 2.0,
397
+ "1747178030": 2.0,
398
+ "1881223807": 2.0,
399
+ "1919198188": 2.0,
400
+ "1991414747": 2.0,
401
+ "1997090716": 2.0,
402
+ "2015956957": 0.5298813670873642,
403
+ "2054899126": 2.0,
404
+ "211245842": 2.0,
405
+ "2141802816": 2.0,
406
+ "256770874": 2.0,
407
+ "278008524": 1.113811194896698,
408
+ "416183106": 2.0,
409
+ "534717141": 2.0,
410
+ "551504635": 2.0,
411
+ "565369392": 1.4266315797964733,
412
+ "565621826": 2.0,
413
+ "585904130": 0.23764699200789133,
414
+ "626486813": 0.6124684102833271,
415
+ "636680137": 2.0,
416
+ "706479600": 2.0,
417
+ "750448296": 0.23580759018659592,
418
+ "75070641": 2.0,
419
+ "767747424": 0.18863565661013126,
420
+ "77147264": 2.0,
421
+ "794181626": 2.0,
422
+ "957101489": 2.0,
423
+ "998761162": 2.0
424
+ }
425
+ },
426
+ "macro_by_task": {
427
+ "ci": {
428
+ "confidence": 0.95,
429
+ "high": 2.0,
430
+ "low": 1.2693931020796299,
431
+ "mean": 1.7564643673598765,
432
+ "n": 5,
433
+ "num_samples": 500
434
+ },
435
+ "mean": 1.7564643673598765,
436
+ "per_task": {
437
+ "LiftPegUpright-v1": 2.0,
438
+ "PickCube-v1": 2.0,
439
+ "PullCube-v1": 0.7823218367993832,
440
+ "PushCube-v1": 2.0,
441
+ "StackCube-v1": 2.0
442
+ }
443
+ },
444
+ "metric": "hidden_chart_oracle_utility_at_8",
445
+ "micro": {
446
+ "confidence": 0.95,
447
+ "high": 1.6062410234109215,
448
+ "low": 1.1855228276572365,
449
+ "mean": 1.3999846732054932,
450
+ "n": 69,
451
+ "num_samples": 500
452
+ },
453
+ "n": 69
454
+ },
455
+ "outcome_ptr_at_8": {
456
+ "macro_by_seed": {
457
+ "ci": {
458
+ "confidence": 0.95,
459
+ "high": 0.5886524822695036,
460
+ "low": 0.3191489361702128,
461
+ "mean": 0.4578014184397163,
462
+ "n": 47,
463
+ "num_samples": 500
464
+ },
465
+ "mean": 0.4578014184397163,
466
+ "per_seed": {
467
+ "1025740212": 1.0,
468
+ "1055157354": 1.0,
469
+ "1055567618": 0.0,
470
+ "1113268940": 1.0,
471
+ "1113371261": 0.0,
472
+ "1176943004": 1.0,
473
+ "1189392917": 1.0,
474
+ "1231780997": 1.0,
475
+ "1271718007": 0.0,
476
+ "1273247072": 1.0,
477
+ "1358421325": 0.0,
478
+ "1375492786": 0.0,
479
+ "141984615": 0.6666666666666666,
480
+ "1475440973": 0.0,
481
+ "1483645336": 0.0,
482
+ "1522272686": 1.0,
483
+ "1528740417": 0.0,
484
+ "1549324805": 0.0,
485
+ "1607773860": 0.0,
486
+ "1665739511": 0.0,
487
+ "1747178030": 0.0,
488
+ "1881223807": 1.0,
489
+ "1919198188": 1.0,
490
+ "1991414747": 0.0,
491
+ "1997090716": 1.0,
492
+ "2015956957": 0.6,
493
+ "2054899126": 0.0,
494
+ "211245842": 0.0,
495
+ "2141802816": 1.0,
496
+ "256770874": 0.0,
497
+ "278008524": 0.5,
498
+ "416183106": 1.0,
499
+ "534717141": 0.0,
500
+ "551504635": 0.0,
501
+ "565369392": 0.0,
502
+ "565621826": 0.0,
503
+ "585904130": 0.0,
504
+ "626486813": 0.5,
505
+ "636680137": 0.0,
506
+ "706479600": 1.0,
507
+ "750448296": 0.0,
508
+ "75070641": 1.0,
509
+ "767747424": 0.75,
510
+ "77147264": 0.5,
511
+ "794181626": 1.0,
512
+ "957101489": 1.0,
513
+ "998761162": 1.0
514
+ }
515
+ },
516
+ "macro_by_task": {
517
+ "ci": {
518
+ "confidence": 0.95,
519
+ "high": 0.8133333333333332,
520
+ "low": 0.4062141779788838,
521
+ "mean": 0.5885671191553545,
522
+ "n": 5,
523
+ "num_samples": 500
524
+ },
525
+ "mean": 0.5885671191553545,
526
+ "per_task": {
527
+ "LiftPegUpright-v1": 0.5,
528
+ "PickCube-v1": 1.0,
529
+ "PullCube-v1": 0.29411764705882354,
530
+ "PushCube-v1": 0.6153846153846154,
531
+ "StackCube-v1": 0.5333333333333333
532
+ }
533
+ },
534
+ "metric": "outcome_ptr_at_8",
535
+ "micro": {
536
+ "confidence": 0.95,
537
+ "high": 0.5362318840579711,
538
+ "low": 0.30434782608695654,
539
+ "mean": 0.43478260869565216,
540
+ "n": 69,
541
+ "num_samples": 500
542
+ },
543
+ "n": 69
544
+ },
545
+ "pairwise_causal_calibration_ece": {
546
+ "macro_by_seed": {
547
+ "ci": {
548
+ "confidence": 0.95,
549
+ "high": 0.18010258206023347,
550
+ "low": 0.13094723444140935,
551
+ "mean": 0.15606796759963956,
552
+ "n": 47,
553
+ "num_samples": 500
554
+ },
555
+ "mean": 0.15606796759963956,
556
+ "per_seed": {
557
+ "1025740212": 0.12992986623548633,
558
+ "1055157354": 0.07450750352404123,
559
+ "1055567618": 0.318234334809661,
560
+ "1113268940": 0.10725283874677279,
561
+ "1113371261": 0.04977883974023356,
562
+ "1176943004": 0.1815411965846294,
563
+ "1189392917": 0.2180235973132697,
564
+ "1231780997": 0.28322114525972475,
565
+ "1271718007": 0.07023931523066618,
566
+ "1273247072": 0.096994407338536,
567
+ "1358421325": 0.046694794435454434,
568
+ "1375492786": 0.14027669602920534,
569
+ "141984615": 0.08533231968443573,
570
+ "1475440973": 0.2443552809335613,
571
+ "1483645336": 0.13752192684190806,
572
+ "1522272686": 0.28555947838938045,
573
+ "1528740417": 0.2757671830981365,
574
+ "1549324805": 0.28625156462411294,
575
+ "1607773860": 0.18975917078230178,
576
+ "1665739511": 0.0,
577
+ "1747178030": 0.3089551740932353,
578
+ "1881223807": 0.13674585276395504,
579
+ "1919198188": 0.13114443223224567,
580
+ "1991414747": 0.1306768270010748,
581
+ "1997090716": 0.2554442550173647,
582
+ "2015956957": 0.16317219295797403,
583
+ "2054899126": 0.0,
584
+ "211245842": 0.07548939908426067,
585
+ "2141802816": 0.07786951284417709,
586
+ "256770874": 0.03135306008748828,
587
+ "278008524": 0.24605586147071373,
588
+ "416183106": 0.07978206783428046,
589
+ "534717141": 0.14369576234023818,
590
+ "551504635": 0.2299325778712667,
591
+ "565369392": 0.13991651305345584,
592
+ "565621826": 0.0,
593
+ "585904130": 0.07276320034061332,
594
+ "626486813": 0.2420589895144555,
595
+ "636680137": 0.07628525991805817,
596
+ "706479600": 0.16443847051324711,
597
+ "750448296": 0.2344413298276417,
598
+ "75070641": 0.18559462278331035,
599
+ "767747424": 0.07531396139335009,
600
+ "77147264": 0.2630803856693186,
601
+ "794181626": 0.28339362113630795,
602
+ "957101489": 0.07643021229779011,
603
+ "998761162": 0.28991947553571784
604
+ }
605
+ },
606
+ "macro_by_task": {
607
+ "ci": {
608
+ "confidence": 0.95,
609
+ "high": 0.19005329411145716,
610
+ "low": 0.12022312647229497,
611
+ "mean": 0.15665925257113028,
612
+ "n": 5,
613
+ "num_samples": 500
614
+ },
615
+ "mean": 0.15665925257113028,
616
+ "per_task": {
617
+ "LiftPegUpright-v1": 0.21174314869082475,
618
+ "PickCube-v1": 0.096994407338536,
619
+ "PullCube-v1": 0.14892514417161398,
620
+ "PushCube-v1": 0.19653272589828236,
621
+ "StackCube-v1": 0.12910083675639442
622
+ }
623
+ },
624
+ "metric": "pairwise_causal_calibration_ece",
625
+ "micro": {
626
+ "confidence": 0.95,
627
+ "high": 0.186245353107692,
628
+ "low": 0.13360163870735103,
629
+ "mean": 0.15829487230930359,
630
+ "n": 69,
631
+ "num_samples": 500
632
+ },
633
+ "n": 69
634
+ },
635
+ "proposal_oracle_success_at_8": {
636
+ "macro_by_seed": {
637
+ "ci": {
638
+ "confidence": 0.95,
639
+ "high": 0.5673758865248226,
640
+ "low": 0.2943262411347518,
641
+ "mean": 0.4326241134751774,
642
+ "n": 47,
643
+ "num_samples": 500
644
+ },
645
+ "mean": 0.4326241134751774,
646
+ "per_seed": {
647
+ "1025740212": 0.0,
648
+ "1055157354": 0.0,
649
+ "1055567618": 1.0,
650
+ "1113268940": 0.0,
651
+ "1113371261": 1.0,
652
+ "1176943004": 1.0,
653
+ "1189392917": 1.0,
654
+ "1231780997": 1.0,
655
+ "1271718007": 0.5,
656
+ "1273247072": 0.0,
657
+ "1358421325": 0.0,
658
+ "1375492786": 0.0,
659
+ "141984615": 0.0,
660
+ "1475440973": 1.0,
661
+ "1483645336": 1.0,
662
+ "1522272686": 0.0,
663
+ "1528740417": 1.0,
664
+ "1549324805": 0.3333333333333333,
665
+ "1607773860": 0.0,
666
+ "1665739511": 1.0,
667
+ "1747178030": 0.0,
668
+ "1881223807": 0.0,
669
+ "1919198188": 0.0,
670
+ "1991414747": 1.0,
671
+ "1997090716": 1.0,
672
+ "2015956957": 0.0,
673
+ "2054899126": 1.0,
674
+ "211245842": 0.0,
675
+ "2141802816": 0.0,
676
+ "256770874": 0.0,
677
+ "278008524": 0.0,
678
+ "416183106": 1.0,
679
+ "534717141": 1.0,
680
+ "551504635": 0.0,
681
+ "565369392": 0.0,
682
+ "565621826": 1.0,
683
+ "585904130": 0.0,
684
+ "626486813": 0.0,
685
+ "636680137": 0.0,
686
+ "706479600": 1.0,
687
+ "750448296": 0.0,
688
+ "75070641": 0.0,
689
+ "767747424": 0.0,
690
+ "77147264": 0.5,
691
+ "794181626": 1.0,
692
+ "957101489": 1.0,
693
+ "998761162": 1.0
694
+ }
695
+ },
696
+ "macro_by_task": {
697
+ "ci": {
698
+ "confidence": 0.95,
699
+ "high": 0.5850678733031673,
700
+ "low": 0.1,
701
+ "mean": 0.3440422322775264,
702
+ "n": 5,
703
+ "num_samples": 500
704
+ },
705
+ "mean": 0.3440422322775264,
706
+ "per_task": {
707
+ "LiftPegUpright-v1": 0.5,
708
+ "PickCube-v1": 0.0,
709
+ "PullCube-v1": 0.11764705882352941,
710
+ "PushCube-v1": 0.7692307692307693,
711
+ "StackCube-v1": 0.3333333333333333
712
+ }
713
+ },
714
+ "metric": "proposal_oracle_success_at_8",
715
+ "micro": {
716
+ "confidence": 0.95,
717
+ "high": 0.43478260869565216,
718
+ "low": 0.21739130434782608,
719
+ "mean": 0.3188405797101449,
720
+ "n": 69,
721
+ "num_samples": 500
722
+ },
723
+ "n": 69
724
+ },
725
+ "proposal_oracle_success_gain_over_base_at_8": {
726
+ "macro_by_seed": {
727
+ "ci": {
728
+ "confidence": 0.95,
729
+ "high": 0.25709219858156024,
730
+ "low": -0.03368794326241135,
731
+ "mean": 0.10283687943262412,
732
+ "n": 47,
733
+ "num_samples": 500
734
+ },
735
+ "mean": 0.10283687943262412,
736
+ "per_seed": {
737
+ "1025740212": 0.0,
738
+ "1055157354": 0.0,
739
+ "1055567618": 0.0,
740
+ "1113268940": 0.0,
741
+ "1113371261": 0.0,
742
+ "1176943004": 1.0,
743
+ "1189392917": 1.0,
744
+ "1231780997": 1.0,
745
+ "1271718007": 0.0,
746
+ "1273247072": 0.0,
747
+ "1358421325": 0.0,
748
+ "1375492786": -1.0,
749
+ "141984615": 0.0,
750
+ "1475440973": 0.0,
751
+ "1483645336": 0.0,
752
+ "1522272686": 0.0,
753
+ "1528740417": 0.0,
754
+ "1549324805": -0.3333333333333333,
755
+ "1607773860": -1.0,
756
+ "1665739511": 0.0,
757
+ "1747178030": 0.0,
758
+ "1881223807": 0.0,
759
+ "1919198188": 0.0,
760
+ "1991414747": 0.0,
761
+ "1997090716": 1.0,
762
+ "2015956957": 0.0,
763
+ "2054899126": 0.0,
764
+ "211245842": 0.0,
765
+ "2141802816": 0.0,
766
+ "256770874": 0.0,
767
+ "278008524": 0.0,
768
+ "416183106": 1.0,
769
+ "534717141": 0.0,
770
+ "551504635": 0.0,
771
+ "565369392": -0.3333333333333333,
772
+ "565621826": 0.0,
773
+ "585904130": 0.0,
774
+ "626486813": -0.25,
775
+ "636680137": -1.0,
776
+ "706479600": 1.0,
777
+ "750448296": 0.0,
778
+ "75070641": 0.0,
779
+ "767747424": -0.25,
780
+ "77147264": 0.0,
781
+ "794181626": 1.0,
782
+ "957101489": 1.0,
783
+ "998761162": 1.0
784
+ }
785
+ },
786
+ "macro_by_task": {
787
+ "ci": {
788
+ "confidence": 0.95,
789
+ "high": 0.2641025641025641,
790
+ "low": -0.07058823529411765,
791
+ "mean": 0.10162895927601809,
792
+ "n": 5,
793
+ "num_samples": 500
794
+ },
795
+ "mean": 0.10162895927601809,
796
+ "per_task": {
797
+ "LiftPegUpright-v1": 0.16666666666666666,
798
+ "PickCube-v1": 0.0,
799
+ "PullCube-v1": -0.17647058823529413,
800
+ "PushCube-v1": 0.38461538461538464,
801
+ "StackCube-v1": 0.13333333333333333
802
+ }
803
+ },
804
+ "metric": "proposal_oracle_success_gain_over_base_at_8",
805
+ "micro": {
806
+ "confidence": 0.95,
807
+ "high": 0.14492753623188406,
808
+ "low": -0.08695652173913043,
809
+ "mean": 0.028985507246376812,
810
+ "n": 69,
811
+ "num_samples": 500
812
+ },
813
+ "n": 69
814
+ },
815
+ "proposal_oracle_utility_at_8": {
816
+ "macro_by_seed": {
817
+ "ci": {
818
+ "confidence": 0.95,
819
+ "high": 1.318000317976896,
820
+ "low": 0.8185871909540939,
821
+ "mean": 1.0621498337636393,
822
+ "n": 47,
823
+ "num_samples": 500
824
+ },
825
+ "mean": 1.0621498337636393,
826
+ "per_seed": {
827
+ "1025740212": 0.6983401775360107,
828
+ "1055157354": 0.8232162594795227,
829
+ "1055567618": 2.0,
830
+ "1113268940": 0.6617497801780701,
831
+ "1113371261": 2.0,
832
+ "1176943004": 2.0,
833
+ "1189392917": 2.0,
834
+ "1231780997": 2.0,
835
+ "1271718007": 1.1110476776957512,
836
+ "1273247072": 0.4997493326663971,
837
+ "1358421325": 0.8275434374809265,
838
+ "1375492786": 0.0017900466918945312,
839
+ "141984615": 0.09240151196718216,
840
+ "1475440973": 2.0,
841
+ "1483645336": 2.0,
842
+ "1522272686": 0.23248937726020813,
843
+ "1528740417": 2.0,
844
+ "1549324805": 0.6785067518552145,
845
+ "1607773860": 0.8267073631286621,
846
+ "1665739511": 2.0,
847
+ "1747178030": 0.11134227365255356,
848
+ "1881223807": 0.034886181354522705,
849
+ "1919198188": 0.046244338154792786,
850
+ "1991414747": 2.0,
851
+ "1997090716": 2.0,
852
+ "2015956957": 0.05629209652543068,
853
+ "2054899126": 2.0,
854
+ "211245842": 0.8358442187309265,
855
+ "2141802816": 0.09589323401451111,
856
+ "256770874": 0.8125,
857
+ "278008524": 0.03147180867381394,
858
+ "416183106": 2.0,
859
+ "534717141": 2.0,
860
+ "551504635": 0.7224790453910828,
861
+ "565369392": 0.17366480827331543,
862
+ "565621826": 2.0,
863
+ "585904130": 0.032918572425842285,
864
+ "626486813": 0.07365635968744755,
865
+ "636680137": 0.013928472995758057,
866
+ "706479600": 2.0,
867
+ "750448296": 0.07342664152383804,
868
+ "75070641": 0.8128874897956848,
869
+ "767747424": 0.11342880013398826,
870
+ "77147264": 1.426636129617691,
871
+ "794181626": 2.0,
872
+ "957101489": 2.0,
873
+ "998761162": 2.0
874
+ }
875
+ },
876
+ "macro_by_task": {
877
+ "ci": {
878
+ "confidence": 0.95,
879
+ "high": 1.3423044667794155,
880
+ "low": 0.537627078428061,
881
+ "mean": 0.9574398233701207,
882
+ "n": 5,
883
+ "num_samples": 500
884
+ },
885
+ "mean": 0.9574398233701207,
886
+ "per_task": {
887
+ "LiftPegUpright-v1": 1.3470948338508606,
888
+ "PickCube-v1": 0.4997493326663971,
889
+ "PullCube-v1": 0.3012250267818351,
890
+ "PushCube-v1": 1.55294325030767,
891
+ "StackCube-v1": 1.0861866732438406
892
+ }
893
+ },
894
+ "metric": "proposal_oracle_utility_at_8",
895
+ "micro": {
896
+ "confidence": 0.95,
897
+ "high": 1.0082228477517872,
898
+ "low": 0.6154177648528223,
899
+ "mean": 0.8015221970871199,
900
+ "n": 69,
901
+ "num_samples": 500
902
+ },
903
+ "n": 69
904
+ },
905
+ "proposal_oracle_utility_gain_over_base_at_8": {
906
+ "macro_by_seed": {
907
+ "ci": {
908
+ "confidence": 0.95,
909
+ "high": 0.4860803771314892,
910
+ "low": -0.031156662437926107,
911
+ "mean": 0.20546402470621525,
912
+ "n": 47,
913
+ "num_samples": 500
914
+ },
915
+ "mean": 0.20546402470621525,
916
+ "per_seed": {
917
+ "1025740212": 0.16996651887893677,
918
+ "1055157354": 0.6467251628637314,
919
+ "1055567618": 0.0,
920
+ "1113268940": 0.2394372522830963,
921
+ "1113371261": 0.0,
922
+ "1176943004": 1.9419668167829514,
923
+ "1189392917": 1.8915813565254211,
924
+ "1231780997": 1.2729938626289368,
925
+ "1271718007": -0.3043334558606148,
926
+ "1273247072": 0.34631967544555664,
927
+ "1358421325": -0.004933059215545654,
928
+ "1375492786": -1.9982099533081055,
929
+ "141984615": -0.001410627116759618,
930
+ "1475440973": 0.0,
931
+ "1483645336": 0.0,
932
+ "1522272686": 0.03404626250267029,
933
+ "1528740417": 0.0,
934
+ "1549324805": -0.7002406964699427,
935
+ "1607773860": -1.173292636871338,
936
+ "1665739511": 0.0,
937
+ "1747178030": -0.034964315593242645,
938
+ "1881223807": 0.0028591901063919067,
939
+ "1919198188": 0.009045466780662537,
940
+ "1991414747": 0.0,
941
+ "1997090716": 1.8607252091169357,
942
+ "2015956957": -0.010827693343162536,
943
+ "2054899126": 0.0,
944
+ "211245842": -0.0029901862144470215,
945
+ "2141802816": 0.02508598566055298,
946
+ "256770874": -0.04324805736541748,
947
+ "278008524": -0.07600005972199142,
948
+ "416183106": 1.7996735870838165,
949
+ "534717141": 0.0,
950
+ "551504635": -0.00224381685256958,
951
+ "565369392": -0.6261008381843567,
952
+ "565621826": 0.0,
953
+ "585904130": -0.04237352559963862,
954
+ "626486813": -0.43178365693893284,
955
+ "636680137": -1.986071527004242,
956
+ "706479600": 1.1649112105369568,
957
+ "750448296": -0.03810691833496094,
958
+ "75070641": 0.6101358532905579,
959
+ "767747424": -0.4340953875798732,
960
+ "77147264": 0.005056649446487427,
961
+ "794181626": 1.8190439492464066,
962
+ "957101489": 1.7962781339883804,
963
+ "998761162": 1.9321834295988083
964
+ }
965
+ },
966
+ "macro_by_task": {
967
+ "ci": {
968
+ "confidence": 0.95,
969
+ "high": 0.5508839118022185,
970
+ "low": -0.08473387408052005,
971
+ "mean": 0.2578969950397495,
972
+ "n": 5,
973
+ "num_samples": 500
974
+ },
975
+ "mean": 0.2578969950397495,
976
+ "per_task": {
977
+ "LiftPegUpright-v1": 0.2800256361564,
978
+ "PickCube-v1": 0.34631967544555664,
979
+ "PullCube-v1": -0.35000489400151896,
980
+ "PushCube-v1": 0.7306246826281915,
981
+ "StackCube-v1": 0.2825198749701182
982
+ }
983
+ },
984
+ "metric": "proposal_oracle_utility_gain_over_base_at_8",
985
+ "micro": {
986
+ "confidence": 0.95,
987
+ "high": 0.2701519333851942,
988
+ "low": -0.15554412837018786,
989
+ "mean": 0.055974291232617004,
990
+ "n": 69,
991
+ "num_samples": 500
992
+ },
993
+ "n": 69
994
+ },
995
+ "selected_success_at_8": {
996
+ "macro_by_seed": {
997
+ "ci": {
998
+ "confidence": 0.95,
999
+ "high": 0.3475177304964539,
1000
+ "low": 0.1347517730496454,
1001
+ "mean": 0.24113475177304963,
1002
+ "n": 47,
1003
+ "num_samples": 500
1004
+ },
1005
+ "mean": 0.24113475177304963,
1006
+ "per_seed": {
1007
+ "1025740212": 0.0,
1008
+ "1055157354": 0.0,
1009
+ "1055567618": 0.0,
1010
+ "1113268940": 0.0,
1011
+ "1113371261": 1.0,
1012
+ "1176943004": 0.0,
1013
+ "1189392917": 1.0,
1014
+ "1231780997": 1.0,
1015
+ "1271718007": 0.5,
1016
+ "1273247072": 0.0,
1017
+ "1358421325": 0.0,
1018
+ "1375492786": 0.0,
1019
+ "141984615": 0.0,
1020
+ "1475440973": 0.0,
1021
+ "1483645336": 0.0,
1022
+ "1522272686": 0.0,
1023
+ "1528740417": 1.0,
1024
+ "1549324805": 0.3333333333333333,
1025
+ "1607773860": 0.0,
1026
+ "1665739511": 1.0,
1027
+ "1747178030": 0.0,
1028
+ "1881223807": 0.0,
1029
+ "1919198188": 0.0,
1030
+ "1991414747": 0.0,
1031
+ "1997090716": 1.0,
1032
+ "2015956957": 0.0,
1033
+ "2054899126": 1.0,
1034
+ "211245842": 0.0,
1035
+ "2141802816": 0.0,
1036
+ "256770874": 0.0,
1037
+ "278008524": 0.0,
1038
+ "416183106": 0.0,
1039
+ "534717141": 0.0,
1040
+ "551504635": 0.0,
1041
+ "565369392": 0.0,
1042
+ "565621826": 1.0,
1043
+ "585904130": 0.0,
1044
+ "626486813": 0.0,
1045
+ "636680137": 0.0,
1046
+ "706479600": 1.0,
1047
+ "750448296": 0.0,
1048
+ "75070641": 0.0,
1049
+ "767747424": 0.0,
1050
+ "77147264": 0.5,
1051
+ "794181626": 1.0,
1052
+ "957101489": 0.0,
1053
+ "998761162": 0.0
1054
+ }
1055
+ },
1056
+ "macro_by_task": {
1057
+ "ci": {
1058
+ "confidence": 0.95,
1059
+ "high": 0.3589743589743589,
1060
+ "low": 0.05333333333333333,
1061
+ "mean": 0.20328808446455504,
1062
+ "n": 5,
1063
+ "num_samples": 500
1064
+ },
1065
+ "mean": 0.20328808446455504,
1066
+ "per_task": {
1067
+ "LiftPegUpright-v1": 0.3333333333333333,
1068
+ "PickCube-v1": 0.0,
1069
+ "PullCube-v1": 0.08823529411764706,
1070
+ "PushCube-v1": 0.46153846153846156,
1071
+ "StackCube-v1": 0.13333333333333333
1072
+ }
1073
+ },
1074
+ "metric": "selected_success_at_8",
1075
+ "micro": {
1076
+ "confidence": 0.95,
1077
+ "high": 0.2898550724637681,
1078
+ "low": 0.11594202898550725,
1079
+ "mean": 0.18840579710144928,
1080
+ "n": 69,
1081
+ "num_samples": 500
1082
+ },
1083
+ "n": 69
1084
+ },
1085
+ "selected_success_gain_over_base_at_8": {
1086
+ "macro_by_seed": {
1087
+ "ci": {
1088
+ "confidence": 0.95,
1089
+ "high": 0.04609929078014185,
1090
+ "low": -0.23404255319148937,
1091
+ "mean": -0.08865248226950355,
1092
+ "n": 47,
1093
+ "num_samples": 500
1094
+ },
1095
+ "mean": -0.08865248226950355,
1096
+ "per_seed": {
1097
+ "1025740212": 0.0,
1098
+ "1055157354": 0.0,
1099
+ "1055567618": -1.0,
1100
+ "1113268940": 0.0,
1101
+ "1113371261": 0.0,
1102
+ "1176943004": 0.0,
1103
+ "1189392917": 1.0,
1104
+ "1231780997": 1.0,
1105
+ "1271718007": 0.0,
1106
+ "1273247072": 0.0,
1107
+ "1358421325": 0.0,
1108
+ "1375492786": -1.0,
1109
+ "141984615": 0.0,
1110
+ "1475440973": -1.0,
1111
+ "1483645336": -1.0,
1112
+ "1522272686": 0.0,
1113
+ "1528740417": 0.0,
1114
+ "1549324805": -0.3333333333333333,
1115
+ "1607773860": -1.0,
1116
+ "1665739511": 0.0,
1117
+ "1747178030": 0.0,
1118
+ "1881223807": 0.0,
1119
+ "1919198188": 0.0,
1120
+ "1991414747": -1.0,
1121
+ "1997090716": 1.0,
1122
+ "2015956957": 0.0,
1123
+ "2054899126": 0.0,
1124
+ "211245842": 0.0,
1125
+ "2141802816": 0.0,
1126
+ "256770874": 0.0,
1127
+ "278008524": 0.0,
1128
+ "416183106": 0.0,
1129
+ "534717141": -1.0,
1130
+ "551504635": 0.0,
1131
+ "565369392": -0.3333333333333333,
1132
+ "565621826": 0.0,
1133
+ "585904130": 0.0,
1134
+ "626486813": -0.25,
1135
+ "636680137": -1.0,
1136
+ "706479600": 1.0,
1137
+ "750448296": 0.0,
1138
+ "75070641": 0.0,
1139
+ "767747424": -0.25,
1140
+ "77147264": 0.0,
1141
+ "794181626": 1.0,
1142
+ "957101489": 0.0,
1143
+ "998761162": 0.0
1144
+ }
1145
+ },
1146
+ "macro_by_task": {
1147
+ "ci": {
1148
+ "confidence": 0.95,
1149
+ "high": 0.03282051282051283,
1150
+ "low": -0.13686274509803922,
1151
+ "mean": -0.03912518853695324,
1152
+ "n": 5,
1153
+ "num_samples": 500
1154
+ },
1155
+ "mean": -0.03912518853695324,
1156
+ "per_task": {
1157
+ "LiftPegUpright-v1": 0.0,
1158
+ "PickCube-v1": 0.0,
1159
+ "PullCube-v1": -0.20588235294117646,
1160
+ "PushCube-v1": 0.07692307692307693,
1161
+ "StackCube-v1": -0.06666666666666667
1162
+ }
1163
+ },
1164
+ "metric": "selected_success_gain_over_base_at_8",
1165
+ "micro": {
1166
+ "confidence": 0.95,
1167
+ "high": 0.0,
1168
+ "low": -0.21739130434782608,
1169
+ "mean": -0.10144927536231885,
1170
+ "n": 69,
1171
+ "num_samples": 500
1172
+ },
1173
+ "n": 69
1174
+ },
1175
+ "selected_utility_at_8": {
1176
+ "macro_by_seed": {
1177
+ "ci": {
1178
+ "confidence": 0.95,
1179
+ "high": 0.7957249261758369,
1180
+ "low": 0.38785451346428745,
1181
+ "mean": 0.5988144476751877,
1182
+ "n": 47,
1183
+ "num_samples": 500
1184
+ },
1185
+ "mean": 0.5988144476751877,
1186
+ "per_seed": {
1187
+ "1025740212": 0.2209487408399582,
1188
+ "1055157354": 0.18621200323104858,
1189
+ "1055567618": 0.17464876174926758,
1190
+ "1113268940": 0.47868117690086365,
1191
+ "1113371261": 2.0,
1192
+ "1176943004": 0.02513018250465393,
1193
+ "1189392917": 2.0,
1194
+ "1231780997": 2.0,
1195
+ "1271718007": 1.10751623660326,
1196
+ "1273247072": 0.04248468950390816,
1197
+ "1358421325": 0.1808367669582367,
1198
+ "1375492786": 0.001317779184319079,
1199
+ "141984615": 0.03509971871972084,
1200
+ "1475440973": 0.710078775882721,
1201
+ "1483645336": 0.06921124458312988,
1202
+ "1522272686": 0.19632326066493988,
1203
+ "1528740417": 2.0,
1204
+ "1549324805": 0.6769271823577583,
1205
+ "1607773860": 0.2231033444404602,
1206
+ "1665739511": 2.0,
1207
+ "1747178030": 0.08418524265289307,
1208
+ "1881223807": 0.025857940316200256,
1209
+ "1919198188": 0.046244338154792786,
1210
+ "1991414747": 0.013612315058708191,
1211
+ "1997090716": 2.0,
1212
+ "2015956957": 0.034096240811049935,
1213
+ "2054899126": 2.0,
1214
+ "211245842": 0.15655378997325897,
1215
+ "2141802816": 0.07612672448158264,
1216
+ "256770874": 0.10803383588790894,
1217
+ "278008524": 0.020062235475052148,
1218
+ "416183106": 0.23312881588935852,
1219
+ "534717141": 0.04410672187805176,
1220
+ "551504635": 0.7224790453910828,
1221
+ "565369392": 0.12740397453308105,
1222
+ "565621826": 2.0,
1223
+ "585904130": 0.02085024770349264,
1224
+ "626486813": 0.04864858591463417,
1225
+ "636680137": 0.013928472995758057,
1226
+ "706479600": 2.0,
1227
+ "750448296": 0.046416282653808594,
1228
+ "75070641": 0.22657974064350128,
1229
+ "767747424": 0.06781683932058513,
1230
+ "77147264": 1.426636129617691,
1231
+ "794181626": 2.0,
1232
+ "957101489": 0.22424602508544922,
1233
+ "998761162": 0.04874563217163086
1234
+ }
1235
+ },
1236
+ "macro_by_task": {
1237
+ "ci": {
1238
+ "confidence": 0.95,
1239
+ "high": 0.8462622490316436,
1240
+ "low": 0.19835559911861575,
1241
+ "mean": 0.5396028765330518,
1242
+ "n": 5,
1243
+ "num_samples": 500
1244
+ },
1245
+ "mean": 0.5396028765330518,
1246
+ "per_task": {
1247
+ "LiftPegUpright-v1": 1.0220312898357709,
1248
+ "PickCube-v1": 0.04248468950390816,
1249
+ "PullCube-v1": 0.2192637944552938,
1250
+ "PushCube-v1": 0.9459535811956112,
1251
+ "StackCube-v1": 0.468281027674675
1252
+ }
1253
+ },
1254
+ "metric": "selected_utility_at_8",
1255
+ "micro": {
1256
+ "confidence": 0.95,
1257
+ "high": 0.6843658976587534,
1258
+ "low": 0.3229842852494018,
1259
+ "mean": 0.4775543972559651,
1260
+ "n": 69,
1261
+ "num_samples": 500
1262
+ },
1263
+ "n": 69
1264
+ },
1265
+ "selected_utility_gain_over_base_at_8": {
1266
+ "macro_by_seed": {
1267
+ "ci": {
1268
+ "confidence": 0.95,
1269
+ "high": -0.010564623637940616,
1270
+ "low": -0.5054863751322652,
1271
+ "mean": -0.25787136138223626,
1272
+ "n": 47,
1273
+ "num_samples": 500
1274
+ },
1275
+ "mean": -0.25787136138223626,
1276
+ "per_seed": {
1277
+ "1025740212": -0.3074249178171158,
1278
+ "1055157354": 0.009720906615257263,
1279
+ "1055567618": -1.8253512382507324,
1280
+ "1113268940": 0.05636864900588989,
1281
+ "1113371261": 0.0,
1282
+ "1176943004": -0.032903000712394714,
1283
+ "1189392917": 1.8915813565254211,
1284
+ "1231780997": 1.2729938626289368,
1285
+ "1271718007": -0.3078648969531059,
1286
+ "1273247072": -0.1109449677169323,
1287
+ "1358421325": -0.6516397297382355,
1288
+ "1375492786": -1.998682220815681,
1289
+ "141984615": -0.05871242036422094,
1290
+ "1475440973": -1.289921224117279,
1291
+ "1483645336": -1.9307887554168701,
1292
+ "1522272686": -0.0021198540925979614,
1293
+ "1528740417": 0.0,
1294
+ "1549324805": -0.7018202659673989,
1295
+ "1607773860": -1.7768966555595398,
1296
+ "1665739511": 0.0,
1297
+ "1747178030": -0.06212134659290314,
1298
+ "1881223807": -0.006169050931930542,
1299
+ "1919198188": 0.009045466780662537,
1300
+ "1991414747": -1.9863876849412918,
1301
+ "1997090716": 1.8607252091169357,
1302
+ "2015956957": -0.033023549057543275,
1303
+ "2054899126": 0.0,
1304
+ "211245842": -0.6822806149721146,
1305
+ "2141802816": 0.005319476127624512,
1306
+ "256770874": -0.7477142214775085,
1307
+ "278008524": -0.08740963292075321,
1308
+ "416183106": 0.03280240297317505,
1309
+ "534717141": -1.9558932781219482,
1310
+ "551504635": -0.00224381685256958,
1311
+ "565369392": -0.6723616719245911,
1312
+ "565621826": 0.0,
1313
+ "585904130": -0.05444185032198826,
1314
+ "626486813": -0.4567914307117462,
1315
+ "636680137": -1.986071527004242,
1316
+ "706479600": 1.1649112105369568,
1317
+ "750448296": -0.06511727720499039,
1318
+ "75070641": 0.02382810413837433,
1319
+ "767747424": -0.47970734839327633,
1320
+ "77147264": 0.005056649446487427,
1321
+ "794181626": 1.8190439492464066,
1322
+ "957101489": 0.02052415907382965,
1323
+ "998761162": -0.019070938229560852
1324
+ }
1325
+ },
1326
+ "macro_by_task": {
1327
+ "ci": {
1328
+ "confidence": 0.95,
1329
+ "high": -0.0019037275818678035,
1330
+ "low": -0.33526441148838354,
1331
+ "mean": -0.1599399517973194,
1332
+ "n": 5,
1333
+ "num_samples": 500
1334
+ },
1335
+ "mean": -0.1599399517973194,
1336
+ "per_task": {
1337
+ "LiftPegUpright-v1": -0.04503790785868963,
1338
+ "PickCube-v1": -0.1109449677169323,
1339
+ "PullCube-v1": -0.4319661263280603,
1340
+ "PushCube-v1": 0.12363501351613265,
1341
+ "StackCube-v1": -0.33538577059904734
1342
+ }
1343
+ },
1344
+ "metric": "selected_utility_gain_over_base_at_8",
1345
+ "micro": {
1346
+ "confidence": 0.95,
1347
+ "high": -0.06954942767143897,
1348
+ "low": -0.4864171527484702,
1349
+ "mean": -0.26799350859853777,
1350
+ "n": 69,
1351
+ "num_samples": 500
1352
+ },
1353
+ "n": 69
1354
+ },
1355
+ "selector_regret_at_8": {
1356
+ "macro_by_seed": {
1357
+ "ci": {
1358
+ "confidence": 0.95,
1359
+ "high": 0.6905135366465427,
1360
+ "low": 0.2873830998237971,
1361
+ "mean": 0.46333538608845154,
1362
+ "n": 47,
1363
+ "num_samples": 500
1364
+ },
1365
+ "mean": 0.46333538608845154,
1366
+ "per_seed": {
1367
+ "1025740212": 0.47739143669605255,
1368
+ "1055157354": 0.6370042562484741,
1369
+ "1055567618": 1.8253512382507324,
1370
+ "1113268940": 0.18306860327720642,
1371
+ "1113371261": 0.0,
1372
+ "1176943004": 1.974869817495346,
1373
+ "1189392917": 0.0,
1374
+ "1231780997": 0.0,
1375
+ "1271718007": 0.00353144109249115,
1376
+ "1273247072": 0.45726464316248894,
1377
+ "1358421325": 0.6467066705226898,
1378
+ "1375492786": 0.00047226750757545233,
1379
+ "141984615": 0.05730179324746132,
1380
+ "1475440973": 1.289921224117279,
1381
+ "1483645336": 1.9307887554168701,
1382
+ "1522272686": 0.03616611659526825,
1383
+ "1528740417": 0.0,
1384
+ "1549324805": 0.0015795694974561532,
1385
+ "1607773860": 0.6036040186882019,
1386
+ "1665739511": 0.0,
1387
+ "1747178030": 0.027157030999660492,
1388
+ "1881223807": 0.009028241038322449,
1389
+ "1919198188": 0.0,
1390
+ "1991414747": 1.9863876849412918,
1391
+ "1997090716": 0.0,
1392
+ "2015956957": 0.022195855714380743,
1393
+ "2054899126": 0.0,
1394
+ "211245842": 0.6792904287576675,
1395
+ "2141802816": 0.019766509532928467,
1396
+ "256770874": 0.7044661641120911,
1397
+ "278008524": 0.011409573198761791,
1398
+ "416183106": 1.7668711841106415,
1399
+ "534717141": 1.9558932781219482,
1400
+ "551504635": 0.0,
1401
+ "565369392": 0.046260833740234375,
1402
+ "565621826": 0.0,
1403
+ "585904130": 0.012068324722349644,
1404
+ "626486813": 0.02500777377281338,
1405
+ "636680137": 0.0,
1406
+ "706479600": 0.0,
1407
+ "750448296": 0.02701035887002945,
1408
+ "75070641": 0.5863077491521835,
1409
+ "767747424": 0.04561196081340313,
1410
+ "77147264": 0.0,
1411
+ "794181626": 0.0,
1412
+ "957101489": 1.7757539749145508,
1413
+ "998761162": 1.9512543678283691
1414
+ }
1415
+ },
1416
+ "macro_by_task": {
1417
+ "ci": {
1418
+ "confidence": 0.95,
1419
+ "high": 0.5835942497964088,
1420
+ "low": 0.22782261933967035,
1421
+ "mean": 0.4178369468370689,
1422
+ "n": 5,
1423
+ "num_samples": 500
1424
+ },
1425
+ "mean": 0.4178369468370689,
1426
+ "per_task": {
1427
+ "LiftPegUpright-v1": 0.3250635440150897,
1428
+ "PickCube-v1": 0.45726464316248894,
1429
+ "PullCube-v1": 0.08196123232654132,
1430
+ "PushCube-v1": 0.6069896691120588,
1431
+ "StackCube-v1": 0.6179056455691655
1432
+ }
1433
+ },
1434
+ "metric": "selector_regret_at_8",
1435
+ "micro": {
1436
+ "confidence": 0.95,
1437
+ "high": 0.48046939117341314,
1438
+ "low": 0.18767418079685583,
1439
+ "mean": 0.32396779983115476,
1440
+ "n": 69,
1441
+ "num_samples": 500
1442
+ },
1443
+ "n": 69
1444
+ },
1445
+ "success_selector_gap_at_8": {
1446
+ "macro_by_seed": {
1447
+ "ci": {
1448
+ "confidence": 0.95,
1449
+ "high": 0.3191489361702128,
1450
+ "low": 0.0851063829787234,
1451
+ "mean": 0.19148936170212766,
1452
+ "n": 47,
1453
+ "num_samples": 500
1454
+ },
1455
+ "mean": 0.19148936170212766,
1456
+ "per_seed": {
1457
+ "1025740212": 0.0,
1458
+ "1055157354": 0.0,
1459
+ "1055567618": 1.0,
1460
+ "1113268940": 0.0,
1461
+ "1113371261": 0.0,
1462
+ "1176943004": 1.0,
1463
+ "1189392917": 0.0,
1464
+ "1231780997": 0.0,
1465
+ "1271718007": 0.0,
1466
+ "1273247072": 0.0,
1467
+ "1358421325": 0.0,
1468
+ "1375492786": 0.0,
1469
+ "141984615": 0.0,
1470
+ "1475440973": 1.0,
1471
+ "1483645336": 1.0,
1472
+ "1522272686": 0.0,
1473
+ "1528740417": 0.0,
1474
+ "1549324805": 0.0,
1475
+ "1607773860": 0.0,
1476
+ "1665739511": 0.0,
1477
+ "1747178030": 0.0,
1478
+ "1881223807": 0.0,
1479
+ "1919198188": 0.0,
1480
+ "1991414747": 1.0,
1481
+ "1997090716": 0.0,
1482
+ "2015956957": 0.0,
1483
+ "2054899126": 0.0,
1484
+ "211245842": 0.0,
1485
+ "2141802816": 0.0,
1486
+ "256770874": 0.0,
1487
+ "278008524": 0.0,
1488
+ "416183106": 1.0,
1489
+ "534717141": 1.0,
1490
+ "551504635": 0.0,
1491
+ "565369392": 0.0,
1492
+ "565621826": 0.0,
1493
+ "585904130": 0.0,
1494
+ "626486813": 0.0,
1495
+ "636680137": 0.0,
1496
+ "706479600": 0.0,
1497
+ "750448296": 0.0,
1498
+ "75070641": 0.0,
1499
+ "767747424": 0.0,
1500
+ "77147264": 0.0,
1501
+ "794181626": 0.0,
1502
+ "957101489": 1.0,
1503
+ "998761162": 1.0
1504
+ }
1505
+ },
1506
+ "macro_by_task": {
1507
+ "ci": {
1508
+ "confidence": 0.95,
1509
+ "high": 0.24307692307692305,
1510
+ "low": 0.045098039215686274,
1511
+ "mean": 0.14075414781297133,
1512
+ "n": 5,
1513
+ "num_samples": 500
1514
+ },
1515
+ "mean": 0.14075414781297133,
1516
+ "per_task": {
1517
+ "LiftPegUpright-v1": 0.16666666666666666,
1518
+ "PickCube-v1": 0.0,
1519
+ "PullCube-v1": 0.029411764705882353,
1520
+ "PushCube-v1": 0.3076923076923077,
1521
+ "StackCube-v1": 0.2
1522
+ }
1523
+ },
1524
+ "metric": "success_selector_gap_at_8",
1525
+ "micro": {
1526
+ "confidence": 0.95,
1527
+ "high": 0.21739130434782608,
1528
+ "low": 0.057971014492753624,
1529
+ "mean": 0.13043478260869565,
1530
+ "n": 69,
1531
+ "num_samples": 500
1532
+ },
1533
+ "n": 69
1534
+ },
1535
+ "success_support_gap_at_8": {
1536
+ "macro_by_seed": {
1537
+ "ci": {
1538
+ "confidence": 0.95,
1539
+ "high": 0.551418439716312,
1540
+ "low": 0.2726950354609929,
1541
+ "mean": 0.4031914893617021,
1542
+ "n": 47,
1543
+ "num_samples": 500
1544
+ },
1545
+ "mean": 0.4031914893617021,
1546
+ "per_seed": {
1547
+ "1025740212": 1.0,
1548
+ "1055157354": 1.0,
1549
+ "1055567618": 0.0,
1550
+ "1113268940": 1.0,
1551
+ "1113371261": 0.0,
1552
+ "1176943004": 0.0,
1553
+ "1189392917": 0.0,
1554
+ "1231780997": 0.0,
1555
+ "1271718007": 0.5,
1556
+ "1273247072": 1.0,
1557
+ "1358421325": 1.0,
1558
+ "1375492786": 0.0,
1559
+ "141984615": 0.0,
1560
+ "1475440973": 0.0,
1561
+ "1483645336": 0.0,
1562
+ "1522272686": 1.0,
1563
+ "1528740417": 0.0,
1564
+ "1549324805": 0.3333333333333333,
1565
+ "1607773860": 1.0,
1566
+ "1665739511": 0.0,
1567
+ "1747178030": 1.0,
1568
+ "1881223807": 1.0,
1569
+ "1919198188": 1.0,
1570
+ "1991414747": 0.0,
1571
+ "1997090716": 0.0,
1572
+ "2015956957": 0.2,
1573
+ "2054899126": 0.0,
1574
+ "211245842": 1.0,
1575
+ "2141802816": 1.0,
1576
+ "256770874": 1.0,
1577
+ "278008524": 0.5,
1578
+ "416183106": 0.0,
1579
+ "534717141": 0.0,
1580
+ "551504635": 1.0,
1581
+ "565369392": 0.6666666666666666,
1582
+ "565621826": 0.0,
1583
+ "585904130": 0.0,
1584
+ "626486813": 0.25,
1585
+ "636680137": 1.0,
1586
+ "706479600": 0.0,
1587
+ "750448296": 0.0,
1588
+ "75070641": 1.0,
1589
+ "767747424": 0.0,
1590
+ "77147264": 0.5,
1591
+ "794181626": 0.0,
1592
+ "957101489": 0.0,
1593
+ "998761162": 0.0
1594
+ }
1595
+ },
1596
+ "macro_by_task": {
1597
+ "ci": {
1598
+ "confidence": 0.95,
1599
+ "high": 0.7794871794871795,
1600
+ "low": 0.2746606334841629,
1601
+ "mean": 0.5206636500754148,
1602
+ "n": 5,
1603
+ "num_samples": 500
1604
+ },
1605
+ "mean": 0.5206636500754148,
1606
+ "per_task": {
1607
+ "LiftPegUpright-v1": 0.5,
1608
+ "PickCube-v1": 1.0,
1609
+ "PullCube-v1": 0.20588235294117646,
1610
+ "PushCube-v1": 0.23076923076923078,
1611
+ "StackCube-v1": 0.6666666666666666
1612
+ }
1613
+ },
1614
+ "metric": "success_support_gap_at_8",
1615
+ "micro": {
1616
+ "confidence": 0.95,
1617
+ "high": 0.463768115942029,
1618
+ "low": 0.2318840579710145,
1619
+ "mean": 0.34782608695652173,
1620
+ "n": 69,
1621
+ "num_samples": 500
1622
+ },
1623
+ "n": 69
1624
+ },
1625
+ "success_total_car_to_hidden_at_8": {
1626
+ "macro_by_seed": {
1627
+ "ci": {
1628
+ "confidence": 0.95,
1629
+ "high": 0.7304964539007093,
1630
+ "low": 0.45886524822695035,
1631
+ "mean": 0.5946808510638298,
1632
+ "n": 47,
1633
+ "num_samples": 500
1634
+ },
1635
+ "mean": 0.5946808510638298,
1636
+ "per_seed": {
1637
+ "1025740212": 1.0,
1638
+ "1055157354": 1.0,
1639
+ "1055567618": 1.0,
1640
+ "1113268940": 1.0,
1641
+ "1113371261": 0.0,
1642
+ "1176943004": 1.0,
1643
+ "1189392917": 0.0,
1644
+ "1231780997": 0.0,
1645
+ "1271718007": 0.5,
1646
+ "1273247072": 1.0,
1647
+ "1358421325": 1.0,
1648
+ "1375492786": 0.0,
1649
+ "141984615": 0.0,
1650
+ "1475440973": 1.0,
1651
+ "1483645336": 1.0,
1652
+ "1522272686": 1.0,
1653
+ "1528740417": 0.0,
1654
+ "1549324805": 0.3333333333333333,
1655
+ "1607773860": 1.0,
1656
+ "1665739511": 0.0,
1657
+ "1747178030": 1.0,
1658
+ "1881223807": 1.0,
1659
+ "1919198188": 1.0,
1660
+ "1991414747": 1.0,
1661
+ "1997090716": 0.0,
1662
+ "2015956957": 0.2,
1663
+ "2054899126": 0.0,
1664
+ "211245842": 1.0,
1665
+ "2141802816": 1.0,
1666
+ "256770874": 1.0,
1667
+ "278008524": 0.5,
1668
+ "416183106": 1.0,
1669
+ "534717141": 1.0,
1670
+ "551504635": 1.0,
1671
+ "565369392": 0.6666666666666666,
1672
+ "565621826": 0.0,
1673
+ "585904130": 0.0,
1674
+ "626486813": 0.25,
1675
+ "636680137": 1.0,
1676
+ "706479600": 0.0,
1677
+ "750448296": 0.0,
1678
+ "75070641": 1.0,
1679
+ "767747424": 0.0,
1680
+ "77147264": 0.5,
1681
+ "794181626": 0.0,
1682
+ "957101489": 1.0,
1683
+ "998761162": 1.0
1684
+ }
1685
+ },
1686
+ "macro_by_task": {
1687
+ "ci": {
1688
+ "confidence": 0.95,
1689
+ "high": 0.8810256410256411,
1690
+ "low": 0.41719457013574657,
1691
+ "mean": 0.6614177978883861,
1692
+ "n": 5,
1693
+ "num_samples": 500
1694
+ },
1695
+ "mean": 0.6614177978883861,
1696
+ "per_task": {
1697
+ "LiftPegUpright-v1": 0.6666666666666666,
1698
+ "PickCube-v1": 1.0,
1699
+ "PullCube-v1": 0.23529411764705882,
1700
+ "PushCube-v1": 0.5384615384615384,
1701
+ "StackCube-v1": 0.8666666666666667
1702
+ }
1703
+ },
1704
+ "metric": "success_total_car_to_hidden_at_8",
1705
+ "micro": {
1706
+ "confidence": 0.95,
1707
+ "high": 0.5942028985507246,
1708
+ "low": 0.36231884057971014,
1709
+ "mean": 0.4782608695652174,
1710
+ "n": 69,
1711
+ "num_samples": 500
1712
+ },
1713
+ "n": 69
1714
+ },
1715
+ "support_gap_at_8": {
1716
+ "macro_by_seed": {
1717
+ "ci": {
1718
+ "confidence": 0.95,
1719
+ "high": 0.8704533781212297,
1720
+ "low": 0.4250871043820726,
1721
+ "mean": 0.6443930889275058,
1722
+ "n": 47,
1723
+ "num_samples": 500
1724
+ },
1725
+ "mean": 0.6443930889275058,
1726
+ "per_seed": {
1727
+ "1025740212": 1.3016598224639893,
1728
+ "1055157354": 1.1767837405204773,
1729
+ "1055567618": 0.0,
1730
+ "1113268940": 1.33825021982193,
1731
+ "1113371261": 0.0,
1732
+ "1176943004": 0.0,
1733
+ "1189392917": 0.0,
1734
+ "1231780997": 0.0,
1735
+ "1271718007": 0.8889523223042488,
1736
+ "1273247072": 1.500250667333603,
1737
+ "1358421325": 1.1724565625190735,
1738
+ "1375492786": 0.2416609227657318,
1739
+ "141984615": 0.11134201536575954,
1740
+ "1475440973": 0.0,
1741
+ "1483645336": 0.0,
1742
+ "1522272686": 1.7675106227397919,
1743
+ "1528740417": 0.0,
1744
+ "1549324805": 0.7242437700430552,
1745
+ "1607773860": 1.173292636871338,
1746
+ "1665739511": 0.0,
1747
+ "1747178030": 1.8886577263474464,
1748
+ "1881223807": 1.9651138186454773,
1749
+ "1919198188": 1.9537556618452072,
1750
+ "1991414747": 0.0,
1751
+ "1997090716": 0.0,
1752
+ "2015956957": 0.4735892705619335,
1753
+ "2054899126": 0.0,
1754
+ "211245842": 1.1641557812690735,
1755
+ "2141802816": 1.904106765985489,
1756
+ "256770874": 1.1875,
1757
+ "278008524": 1.082339386222884,
1758
+ "416183106": 0.0,
1759
+ "534717141": 0.0,
1760
+ "551504635": 1.2775209546089172,
1761
+ "565369392": 1.2529667715231578,
1762
+ "565621826": 0.0,
1763
+ "585904130": 0.20472841958204904,
1764
+ "626486813": 0.5414478182792664,
1765
+ "636680137": 1.986071527004242,
1766
+ "706479600": 0.0,
1767
+ "750448296": 0.16238094866275787,
1768
+ "75070641": 1.1871125102043152,
1769
+ "767747424": 0.08526064571924508,
1770
+ "77147264": 0.573363870382309,
1771
+ "794181626": 0.0,
1772
+ "957101489": 0.0,
1773
+ "998761162": 0.0
1774
+ }
1775
+ },
1776
+ "macro_by_task": {
1777
+ "ci": {
1778
+ "confidence": 0.95,
1779
+ "high": 1.1794891647094239,
1780
+ "low": 0.5025559310228205,
1781
+ "mean": 0.7993812809442767,
1782
+ "n": 5,
1783
+ "num_samples": 500
1784
+ },
1785
+ "mean": 0.7993812809442767,
1786
+ "per_task": {
1787
+ "LiftPegUpright-v1": 0.6529051661491394,
1788
+ "PickCube-v1": 1.500250667333603,
1789
+ "PullCube-v1": 0.4828804947901517,
1790
+ "PushCube-v1": 0.44705674969233,
1791
+ "StackCube-v1": 0.9138133267561594
1792
+ }
1793
+ },
1794
+ "metric": "support_gap_at_8",
1795
+ "micro": {
1796
+ "confidence": 0.95,
1797
+ "high": 0.7763315162122033,
1798
+ "low": 0.4135484881631598,
1799
+ "mean": 0.5993413932526998,
1800
+ "n": 69,
1801
+ "num_samples": 500
1802
+ },
1803
+ "n": 69
1804
+ },
1805
+ "total_car_to_hidden_at_8": {
1806
+ "macro_by_seed": {
1807
+ "ci": {
1808
+ "confidence": 0.95,
1809
+ "high": 1.339665490579098,
1810
+ "low": 0.8604674879511063,
1811
+ "mean": 1.1073883634887087,
1812
+ "n": 47,
1813
+ "num_samples": 500
1814
+ },
1815
+ "mean": 1.1073883634887087,
1816
+ "per_seed": {
1817
+ "1025740212": 1.7790512591600418,
1818
+ "1055157354": 1.8137879967689514,
1819
+ "1055567618": 1.8253512382507324,
1820
+ "1113268940": 1.5213188230991364,
1821
+ "1113371261": 0.0,
1822
+ "1176943004": 1.974869817495346,
1823
+ "1189392917": 0.0,
1824
+ "1231780997": 0.0,
1825
+ "1271718007": 0.89248376339674,
1826
+ "1273247072": 1.9575153104960918,
1827
+ "1358421325": 1.8191632330417633,
1828
+ "1375492786": 0.24213319027330726,
1829
+ "141984615": 0.1653481237590313,
1830
+ "1475440973": 1.289921224117279,
1831
+ "1483645336": 1.9307887554168701,
1832
+ "1522272686": 1.8036767393350601,
1833
+ "1528740417": 0.0,
1834
+ "1549324805": 0.7258233395405114,
1835
+ "1607773860": 1.7768966555595398,
1836
+ "1665739511": 0.0,
1837
+ "1747178030": 1.915814757347107,
1838
+ "1881223807": 1.9741420596837997,
1839
+ "1919198188": 1.9537556618452072,
1840
+ "1991414747": 1.9863876849412918,
1841
+ "1997090716": 0.0,
1842
+ "2015956957": 0.49578512627631427,
1843
+ "2054899126": 0.0,
1844
+ "211245842": 1.843446210026741,
1845
+ "2141802816": 1.9238732755184174,
1846
+ "256770874": 1.891966164112091,
1847
+ "278008524": 1.0937489594216458,
1848
+ "416183106": 1.7668711841106415,
1849
+ "534717141": 1.9558932781219482,
1850
+ "551504635": 1.2775209546089172,
1851
+ "565369392": 1.2992276052633922,
1852
+ "565621826": 0.0,
1853
+ "585904130": 0.2167967443043987,
1854
+ "626486813": 0.5638198243686929,
1855
+ "636680137": 1.986071527004242,
1856
+ "706479600": 0.0,
1857
+ "750448296": 0.18939130753278732,
1858
+ "75070641": 1.7734202593564987,
1859
+ "767747424": 0.12081881728954613,
1860
+ "77147264": 0.573363870382309,
1861
+ "794181626": 0.0,
1862
+ "957101489": 1.7757539749145508,
1863
+ "998761162": 1.9512543678283691
1864
+ }
1865
+ },
1866
+ "macro_by_task": {
1867
+ "ci": {
1868
+ "confidence": 0.95,
1869
+ "high": 1.676446722795566,
1870
+ "low": 0.8272199847642051,
1871
+ "mean": 1.2168614908268247,
1872
+ "n": 5,
1873
+ "num_samples": 500
1874
+ },
1875
+ "mean": 1.2168614908268247,
1876
+ "per_task": {
1877
+ "LiftPegUpright-v1": 0.977968710164229,
1878
+ "PickCube-v1": 1.9575153104960918,
1879
+ "PullCube-v1": 0.5630580423440894,
1880
+ "PushCube-v1": 1.0540464188043888,
1881
+ "StackCube-v1": 1.531718972325325
1882
+ }
1883
+ },
1884
+ "metric": "total_car_to_hidden_at_8",
1885
+ "micro": {
1886
+ "confidence": 0.95,
1887
+ "high": 1.1262573117906551,
1888
+ "low": 0.7123256691755133,
1889
+ "mean": 0.9224302759495281,
1890
+ "n": 69,
1891
+ "num_samples": 500
1892
+ },
1893
+ "n": 69
1894
+ }
1895
+ },
1896
+ "num_rows": 69,
1897
+ "report_type": "ctt_generated_measured_rollout",
1898
+ "schema_version": 1,
1899
+ "source_content_hash": "90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957",
1900
+ "source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372",
1901
+ "success_summary": {
1902
+ "base_success_rate": 0.2898550724637681,
1903
+ "base_utility_mean": 0.7455479058545029,
1904
+ "hidden_chart_oracle_success_rate": 0.6666666666666666,
1905
+ "hidden_chart_oracle_utility_mean": 1.3999846732054932,
1906
+ "max_restore_error": 1.4901161193847656e-08,
1907
+ "proposal_oracle_success_gain_over_base": 0.028985507246376812,
1908
+ "proposal_oracle_success_rate": 0.3188405797101449,
1909
+ "proposal_oracle_utility_mean": 0.8015221970871199,
1910
+ "selected_success_gain_over_base": -0.10144927536231885,
1911
+ "selected_success_rate": 0.18840579710144928,
1912
+ "selected_utility_mean": 0.4775543972559651,
1913
+ "success_selector_gap": 0.13043478260869565,
1914
+ "success_support_gap": 0.34782608695652173
1915
+ },
1916
+ "target_content_hash": "fa0fe0e3881ee70ce91492445871614d01b3a47fc7c576659c53ff17ddee4e42",
1917
+ "target_split_hash": "9d1dc1c10868f76fa080895df0f2730c173f12b6db14932599d55c2b60859381"
1918
+ }
workspace/runs/ctt_residual_rollout_val69_seed1/metrics_by_seed.json ADDED
@@ -0,0 +1,1083 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "1025740212": {
3
+ "base_success": 0.0,
4
+ "base_utility": 0.528373658657074,
5
+ "branch_car_at_8": 0.47739143669605255,
6
+ "hidden_chart_oracle_success_at_8": 1.0,
7
+ "hidden_chart_oracle_utility_at_8": 2.0,
8
+ "outcome_ptr_at_8": 1.0,
9
+ "pairwise_causal_calibration_ece": 0.12992986623548633,
10
+ "proposal_oracle_success_at_8": 0.0,
11
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
12
+ "proposal_oracle_utility_at_8": 0.6983401775360107,
13
+ "proposal_oracle_utility_gain_over_base_at_8": 0.16996651887893677,
14
+ "selected_success_at_8": 0.0,
15
+ "selected_success_gain_over_base_at_8": 0.0,
16
+ "selected_utility_at_8": 0.2209487408399582,
17
+ "selected_utility_gain_over_base_at_8": -0.3074249178171158,
18
+ "selector_regret_at_8": 0.47739143669605255,
19
+ "success_selector_gap_at_8": 0.0,
20
+ "success_support_gap_at_8": 1.0,
21
+ "success_total_car_to_hidden_at_8": 1.0,
22
+ "support_gap_at_8": 1.3016598224639893,
23
+ "total_car_to_hidden_at_8": 1.7790512591600418
24
+ },
25
+ "1055157354": {
26
+ "base_success": 0.0,
27
+ "base_utility": 0.17649109661579132,
28
+ "branch_car_at_8": 0.6370042562484741,
29
+ "hidden_chart_oracle_success_at_8": 1.0,
30
+ "hidden_chart_oracle_utility_at_8": 2.0,
31
+ "outcome_ptr_at_8": 1.0,
32
+ "pairwise_causal_calibration_ece": 0.07450750352404123,
33
+ "proposal_oracle_success_at_8": 0.0,
34
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
35
+ "proposal_oracle_utility_at_8": 0.8232162594795227,
36
+ "proposal_oracle_utility_gain_over_base_at_8": 0.6467251628637314,
37
+ "selected_success_at_8": 0.0,
38
+ "selected_success_gain_over_base_at_8": 0.0,
39
+ "selected_utility_at_8": 0.18621200323104858,
40
+ "selected_utility_gain_over_base_at_8": 0.009720906615257263,
41
+ "selector_regret_at_8": 0.6370042562484741,
42
+ "success_selector_gap_at_8": 0.0,
43
+ "success_support_gap_at_8": 1.0,
44
+ "success_total_car_to_hidden_at_8": 1.0,
45
+ "support_gap_at_8": 1.1767837405204773,
46
+ "total_car_to_hidden_at_8": 1.8137879967689514
47
+ },
48
+ "1055567618": {
49
+ "base_success": 1.0,
50
+ "base_utility": 2.0,
51
+ "branch_car_at_8": 1.8253512382507324,
52
+ "hidden_chart_oracle_success_at_8": 1.0,
53
+ "hidden_chart_oracle_utility_at_8": 2.0,
54
+ "outcome_ptr_at_8": 0.0,
55
+ "pairwise_causal_calibration_ece": 0.318234334809661,
56
+ "proposal_oracle_success_at_8": 1.0,
57
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
58
+ "proposal_oracle_utility_at_8": 2.0,
59
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
60
+ "selected_success_at_8": 0.0,
61
+ "selected_success_gain_over_base_at_8": -1.0,
62
+ "selected_utility_at_8": 0.17464876174926758,
63
+ "selected_utility_gain_over_base_at_8": -1.8253512382507324,
64
+ "selector_regret_at_8": 1.8253512382507324,
65
+ "success_selector_gap_at_8": 1.0,
66
+ "success_support_gap_at_8": 0.0,
67
+ "success_total_car_to_hidden_at_8": 1.0,
68
+ "support_gap_at_8": 0.0,
69
+ "total_car_to_hidden_at_8": 1.8253512382507324
70
+ },
71
+ "1113268940": {
72
+ "base_success": 0.0,
73
+ "base_utility": 0.42231252789497375,
74
+ "branch_car_at_8": 0.18306860327720642,
75
+ "hidden_chart_oracle_success_at_8": 1.0,
76
+ "hidden_chart_oracle_utility_at_8": 2.0,
77
+ "outcome_ptr_at_8": 1.0,
78
+ "pairwise_causal_calibration_ece": 0.10725283874677279,
79
+ "proposal_oracle_success_at_8": 0.0,
80
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
81
+ "proposal_oracle_utility_at_8": 0.6617497801780701,
82
+ "proposal_oracle_utility_gain_over_base_at_8": 0.2394372522830963,
83
+ "selected_success_at_8": 0.0,
84
+ "selected_success_gain_over_base_at_8": 0.0,
85
+ "selected_utility_at_8": 0.47868117690086365,
86
+ "selected_utility_gain_over_base_at_8": 0.05636864900588989,
87
+ "selector_regret_at_8": 0.18306860327720642,
88
+ "success_selector_gap_at_8": 0.0,
89
+ "success_support_gap_at_8": 1.0,
90
+ "success_total_car_to_hidden_at_8": 1.0,
91
+ "support_gap_at_8": 1.33825021982193,
92
+ "total_car_to_hidden_at_8": 1.5213188230991364
93
+ },
94
+ "1113371261": {
95
+ "base_success": 1.0,
96
+ "base_utility": 2.0,
97
+ "branch_car_at_8": 0.0,
98
+ "hidden_chart_oracle_success_at_8": 1.0,
99
+ "hidden_chart_oracle_utility_at_8": 2.0,
100
+ "outcome_ptr_at_8": 0.0,
101
+ "pairwise_causal_calibration_ece": 0.04977883974023356,
102
+ "proposal_oracle_success_at_8": 1.0,
103
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
104
+ "proposal_oracle_utility_at_8": 2.0,
105
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
106
+ "selected_success_at_8": 1.0,
107
+ "selected_success_gain_over_base_at_8": 0.0,
108
+ "selected_utility_at_8": 2.0,
109
+ "selected_utility_gain_over_base_at_8": 0.0,
110
+ "selector_regret_at_8": 0.0,
111
+ "success_selector_gap_at_8": 0.0,
112
+ "success_support_gap_at_8": 0.0,
113
+ "success_total_car_to_hidden_at_8": 0.0,
114
+ "support_gap_at_8": 0.0,
115
+ "total_car_to_hidden_at_8": 0.0
116
+ },
117
+ "1176943004": {
118
+ "base_success": 0.0,
119
+ "base_utility": 0.058033183217048645,
120
+ "branch_car_at_8": 1.974869817495346,
121
+ "hidden_chart_oracle_success_at_8": 1.0,
122
+ "hidden_chart_oracle_utility_at_8": 2.0,
123
+ "outcome_ptr_at_8": 1.0,
124
+ "pairwise_causal_calibration_ece": 0.1815411965846294,
125
+ "proposal_oracle_success_at_8": 1.0,
126
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
127
+ "proposal_oracle_utility_at_8": 2.0,
128
+ "proposal_oracle_utility_gain_over_base_at_8": 1.9419668167829514,
129
+ "selected_success_at_8": 0.0,
130
+ "selected_success_gain_over_base_at_8": 0.0,
131
+ "selected_utility_at_8": 0.02513018250465393,
132
+ "selected_utility_gain_over_base_at_8": -0.032903000712394714,
133
+ "selector_regret_at_8": 1.974869817495346,
134
+ "success_selector_gap_at_8": 1.0,
135
+ "success_support_gap_at_8": 0.0,
136
+ "success_total_car_to_hidden_at_8": 1.0,
137
+ "support_gap_at_8": 0.0,
138
+ "total_car_to_hidden_at_8": 1.974869817495346
139
+ },
140
+ "1189392917": {
141
+ "base_success": 0.0,
142
+ "base_utility": 0.10841864347457886,
143
+ "branch_car_at_8": 0.0,
144
+ "hidden_chart_oracle_success_at_8": 1.0,
145
+ "hidden_chart_oracle_utility_at_8": 2.0,
146
+ "outcome_ptr_at_8": 1.0,
147
+ "pairwise_causal_calibration_ece": 0.2180235973132697,
148
+ "proposal_oracle_success_at_8": 1.0,
149
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
150
+ "proposal_oracle_utility_at_8": 2.0,
151
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211,
152
+ "selected_success_at_8": 1.0,
153
+ "selected_success_gain_over_base_at_8": 1.0,
154
+ "selected_utility_at_8": 2.0,
155
+ "selected_utility_gain_over_base_at_8": 1.8915813565254211,
156
+ "selector_regret_at_8": 0.0,
157
+ "success_selector_gap_at_8": 0.0,
158
+ "success_support_gap_at_8": 0.0,
159
+ "success_total_car_to_hidden_at_8": 0.0,
160
+ "support_gap_at_8": 0.0,
161
+ "total_car_to_hidden_at_8": 0.0
162
+ },
163
+ "1231780997": {
164
+ "base_success": 0.0,
165
+ "base_utility": 0.7270061373710632,
166
+ "branch_car_at_8": 0.0,
167
+ "hidden_chart_oracle_success_at_8": 1.0,
168
+ "hidden_chart_oracle_utility_at_8": 2.0,
169
+ "outcome_ptr_at_8": 1.0,
170
+ "pairwise_causal_calibration_ece": 0.28322114525972475,
171
+ "proposal_oracle_success_at_8": 1.0,
172
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
173
+ "proposal_oracle_utility_at_8": 2.0,
174
+ "proposal_oracle_utility_gain_over_base_at_8": 1.2729938626289368,
175
+ "selected_success_at_8": 1.0,
176
+ "selected_success_gain_over_base_at_8": 1.0,
177
+ "selected_utility_at_8": 2.0,
178
+ "selected_utility_gain_over_base_at_8": 1.2729938626289368,
179
+ "selector_regret_at_8": 0.0,
180
+ "success_selector_gap_at_8": 0.0,
181
+ "success_support_gap_at_8": 0.0,
182
+ "success_total_car_to_hidden_at_8": 0.0,
183
+ "support_gap_at_8": 0.0,
184
+ "total_car_to_hidden_at_8": 0.0
185
+ },
186
+ "1271718007": {
187
+ "base_success": 0.5,
188
+ "base_utility": 1.415381133556366,
189
+ "branch_car_at_8": 0.00353144109249115,
190
+ "hidden_chart_oracle_success_at_8": 1.0,
191
+ "hidden_chart_oracle_utility_at_8": 2.0,
192
+ "outcome_ptr_at_8": 0.0,
193
+ "pairwise_causal_calibration_ece": 0.07023931523066618,
194
+ "proposal_oracle_success_at_8": 0.5,
195
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
196
+ "proposal_oracle_utility_at_8": 1.1110476776957512,
197
+ "proposal_oracle_utility_gain_over_base_at_8": -0.3043334558606148,
198
+ "selected_success_at_8": 0.5,
199
+ "selected_success_gain_over_base_at_8": 0.0,
200
+ "selected_utility_at_8": 1.10751623660326,
201
+ "selected_utility_gain_over_base_at_8": -0.3078648969531059,
202
+ "selector_regret_at_8": 0.00353144109249115,
203
+ "success_selector_gap_at_8": 0.0,
204
+ "success_support_gap_at_8": 0.5,
205
+ "success_total_car_to_hidden_at_8": 0.5,
206
+ "support_gap_at_8": 0.8889523223042488,
207
+ "total_car_to_hidden_at_8": 0.89248376339674
208
+ },
209
+ "1273247072": {
210
+ "base_success": 0.0,
211
+ "base_utility": 0.15342965722084045,
212
+ "branch_car_at_8": 0.45726464316248894,
213
+ "hidden_chart_oracle_success_at_8": 1.0,
214
+ "hidden_chart_oracle_utility_at_8": 2.0,
215
+ "outcome_ptr_at_8": 1.0,
216
+ "pairwise_causal_calibration_ece": 0.096994407338536,
217
+ "proposal_oracle_success_at_8": 0.0,
218
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
219
+ "proposal_oracle_utility_at_8": 0.4997493326663971,
220
+ "proposal_oracle_utility_gain_over_base_at_8": 0.34631967544555664,
221
+ "selected_success_at_8": 0.0,
222
+ "selected_success_gain_over_base_at_8": 0.0,
223
+ "selected_utility_at_8": 0.04248468950390816,
224
+ "selected_utility_gain_over_base_at_8": -0.1109449677169323,
225
+ "selector_regret_at_8": 0.45726464316248894,
226
+ "success_selector_gap_at_8": 0.0,
227
+ "success_support_gap_at_8": 1.0,
228
+ "success_total_car_to_hidden_at_8": 1.0,
229
+ "support_gap_at_8": 1.500250667333603,
230
+ "total_car_to_hidden_at_8": 1.9575153104960918
231
+ },
232
+ "1358421325": {
233
+ "base_success": 0.0,
234
+ "base_utility": 0.8324764966964722,
235
+ "branch_car_at_8": 0.6467066705226898,
236
+ "hidden_chart_oracle_success_at_8": 1.0,
237
+ "hidden_chart_oracle_utility_at_8": 2.0,
238
+ "outcome_ptr_at_8": 0.0,
239
+ "pairwise_causal_calibration_ece": 0.046694794435454434,
240
+ "proposal_oracle_success_at_8": 0.0,
241
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
242
+ "proposal_oracle_utility_at_8": 0.8275434374809265,
243
+ "proposal_oracle_utility_gain_over_base_at_8": -0.004933059215545654,
244
+ "selected_success_at_8": 0.0,
245
+ "selected_success_gain_over_base_at_8": 0.0,
246
+ "selected_utility_at_8": 0.1808367669582367,
247
+ "selected_utility_gain_over_base_at_8": -0.6516397297382355,
248
+ "selector_regret_at_8": 0.6467066705226898,
249
+ "success_selector_gap_at_8": 0.0,
250
+ "success_support_gap_at_8": 1.0,
251
+ "success_total_car_to_hidden_at_8": 1.0,
252
+ "support_gap_at_8": 1.1724565625190735,
253
+ "total_car_to_hidden_at_8": 1.8191632330417633
254
+ },
255
+ "1375492786": {
256
+ "base_success": 1.0,
257
+ "base_utility": 2.0,
258
+ "branch_car_at_8": 0.00047226750757545233,
259
+ "hidden_chart_oracle_success_at_8": 0.0,
260
+ "hidden_chart_oracle_utility_at_8": 0.24345096945762634,
261
+ "outcome_ptr_at_8": 0.0,
262
+ "pairwise_causal_calibration_ece": 0.14027669602920534,
263
+ "proposal_oracle_success_at_8": 0.0,
264
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
265
+ "proposal_oracle_utility_at_8": 0.0017900466918945312,
266
+ "proposal_oracle_utility_gain_over_base_at_8": -1.9982099533081055,
267
+ "selected_success_at_8": 0.0,
268
+ "selected_success_gain_over_base_at_8": -1.0,
269
+ "selected_utility_at_8": 0.001317779184319079,
270
+ "selected_utility_gain_over_base_at_8": -1.998682220815681,
271
+ "selector_regret_at_8": 0.00047226750757545233,
272
+ "success_selector_gap_at_8": 0.0,
273
+ "success_support_gap_at_8": 0.0,
274
+ "success_total_car_to_hidden_at_8": 0.0,
275
+ "support_gap_at_8": 0.2416609227657318,
276
+ "total_car_to_hidden_at_8": 0.24213319027330726
277
+ },
278
+ "141984615": {
279
+ "base_success": 0.0,
280
+ "base_utility": 0.09381213908394177,
281
+ "branch_car_at_8": 0.05730179324746132,
282
+ "hidden_chart_oracle_success_at_8": 0.0,
283
+ "hidden_chart_oracle_utility_at_8": 0.20044784247875214,
284
+ "outcome_ptr_at_8": 0.6666666666666666,
285
+ "pairwise_causal_calibration_ece": 0.08533231968443573,
286
+ "proposal_oracle_success_at_8": 0.0,
287
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
288
+ "proposal_oracle_utility_at_8": 0.09240151196718216,
289
+ "proposal_oracle_utility_gain_over_base_at_8": -0.001410627116759618,
290
+ "selected_success_at_8": 0.0,
291
+ "selected_success_gain_over_base_at_8": 0.0,
292
+ "selected_utility_at_8": 0.03509971871972084,
293
+ "selected_utility_gain_over_base_at_8": -0.05871242036422094,
294
+ "selector_regret_at_8": 0.05730179324746132,
295
+ "success_selector_gap_at_8": 0.0,
296
+ "success_support_gap_at_8": 0.0,
297
+ "success_total_car_to_hidden_at_8": 0.0,
298
+ "support_gap_at_8": 0.11134201536575954,
299
+ "total_car_to_hidden_at_8": 0.1653481237590313
300
+ },
301
+ "1475440973": {
302
+ "base_success": 1.0,
303
+ "base_utility": 2.0,
304
+ "branch_car_at_8": 1.289921224117279,
305
+ "hidden_chart_oracle_success_at_8": 1.0,
306
+ "hidden_chart_oracle_utility_at_8": 2.0,
307
+ "outcome_ptr_at_8": 0.0,
308
+ "pairwise_causal_calibration_ece": 0.2443552809335613,
309
+ "proposal_oracle_success_at_8": 1.0,
310
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
311
+ "proposal_oracle_utility_at_8": 2.0,
312
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
313
+ "selected_success_at_8": 0.0,
314
+ "selected_success_gain_over_base_at_8": -1.0,
315
+ "selected_utility_at_8": 0.710078775882721,
316
+ "selected_utility_gain_over_base_at_8": -1.289921224117279,
317
+ "selector_regret_at_8": 1.289921224117279,
318
+ "success_selector_gap_at_8": 1.0,
319
+ "success_support_gap_at_8": 0.0,
320
+ "success_total_car_to_hidden_at_8": 1.0,
321
+ "support_gap_at_8": 0.0,
322
+ "total_car_to_hidden_at_8": 1.289921224117279
323
+ },
324
+ "1483645336": {
325
+ "base_success": 1.0,
326
+ "base_utility": 2.0,
327
+ "branch_car_at_8": 1.9307887554168701,
328
+ "hidden_chart_oracle_success_at_8": 1.0,
329
+ "hidden_chart_oracle_utility_at_8": 2.0,
330
+ "outcome_ptr_at_8": 0.0,
331
+ "pairwise_causal_calibration_ece": 0.13752192684190806,
332
+ "proposal_oracle_success_at_8": 1.0,
333
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
334
+ "proposal_oracle_utility_at_8": 2.0,
335
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
336
+ "selected_success_at_8": 0.0,
337
+ "selected_success_gain_over_base_at_8": -1.0,
338
+ "selected_utility_at_8": 0.06921124458312988,
339
+ "selected_utility_gain_over_base_at_8": -1.9307887554168701,
340
+ "selector_regret_at_8": 1.9307887554168701,
341
+ "success_selector_gap_at_8": 1.0,
342
+ "success_support_gap_at_8": 0.0,
343
+ "success_total_car_to_hidden_at_8": 1.0,
344
+ "support_gap_at_8": 0.0,
345
+ "total_car_to_hidden_at_8": 1.9307887554168701
346
+ },
347
+ "1522272686": {
348
+ "base_success": 0.0,
349
+ "base_utility": 0.19844311475753784,
350
+ "branch_car_at_8": 0.03616611659526825,
351
+ "hidden_chart_oracle_success_at_8": 1.0,
352
+ "hidden_chart_oracle_utility_at_8": 2.0,
353
+ "outcome_ptr_at_8": 1.0,
354
+ "pairwise_causal_calibration_ece": 0.28555947838938045,
355
+ "proposal_oracle_success_at_8": 0.0,
356
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
357
+ "proposal_oracle_utility_at_8": 0.23248937726020813,
358
+ "proposal_oracle_utility_gain_over_base_at_8": 0.03404626250267029,
359
+ "selected_success_at_8": 0.0,
360
+ "selected_success_gain_over_base_at_8": 0.0,
361
+ "selected_utility_at_8": 0.19632326066493988,
362
+ "selected_utility_gain_over_base_at_8": -0.0021198540925979614,
363
+ "selector_regret_at_8": 0.03616611659526825,
364
+ "success_selector_gap_at_8": 0.0,
365
+ "success_support_gap_at_8": 1.0,
366
+ "success_total_car_to_hidden_at_8": 1.0,
367
+ "support_gap_at_8": 1.7675106227397919,
368
+ "total_car_to_hidden_at_8": 1.8036767393350601
369
+ },
370
+ "1528740417": {
371
+ "base_success": 1.0,
372
+ "base_utility": 2.0,
373
+ "branch_car_at_8": 0.0,
374
+ "hidden_chart_oracle_success_at_8": 1.0,
375
+ "hidden_chart_oracle_utility_at_8": 2.0,
376
+ "outcome_ptr_at_8": 0.0,
377
+ "pairwise_causal_calibration_ece": 0.2757671830981365,
378
+ "proposal_oracle_success_at_8": 1.0,
379
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
380
+ "proposal_oracle_utility_at_8": 2.0,
381
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
382
+ "selected_success_at_8": 1.0,
383
+ "selected_success_gain_over_base_at_8": 0.0,
384
+ "selected_utility_at_8": 2.0,
385
+ "selected_utility_gain_over_base_at_8": 0.0,
386
+ "selector_regret_at_8": 0.0,
387
+ "success_selector_gap_at_8": 0.0,
388
+ "success_support_gap_at_8": 0.0,
389
+ "success_total_car_to_hidden_at_8": 0.0,
390
+ "support_gap_at_8": 0.0,
391
+ "total_car_to_hidden_at_8": 0.0
392
+ },
393
+ "1549324805": {
394
+ "base_success": 0.6666666666666666,
395
+ "base_utility": 1.3787474483251572,
396
+ "branch_car_at_8": 0.0015795694974561532,
397
+ "hidden_chart_oracle_success_at_8": 0.6666666666666666,
398
+ "hidden_chart_oracle_utility_at_8": 1.4027505218982697,
399
+ "outcome_ptr_at_8": 0.0,
400
+ "pairwise_causal_calibration_ece": 0.28625156462411294,
401
+ "proposal_oracle_success_at_8": 0.3333333333333333,
402
+ "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333,
403
+ "proposal_oracle_utility_at_8": 0.6785067518552145,
404
+ "proposal_oracle_utility_gain_over_base_at_8": -0.7002406964699427,
405
+ "selected_success_at_8": 0.3333333333333333,
406
+ "selected_success_gain_over_base_at_8": -0.3333333333333333,
407
+ "selected_utility_at_8": 0.6769271823577583,
408
+ "selected_utility_gain_over_base_at_8": -0.7018202659673989,
409
+ "selector_regret_at_8": 0.0015795694974561532,
410
+ "success_selector_gap_at_8": 0.0,
411
+ "success_support_gap_at_8": 0.3333333333333333,
412
+ "success_total_car_to_hidden_at_8": 0.3333333333333333,
413
+ "support_gap_at_8": 0.7242437700430552,
414
+ "total_car_to_hidden_at_8": 0.7258233395405114
415
+ },
416
+ "1607773860": {
417
+ "base_success": 1.0,
418
+ "base_utility": 2.0,
419
+ "branch_car_at_8": 0.6036040186882019,
420
+ "hidden_chart_oracle_success_at_8": 1.0,
421
+ "hidden_chart_oracle_utility_at_8": 2.0,
422
+ "outcome_ptr_at_8": 0.0,
423
+ "pairwise_causal_calibration_ece": 0.18975917078230178,
424
+ "proposal_oracle_success_at_8": 0.0,
425
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
426
+ "proposal_oracle_utility_at_8": 0.8267073631286621,
427
+ "proposal_oracle_utility_gain_over_base_at_8": -1.173292636871338,
428
+ "selected_success_at_8": 0.0,
429
+ "selected_success_gain_over_base_at_8": -1.0,
430
+ "selected_utility_at_8": 0.2231033444404602,
431
+ "selected_utility_gain_over_base_at_8": -1.7768966555595398,
432
+ "selector_regret_at_8": 0.6036040186882019,
433
+ "success_selector_gap_at_8": 0.0,
434
+ "success_support_gap_at_8": 1.0,
435
+ "success_total_car_to_hidden_at_8": 1.0,
436
+ "support_gap_at_8": 1.173292636871338,
437
+ "total_car_to_hidden_at_8": 1.7768966555595398
438
+ },
439
+ "1665739511": {
440
+ "base_success": 1.0,
441
+ "base_utility": 2.0,
442
+ "branch_car_at_8": 0.0,
443
+ "hidden_chart_oracle_success_at_8": 1.0,
444
+ "hidden_chart_oracle_utility_at_8": 2.0,
445
+ "outcome_ptr_at_8": 0.0,
446
+ "pairwise_causal_calibration_ece": 0.0,
447
+ "proposal_oracle_success_at_8": 1.0,
448
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
449
+ "proposal_oracle_utility_at_8": 2.0,
450
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
451
+ "selected_success_at_8": 1.0,
452
+ "selected_success_gain_over_base_at_8": 0.0,
453
+ "selected_utility_at_8": 2.0,
454
+ "selected_utility_gain_over_base_at_8": 0.0,
455
+ "selector_regret_at_8": 0.0,
456
+ "success_selector_gap_at_8": 0.0,
457
+ "success_support_gap_at_8": 0.0,
458
+ "success_total_car_to_hidden_at_8": 0.0,
459
+ "support_gap_at_8": 0.0,
460
+ "total_car_to_hidden_at_8": 0.0
461
+ },
462
+ "1747178030": {
463
+ "base_success": 0.0,
464
+ "base_utility": 0.1463065892457962,
465
+ "branch_car_at_8": 0.027157030999660492,
466
+ "hidden_chart_oracle_success_at_8": 1.0,
467
+ "hidden_chart_oracle_utility_at_8": 2.0,
468
+ "outcome_ptr_at_8": 0.0,
469
+ "pairwise_causal_calibration_ece": 0.3089551740932353,
470
+ "proposal_oracle_success_at_8": 0.0,
471
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
472
+ "proposal_oracle_utility_at_8": 0.11134227365255356,
473
+ "proposal_oracle_utility_gain_over_base_at_8": -0.034964315593242645,
474
+ "selected_success_at_8": 0.0,
475
+ "selected_success_gain_over_base_at_8": 0.0,
476
+ "selected_utility_at_8": 0.08418524265289307,
477
+ "selected_utility_gain_over_base_at_8": -0.06212134659290314,
478
+ "selector_regret_at_8": 0.027157030999660492,
479
+ "success_selector_gap_at_8": 0.0,
480
+ "success_support_gap_at_8": 1.0,
481
+ "success_total_car_to_hidden_at_8": 1.0,
482
+ "support_gap_at_8": 1.8886577263474464,
483
+ "total_car_to_hidden_at_8": 1.915814757347107
484
+ },
485
+ "1881223807": {
486
+ "base_success": 0.0,
487
+ "base_utility": 0.0320269912481308,
488
+ "branch_car_at_8": 0.009028241038322449,
489
+ "hidden_chart_oracle_success_at_8": 1.0,
490
+ "hidden_chart_oracle_utility_at_8": 2.0,
491
+ "outcome_ptr_at_8": 1.0,
492
+ "pairwise_causal_calibration_ece": 0.13674585276395504,
493
+ "proposal_oracle_success_at_8": 0.0,
494
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
495
+ "proposal_oracle_utility_at_8": 0.034886181354522705,
496
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0028591901063919067,
497
+ "selected_success_at_8": 0.0,
498
+ "selected_success_gain_over_base_at_8": 0.0,
499
+ "selected_utility_at_8": 0.025857940316200256,
500
+ "selected_utility_gain_over_base_at_8": -0.006169050931930542,
501
+ "selector_regret_at_8": 0.009028241038322449,
502
+ "success_selector_gap_at_8": 0.0,
503
+ "success_support_gap_at_8": 1.0,
504
+ "success_total_car_to_hidden_at_8": 1.0,
505
+ "support_gap_at_8": 1.9651138186454773,
506
+ "total_car_to_hidden_at_8": 1.9741420596837997
507
+ },
508
+ "1919198188": {
509
+ "base_success": 0.0,
510
+ "base_utility": 0.03719887137413025,
511
+ "branch_car_at_8": 0.0,
512
+ "hidden_chart_oracle_success_at_8": 1.0,
513
+ "hidden_chart_oracle_utility_at_8": 2.0,
514
+ "outcome_ptr_at_8": 1.0,
515
+ "pairwise_causal_calibration_ece": 0.13114443223224567,
516
+ "proposal_oracle_success_at_8": 0.0,
517
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
518
+ "proposal_oracle_utility_at_8": 0.046244338154792786,
519
+ "proposal_oracle_utility_gain_over_base_at_8": 0.009045466780662537,
520
+ "selected_success_at_8": 0.0,
521
+ "selected_success_gain_over_base_at_8": 0.0,
522
+ "selected_utility_at_8": 0.046244338154792786,
523
+ "selected_utility_gain_over_base_at_8": 0.009045466780662537,
524
+ "selector_regret_at_8": 0.0,
525
+ "success_selector_gap_at_8": 0.0,
526
+ "success_support_gap_at_8": 1.0,
527
+ "success_total_car_to_hidden_at_8": 1.0,
528
+ "support_gap_at_8": 1.9537556618452072,
529
+ "total_car_to_hidden_at_8": 1.9537556618452072
530
+ },
531
+ "1991414747": {
532
+ "base_success": 1.0,
533
+ "base_utility": 2.0,
534
+ "branch_car_at_8": 1.9863876849412918,
535
+ "hidden_chart_oracle_success_at_8": 1.0,
536
+ "hidden_chart_oracle_utility_at_8": 2.0,
537
+ "outcome_ptr_at_8": 0.0,
538
+ "pairwise_causal_calibration_ece": 0.1306768270010748,
539
+ "proposal_oracle_success_at_8": 1.0,
540
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
541
+ "proposal_oracle_utility_at_8": 2.0,
542
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
543
+ "selected_success_at_8": 0.0,
544
+ "selected_success_gain_over_base_at_8": -1.0,
545
+ "selected_utility_at_8": 0.013612315058708191,
546
+ "selected_utility_gain_over_base_at_8": -1.9863876849412918,
547
+ "selector_regret_at_8": 1.9863876849412918,
548
+ "success_selector_gap_at_8": 1.0,
549
+ "success_support_gap_at_8": 0.0,
550
+ "success_total_car_to_hidden_at_8": 1.0,
551
+ "support_gap_at_8": 0.0,
552
+ "total_car_to_hidden_at_8": 1.9863876849412918
553
+ },
554
+ "1997090716": {
555
+ "base_success": 0.0,
556
+ "base_utility": 0.13927479088306427,
557
+ "branch_car_at_8": 0.0,
558
+ "hidden_chart_oracle_success_at_8": 1.0,
559
+ "hidden_chart_oracle_utility_at_8": 2.0,
560
+ "outcome_ptr_at_8": 1.0,
561
+ "pairwise_causal_calibration_ece": 0.2554442550173647,
562
+ "proposal_oracle_success_at_8": 1.0,
563
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
564
+ "proposal_oracle_utility_at_8": 2.0,
565
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357,
566
+ "selected_success_at_8": 1.0,
567
+ "selected_success_gain_over_base_at_8": 1.0,
568
+ "selected_utility_at_8": 2.0,
569
+ "selected_utility_gain_over_base_at_8": 1.8607252091169357,
570
+ "selector_regret_at_8": 0.0,
571
+ "success_selector_gap_at_8": 0.0,
572
+ "success_support_gap_at_8": 0.0,
573
+ "success_total_car_to_hidden_at_8": 0.0,
574
+ "support_gap_at_8": 0.0,
575
+ "total_car_to_hidden_at_8": 0.0
576
+ },
577
+ "2015956957": {
578
+ "base_success": 0.0,
579
+ "base_utility": 0.06711978986859321,
580
+ "branch_car_at_8": 0.022195855714380743,
581
+ "hidden_chart_oracle_success_at_8": 0.2,
582
+ "hidden_chart_oracle_utility_at_8": 0.5298813670873642,
583
+ "outcome_ptr_at_8": 0.6,
584
+ "pairwise_causal_calibration_ece": 0.16317219295797403,
585
+ "proposal_oracle_success_at_8": 0.0,
586
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
587
+ "proposal_oracle_utility_at_8": 0.05629209652543068,
588
+ "proposal_oracle_utility_gain_over_base_at_8": -0.010827693343162536,
589
+ "selected_success_at_8": 0.0,
590
+ "selected_success_gain_over_base_at_8": 0.0,
591
+ "selected_utility_at_8": 0.034096240811049935,
592
+ "selected_utility_gain_over_base_at_8": -0.033023549057543275,
593
+ "selector_regret_at_8": 0.022195855714380743,
594
+ "success_selector_gap_at_8": 0.0,
595
+ "success_support_gap_at_8": 0.2,
596
+ "success_total_car_to_hidden_at_8": 0.2,
597
+ "support_gap_at_8": 0.4735892705619335,
598
+ "total_car_to_hidden_at_8": 0.49578512627631427
599
+ },
600
+ "2054899126": {
601
+ "base_success": 1.0,
602
+ "base_utility": 2.0,
603
+ "branch_car_at_8": 0.0,
604
+ "hidden_chart_oracle_success_at_8": 1.0,
605
+ "hidden_chart_oracle_utility_at_8": 2.0,
606
+ "outcome_ptr_at_8": 0.0,
607
+ "pairwise_causal_calibration_ece": 0.0,
608
+ "proposal_oracle_success_at_8": 1.0,
609
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
610
+ "proposal_oracle_utility_at_8": 2.0,
611
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
612
+ "selected_success_at_8": 1.0,
613
+ "selected_success_gain_over_base_at_8": 0.0,
614
+ "selected_utility_at_8": 2.0,
615
+ "selected_utility_gain_over_base_at_8": 0.0,
616
+ "selector_regret_at_8": 0.0,
617
+ "success_selector_gap_at_8": 0.0,
618
+ "success_support_gap_at_8": 0.0,
619
+ "success_total_car_to_hidden_at_8": 0.0,
620
+ "support_gap_at_8": 0.0,
621
+ "total_car_to_hidden_at_8": 0.0
622
+ },
623
+ "211245842": {
624
+ "base_success": 0.0,
625
+ "base_utility": 0.8388344049453735,
626
+ "branch_car_at_8": 0.6792904287576675,
627
+ "hidden_chart_oracle_success_at_8": 1.0,
628
+ "hidden_chart_oracle_utility_at_8": 2.0,
629
+ "outcome_ptr_at_8": 0.0,
630
+ "pairwise_causal_calibration_ece": 0.07548939908426067,
631
+ "proposal_oracle_success_at_8": 0.0,
632
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
633
+ "proposal_oracle_utility_at_8": 0.8358442187309265,
634
+ "proposal_oracle_utility_gain_over_base_at_8": -0.0029901862144470215,
635
+ "selected_success_at_8": 0.0,
636
+ "selected_success_gain_over_base_at_8": 0.0,
637
+ "selected_utility_at_8": 0.15655378997325897,
638
+ "selected_utility_gain_over_base_at_8": -0.6822806149721146,
639
+ "selector_regret_at_8": 0.6792904287576675,
640
+ "success_selector_gap_at_8": 0.0,
641
+ "success_support_gap_at_8": 1.0,
642
+ "success_total_car_to_hidden_at_8": 1.0,
643
+ "support_gap_at_8": 1.1641557812690735,
644
+ "total_car_to_hidden_at_8": 1.843446210026741
645
+ },
646
+ "2141802816": {
647
+ "base_success": 0.0,
648
+ "base_utility": 0.07080724835395813,
649
+ "branch_car_at_8": 0.019766509532928467,
650
+ "hidden_chart_oracle_success_at_8": 1.0,
651
+ "hidden_chart_oracle_utility_at_8": 2.0,
652
+ "outcome_ptr_at_8": 1.0,
653
+ "pairwise_causal_calibration_ece": 0.07786951284417709,
654
+ "proposal_oracle_success_at_8": 0.0,
655
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
656
+ "proposal_oracle_utility_at_8": 0.09589323401451111,
657
+ "proposal_oracle_utility_gain_over_base_at_8": 0.02508598566055298,
658
+ "selected_success_at_8": 0.0,
659
+ "selected_success_gain_over_base_at_8": 0.0,
660
+ "selected_utility_at_8": 0.07612672448158264,
661
+ "selected_utility_gain_over_base_at_8": 0.005319476127624512,
662
+ "selector_regret_at_8": 0.019766509532928467,
663
+ "success_selector_gap_at_8": 0.0,
664
+ "success_support_gap_at_8": 1.0,
665
+ "success_total_car_to_hidden_at_8": 1.0,
666
+ "support_gap_at_8": 1.904106765985489,
667
+ "total_car_to_hidden_at_8": 1.9238732755184174
668
+ },
669
+ "256770874": {
670
+ "base_success": 0.0,
671
+ "base_utility": 0.8557480573654175,
672
+ "branch_car_at_8": 0.7044661641120911,
673
+ "hidden_chart_oracle_success_at_8": 1.0,
674
+ "hidden_chart_oracle_utility_at_8": 2.0,
675
+ "outcome_ptr_at_8": 0.0,
676
+ "pairwise_causal_calibration_ece": 0.03135306008748828,
677
+ "proposal_oracle_success_at_8": 0.0,
678
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
679
+ "proposal_oracle_utility_at_8": 0.8125,
680
+ "proposal_oracle_utility_gain_over_base_at_8": -0.04324805736541748,
681
+ "selected_success_at_8": 0.0,
682
+ "selected_success_gain_over_base_at_8": 0.0,
683
+ "selected_utility_at_8": 0.10803383588790894,
684
+ "selected_utility_gain_over_base_at_8": -0.7477142214775085,
685
+ "selector_regret_at_8": 0.7044661641120911,
686
+ "success_selector_gap_at_8": 0.0,
687
+ "success_support_gap_at_8": 1.0,
688
+ "success_total_car_to_hidden_at_8": 1.0,
689
+ "support_gap_at_8": 1.1875,
690
+ "total_car_to_hidden_at_8": 1.891966164112091
691
+ },
692
+ "278008524": {
693
+ "base_success": 0.0,
694
+ "base_utility": 0.10747186839580536,
695
+ "branch_car_at_8": 0.011409573198761791,
696
+ "hidden_chart_oracle_success_at_8": 0.5,
697
+ "hidden_chart_oracle_utility_at_8": 1.113811194896698,
698
+ "outcome_ptr_at_8": 0.5,
699
+ "pairwise_causal_calibration_ece": 0.24605586147071373,
700
+ "proposal_oracle_success_at_8": 0.0,
701
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
702
+ "proposal_oracle_utility_at_8": 0.03147180867381394,
703
+ "proposal_oracle_utility_gain_over_base_at_8": -0.07600005972199142,
704
+ "selected_success_at_8": 0.0,
705
+ "selected_success_gain_over_base_at_8": 0.0,
706
+ "selected_utility_at_8": 0.020062235475052148,
707
+ "selected_utility_gain_over_base_at_8": -0.08740963292075321,
708
+ "selector_regret_at_8": 0.011409573198761791,
709
+ "success_selector_gap_at_8": 0.0,
710
+ "success_support_gap_at_8": 0.5,
711
+ "success_total_car_to_hidden_at_8": 0.5,
712
+ "support_gap_at_8": 1.082339386222884,
713
+ "total_car_to_hidden_at_8": 1.0937489594216458
714
+ },
715
+ "416183106": {
716
+ "base_success": 0.0,
717
+ "base_utility": 0.20032641291618347,
718
+ "branch_car_at_8": 1.7668711841106415,
719
+ "hidden_chart_oracle_success_at_8": 1.0,
720
+ "hidden_chart_oracle_utility_at_8": 2.0,
721
+ "outcome_ptr_at_8": 1.0,
722
+ "pairwise_causal_calibration_ece": 0.07978206783428046,
723
+ "proposal_oracle_success_at_8": 1.0,
724
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
725
+ "proposal_oracle_utility_at_8": 2.0,
726
+ "proposal_oracle_utility_gain_over_base_at_8": 1.7996735870838165,
727
+ "selected_success_at_8": 0.0,
728
+ "selected_success_gain_over_base_at_8": 0.0,
729
+ "selected_utility_at_8": 0.23312881588935852,
730
+ "selected_utility_gain_over_base_at_8": 0.03280240297317505,
731
+ "selector_regret_at_8": 1.7668711841106415,
732
+ "success_selector_gap_at_8": 1.0,
733
+ "success_support_gap_at_8": 0.0,
734
+ "success_total_car_to_hidden_at_8": 1.0,
735
+ "support_gap_at_8": 0.0,
736
+ "total_car_to_hidden_at_8": 1.7668711841106415
737
+ },
738
+ "534717141": {
739
+ "base_success": 1.0,
740
+ "base_utility": 2.0,
741
+ "branch_car_at_8": 1.9558932781219482,
742
+ "hidden_chart_oracle_success_at_8": 1.0,
743
+ "hidden_chart_oracle_utility_at_8": 2.0,
744
+ "outcome_ptr_at_8": 0.0,
745
+ "pairwise_causal_calibration_ece": 0.14369576234023818,
746
+ "proposal_oracle_success_at_8": 1.0,
747
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
748
+ "proposal_oracle_utility_at_8": 2.0,
749
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
750
+ "selected_success_at_8": 0.0,
751
+ "selected_success_gain_over_base_at_8": -1.0,
752
+ "selected_utility_at_8": 0.04410672187805176,
753
+ "selected_utility_gain_over_base_at_8": -1.9558932781219482,
754
+ "selector_regret_at_8": 1.9558932781219482,
755
+ "success_selector_gap_at_8": 1.0,
756
+ "success_support_gap_at_8": 0.0,
757
+ "success_total_car_to_hidden_at_8": 1.0,
758
+ "support_gap_at_8": 0.0,
759
+ "total_car_to_hidden_at_8": 1.9558932781219482
760
+ },
761
+ "551504635": {
762
+ "base_success": 0.0,
763
+ "base_utility": 0.7247228622436523,
764
+ "branch_car_at_8": 0.0,
765
+ "hidden_chart_oracle_success_at_8": 1.0,
766
+ "hidden_chart_oracle_utility_at_8": 2.0,
767
+ "outcome_ptr_at_8": 0.0,
768
+ "pairwise_causal_calibration_ece": 0.2299325778712667,
769
+ "proposal_oracle_success_at_8": 0.0,
770
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
771
+ "proposal_oracle_utility_at_8": 0.7224790453910828,
772
+ "proposal_oracle_utility_gain_over_base_at_8": -0.00224381685256958,
773
+ "selected_success_at_8": 0.0,
774
+ "selected_success_gain_over_base_at_8": 0.0,
775
+ "selected_utility_at_8": 0.7224790453910828,
776
+ "selected_utility_gain_over_base_at_8": -0.00224381685256958,
777
+ "selector_regret_at_8": 0.0,
778
+ "success_selector_gap_at_8": 0.0,
779
+ "success_support_gap_at_8": 1.0,
780
+ "success_total_car_to_hidden_at_8": 1.0,
781
+ "support_gap_at_8": 1.2775209546089172,
782
+ "total_car_to_hidden_at_8": 1.2775209546089172
783
+ },
784
+ "565369392": {
785
+ "base_success": 0.3333333333333333,
786
+ "base_utility": 0.7997656464576721,
787
+ "branch_car_at_8": 0.046260833740234375,
788
+ "hidden_chart_oracle_success_at_8": 0.6666666666666666,
789
+ "hidden_chart_oracle_utility_at_8": 1.4266315797964733,
790
+ "outcome_ptr_at_8": 0.0,
791
+ "pairwise_causal_calibration_ece": 0.13991651305345584,
792
+ "proposal_oracle_success_at_8": 0.0,
793
+ "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333,
794
+ "proposal_oracle_utility_at_8": 0.17366480827331543,
795
+ "proposal_oracle_utility_gain_over_base_at_8": -0.6261008381843567,
796
+ "selected_success_at_8": 0.0,
797
+ "selected_success_gain_over_base_at_8": -0.3333333333333333,
798
+ "selected_utility_at_8": 0.12740397453308105,
799
+ "selected_utility_gain_over_base_at_8": -0.6723616719245911,
800
+ "selector_regret_at_8": 0.046260833740234375,
801
+ "success_selector_gap_at_8": 0.0,
802
+ "success_support_gap_at_8": 0.6666666666666666,
803
+ "success_total_car_to_hidden_at_8": 0.6666666666666666,
804
+ "support_gap_at_8": 1.2529667715231578,
805
+ "total_car_to_hidden_at_8": 1.2992276052633922
806
+ },
807
+ "565621826": {
808
+ "base_success": 1.0,
809
+ "base_utility": 2.0,
810
+ "branch_car_at_8": 0.0,
811
+ "hidden_chart_oracle_success_at_8": 1.0,
812
+ "hidden_chart_oracle_utility_at_8": 2.0,
813
+ "outcome_ptr_at_8": 0.0,
814
+ "pairwise_causal_calibration_ece": 0.0,
815
+ "proposal_oracle_success_at_8": 1.0,
816
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
817
+ "proposal_oracle_utility_at_8": 2.0,
818
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
819
+ "selected_success_at_8": 1.0,
820
+ "selected_success_gain_over_base_at_8": 0.0,
821
+ "selected_utility_at_8": 2.0,
822
+ "selected_utility_gain_over_base_at_8": 0.0,
823
+ "selector_regret_at_8": 0.0,
824
+ "success_selector_gap_at_8": 0.0,
825
+ "success_support_gap_at_8": 0.0,
826
+ "success_total_car_to_hidden_at_8": 0.0,
827
+ "support_gap_at_8": 0.0,
828
+ "total_car_to_hidden_at_8": 0.0
829
+ },
830
+ "585904130": {
831
+ "base_success": 0.0,
832
+ "base_utility": 0.07529209802548091,
833
+ "branch_car_at_8": 0.012068324722349644,
834
+ "hidden_chart_oracle_success_at_8": 0.0,
835
+ "hidden_chart_oracle_utility_at_8": 0.23764699200789133,
836
+ "outcome_ptr_at_8": 0.0,
837
+ "pairwise_causal_calibration_ece": 0.07276320034061332,
838
+ "proposal_oracle_success_at_8": 0.0,
839
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
840
+ "proposal_oracle_utility_at_8": 0.032918572425842285,
841
+ "proposal_oracle_utility_gain_over_base_at_8": -0.04237352559963862,
842
+ "selected_success_at_8": 0.0,
843
+ "selected_success_gain_over_base_at_8": 0.0,
844
+ "selected_utility_at_8": 0.02085024770349264,
845
+ "selected_utility_gain_over_base_at_8": -0.05444185032198826,
846
+ "selector_regret_at_8": 0.012068324722349644,
847
+ "success_selector_gap_at_8": 0.0,
848
+ "success_support_gap_at_8": 0.0,
849
+ "success_total_car_to_hidden_at_8": 0.0,
850
+ "support_gap_at_8": 0.20472841958204904,
851
+ "total_car_to_hidden_at_8": 0.2167967443043987
852
+ },
853
+ "626486813": {
854
+ "base_success": 0.25,
855
+ "base_utility": 0.5054400166263804,
856
+ "branch_car_at_8": 0.02500777377281338,
857
+ "hidden_chart_oracle_success_at_8": 0.25,
858
+ "hidden_chart_oracle_utility_at_8": 0.6124684102833271,
859
+ "outcome_ptr_at_8": 0.5,
860
+ "pairwise_causal_calibration_ece": 0.2420589895144555,
861
+ "proposal_oracle_success_at_8": 0.0,
862
+ "proposal_oracle_success_gain_over_base_at_8": -0.25,
863
+ "proposal_oracle_utility_at_8": 0.07365635968744755,
864
+ "proposal_oracle_utility_gain_over_base_at_8": -0.43178365693893284,
865
+ "selected_success_at_8": 0.0,
866
+ "selected_success_gain_over_base_at_8": -0.25,
867
+ "selected_utility_at_8": 0.04864858591463417,
868
+ "selected_utility_gain_over_base_at_8": -0.4567914307117462,
869
+ "selector_regret_at_8": 0.02500777377281338,
870
+ "success_selector_gap_at_8": 0.0,
871
+ "success_support_gap_at_8": 0.25,
872
+ "success_total_car_to_hidden_at_8": 0.25,
873
+ "support_gap_at_8": 0.5414478182792664,
874
+ "total_car_to_hidden_at_8": 0.5638198243686929
875
+ },
876
+ "636680137": {
877
+ "base_success": 1.0,
878
+ "base_utility": 2.0,
879
+ "branch_car_at_8": 0.0,
880
+ "hidden_chart_oracle_success_at_8": 1.0,
881
+ "hidden_chart_oracle_utility_at_8": 2.0,
882
+ "outcome_ptr_at_8": 0.0,
883
+ "pairwise_causal_calibration_ece": 0.07628525991805817,
884
+ "proposal_oracle_success_at_8": 0.0,
885
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
886
+ "proposal_oracle_utility_at_8": 0.013928472995758057,
887
+ "proposal_oracle_utility_gain_over_base_at_8": -1.986071527004242,
888
+ "selected_success_at_8": 0.0,
889
+ "selected_success_gain_over_base_at_8": -1.0,
890
+ "selected_utility_at_8": 0.013928472995758057,
891
+ "selected_utility_gain_over_base_at_8": -1.986071527004242,
892
+ "selector_regret_at_8": 0.0,
893
+ "success_selector_gap_at_8": 0.0,
894
+ "success_support_gap_at_8": 1.0,
895
+ "success_total_car_to_hidden_at_8": 1.0,
896
+ "support_gap_at_8": 1.986071527004242,
897
+ "total_car_to_hidden_at_8": 1.986071527004242
898
+ },
899
+ "706479600": {
900
+ "base_success": 0.0,
901
+ "base_utility": 0.8350887894630432,
902
+ "branch_car_at_8": 0.0,
903
+ "hidden_chart_oracle_success_at_8": 1.0,
904
+ "hidden_chart_oracle_utility_at_8": 2.0,
905
+ "outcome_ptr_at_8": 1.0,
906
+ "pairwise_causal_calibration_ece": 0.16443847051324711,
907
+ "proposal_oracle_success_at_8": 1.0,
908
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
909
+ "proposal_oracle_utility_at_8": 2.0,
910
+ "proposal_oracle_utility_gain_over_base_at_8": 1.1649112105369568,
911
+ "selected_success_at_8": 1.0,
912
+ "selected_success_gain_over_base_at_8": 1.0,
913
+ "selected_utility_at_8": 2.0,
914
+ "selected_utility_gain_over_base_at_8": 1.1649112105369568,
915
+ "selector_regret_at_8": 0.0,
916
+ "success_selector_gap_at_8": 0.0,
917
+ "success_support_gap_at_8": 0.0,
918
+ "success_total_car_to_hidden_at_8": 0.0,
919
+ "support_gap_at_8": 0.0,
920
+ "total_car_to_hidden_at_8": 0.0
921
+ },
922
+ "750448296": {
923
+ "base_success": 0.0,
924
+ "base_utility": 0.11153355985879898,
925
+ "branch_car_at_8": 0.02701035887002945,
926
+ "hidden_chart_oracle_success_at_8": 0.0,
927
+ "hidden_chart_oracle_utility_at_8": 0.23580759018659592,
928
+ "outcome_ptr_at_8": 0.0,
929
+ "pairwise_causal_calibration_ece": 0.2344413298276417,
930
+ "proposal_oracle_success_at_8": 0.0,
931
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
932
+ "proposal_oracle_utility_at_8": 0.07342664152383804,
933
+ "proposal_oracle_utility_gain_over_base_at_8": -0.03810691833496094,
934
+ "selected_success_at_8": 0.0,
935
+ "selected_success_gain_over_base_at_8": 0.0,
936
+ "selected_utility_at_8": 0.046416282653808594,
937
+ "selected_utility_gain_over_base_at_8": -0.06511727720499039,
938
+ "selector_regret_at_8": 0.02701035887002945,
939
+ "success_selector_gap_at_8": 0.0,
940
+ "success_support_gap_at_8": 0.0,
941
+ "success_total_car_to_hidden_at_8": 0.0,
942
+ "support_gap_at_8": 0.16238094866275787,
943
+ "total_car_to_hidden_at_8": 0.18939130753278732
944
+ },
945
+ "75070641": {
946
+ "base_success": 0.0,
947
+ "base_utility": 0.20275163650512695,
948
+ "branch_car_at_8": 0.5863077491521835,
949
+ "hidden_chart_oracle_success_at_8": 1.0,
950
+ "hidden_chart_oracle_utility_at_8": 2.0,
951
+ "outcome_ptr_at_8": 1.0,
952
+ "pairwise_causal_calibration_ece": 0.18559462278331035,
953
+ "proposal_oracle_success_at_8": 0.0,
954
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
955
+ "proposal_oracle_utility_at_8": 0.8128874897956848,
956
+ "proposal_oracle_utility_gain_over_base_at_8": 0.6101358532905579,
957
+ "selected_success_at_8": 0.0,
958
+ "selected_success_gain_over_base_at_8": 0.0,
959
+ "selected_utility_at_8": 0.22657974064350128,
960
+ "selected_utility_gain_over_base_at_8": 0.02382810413837433,
961
+ "selector_regret_at_8": 0.5863077491521835,
962
+ "success_selector_gap_at_8": 0.0,
963
+ "success_support_gap_at_8": 1.0,
964
+ "success_total_car_to_hidden_at_8": 1.0,
965
+ "support_gap_at_8": 1.1871125102043152,
966
+ "total_car_to_hidden_at_8": 1.7734202593564987
967
+ },
968
+ "767747424": {
969
+ "base_success": 0.25,
970
+ "base_utility": 0.5475241877138615,
971
+ "branch_car_at_8": 0.04561196081340313,
972
+ "hidden_chart_oracle_success_at_8": 0.0,
973
+ "hidden_chart_oracle_utility_at_8": 0.18863565661013126,
974
+ "outcome_ptr_at_8": 0.75,
975
+ "pairwise_causal_calibration_ece": 0.07531396139335009,
976
+ "proposal_oracle_success_at_8": 0.0,
977
+ "proposal_oracle_success_gain_over_base_at_8": -0.25,
978
+ "proposal_oracle_utility_at_8": 0.11342880013398826,
979
+ "proposal_oracle_utility_gain_over_base_at_8": -0.4340953875798732,
980
+ "selected_success_at_8": 0.0,
981
+ "selected_success_gain_over_base_at_8": -0.25,
982
+ "selected_utility_at_8": 0.06781683932058513,
983
+ "selected_utility_gain_over_base_at_8": -0.47970734839327633,
984
+ "selector_regret_at_8": 0.04561196081340313,
985
+ "success_selector_gap_at_8": 0.0,
986
+ "success_support_gap_at_8": 0.0,
987
+ "success_total_car_to_hidden_at_8": 0.0,
988
+ "support_gap_at_8": 0.08526064571924508,
989
+ "total_car_to_hidden_at_8": 0.12081881728954613
990
+ },
991
+ "77147264": {
992
+ "base_success": 0.5,
993
+ "base_utility": 1.4215794801712036,
994
+ "branch_car_at_8": 0.0,
995
+ "hidden_chart_oracle_success_at_8": 1.0,
996
+ "hidden_chart_oracle_utility_at_8": 2.0,
997
+ "outcome_ptr_at_8": 0.5,
998
+ "pairwise_causal_calibration_ece": 0.2630803856693186,
999
+ "proposal_oracle_success_at_8": 0.5,
1000
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
1001
+ "proposal_oracle_utility_at_8": 1.426636129617691,
1002
+ "proposal_oracle_utility_gain_over_base_at_8": 0.005056649446487427,
1003
+ "selected_success_at_8": 0.5,
1004
+ "selected_success_gain_over_base_at_8": 0.0,
1005
+ "selected_utility_at_8": 1.426636129617691,
1006
+ "selected_utility_gain_over_base_at_8": 0.005056649446487427,
1007
+ "selector_regret_at_8": 0.0,
1008
+ "success_selector_gap_at_8": 0.0,
1009
+ "success_support_gap_at_8": 0.5,
1010
+ "success_total_car_to_hidden_at_8": 0.5,
1011
+ "support_gap_at_8": 0.573363870382309,
1012
+ "total_car_to_hidden_at_8": 0.573363870382309
1013
+ },
1014
+ "794181626": {
1015
+ "base_success": 0.0,
1016
+ "base_utility": 0.18095605075359344,
1017
+ "branch_car_at_8": 0.0,
1018
+ "hidden_chart_oracle_success_at_8": 1.0,
1019
+ "hidden_chart_oracle_utility_at_8": 2.0,
1020
+ "outcome_ptr_at_8": 1.0,
1021
+ "pairwise_causal_calibration_ece": 0.28339362113630795,
1022
+ "proposal_oracle_success_at_8": 1.0,
1023
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
1024
+ "proposal_oracle_utility_at_8": 2.0,
1025
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8190439492464066,
1026
+ "selected_success_at_8": 1.0,
1027
+ "selected_success_gain_over_base_at_8": 1.0,
1028
+ "selected_utility_at_8": 2.0,
1029
+ "selected_utility_gain_over_base_at_8": 1.8190439492464066,
1030
+ "selector_regret_at_8": 0.0,
1031
+ "success_selector_gap_at_8": 0.0,
1032
+ "success_support_gap_at_8": 0.0,
1033
+ "success_total_car_to_hidden_at_8": 0.0,
1034
+ "support_gap_at_8": 0.0,
1035
+ "total_car_to_hidden_at_8": 0.0
1036
+ },
1037
+ "957101489": {
1038
+ "base_success": 0.0,
1039
+ "base_utility": 0.20372186601161957,
1040
+ "branch_car_at_8": 1.7757539749145508,
1041
+ "hidden_chart_oracle_success_at_8": 1.0,
1042
+ "hidden_chart_oracle_utility_at_8": 2.0,
1043
+ "outcome_ptr_at_8": 1.0,
1044
+ "pairwise_causal_calibration_ece": 0.07643021229779011,
1045
+ "proposal_oracle_success_at_8": 1.0,
1046
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
1047
+ "proposal_oracle_utility_at_8": 2.0,
1048
+ "proposal_oracle_utility_gain_over_base_at_8": 1.7962781339883804,
1049
+ "selected_success_at_8": 0.0,
1050
+ "selected_success_gain_over_base_at_8": 0.0,
1051
+ "selected_utility_at_8": 0.22424602508544922,
1052
+ "selected_utility_gain_over_base_at_8": 0.02052415907382965,
1053
+ "selector_regret_at_8": 1.7757539749145508,
1054
+ "success_selector_gap_at_8": 1.0,
1055
+ "success_support_gap_at_8": 0.0,
1056
+ "success_total_car_to_hidden_at_8": 1.0,
1057
+ "support_gap_at_8": 0.0,
1058
+ "total_car_to_hidden_at_8": 1.7757539749145508
1059
+ },
1060
+ "998761162": {
1061
+ "base_success": 0.0,
1062
+ "base_utility": 0.06781657040119171,
1063
+ "branch_car_at_8": 1.9512543678283691,
1064
+ "hidden_chart_oracle_success_at_8": 1.0,
1065
+ "hidden_chart_oracle_utility_at_8": 2.0,
1066
+ "outcome_ptr_at_8": 1.0,
1067
+ "pairwise_causal_calibration_ece": 0.28991947553571784,
1068
+ "proposal_oracle_success_at_8": 1.0,
1069
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
1070
+ "proposal_oracle_utility_at_8": 2.0,
1071
+ "proposal_oracle_utility_gain_over_base_at_8": 1.9321834295988083,
1072
+ "selected_success_at_8": 0.0,
1073
+ "selected_success_gain_over_base_at_8": 0.0,
1074
+ "selected_utility_at_8": 0.04874563217163086,
1075
+ "selected_utility_gain_over_base_at_8": -0.019070938229560852,
1076
+ "selector_regret_at_8": 1.9512543678283691,
1077
+ "success_selector_gap_at_8": 1.0,
1078
+ "success_support_gap_at_8": 0.0,
1079
+ "success_total_car_to_hidden_at_8": 1.0,
1080
+ "support_gap_at_8": 0.0,
1081
+ "total_car_to_hidden_at_8": 1.9512543678283691
1082
+ }
1083
+ }
workspace/runs/ctt_residual_rollout_val69_seed1/metrics_by_task.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "LiftPegUpright-v1": {
3
+ "base_success": 0.3333333333333333,
4
+ "base_utility": 1.0670691976944606,
5
+ "branch_car_at_8": 0.3250635440150897,
6
+ "hidden_chart_oracle_success_at_8": 1.0,
7
+ "hidden_chart_oracle_utility_at_8": 2.0,
8
+ "outcome_ptr_at_8": 0.5,
9
+ "pairwise_causal_calibration_ece": 0.21174314869082475,
10
+ "proposal_oracle_success_at_8": 0.5,
11
+ "proposal_oracle_success_gain_over_base_at_8": 0.16666666666666666,
12
+ "proposal_oracle_utility_at_8": 1.3470948338508606,
13
+ "proposal_oracle_utility_gain_over_base_at_8": 0.2800256361564,
14
+ "selected_success_at_8": 0.3333333333333333,
15
+ "selected_success_gain_over_base_at_8": 0.0,
16
+ "selected_utility_at_8": 1.0220312898357709,
17
+ "selected_utility_gain_over_base_at_8": -0.04503790785868963,
18
+ "selector_regret_at_8": 0.3250635440150897,
19
+ "success_selector_gap_at_8": 0.16666666666666666,
20
+ "success_support_gap_at_8": 0.5,
21
+ "success_total_car_to_hidden_at_8": 0.6666666666666666,
22
+ "support_gap_at_8": 0.6529051661491394,
23
+ "total_car_to_hidden_at_8": 0.977968710164229
24
+ },
25
+ "PickCube-v1": {
26
+ "base_success": 0.0,
27
+ "base_utility": 0.15342965722084045,
28
+ "branch_car_at_8": 0.45726464316248894,
29
+ "hidden_chart_oracle_success_at_8": 1.0,
30
+ "hidden_chart_oracle_utility_at_8": 2.0,
31
+ "outcome_ptr_at_8": 1.0,
32
+ "pairwise_causal_calibration_ece": 0.096994407338536,
33
+ "proposal_oracle_success_at_8": 0.0,
34
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
35
+ "proposal_oracle_utility_at_8": 0.4997493326663971,
36
+ "proposal_oracle_utility_gain_over_base_at_8": 0.34631967544555664,
37
+ "selected_success_at_8": 0.0,
38
+ "selected_success_gain_over_base_at_8": 0.0,
39
+ "selected_utility_at_8": 0.04248468950390816,
40
+ "selected_utility_gain_over_base_at_8": -0.1109449677169323,
41
+ "selector_regret_at_8": 0.45726464316248894,
42
+ "success_selector_gap_at_8": 0.0,
43
+ "success_support_gap_at_8": 1.0,
44
+ "success_total_car_to_hidden_at_8": 1.0,
45
+ "support_gap_at_8": 1.500250667333603,
46
+ "total_car_to_hidden_at_8": 1.9575153104960918
47
+ },
48
+ "PullCube-v1": {
49
+ "base_success": 0.29411764705882354,
50
+ "base_utility": 0.6512299207833541,
51
+ "branch_car_at_8": 0.08196123232654132,
52
+ "hidden_chart_oracle_success_at_8": 0.3235294117647059,
53
+ "hidden_chart_oracle_utility_at_8": 0.7823218367993832,
54
+ "outcome_ptr_at_8": 0.29411764705882354,
55
+ "pairwise_causal_calibration_ece": 0.14892514417161398,
56
+ "proposal_oracle_success_at_8": 0.11764705882352941,
57
+ "proposal_oracle_success_gain_over_base_at_8": -0.17647058823529413,
58
+ "proposal_oracle_utility_at_8": 0.3012250267818351,
59
+ "proposal_oracle_utility_gain_over_base_at_8": -0.35000489400151896,
60
+ "selected_success_at_8": 0.08823529411764706,
61
+ "selected_success_gain_over_base_at_8": -0.20588235294117646,
62
+ "selected_utility_at_8": 0.2192637944552938,
63
+ "selected_utility_gain_over_base_at_8": -0.4319661263280603,
64
+ "selector_regret_at_8": 0.08196123232654132,
65
+ "success_selector_gap_at_8": 0.029411764705882353,
66
+ "success_support_gap_at_8": 0.20588235294117646,
67
+ "success_total_car_to_hidden_at_8": 0.23529411764705882,
68
+ "support_gap_at_8": 0.4828804947901517,
69
+ "total_car_to_hidden_at_8": 0.5630580423440894
70
+ },
71
+ "PushCube-v1": {
72
+ "base_success": 0.38461538461538464,
73
+ "base_utility": 0.8223185676794785,
74
+ "branch_car_at_8": 0.6069896691120588,
75
+ "hidden_chart_oracle_success_at_8": 1.0,
76
+ "hidden_chart_oracle_utility_at_8": 2.0,
77
+ "outcome_ptr_at_8": 0.6153846153846154,
78
+ "pairwise_causal_calibration_ece": 0.19653272589828236,
79
+ "proposal_oracle_success_at_8": 0.7692307692307693,
80
+ "proposal_oracle_success_gain_over_base_at_8": 0.38461538461538464,
81
+ "proposal_oracle_utility_at_8": 1.55294325030767,
82
+ "proposal_oracle_utility_gain_over_base_at_8": 0.7306246826281915,
83
+ "selected_success_at_8": 0.46153846153846156,
84
+ "selected_success_gain_over_base_at_8": 0.07692307692307693,
85
+ "selected_utility_at_8": 0.9459535811956112,
86
+ "selected_utility_gain_over_base_at_8": 0.12363501351613265,
87
+ "selector_regret_at_8": 0.6069896691120588,
88
+ "success_selector_gap_at_8": 0.3076923076923077,
89
+ "success_support_gap_at_8": 0.23076923076923078,
90
+ "success_total_car_to_hidden_at_8": 0.5384615384615384,
91
+ "support_gap_at_8": 0.44705674969233,
92
+ "total_car_to_hidden_at_8": 1.0540464188043888
93
+ },
94
+ "StackCube-v1": {
95
+ "base_success": 0.2,
96
+ "base_utility": 0.8036667982737223,
97
+ "branch_car_at_8": 0.6179056455691655,
98
+ "hidden_chart_oracle_success_at_8": 1.0,
99
+ "hidden_chart_oracle_utility_at_8": 2.0,
100
+ "outcome_ptr_at_8": 0.5333333333333333,
101
+ "pairwise_causal_calibration_ece": 0.12910083675639442,
102
+ "proposal_oracle_success_at_8": 0.3333333333333333,
103
+ "proposal_oracle_success_gain_over_base_at_8": 0.13333333333333333,
104
+ "proposal_oracle_utility_at_8": 1.0861866732438406,
105
+ "proposal_oracle_utility_gain_over_base_at_8": 0.2825198749701182,
106
+ "selected_success_at_8": 0.13333333333333333,
107
+ "selected_success_gain_over_base_at_8": -0.06666666666666667,
108
+ "selected_utility_at_8": 0.468281027674675,
109
+ "selected_utility_gain_over_base_at_8": -0.33538577059904734,
110
+ "selector_regret_at_8": 0.6179056455691655,
111
+ "success_selector_gap_at_8": 0.2,
112
+ "success_support_gap_at_8": 0.6666666666666666,
113
+ "success_total_car_to_hidden_at_8": 0.8666666666666667,
114
+ "support_gap_at_8": 0.9138133267561594,
115
+ "total_car_to_hidden_at_8": 1.531718972325325
116
+ }
117
+ }
workspace/runs/ctt_residual_rollout_val69_seed1/source_data_hash.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957
workspace/runs/ctt_residual_rollout_val69_seed1/source_split_hash.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372
workspace/runs/ctt_residual_rollout_val69_seed1/split_hash.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 9d1dc1c10868f76fa080895df0f2730c173f12b6db14932599d55c2b60859381
workspace/runs/ctt_residual_rollout_val69_seed1/table.tex ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ % Auto-generated by scripts/eval_metrics.py
2
+ \begin{tabular}{lrrrr}
3
+ \toprule
4
+ Metric & N & Micro mean & CI low & CI high \\
5
+ \midrule
6
+ base\_success & 69 & 0.2899 & 0.1884 & 0.4203 \\
7
+ base\_utility & 69 & 0.7455 & 0.5693 & 0.9621 \\
8
+ branch\_car\_at\_8 & 69 & 0.3240 & 0.1877 & 0.4805 \\
9
+ hidden\_chart\_oracle\_success\_at\_8 & 69 & 0.6667 & 0.5507 & 0.7826 \\
10
+ hidden\_chart\_oracle\_utility\_at\_8 & 69 & 1.4000 & 1.1855 & 1.6062 \\
11
+ outcome\_ptr\_at\_8 & 69 & 0.4348 & 0.3043 & 0.5362 \\
12
+ pairwise\_causal\_calibration\_ece & 69 & 0.1583 & 0.1336 & 0.1862 \\
13
+ proposal\_oracle\_success\_at\_8 & 69 & 0.3188 & 0.2174 & 0.4348 \\
14
+ proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 69 & 0.0290 & -0.0870 & 0.1449 \\
15
+ proposal\_oracle\_utility\_at\_8 & 69 & 0.8015 & 0.6154 & 1.0082 \\
16
+ proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 69 & 0.0560 & -0.1555 & 0.2702 \\
17
+ selected\_success\_at\_8 & 69 & 0.1884 & 0.1159 & 0.2899 \\
18
+ selected\_success\_gain\_over\_base\_at\_8 & 69 & -0.1014 & -0.2174 & 0.0000 \\
19
+ selected\_utility\_at\_8 & 69 & 0.4776 & 0.3230 & 0.6844 \\
20
+ selected\_utility\_gain\_over\_base\_at\_8 & 69 & -0.2680 & -0.4864 & -0.0695 \\
21
+ selector\_regret\_at\_8 & 69 & 0.3240 & 0.1877 & 0.4805 \\
22
+ success\_selector\_gap\_at\_8 & 69 & 0.1304 & 0.0580 & 0.2174 \\
23
+ success\_support\_gap\_at\_8 & 69 & 0.3478 & 0.2319 & 0.4638 \\
24
+ success\_total\_car\_to\_hidden\_at\_8 & 69 & 0.4783 & 0.3623 & 0.5942 \\
25
+ support\_gap\_at\_8 & 69 & 0.5993 & 0.4135 & 0.7763 \\
26
+ total\_car\_to\_hidden\_at\_8 & 69 & 0.9224 & 0.7123 & 1.1263 \\
27
+ \bottomrule
28
+ \end{tabular}
workspace/runs/ctt_residual_rollout_val69_seed1/train.log ADDED
@@ -0,0 +1 @@
 
 
1
+ CTT checkpoint trained separately; rollout used checkpoint=runs/ctt_residual_full_seed1/model.pt
workspace/runs/ctt_residual_rollout_val69_seed2/data_hash.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ fa0fe0e3881ee70ce91492445871614d01b3a47fc7c576659c53ff17ddee4e42
workspace/runs/ctt_residual_rollout_val69_seed2/eval.log ADDED
@@ -0,0 +1,668 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ start
2
+ importing gymnasium/mani_skill
3
+ imported gymnasium/mani_skill
4
+ loaded checkpoint=runs/ctt_residual_full_seed2/model.pt
5
+ loaded charts source=301 target=69
6
+ generated proposals rows=69 total=552
7
+ env init sequential task=LiftPegUpright-v1 batch=4 sim=physx_cpu render=cpu
8
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=0 shape=(1, 1, 16, 7)
9
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=1 shape=(1, 1, 16, 7)
10
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=2 shape=(1, 1, 16, 7)
11
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=3 shape=(1, 1, 16, 7)
12
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=4 shape=(1, 1, 16, 7)
13
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=5 shape=(1, 1, 16, 7)
14
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=6 shape=(1, 1, 16, 7)
15
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=7 shape=(1, 1, 16, 7)
16
+ execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=8 shape=(1, 1, 16, 7)
17
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=0 shape=(1, 1, 16, 7)
18
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=1 shape=(1, 1, 16, 7)
19
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=2 shape=(1, 1, 16, 7)
20
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=3 shape=(1, 1, 16, 7)
21
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=4 shape=(1, 1, 16, 7)
22
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=5 shape=(1, 1, 16, 7)
23
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=6 shape=(1, 1, 16, 7)
24
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=7 shape=(1, 1, 16, 7)
25
+ execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=8 shape=(1, 1, 16, 7)
26
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=0 shape=(1, 1, 16, 7)
27
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=1 shape=(1, 1, 16, 7)
28
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=2 shape=(1, 1, 16, 7)
29
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=3 shape=(1, 1, 16, 7)
30
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=4 shape=(1, 1, 16, 7)
31
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=5 shape=(1, 1, 16, 7)
32
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=6 shape=(1, 1, 16, 7)
33
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=7 shape=(1, 1, 16, 7)
34
+ execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=8 shape=(1, 1, 16, 7)
35
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=0 shape=(1, 1, 16, 7)
36
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=1 shape=(1, 1, 16, 7)
37
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=2 shape=(1, 1, 16, 7)
38
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=3 shape=(1, 1, 16, 7)
39
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=4 shape=(1, 1, 16, 7)
40
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=5 shape=(1, 1, 16, 7)
41
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=6 shape=(1, 1, 16, 7)
42
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=7 shape=(1, 1, 16, 7)
43
+ execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=8 shape=(1, 1, 16, 7)
44
+ batch done sequential task=LiftPegUpright-v1
45
+ env init sequential task=LiftPegUpright-v1 batch=2 sim=physx_cpu render=cpu
46
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=0 shape=(1, 1, 16, 7)
47
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=1 shape=(1, 1, 16, 7)
48
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=2 shape=(1, 1, 16, 7)
49
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=3 shape=(1, 1, 16, 7)
50
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=4 shape=(1, 1, 16, 7)
51
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=5 shape=(1, 1, 16, 7)
52
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=6 shape=(1, 1, 16, 7)
53
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=7 shape=(1, 1, 16, 7)
54
+ execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=8 shape=(1, 1, 16, 7)
55
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=0 shape=(1, 1, 16, 7)
56
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=1 shape=(1, 1, 16, 7)
57
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=2 shape=(1, 1, 16, 7)
58
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=3 shape=(1, 1, 16, 7)
59
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=4 shape=(1, 1, 16, 7)
60
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=5 shape=(1, 1, 16, 7)
61
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=6 shape=(1, 1, 16, 7)
62
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=7 shape=(1, 1, 16, 7)
63
+ execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=8 shape=(1, 1, 16, 7)
64
+ batch done sequential task=LiftPegUpright-v1
65
+ env init sequential task=PickCube-v1 batch=1 sim=physx_cpu render=cpu
66
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=0 shape=(1, 1, 16, 7)
67
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=1 shape=(1, 1, 16, 7)
68
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=2 shape=(1, 1, 16, 7)
69
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=3 shape=(1, 1, 16, 7)
70
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=4 shape=(1, 1, 16, 7)
71
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=5 shape=(1, 1, 16, 7)
72
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=6 shape=(1, 1, 16, 7)
73
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=7 shape=(1, 1, 16, 7)
74
+ execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=8 shape=(1, 1, 16, 7)
75
+ batch done sequential task=PickCube-v1
76
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
77
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=0 shape=(1, 1, 16, 7)
78
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=1 shape=(1, 1, 16, 7)
79
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=2 shape=(1, 1, 16, 7)
80
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=3 shape=(1, 1, 16, 7)
81
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=4 shape=(1, 1, 16, 7)
82
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=5 shape=(1, 1, 16, 7)
83
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=6 shape=(1, 1, 16, 7)
84
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=7 shape=(1, 1, 16, 7)
85
+ execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=8 shape=(1, 1, 16, 7)
86
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=0 shape=(1, 1, 16, 7)
87
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=1 shape=(1, 1, 16, 7)
88
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=2 shape=(1, 1, 16, 7)
89
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=3 shape=(1, 1, 16, 7)
90
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=4 shape=(1, 1, 16, 7)
91
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=5 shape=(1, 1, 16, 7)
92
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=6 shape=(1, 1, 16, 7)
93
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=7 shape=(1, 1, 16, 7)
94
+ execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=8 shape=(1, 1, 16, 7)
95
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=0 shape=(1, 1, 16, 7)
96
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=1 shape=(1, 1, 16, 7)
97
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=2 shape=(1, 1, 16, 7)
98
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=3 shape=(1, 1, 16, 7)
99
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=4 shape=(1, 1, 16, 7)
100
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=5 shape=(1, 1, 16, 7)
101
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=6 shape=(1, 1, 16, 7)
102
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=7 shape=(1, 1, 16, 7)
103
+ execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=8 shape=(1, 1, 16, 7)
104
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=0 shape=(1, 1, 16, 7)
105
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=1 shape=(1, 1, 16, 7)
106
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=2 shape=(1, 1, 16, 7)
107
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=3 shape=(1, 1, 16, 7)
108
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=4 shape=(1, 1, 16, 7)
109
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=5 shape=(1, 1, 16, 7)
110
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=6 shape=(1, 1, 16, 7)
111
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=7 shape=(1, 1, 16, 7)
112
+ execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=8 shape=(1, 1, 16, 7)
113
+ batch done sequential task=PullCube-v1
114
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
115
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=0 shape=(1, 1, 16, 7)
116
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=1 shape=(1, 1, 16, 7)
117
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=2 shape=(1, 1, 16, 7)
118
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=3 shape=(1, 1, 16, 7)
119
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=4 shape=(1, 1, 16, 7)
120
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=5 shape=(1, 1, 16, 7)
121
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=6 shape=(1, 1, 16, 7)
122
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=7 shape=(1, 1, 16, 7)
123
+ execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=8 shape=(1, 1, 16, 7)
124
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=0 shape=(1, 1, 16, 7)
125
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=1 shape=(1, 1, 16, 7)
126
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=2 shape=(1, 1, 16, 7)
127
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=3 shape=(1, 1, 16, 7)
128
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=4 shape=(1, 1, 16, 7)
129
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=5 shape=(1, 1, 16, 7)
130
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=6 shape=(1, 1, 16, 7)
131
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=7 shape=(1, 1, 16, 7)
132
+ execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=8 shape=(1, 1, 16, 7)
133
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=0 shape=(1, 1, 16, 7)
134
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=1 shape=(1, 1, 16, 7)
135
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=2 shape=(1, 1, 16, 7)
136
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=3 shape=(1, 1, 16, 7)
137
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=4 shape=(1, 1, 16, 7)
138
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=5 shape=(1, 1, 16, 7)
139
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=6 shape=(1, 1, 16, 7)
140
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=7 shape=(1, 1, 16, 7)
141
+ execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=8 shape=(1, 1, 16, 7)
142
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=0 shape=(1, 1, 16, 7)
143
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=1 shape=(1, 1, 16, 7)
144
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=2 shape=(1, 1, 16, 7)
145
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=3 shape=(1, 1, 16, 7)
146
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=4 shape=(1, 1, 16, 7)
147
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=5 shape=(1, 1, 16, 7)
148
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=6 shape=(1, 1, 16, 7)
149
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=7 shape=(1, 1, 16, 7)
150
+ execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=8 shape=(1, 1, 16, 7)
151
+ batch done sequential task=PullCube-v1
152
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
153
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=0 shape=(1, 1, 16, 7)
154
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=1 shape=(1, 1, 16, 7)
155
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=2 shape=(1, 1, 16, 7)
156
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=3 shape=(1, 1, 16, 7)
157
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=4 shape=(1, 1, 16, 7)
158
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=5 shape=(1, 1, 16, 7)
159
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=6 shape=(1, 1, 16, 7)
160
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=7 shape=(1, 1, 16, 7)
161
+ execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=8 shape=(1, 1, 16, 7)
162
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=0 shape=(1, 1, 16, 7)
163
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=1 shape=(1, 1, 16, 7)
164
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=2 shape=(1, 1, 16, 7)
165
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=3 shape=(1, 1, 16, 7)
166
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=4 shape=(1, 1, 16, 7)
167
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=5 shape=(1, 1, 16, 7)
168
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=6 shape=(1, 1, 16, 7)
169
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=7 shape=(1, 1, 16, 7)
170
+ execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=8 shape=(1, 1, 16, 7)
171
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=0 shape=(1, 1, 16, 7)
172
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=1 shape=(1, 1, 16, 7)
173
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=2 shape=(1, 1, 16, 7)
174
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=3 shape=(1, 1, 16, 7)
175
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=4 shape=(1, 1, 16, 7)
176
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=5 shape=(1, 1, 16, 7)
177
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=6 shape=(1, 1, 16, 7)
178
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=7 shape=(1, 1, 16, 7)
179
+ execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=8 shape=(1, 1, 16, 7)
180
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=0 shape=(1, 1, 16, 7)
181
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=1 shape=(1, 1, 16, 7)
182
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=2 shape=(1, 1, 16, 7)
183
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=3 shape=(1, 1, 16, 7)
184
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=4 shape=(1, 1, 16, 7)
185
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=5 shape=(1, 1, 16, 7)
186
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=6 shape=(1, 1, 16, 7)
187
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=7 shape=(1, 1, 16, 7)
188
+ execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=8 shape=(1, 1, 16, 7)
189
+ batch done sequential task=PullCube-v1
190
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
191
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=0 shape=(1, 1, 16, 7)
192
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=1 shape=(1, 1, 16, 7)
193
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=2 shape=(1, 1, 16, 7)
194
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=3 shape=(1, 1, 16, 7)
195
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=4 shape=(1, 1, 16, 7)
196
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=5 shape=(1, 1, 16, 7)
197
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=6 shape=(1, 1, 16, 7)
198
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=7 shape=(1, 1, 16, 7)
199
+ execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=8 shape=(1, 1, 16, 7)
200
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=0 shape=(1, 1, 16, 7)
201
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=1 shape=(1, 1, 16, 7)
202
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=2 shape=(1, 1, 16, 7)
203
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=3 shape=(1, 1, 16, 7)
204
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=4 shape=(1, 1, 16, 7)
205
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=5 shape=(1, 1, 16, 7)
206
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=6 shape=(1, 1, 16, 7)
207
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=7 shape=(1, 1, 16, 7)
208
+ execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=8 shape=(1, 1, 16, 7)
209
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=0 shape=(1, 1, 16, 7)
210
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=1 shape=(1, 1, 16, 7)
211
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=2 shape=(1, 1, 16, 7)
212
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=3 shape=(1, 1, 16, 7)
213
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=4 shape=(1, 1, 16, 7)
214
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=5 shape=(1, 1, 16, 7)
215
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=6 shape=(1, 1, 16, 7)
216
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=7 shape=(1, 1, 16, 7)
217
+ execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=8 shape=(1, 1, 16, 7)
218
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=0 shape=(1, 1, 16, 7)
219
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=1 shape=(1, 1, 16, 7)
220
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=2 shape=(1, 1, 16, 7)
221
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=3 shape=(1, 1, 16, 7)
222
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=4 shape=(1, 1, 16, 7)
223
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=5 shape=(1, 1, 16, 7)
224
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=6 shape=(1, 1, 16, 7)
225
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=7 shape=(1, 1, 16, 7)
226
+ execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=8 shape=(1, 1, 16, 7)
227
+ batch done sequential task=PullCube-v1
228
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
229
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=0 shape=(1, 1, 16, 7)
230
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=1 shape=(1, 1, 16, 7)
231
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=2 shape=(1, 1, 16, 7)
232
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=3 shape=(1, 1, 16, 7)
233
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=4 shape=(1, 1, 16, 7)
234
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=5 shape=(1, 1, 16, 7)
235
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=6 shape=(1, 1, 16, 7)
236
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=7 shape=(1, 1, 16, 7)
237
+ execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=8 shape=(1, 1, 16, 7)
238
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=0 shape=(1, 1, 16, 7)
239
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=1 shape=(1, 1, 16, 7)
240
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=2 shape=(1, 1, 16, 7)
241
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=3 shape=(1, 1, 16, 7)
242
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=4 shape=(1, 1, 16, 7)
243
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=5 shape=(1, 1, 16, 7)
244
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=6 shape=(1, 1, 16, 7)
245
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=7 shape=(1, 1, 16, 7)
246
+ execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=8 shape=(1, 1, 16, 7)
247
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=0 shape=(1, 1, 16, 7)
248
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=1 shape=(1, 1, 16, 7)
249
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=2 shape=(1, 1, 16, 7)
250
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=3 shape=(1, 1, 16, 7)
251
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=4 shape=(1, 1, 16, 7)
252
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=5 shape=(1, 1, 16, 7)
253
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=6 shape=(1, 1, 16, 7)
254
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=7 shape=(1, 1, 16, 7)
255
+ execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=8 shape=(1, 1, 16, 7)
256
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=0 shape=(1, 1, 16, 7)
257
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=1 shape=(1, 1, 16, 7)
258
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=2 shape=(1, 1, 16, 7)
259
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=3 shape=(1, 1, 16, 7)
260
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=4 shape=(1, 1, 16, 7)
261
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=5 shape=(1, 1, 16, 7)
262
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=6 shape=(1, 1, 16, 7)
263
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=7 shape=(1, 1, 16, 7)
264
+ execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=8 shape=(1, 1, 16, 7)
265
+ batch done sequential task=PullCube-v1
266
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
267
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=0 shape=(1, 1, 16, 7)
268
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=1 shape=(1, 1, 16, 7)
269
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=2 shape=(1, 1, 16, 7)
270
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=3 shape=(1, 1, 16, 7)
271
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=4 shape=(1, 1, 16, 7)
272
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=5 shape=(1, 1, 16, 7)
273
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=6 shape=(1, 1, 16, 7)
274
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=7 shape=(1, 1, 16, 7)
275
+ execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=8 shape=(1, 1, 16, 7)
276
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=0 shape=(1, 1, 16, 7)
277
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=1 shape=(1, 1, 16, 7)
278
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=2 shape=(1, 1, 16, 7)
279
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=3 shape=(1, 1, 16, 7)
280
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=4 shape=(1, 1, 16, 7)
281
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=5 shape=(1, 1, 16, 7)
282
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=6 shape=(1, 1, 16, 7)
283
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=7 shape=(1, 1, 16, 7)
284
+ execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=8 shape=(1, 1, 16, 7)
285
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=0 shape=(1, 1, 16, 7)
286
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=1 shape=(1, 1, 16, 7)
287
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=2 shape=(1, 1, 16, 7)
288
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=3 shape=(1, 1, 16, 7)
289
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=4 shape=(1, 1, 16, 7)
290
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=5 shape=(1, 1, 16, 7)
291
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=6 shape=(1, 1, 16, 7)
292
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=7 shape=(1, 1, 16, 7)
293
+ execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=8 shape=(1, 1, 16, 7)
294
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=0 shape=(1, 1, 16, 7)
295
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=1 shape=(1, 1, 16, 7)
296
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=2 shape=(1, 1, 16, 7)
297
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=3 shape=(1, 1, 16, 7)
298
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=4 shape=(1, 1, 16, 7)
299
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=5 shape=(1, 1, 16, 7)
300
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=6 shape=(1, 1, 16, 7)
301
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=7 shape=(1, 1, 16, 7)
302
+ execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=8 shape=(1, 1, 16, 7)
303
+ batch done sequential task=PullCube-v1
304
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
305
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=0 shape=(1, 1, 16, 7)
306
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=1 shape=(1, 1, 16, 7)
307
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=2 shape=(1, 1, 16, 7)
308
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=3 shape=(1, 1, 16, 7)
309
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=4 shape=(1, 1, 16, 7)
310
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=5 shape=(1, 1, 16, 7)
311
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=6 shape=(1, 1, 16, 7)
312
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=7 shape=(1, 1, 16, 7)
313
+ execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=8 shape=(1, 1, 16, 7)
314
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=0 shape=(1, 1, 16, 7)
315
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=1 shape=(1, 1, 16, 7)
316
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=2 shape=(1, 1, 16, 7)
317
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=3 shape=(1, 1, 16, 7)
318
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=4 shape=(1, 1, 16, 7)
319
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=5 shape=(1, 1, 16, 7)
320
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=6 shape=(1, 1, 16, 7)
321
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=7 shape=(1, 1, 16, 7)
322
+ execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=8 shape=(1, 1, 16, 7)
323
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=0 shape=(1, 1, 16, 7)
324
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=1 shape=(1, 1, 16, 7)
325
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=2 shape=(1, 1, 16, 7)
326
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=3 shape=(1, 1, 16, 7)
327
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=4 shape=(1, 1, 16, 7)
328
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=5 shape=(1, 1, 16, 7)
329
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=6 shape=(1, 1, 16, 7)
330
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=7 shape=(1, 1, 16, 7)
331
+ execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=8 shape=(1, 1, 16, 7)
332
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=0 shape=(1, 1, 16, 7)
333
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=1 shape=(1, 1, 16, 7)
334
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=2 shape=(1, 1, 16, 7)
335
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=3 shape=(1, 1, 16, 7)
336
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=4 shape=(1, 1, 16, 7)
337
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=5 shape=(1, 1, 16, 7)
338
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=6 shape=(1, 1, 16, 7)
339
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=7 shape=(1, 1, 16, 7)
340
+ execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=8 shape=(1, 1, 16, 7)
341
+ batch done sequential task=PullCube-v1
342
+ env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu
343
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=0 shape=(1, 1, 16, 7)
344
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=1 shape=(1, 1, 16, 7)
345
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=2 shape=(1, 1, 16, 7)
346
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=3 shape=(1, 1, 16, 7)
347
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=4 shape=(1, 1, 16, 7)
348
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=5 shape=(1, 1, 16, 7)
349
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=6 shape=(1, 1, 16, 7)
350
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=7 shape=(1, 1, 16, 7)
351
+ execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=8 shape=(1, 1, 16, 7)
352
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=0 shape=(1, 1, 16, 7)
353
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=1 shape=(1, 1, 16, 7)
354
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=2 shape=(1, 1, 16, 7)
355
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=3 shape=(1, 1, 16, 7)
356
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=4 shape=(1, 1, 16, 7)
357
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=5 shape=(1, 1, 16, 7)
358
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=6 shape=(1, 1, 16, 7)
359
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=7 shape=(1, 1, 16, 7)
360
+ execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=8 shape=(1, 1, 16, 7)
361
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=0 shape=(1, 1, 16, 7)
362
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=1 shape=(1, 1, 16, 7)
363
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=2 shape=(1, 1, 16, 7)
364
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=3 shape=(1, 1, 16, 7)
365
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=4 shape=(1, 1, 16, 7)
366
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=5 shape=(1, 1, 16, 7)
367
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=6 shape=(1, 1, 16, 7)
368
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=7 shape=(1, 1, 16, 7)
369
+ execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=8 shape=(1, 1, 16, 7)
370
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=0 shape=(1, 1, 16, 7)
371
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=1 shape=(1, 1, 16, 7)
372
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=2 shape=(1, 1, 16, 7)
373
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=3 shape=(1, 1, 16, 7)
374
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=4 shape=(1, 1, 16, 7)
375
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=5 shape=(1, 1, 16, 7)
376
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=6 shape=(1, 1, 16, 7)
377
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=7 shape=(1, 1, 16, 7)
378
+ execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=8 shape=(1, 1, 16, 7)
379
+ batch done sequential task=PullCube-v1
380
+ env init sequential task=PullCube-v1 batch=2 sim=physx_cpu render=cpu
381
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=0 shape=(1, 1, 16, 7)
382
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=1 shape=(1, 1, 16, 7)
383
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=2 shape=(1, 1, 16, 7)
384
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=3 shape=(1, 1, 16, 7)
385
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=4 shape=(1, 1, 16, 7)
386
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=5 shape=(1, 1, 16, 7)
387
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=6 shape=(1, 1, 16, 7)
388
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=7 shape=(1, 1, 16, 7)
389
+ execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=8 shape=(1, 1, 16, 7)
390
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=0 shape=(1, 1, 16, 7)
391
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=1 shape=(1, 1, 16, 7)
392
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=2 shape=(1, 1, 16, 7)
393
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=3 shape=(1, 1, 16, 7)
394
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=4 shape=(1, 1, 16, 7)
395
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=5 shape=(1, 1, 16, 7)
396
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=6 shape=(1, 1, 16, 7)
397
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=7 shape=(1, 1, 16, 7)
398
+ execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=8 shape=(1, 1, 16, 7)
399
+ batch done sequential task=PullCube-v1
400
+ env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu
401
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=0 shape=(1, 1, 16, 7)
402
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=1 shape=(1, 1, 16, 7)
403
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=2 shape=(1, 1, 16, 7)
404
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=3 shape=(1, 1, 16, 7)
405
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=4 shape=(1, 1, 16, 7)
406
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=5 shape=(1, 1, 16, 7)
407
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=6 shape=(1, 1, 16, 7)
408
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=7 shape=(1, 1, 16, 7)
409
+ execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=8 shape=(1, 1, 16, 7)
410
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=0 shape=(1, 1, 16, 7)
411
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=1 shape=(1, 1, 16, 7)
412
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=2 shape=(1, 1, 16, 7)
413
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=3 shape=(1, 1, 16, 7)
414
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=4 shape=(1, 1, 16, 7)
415
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=5 shape=(1, 1, 16, 7)
416
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=6 shape=(1, 1, 16, 7)
417
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=7 shape=(1, 1, 16, 7)
418
+ execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=8 shape=(1, 1, 16, 7)
419
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=0 shape=(1, 1, 16, 7)
420
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=1 shape=(1, 1, 16, 7)
421
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=2 shape=(1, 1, 16, 7)
422
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=3 shape=(1, 1, 16, 7)
423
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=4 shape=(1, 1, 16, 7)
424
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=5 shape=(1, 1, 16, 7)
425
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=6 shape=(1, 1, 16, 7)
426
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=7 shape=(1, 1, 16, 7)
427
+ execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=8 shape=(1, 1, 16, 7)
428
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=0 shape=(1, 1, 16, 7)
429
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=1 shape=(1, 1, 16, 7)
430
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=2 shape=(1, 1, 16, 7)
431
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=3 shape=(1, 1, 16, 7)
432
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=4 shape=(1, 1, 16, 7)
433
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=5 shape=(1, 1, 16, 7)
434
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=6 shape=(1, 1, 16, 7)
435
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=7 shape=(1, 1, 16, 7)
436
+ execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=8 shape=(1, 1, 16, 7)
437
+ batch done sequential task=PushCube-v1
438
+ env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu
439
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=0 shape=(1, 1, 16, 7)
440
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=1 shape=(1, 1, 16, 7)
441
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=2 shape=(1, 1, 16, 7)
442
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=3 shape=(1, 1, 16, 7)
443
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=4 shape=(1, 1, 16, 7)
444
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=5 shape=(1, 1, 16, 7)
445
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=6 shape=(1, 1, 16, 7)
446
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=7 shape=(1, 1, 16, 7)
447
+ execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=8 shape=(1, 1, 16, 7)
448
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=0 shape=(1, 1, 16, 7)
449
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=1 shape=(1, 1, 16, 7)
450
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=2 shape=(1, 1, 16, 7)
451
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=3 shape=(1, 1, 16, 7)
452
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=4 shape=(1, 1, 16, 7)
453
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=5 shape=(1, 1, 16, 7)
454
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=6 shape=(1, 1, 16, 7)
455
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=7 shape=(1, 1, 16, 7)
456
+ execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=8 shape=(1, 1, 16, 7)
457
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=0 shape=(1, 1, 16, 7)
458
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=1 shape=(1, 1, 16, 7)
459
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=2 shape=(1, 1, 16, 7)
460
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=3 shape=(1, 1, 16, 7)
461
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=4 shape=(1, 1, 16, 7)
462
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=5 shape=(1, 1, 16, 7)
463
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=6 shape=(1, 1, 16, 7)
464
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=7 shape=(1, 1, 16, 7)
465
+ execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=8 shape=(1, 1, 16, 7)
466
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=0 shape=(1, 1, 16, 7)
467
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=1 shape=(1, 1, 16, 7)
468
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=2 shape=(1, 1, 16, 7)
469
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=3 shape=(1, 1, 16, 7)
470
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=4 shape=(1, 1, 16, 7)
471
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=5 shape=(1, 1, 16, 7)
472
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=6 shape=(1, 1, 16, 7)
473
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=7 shape=(1, 1, 16, 7)
474
+ execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=8 shape=(1, 1, 16, 7)
475
+ batch done sequential task=PushCube-v1
476
+ env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu
477
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=0 shape=(1, 1, 16, 7)
478
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=1 shape=(1, 1, 16, 7)
479
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=2 shape=(1, 1, 16, 7)
480
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=3 shape=(1, 1, 16, 7)
481
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=4 shape=(1, 1, 16, 7)
482
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=5 shape=(1, 1, 16, 7)
483
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=6 shape=(1, 1, 16, 7)
484
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=7 shape=(1, 1, 16, 7)
485
+ execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=8 shape=(1, 1, 16, 7)
486
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=0 shape=(1, 1, 16, 7)
487
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=1 shape=(1, 1, 16, 7)
488
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=2 shape=(1, 1, 16, 7)
489
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=3 shape=(1, 1, 16, 7)
490
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=4 shape=(1, 1, 16, 7)
491
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=5 shape=(1, 1, 16, 7)
492
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=6 shape=(1, 1, 16, 7)
493
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=7 shape=(1, 1, 16, 7)
494
+ execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=8 shape=(1, 1, 16, 7)
495
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=0 shape=(1, 1, 16, 7)
496
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=1 shape=(1, 1, 16, 7)
497
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=2 shape=(1, 1, 16, 7)
498
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=3 shape=(1, 1, 16, 7)
499
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=4 shape=(1, 1, 16, 7)
500
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=5 shape=(1, 1, 16, 7)
501
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=6 shape=(1, 1, 16, 7)
502
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=7 shape=(1, 1, 16, 7)
503
+ execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=8 shape=(1, 1, 16, 7)
504
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=0 shape=(1, 1, 16, 7)
505
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=1 shape=(1, 1, 16, 7)
506
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=2 shape=(1, 1, 16, 7)
507
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=3 shape=(1, 1, 16, 7)
508
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=4 shape=(1, 1, 16, 7)
509
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=5 shape=(1, 1, 16, 7)
510
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=6 shape=(1, 1, 16, 7)
511
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=7 shape=(1, 1, 16, 7)
512
+ execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=8 shape=(1, 1, 16, 7)
513
+ batch done sequential task=PushCube-v1
514
+ env init sequential task=PushCube-v1 batch=1 sim=physx_cpu render=cpu
515
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=0 shape=(1, 1, 16, 7)
516
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=1 shape=(1, 1, 16, 7)
517
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=2 shape=(1, 1, 16, 7)
518
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=3 shape=(1, 1, 16, 7)
519
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=4 shape=(1, 1, 16, 7)
520
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=5 shape=(1, 1, 16, 7)
521
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=6 shape=(1, 1, 16, 7)
522
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=7 shape=(1, 1, 16, 7)
523
+ execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=8 shape=(1, 1, 16, 7)
524
+ batch done sequential task=PushCube-v1
525
+ env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu
526
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=0 shape=(1, 1, 16, 7)
527
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=1 shape=(1, 1, 16, 7)
528
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=2 shape=(1, 1, 16, 7)
529
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=3 shape=(1, 1, 16, 7)
530
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=4 shape=(1, 1, 16, 7)
531
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=5 shape=(1, 1, 16, 7)
532
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=6 shape=(1, 1, 16, 7)
533
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=7 shape=(1, 1, 16, 7)
534
+ execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=8 shape=(1, 1, 16, 7)
535
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=0 shape=(1, 1, 16, 7)
536
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=1 shape=(1, 1, 16, 7)
537
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=2 shape=(1, 1, 16, 7)
538
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=3 shape=(1, 1, 16, 7)
539
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=4 shape=(1, 1, 16, 7)
540
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=5 shape=(1, 1, 16, 7)
541
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=6 shape=(1, 1, 16, 7)
542
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=7 shape=(1, 1, 16, 7)
543
+ execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=8 shape=(1, 1, 16, 7)
544
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=0 shape=(1, 1, 16, 7)
545
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=1 shape=(1, 1, 16, 7)
546
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=2 shape=(1, 1, 16, 7)
547
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=3 shape=(1, 1, 16, 7)
548
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=4 shape=(1, 1, 16, 7)
549
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=5 shape=(1, 1, 16, 7)
550
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=6 shape=(1, 1, 16, 7)
551
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=7 shape=(1, 1, 16, 7)
552
+ execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=8 shape=(1, 1, 16, 7)
553
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=0 shape=(1, 1, 16, 7)
554
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=1 shape=(1, 1, 16, 7)
555
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=2 shape=(1, 1, 16, 7)
556
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=3 shape=(1, 1, 16, 7)
557
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=4 shape=(1, 1, 16, 7)
558
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=5 shape=(1, 1, 16, 7)
559
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=6 shape=(1, 1, 16, 7)
560
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=7 shape=(1, 1, 16, 7)
561
+ execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=8 shape=(1, 1, 16, 7)
562
+ batch done sequential task=StackCube-v1
563
+ env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu
564
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=0 shape=(1, 1, 16, 7)
565
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=1 shape=(1, 1, 16, 7)
566
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=2 shape=(1, 1, 16, 7)
567
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=3 shape=(1, 1, 16, 7)
568
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=4 shape=(1, 1, 16, 7)
569
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=5 shape=(1, 1, 16, 7)
570
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=6 shape=(1, 1, 16, 7)
571
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=7 shape=(1, 1, 16, 7)
572
+ execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=8 shape=(1, 1, 16, 7)
573
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=0 shape=(1, 1, 16, 7)
574
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=1 shape=(1, 1, 16, 7)
575
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=2 shape=(1, 1, 16, 7)
576
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=3 shape=(1, 1, 16, 7)
577
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=4 shape=(1, 1, 16, 7)
578
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=5 shape=(1, 1, 16, 7)
579
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=6 shape=(1, 1, 16, 7)
580
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=7 shape=(1, 1, 16, 7)
581
+ execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=8 shape=(1, 1, 16, 7)
582
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=0 shape=(1, 1, 16, 7)
583
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=1 shape=(1, 1, 16, 7)
584
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=2 shape=(1, 1, 16, 7)
585
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=3 shape=(1, 1, 16, 7)
586
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=4 shape=(1, 1, 16, 7)
587
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=5 shape=(1, 1, 16, 7)
588
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=6 shape=(1, 1, 16, 7)
589
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=7 shape=(1, 1, 16, 7)
590
+ execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=8 shape=(1, 1, 16, 7)
591
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=0 shape=(1, 1, 16, 7)
592
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=1 shape=(1, 1, 16, 7)
593
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=2 shape=(1, 1, 16, 7)
594
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=3 shape=(1, 1, 16, 7)
595
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=4 shape=(1, 1, 16, 7)
596
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=5 shape=(1, 1, 16, 7)
597
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=6 shape=(1, 1, 16, 7)
598
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=7 shape=(1, 1, 16, 7)
599
+ execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=8 shape=(1, 1, 16, 7)
600
+ batch done sequential task=StackCube-v1
601
+ env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu
602
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=0 shape=(1, 1, 16, 7)
603
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=1 shape=(1, 1, 16, 7)
604
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=2 shape=(1, 1, 16, 7)
605
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=3 shape=(1, 1, 16, 7)
606
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=4 shape=(1, 1, 16, 7)
607
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=5 shape=(1, 1, 16, 7)
608
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=6 shape=(1, 1, 16, 7)
609
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=7 shape=(1, 1, 16, 7)
610
+ execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=8 shape=(1, 1, 16, 7)
611
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=0 shape=(1, 1, 16, 7)
612
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=1 shape=(1, 1, 16, 7)
613
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=2 shape=(1, 1, 16, 7)
614
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=3 shape=(1, 1, 16, 7)
615
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=4 shape=(1, 1, 16, 7)
616
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=5 shape=(1, 1, 16, 7)
617
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=6 shape=(1, 1, 16, 7)
618
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=7 shape=(1, 1, 16, 7)
619
+ execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=8 shape=(1, 1, 16, 7)
620
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=0 shape=(1, 1, 16, 7)
621
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=1 shape=(1, 1, 16, 7)
622
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=2 shape=(1, 1, 16, 7)
623
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=3 shape=(1, 1, 16, 7)
624
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=4 shape=(1, 1, 16, 7)
625
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=5 shape=(1, 1, 16, 7)
626
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=6 shape=(1, 1, 16, 7)
627
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=7 shape=(1, 1, 16, 7)
628
+ execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=8 shape=(1, 1, 16, 7)
629
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=0 shape=(1, 1, 16, 7)
630
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=1 shape=(1, 1, 16, 7)
631
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=2 shape=(1, 1, 16, 7)
632
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=3 shape=(1, 1, 16, 7)
633
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=4 shape=(1, 1, 16, 7)
634
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=5 shape=(1, 1, 16, 7)
635
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=6 shape=(1, 1, 16, 7)
636
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=7 shape=(1, 1, 16, 7)
637
+ execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=8 shape=(1, 1, 16, 7)
638
+ batch done sequential task=StackCube-v1
639
+ env init sequential task=StackCube-v1 batch=3 sim=physx_cpu render=cpu
640
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=0 shape=(1, 1, 16, 7)
641
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=1 shape=(1, 1, 16, 7)
642
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=2 shape=(1, 1, 16, 7)
643
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=3 shape=(1, 1, 16, 7)
644
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=4 shape=(1, 1, 16, 7)
645
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=5 shape=(1, 1, 16, 7)
646
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=6 shape=(1, 1, 16, 7)
647
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=7 shape=(1, 1, 16, 7)
648
+ execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=8 shape=(1, 1, 16, 7)
649
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=0 shape=(1, 1, 16, 7)
650
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=1 shape=(1, 1, 16, 7)
651
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=2 shape=(1, 1, 16, 7)
652
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=3 shape=(1, 1, 16, 7)
653
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=4 shape=(1, 1, 16, 7)
654
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=5 shape=(1, 1, 16, 7)
655
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=6 shape=(1, 1, 16, 7)
656
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=7 shape=(1, 1, 16, 7)
657
+ execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=8 shape=(1, 1, 16, 7)
658
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=0 shape=(1, 1, 16, 7)
659
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=1 shape=(1, 1, 16, 7)
660
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=2 shape=(1, 1, 16, 7)
661
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=3 shape=(1, 1, 16, 7)
662
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=4 shape=(1, 1, 16, 7)
663
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=5 shape=(1, 1, 16, 7)
664
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=6 shape=(1, 1, 16, 7)
665
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=7 shape=(1, 1, 16, 7)
666
+ execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=8 shape=(1, 1, 16, 7)
667
+ batch done sequential task=StackCube-v1
668
+ rollout complete rows=69
workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics.json CHANGED
The diff for this file is too large to render. See raw diff
 
workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics_by_seed.json CHANGED
@@ -1,331 +1,1083 @@
1
  {
2
  "1025740212": {
 
 
3
  "branch_car_at_8": 1.772506669163704,
 
 
4
  "outcome_ptr_at_8": 1.0,
5
  "pairwise_causal_calibration_ece": 0.17388092747958847,
 
 
 
 
 
 
 
 
6
  "selector_regret_at_8": 1.772506669163704,
7
- "support_gap_at_8": 0.0
 
 
 
 
8
  },
9
  "1055157354": {
 
 
10
  "branch_car_at_8": 0.03480546176433563,
 
 
11
  "outcome_ptr_at_8": 1.0,
12
  "pairwise_causal_calibration_ece": 0.1805709977888097,
 
 
 
 
 
 
 
 
13
  "selector_regret_at_8": 0.03480546176433563,
14
- "support_gap_at_8": 1.775326356291771
 
 
 
 
15
  },
16
  "1055567618": {
 
 
17
  "branch_car_at_8": 1.9218642711639404,
 
 
18
  "outcome_ptr_at_8": 0.0,
19
  "pairwise_causal_calibration_ece": 0.19289744165908876,
 
 
 
 
 
 
 
 
20
  "selector_regret_at_8": 1.9218642711639404,
21
- "support_gap_at_8": 0.0
 
 
 
 
22
  },
23
  "1113268940": {
 
 
24
  "branch_car_at_8": 0.06608206033706665,
 
 
25
  "outcome_ptr_at_8": 1.0,
26
  "pairwise_causal_calibration_ece": 0.032532089835823584,
 
 
 
 
 
 
 
 
27
  "selector_regret_at_8": 0.06608206033706665,
28
- "support_gap_at_8": 1.2974750995635986
 
 
 
 
29
  },
30
  "1113371261": {
 
 
31
  "branch_car_at_8": 0.0,
 
 
32
  "outcome_ptr_at_8": 0.0,
33
  "pairwise_causal_calibration_ece": 0.233029795067053,
 
 
 
 
 
 
 
 
34
  "selector_regret_at_8": 0.0,
35
- "support_gap_at_8": 0.0
 
 
 
 
36
  },
37
  "1176943004": {
 
 
38
  "branch_car_at_8": 0.0,
 
 
39
  "outcome_ptr_at_8": 1.0,
40
  "pairwise_causal_calibration_ece": 0.17468938592004596,
 
 
 
 
 
 
 
 
41
  "selector_regret_at_8": 0.0,
42
- "support_gap_at_8": 0.0
 
 
 
 
43
  },
44
  "1189392917": {
 
 
45
  "branch_car_at_8": 0.0,
 
 
46
  "outcome_ptr_at_8": 1.0,
47
  "pairwise_causal_calibration_ece": 0.0,
 
 
 
 
 
 
 
 
48
  "selector_regret_at_8": 0.0,
49
- "support_gap_at_8": 0.0
 
 
 
 
50
  },
51
  "1231780997": {
 
 
52
  "branch_car_at_8": 0.5006794333457947,
 
 
53
  "outcome_ptr_at_8": 1.0,
54
  "pairwise_causal_calibration_ece": 0.10207358838016511,
 
 
 
 
 
 
 
 
55
  "selector_regret_at_8": 0.5006794333457947,
56
- "support_gap_at_8": 1.2721588015556335
 
 
 
 
57
  },
58
  "1271718007": {
 
 
59
  "branch_car_at_8": 0.19734752923250198,
 
 
60
  "outcome_ptr_at_8": 0.0,
61
  "pairwise_causal_calibration_ece": 0.2350719661460603,
 
 
 
 
 
 
 
 
62
  "selector_regret_at_8": 0.19734752923250198,
63
- "support_gap_at_8": 0.7364327311515808
 
 
 
 
64
  },
65
  "1273247072": {
 
 
66
  "branch_car_at_8": 0.35906313359737396,
 
 
67
  "outcome_ptr_at_8": 1.0,
68
  "pairwise_causal_calibration_ece": 0.2793744975404514,
 
 
 
 
 
 
 
 
69
  "selector_regret_at_8": 0.35906313359737396,
70
- "support_gap_at_8": 1.5362297892570496
 
 
 
 
71
  },
72
  "1358421325": {
 
 
73
  "branch_car_at_8": 1.90096914768219,
 
 
74
  "outcome_ptr_at_8": 1.0,
75
  "pairwise_causal_calibration_ece": 0.32948159974174274,
 
 
 
 
 
 
 
 
76
  "selector_regret_at_8": 1.90096914768219,
77
- "support_gap_at_8": 0.0
 
 
 
 
78
  },
79
  "1375492786": {
 
 
80
  "branch_car_at_8": 0.0002880295505747199,
 
 
81
  "outcome_ptr_at_8": 0.0,
82
  "pairwise_causal_calibration_ece": 0.008822272371479922,
 
 
 
 
 
 
 
 
83
  "selector_regret_at_8": 0.0002880295505747199,
84
- "support_gap_at_8": 0.24256371456431225
 
 
 
 
85
  },
86
  "141984615": {
 
 
87
  "branch_car_at_8": 0.013830550014972687,
 
 
88
  "outcome_ptr_at_8": 0.3333333333333333,
89
  "pairwise_causal_calibration_ece": 0.19434281311769708,
 
 
 
 
 
 
 
 
90
  "selector_regret_at_8": 0.013830550014972687,
91
- "support_gap_at_8": 0.14621367553869882
 
 
 
 
92
  },
93
  "1475440973": {
 
 
94
  "branch_car_at_8": 1.28389310836792,
 
 
95
  "outcome_ptr_at_8": 0.0,
96
  "pairwise_causal_calibration_ece": 0.22214884767353088,
 
 
 
 
 
 
 
 
97
  "selector_regret_at_8": 1.28389310836792,
98
- "support_gap_at_8": 0.0
 
 
 
 
99
  },
100
  "1483645336": {
 
 
101
  "branch_car_at_8": 1.9686058908700943,
 
 
102
  "outcome_ptr_at_8": 0.0,
103
  "pairwise_causal_calibration_ece": 0.5995435922806452,
 
 
 
 
 
 
 
 
104
  "selector_regret_at_8": 1.9686058908700943,
105
- "support_gap_at_8": 0.0
 
 
 
 
106
  },
107
  "1522272686": {
 
 
108
  "branch_car_at_8": 0.017972871661186218,
 
 
109
  "outcome_ptr_at_8": 1.0,
110
  "pairwise_causal_calibration_ece": 0.3276128524355476,
 
 
 
 
 
 
 
 
111
  "selector_regret_at_8": 0.017972871661186218,
112
- "support_gap_at_8": 1.8050999492406845
 
 
 
 
113
  },
114
  "1528740417": {
 
 
115
  "branch_car_at_8": 1.280472755432129,
 
 
116
  "outcome_ptr_at_8": 0.0,
117
  "pairwise_causal_calibration_ece": 0.16399362968800954,
 
 
 
 
 
 
 
 
118
  "selector_regret_at_8": 1.280472755432129,
119
- "support_gap_at_8": 0.0
 
 
 
 
120
  },
121
  "1549324805": {
 
 
122
  "branch_car_at_8": 0.01237945475926002,
 
 
123
  "outcome_ptr_at_8": 0.0,
124
  "pairwise_causal_calibration_ece": 0.23864987006465407,
 
 
 
 
 
 
 
 
125
  "selector_regret_at_8": 0.01237945475926002,
126
- "support_gap_at_8": 0.7194042829796672
 
 
 
 
127
  },
128
  "1607773860": {
 
 
129
  "branch_car_at_8": 0.0,
 
 
130
  "outcome_ptr_at_8": 0.0,
131
  "pairwise_causal_calibration_ece": 0.24177237819615016,
 
 
 
 
 
 
 
 
132
  "selector_regret_at_8": 0.0,
133
- "support_gap_at_8": 0.0
 
 
 
 
134
  },
135
  "1665739511": {
 
 
136
  "branch_car_at_8": 0.0,
 
 
137
  "outcome_ptr_at_8": 0.0,
138
  "pairwise_causal_calibration_ece": 0.2508232858898205,
 
 
 
 
 
 
 
 
139
  "selector_regret_at_8": 0.0,
140
- "support_gap_at_8": 0.0
 
 
 
 
141
  },
142
  "1747178030": {
 
 
143
  "branch_car_at_8": 0.034092940390110016,
 
 
144
  "outcome_ptr_at_8": 0.0,
145
  "pairwise_causal_calibration_ece": 0.08595818214364334,
 
 
 
 
 
 
 
 
146
  "selector_regret_at_8": 0.034092940390110016,
147
- "support_gap_at_8": 1.8751946315169334
 
 
 
 
148
  },
149
  "1881223807": {
 
 
150
  "branch_car_at_8": 0.024854734539985657,
 
 
151
  "outcome_ptr_at_8": 1.0,
152
  "pairwise_causal_calibration_ece": 0.2146835718150399,
 
 
 
 
 
 
 
 
153
  "selector_regret_at_8": 0.024854734539985657,
154
- "support_gap_at_8": 1.956517294049263
 
 
 
 
155
  },
156
  "1919198188": {
 
 
157
  "branch_car_at_8": 0.018839523196220398,
 
 
158
  "outcome_ptr_at_8": 1.0,
159
  "pairwise_causal_calibration_ece": 0.1669540555093941,
 
 
 
 
 
 
 
 
160
  "selector_regret_at_8": 0.018839523196220398,
161
- "support_gap_at_8": 1.9372489601373672
 
 
 
 
162
  },
163
  "1991414747": {
 
 
164
  "branch_car_at_8": 0.0,
 
 
165
  "outcome_ptr_at_8": 0.0,
166
  "pairwise_causal_calibration_ece": 0.11896663871153935,
 
 
 
 
 
 
 
 
167
  "selector_regret_at_8": 0.0,
168
- "support_gap_at_8": 0.0
 
 
 
 
169
  },
170
  "1997090716": {
 
 
171
  "branch_car_at_8": 0.0,
 
 
172
  "outcome_ptr_at_8": 1.0,
173
  "pairwise_causal_calibration_ece": 0.14617959762414967,
 
 
 
 
 
 
 
 
174
  "selector_regret_at_8": 0.0,
175
- "support_gap_at_8": 0.0
 
 
 
 
176
  },
177
  "2015956957": {
 
 
178
  "branch_car_at_8": 0.005573729798197746,
 
 
179
  "outcome_ptr_at_8": 0.6,
180
  "pairwise_causal_calibration_ece": 0.1431252893441326,
 
 
 
 
 
 
 
 
181
  "selector_regret_at_8": 0.005573729798197746,
182
- "support_gap_at_8": 0.09007053077220917
 
 
 
 
183
  },
184
  "2054899126": {
 
 
185
  "branch_car_at_8": 0.0,
 
 
186
  "outcome_ptr_at_8": 0.0,
187
  "pairwise_causal_calibration_ece": 0.0,
 
 
 
 
 
 
 
 
188
  "selector_regret_at_8": 0.0,
189
- "support_gap_at_8": 0.0
 
 
 
 
190
  },
191
  "211245842": {
 
 
192
  "branch_car_at_8": 0.0,
 
 
193
  "outcome_ptr_at_8": 1.0,
194
  "pairwise_causal_calibration_ece": 0.13508468824583414,
 
 
 
 
 
 
 
 
195
  "selector_regret_at_8": 0.0,
196
- "support_gap_at_8": 0.0
 
 
 
 
197
  },
198
  "2141802816": {
 
 
199
  "branch_car_at_8": 1.861636370420456,
 
 
200
  "outcome_ptr_at_8": 1.0,
201
  "pairwise_causal_calibration_ece": 0.09903289408743085,
 
 
 
 
 
 
 
 
202
  "selector_regret_at_8": 1.861636370420456,
203
- "support_gap_at_8": 0.0
 
 
 
 
204
  },
205
  "256770874": {
 
 
206
  "branch_car_at_8": 0.046216949820518494,
 
 
207
  "outcome_ptr_at_8": 0.0,
208
  "pairwise_causal_calibration_ece": 0.046630940401554466,
 
 
 
 
 
 
 
 
209
  "selector_regret_at_8": 0.046216949820518494,
210
- "support_gap_at_8": 1.7893850058317184
 
 
 
 
211
  },
212
  "278008524": {
 
 
213
  "branch_car_at_8": 0.020201382460072637,
 
 
214
  "outcome_ptr_at_8": 0.5,
215
  "pairwise_causal_calibration_ece": 0.15535141304867436,
 
 
 
 
 
 
 
 
216
  "selector_regret_at_8": 0.020201382460072637,
217
- "support_gap_at_8": 1.0809455465059727
 
 
 
 
218
  },
219
  "416183106": {
 
 
220
  "branch_car_at_8": 1.812951534986496,
 
 
221
  "outcome_ptr_at_8": 1.0,
222
  "pairwise_causal_calibration_ece": 0.18072825209943494,
 
 
 
 
 
 
 
 
223
  "selector_regret_at_8": 1.812951534986496,
224
- "support_gap_at_8": 0.0
 
 
 
 
225
  },
226
  "534717141": {
 
 
227
  "branch_car_at_8": 0.0,
 
 
228
  "outcome_ptr_at_8": 0.0,
229
  "pairwise_causal_calibration_ece": 0.2870871533378466,
 
 
 
 
 
 
 
 
230
  "selector_regret_at_8": 0.0,
231
- "support_gap_at_8": 0.0
 
 
 
 
232
  },
233
  "551504635": {
 
 
234
  "branch_car_at_8": 1.273692786693573,
 
 
235
  "outcome_ptr_at_8": 1.0,
236
  "pairwise_causal_calibration_ece": 0.2702494805053396,
 
 
 
 
 
 
 
 
237
  "selector_regret_at_8": 1.273692786693573,
238
- "support_gap_at_8": 0.0
 
 
 
 
239
  },
240
  "565369392": {
 
 
241
  "branch_car_at_8": 0.04557279869914055,
 
 
242
  "outcome_ptr_at_8": 0.0,
243
  "pairwise_causal_calibration_ece": 0.17390197648626904,
 
 
 
 
 
 
 
 
244
  "selector_regret_at_8": 0.04557279869914055,
245
- "support_gap_at_8": 1.2939611425002415
 
 
 
 
246
  },
247
  "565621826": {
 
 
248
  "branch_car_at_8": 0.0,
 
 
249
  "outcome_ptr_at_8": 0.0,
250
  "pairwise_causal_calibration_ece": 0.0,
 
 
 
 
 
 
 
 
251
  "selector_regret_at_8": 0.0,
252
- "support_gap_at_8": 0.0
 
 
 
 
253
  },
254
  "585904130": {
 
 
255
  "branch_car_at_8": 0.011032561771571636,
 
 
256
  "outcome_ptr_at_8": 0.6666666666666666,
257
  "pairwise_causal_calibration_ece": 0.2625659187813512,
 
 
 
 
 
 
 
 
258
  "selector_regret_at_8": 0.011032561771571636,
259
- "support_gap_at_8": 0.15916637579600015
 
 
 
 
260
  },
261
  "626486813": {
 
 
262
  "branch_car_at_8": 0.031722720130346715,
 
 
263
  "outcome_ptr_at_8": 0.75,
264
  "pairwise_causal_calibration_ece": 0.23561688988950158,
 
 
 
 
 
 
 
 
265
  "selector_regret_at_8": 0.031722720130346715,
266
- "support_gap_at_8": 0.5432130172848701
 
 
 
 
267
  },
268
  "636680137": {
 
 
269
  "branch_car_at_8": 0.006491700187325478,
 
 
270
  "outcome_ptr_at_8": 0.0,
271
  "pairwise_causal_calibration_ece": 0.12019361687075947,
 
 
 
 
 
 
 
 
272
  "selector_regret_at_8": 0.006491700187325478,
273
- "support_gap_at_8": 1.9756397809833288
 
 
 
 
274
  },
275
  "706479600": {
 
 
276
  "branch_car_at_8": 0.0,
 
 
277
  "outcome_ptr_at_8": 1.0,
278
  "pairwise_causal_calibration_ece": 0.22468613298280815,
 
 
 
 
 
 
 
 
279
  "selector_regret_at_8": 0.0,
280
- "support_gap_at_8": 0.0
 
 
 
 
281
  },
282
  "750448296": {
 
 
283
  "branch_car_at_8": 0.01770863775163889,
 
 
284
  "outcome_ptr_at_8": 0.0,
285
  "pairwise_causal_calibration_ece": 0.14149358436045273,
 
 
 
 
 
 
 
 
286
  "selector_regret_at_8": 0.01770863775163889,
287
- "support_gap_at_8": 0.1688647512346506
 
 
 
 
288
  },
289
  "75070641": {
 
 
290
  "branch_car_at_8": 0.3710349500179291,
 
 
291
  "outcome_ptr_at_8": 1.0,
292
  "pairwise_causal_calibration_ece": 0.09051828052202912,
 
 
 
 
 
 
 
 
293
  "selector_regret_at_8": 0.3710349500179291,
294
- "support_gap_at_8": 1.434916377067566
 
 
 
 
295
  },
296
  "767747424": {
 
 
297
  "branch_car_at_8": 0.024658456211909652,
 
 
298
  "outcome_ptr_at_8": 0.75,
299
  "pairwise_causal_calibration_ece": 0.14269822176062383,
 
 
 
 
 
 
 
 
300
  "selector_regret_at_8": 0.024658456211909652,
301
- "support_gap_at_8": 0.07677002181299031
 
 
 
 
302
  },
303
  "77147264": {
 
 
304
  "branch_car_at_8": 0.0,
 
 
305
  "outcome_ptr_at_8": 0.5,
306
  "pairwise_causal_calibration_ece": 0.12309184879211332,
 
 
 
 
 
 
 
 
307
  "selector_regret_at_8": 0.0,
308
- "support_gap_at_8": 0.0
 
 
 
 
309
  },
310
  "794181626": {
 
 
311
  "branch_car_at_8": 0.0,
 
 
312
  "outcome_ptr_at_8": 1.0,
313
  "pairwise_causal_calibration_ece": 0.0,
 
 
 
 
 
 
 
 
314
  "selector_regret_at_8": 0.0,
315
- "support_gap_at_8": 0.0
 
 
 
 
316
  },
317
  "957101489": {
 
 
318
  "branch_car_at_8": 1.432861328125,
 
 
319
  "outcome_ptr_at_8": 1.0,
320
  "pairwise_causal_calibration_ece": 0.21219243624964876,
 
 
 
 
 
 
 
 
321
  "selector_regret_at_8": 1.432861328125,
322
- "support_gap_at_8": 0.0
 
 
 
 
323
  },
324
  "998761162": {
 
 
325
  "branch_car_at_8": 1.939187914133072,
 
 
326
  "outcome_ptr_at_8": 1.0,
327
  "pairwise_causal_calibration_ece": 0.11706618974004251,
 
 
 
 
 
 
 
 
328
  "selector_regret_at_8": 1.939187914133072,
329
- "support_gap_at_8": 0.0
 
 
 
 
330
  }
331
  }
 
1
  {
2
  "1025740212": {
3
+ "base_success": 0.0,
4
+ "base_utility": 0.528373658657074,
5
  "branch_car_at_8": 1.772506669163704,
6
+ "hidden_chart_oracle_success_at_8": 1.0,
7
+ "hidden_chart_oracle_utility_at_8": 2.0,
8
  "outcome_ptr_at_8": 1.0,
9
  "pairwise_causal_calibration_ece": 0.17388092747958847,
10
+ "proposal_oracle_success_at_8": 1.0,
11
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
12
+ "proposal_oracle_utility_at_8": 2.0,
13
+ "proposal_oracle_utility_gain_over_base_at_8": 1.471626341342926,
14
+ "selected_success_at_8": 0.0,
15
+ "selected_success_gain_over_base_at_8": 0.0,
16
+ "selected_utility_at_8": 0.22749333083629608,
17
+ "selected_utility_gain_over_base_at_8": -0.3008803278207779,
18
  "selector_regret_at_8": 1.772506669163704,
19
+ "success_selector_gap_at_8": 1.0,
20
+ "success_support_gap_at_8": 0.0,
21
+ "success_total_car_to_hidden_at_8": 1.0,
22
+ "support_gap_at_8": 0.0,
23
+ "total_car_to_hidden_at_8": 1.772506669163704
24
  },
25
  "1055157354": {
26
+ "base_success": 0.0,
27
+ "base_utility": 0.19648407399654388,
28
  "branch_car_at_8": 0.03480546176433563,
29
+ "hidden_chart_oracle_success_at_8": 1.0,
30
+ "hidden_chart_oracle_utility_at_8": 2.0,
31
  "outcome_ptr_at_8": 1.0,
32
  "pairwise_causal_calibration_ece": 0.1805709977888097,
33
+ "proposal_oracle_success_at_8": 0.0,
34
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
35
+ "proposal_oracle_utility_at_8": 0.22467364370822906,
36
+ "proposal_oracle_utility_gain_over_base_at_8": 0.02818956971168518,
37
+ "selected_success_at_8": 0.0,
38
+ "selected_success_gain_over_base_at_8": 0.0,
39
+ "selected_utility_at_8": 0.18986818194389343,
40
+ "selected_utility_gain_over_base_at_8": -0.006615892052650452,
41
  "selector_regret_at_8": 0.03480546176433563,
42
+ "success_selector_gap_at_8": 0.0,
43
+ "success_support_gap_at_8": 1.0,
44
+ "success_total_car_to_hidden_at_8": 1.0,
45
+ "support_gap_at_8": 1.775326356291771,
46
+ "total_car_to_hidden_at_8": 1.8101318180561066
47
  },
48
  "1055567618": {
49
+ "base_success": 1.0,
50
+ "base_utility": 2.0,
51
  "branch_car_at_8": 1.9218642711639404,
52
+ "hidden_chart_oracle_success_at_8": 1.0,
53
+ "hidden_chart_oracle_utility_at_8": 2.0,
54
  "outcome_ptr_at_8": 0.0,
55
  "pairwise_causal_calibration_ece": 0.19289744165908876,
56
+ "proposal_oracle_success_at_8": 1.0,
57
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
58
+ "proposal_oracle_utility_at_8": 2.0,
59
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
60
+ "selected_success_at_8": 0.0,
61
+ "selected_success_gain_over_base_at_8": -1.0,
62
+ "selected_utility_at_8": 0.07813572883605957,
63
+ "selected_utility_gain_over_base_at_8": -1.9218642711639404,
64
  "selector_regret_at_8": 1.9218642711639404,
65
+ "success_selector_gap_at_8": 1.0,
66
+ "success_support_gap_at_8": 0.0,
67
+ "success_total_car_to_hidden_at_8": 1.0,
68
+ "support_gap_at_8": 0.0,
69
+ "total_car_to_hidden_at_8": 1.9218642711639404
70
  },
71
  "1113268940": {
72
+ "base_success": 0.0,
73
+ "base_utility": 0.42231252789497375,
74
  "branch_car_at_8": 0.06608206033706665,
75
+ "hidden_chart_oracle_success_at_8": 1.0,
76
+ "hidden_chart_oracle_utility_at_8": 2.0,
77
  "outcome_ptr_at_8": 1.0,
78
  "pairwise_causal_calibration_ece": 0.032532089835823584,
79
+ "proposal_oracle_success_at_8": 0.0,
80
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
81
+ "proposal_oracle_utility_at_8": 0.7025249004364014,
82
+ "proposal_oracle_utility_gain_over_base_at_8": 0.2802123725414276,
83
+ "selected_success_at_8": 0.0,
84
+ "selected_success_gain_over_base_at_8": 0.0,
85
+ "selected_utility_at_8": 0.6364428400993347,
86
+ "selected_utility_gain_over_base_at_8": 0.21413031220436096,
87
  "selector_regret_at_8": 0.06608206033706665,
88
+ "success_selector_gap_at_8": 0.0,
89
+ "success_support_gap_at_8": 1.0,
90
+ "success_total_car_to_hidden_at_8": 1.0,
91
+ "support_gap_at_8": 1.2974750995635986,
92
+ "total_car_to_hidden_at_8": 1.3635571599006653
93
  },
94
  "1113371261": {
95
+ "base_success": 1.0,
96
+ "base_utility": 2.0,
97
  "branch_car_at_8": 0.0,
98
+ "hidden_chart_oracle_success_at_8": 1.0,
99
+ "hidden_chart_oracle_utility_at_8": 2.0,
100
  "outcome_ptr_at_8": 0.0,
101
  "pairwise_causal_calibration_ece": 0.233029795067053,
102
+ "proposal_oracle_success_at_8": 1.0,
103
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
104
+ "proposal_oracle_utility_at_8": 2.0,
105
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
106
+ "selected_success_at_8": 1.0,
107
+ "selected_success_gain_over_base_at_8": 0.0,
108
+ "selected_utility_at_8": 2.0,
109
+ "selected_utility_gain_over_base_at_8": 0.0,
110
  "selector_regret_at_8": 0.0,
111
+ "success_selector_gap_at_8": 0.0,
112
+ "success_support_gap_at_8": 0.0,
113
+ "success_total_car_to_hidden_at_8": 0.0,
114
+ "support_gap_at_8": 0.0,
115
+ "total_car_to_hidden_at_8": 0.0
116
  },
117
  "1176943004": {
118
+ "base_success": 0.0,
119
+ "base_utility": 0.05094052851200104,
120
  "branch_car_at_8": 0.0,
121
+ "hidden_chart_oracle_success_at_8": 1.0,
122
+ "hidden_chart_oracle_utility_at_8": 2.0,
123
  "outcome_ptr_at_8": 1.0,
124
  "pairwise_causal_calibration_ece": 0.17468938592004596,
125
+ "proposal_oracle_success_at_8": 1.0,
126
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
127
+ "proposal_oracle_utility_at_8": 2.0,
128
+ "proposal_oracle_utility_gain_over_base_at_8": 1.949059471487999,
129
+ "selected_success_at_8": 1.0,
130
+ "selected_success_gain_over_base_at_8": 1.0,
131
+ "selected_utility_at_8": 2.0,
132
+ "selected_utility_gain_over_base_at_8": 1.949059471487999,
133
  "selector_regret_at_8": 0.0,
134
+ "success_selector_gap_at_8": 0.0,
135
+ "success_support_gap_at_8": 0.0,
136
+ "success_total_car_to_hidden_at_8": 0.0,
137
+ "support_gap_at_8": 0.0,
138
+ "total_car_to_hidden_at_8": 0.0
139
  },
140
  "1189392917": {
141
+ "base_success": 0.0,
142
+ "base_utility": 0.10841864347457886,
143
  "branch_car_at_8": 0.0,
144
+ "hidden_chart_oracle_success_at_8": 1.0,
145
+ "hidden_chart_oracle_utility_at_8": 2.0,
146
  "outcome_ptr_at_8": 1.0,
147
  "pairwise_causal_calibration_ece": 0.0,
148
+ "proposal_oracle_success_at_8": 1.0,
149
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
150
+ "proposal_oracle_utility_at_8": 2.0,
151
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211,
152
+ "selected_success_at_8": 1.0,
153
+ "selected_success_gain_over_base_at_8": 1.0,
154
+ "selected_utility_at_8": 2.0,
155
+ "selected_utility_gain_over_base_at_8": 1.8915813565254211,
156
  "selector_regret_at_8": 0.0,
157
+ "success_selector_gap_at_8": 0.0,
158
+ "success_support_gap_at_8": 0.0,
159
+ "success_total_car_to_hidden_at_8": 0.0,
160
+ "support_gap_at_8": 0.0,
161
+ "total_car_to_hidden_at_8": 0.0
162
  },
163
  "1231780997": {
164
+ "base_success": 0.0,
165
+ "base_utility": 0.7270061373710632,
166
  "branch_car_at_8": 0.5006794333457947,
167
+ "hidden_chart_oracle_success_at_8": 1.0,
168
+ "hidden_chart_oracle_utility_at_8": 2.0,
169
  "outcome_ptr_at_8": 1.0,
170
  "pairwise_causal_calibration_ece": 0.10207358838016511,
171
+ "proposal_oracle_success_at_8": 0.0,
172
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
173
+ "proposal_oracle_utility_at_8": 0.7278411984443665,
174
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0008350610733032227,
175
+ "selected_success_at_8": 0.0,
176
+ "selected_success_gain_over_base_at_8": 0.0,
177
+ "selected_utility_at_8": 0.22716176509857178,
178
+ "selected_utility_gain_over_base_at_8": -0.49984437227249146,
179
  "selector_regret_at_8": 0.5006794333457947,
180
+ "success_selector_gap_at_8": 0.0,
181
+ "success_support_gap_at_8": 1.0,
182
+ "success_total_car_to_hidden_at_8": 1.0,
183
+ "support_gap_at_8": 1.2721588015556335,
184
+ "total_car_to_hidden_at_8": 1.7728382349014282
185
  },
186
  "1271718007": {
187
+ "base_success": 0.5,
188
+ "base_utility": 1.415381133556366,
189
  "branch_car_at_8": 0.19734752923250198,
190
+ "hidden_chart_oracle_success_at_8": 1.0,
191
+ "hidden_chart_oracle_utility_at_8": 2.0,
192
  "outcome_ptr_at_8": 0.0,
193
  "pairwise_causal_calibration_ece": 0.2350719661460603,
194
+ "proposal_oracle_success_at_8": 0.5,
195
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
196
+ "proposal_oracle_utility_at_8": 1.2635672688484192,
197
+ "proposal_oracle_utility_gain_over_base_at_8": -0.15181386470794678,
198
+ "selected_success_at_8": 0.5,
199
+ "selected_success_gain_over_base_at_8": 0.0,
200
+ "selected_utility_at_8": 1.0662197396159172,
201
+ "selected_utility_gain_over_base_at_8": -0.34916139394044876,
202
  "selector_regret_at_8": 0.19734752923250198,
203
+ "success_selector_gap_at_8": 0.0,
204
+ "success_support_gap_at_8": 0.5,
205
+ "success_total_car_to_hidden_at_8": 0.5,
206
+ "support_gap_at_8": 0.7364327311515808,
207
+ "total_car_to_hidden_at_8": 0.9337802603840828
208
  },
209
  "1273247072": {
210
+ "base_success": 0.0,
211
+ "base_utility": 0.15342965722084045,
212
  "branch_car_at_8": 0.35906313359737396,
213
+ "hidden_chart_oracle_success_at_8": 1.0,
214
+ "hidden_chart_oracle_utility_at_8": 2.0,
215
  "outcome_ptr_at_8": 1.0,
216
  "pairwise_causal_calibration_ece": 0.2793744975404514,
217
+ "proposal_oracle_success_at_8": 0.0,
218
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
219
+ "proposal_oracle_utility_at_8": 0.46377021074295044,
220
+ "proposal_oracle_utility_gain_over_base_at_8": 0.31034055352211,
221
+ "selected_success_at_8": 0.0,
222
+ "selected_success_gain_over_base_at_8": 0.0,
223
+ "selected_utility_at_8": 0.10470707714557648,
224
+ "selected_utility_gain_over_base_at_8": -0.04872258007526398,
225
  "selector_regret_at_8": 0.35906313359737396,
226
+ "success_selector_gap_at_8": 0.0,
227
+ "success_support_gap_at_8": 1.0,
228
+ "success_total_car_to_hidden_at_8": 1.0,
229
+ "support_gap_at_8": 1.5362297892570496,
230
+ "total_car_to_hidden_at_8": 1.8952929228544235
231
  },
232
  "1358421325": {
233
+ "base_success": 0.0,
234
+ "base_utility": 0.8327022790908813,
235
  "branch_car_at_8": 1.90096914768219,
236
+ "hidden_chart_oracle_success_at_8": 1.0,
237
+ "hidden_chart_oracle_utility_at_8": 2.0,
238
  "outcome_ptr_at_8": 1.0,
239
  "pairwise_causal_calibration_ece": 0.32948159974174274,
240
+ "proposal_oracle_success_at_8": 1.0,
241
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
242
+ "proposal_oracle_utility_at_8": 2.0,
243
+ "proposal_oracle_utility_gain_over_base_at_8": 1.1672977209091187,
244
+ "selected_success_at_8": 0.0,
245
+ "selected_success_gain_over_base_at_8": 0.0,
246
+ "selected_utility_at_8": 0.09903085231781006,
247
+ "selected_utility_gain_over_base_at_8": -0.7336714267730713,
248
  "selector_regret_at_8": 1.90096914768219,
249
+ "success_selector_gap_at_8": 1.0,
250
+ "success_support_gap_at_8": 0.0,
251
+ "success_total_car_to_hidden_at_8": 1.0,
252
+ "support_gap_at_8": 0.0,
253
+ "total_car_to_hidden_at_8": 1.90096914768219
254
  },
255
  "1375492786": {
256
+ "base_success": 1.0,
257
+ "base_utility": 2.0,
258
  "branch_car_at_8": 0.0002880295505747199,
259
+ "hidden_chart_oracle_success_at_8": 0.0,
260
+ "hidden_chart_oracle_utility_at_8": 0.24345096945762634,
261
  "outcome_ptr_at_8": 0.0,
262
  "pairwise_causal_calibration_ece": 0.008822272371479922,
263
+ "proposal_oracle_success_at_8": 0.0,
264
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
265
+ "proposal_oracle_utility_at_8": 0.0008872548933140934,
266
+ "proposal_oracle_utility_gain_over_base_at_8": -1.999112745106686,
267
+ "selected_success_at_8": 0.0,
268
+ "selected_success_gain_over_base_at_8": -1.0,
269
+ "selected_utility_at_8": 0.0005992253427393734,
270
+ "selected_utility_gain_over_base_at_8": -1.9994007746572606,
271
  "selector_regret_at_8": 0.0002880295505747199,
272
+ "success_selector_gap_at_8": 0.0,
273
+ "success_support_gap_at_8": 0.0,
274
+ "success_total_car_to_hidden_at_8": 0.0,
275
+ "support_gap_at_8": 0.24256371456431225,
276
+ "total_car_to_hidden_at_8": 0.24285174411488697
277
  },
278
  "141984615": {
279
+ "base_success": 0.0,
280
+ "base_utility": 0.09381213908394177,
281
  "branch_car_at_8": 0.013830550014972687,
282
+ "hidden_chart_oracle_success_at_8": 0.0,
283
+ "hidden_chart_oracle_utility_at_8": 0.20044784247875214,
284
  "outcome_ptr_at_8": 0.3333333333333333,
285
  "pairwise_causal_calibration_ece": 0.19434281311769708,
286
+ "proposal_oracle_success_at_8": 0.0,
287
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
288
+ "proposal_oracle_utility_at_8": 0.054234166940053306,
289
+ "proposal_oracle_utility_gain_over_base_at_8": -0.039577972143888474,
290
+ "selected_success_at_8": 0.0,
291
+ "selected_success_gain_over_base_at_8": 0.0,
292
+ "selected_utility_at_8": 0.04040361692508062,
293
+ "selected_utility_gain_over_base_at_8": -0.05340852215886116,
294
  "selector_regret_at_8": 0.013830550014972687,
295
+ "success_selector_gap_at_8": 0.0,
296
+ "success_support_gap_at_8": 0.0,
297
+ "success_total_car_to_hidden_at_8": 0.0,
298
+ "support_gap_at_8": 0.14621367553869882,
299
+ "total_car_to_hidden_at_8": 0.1600442255536715
300
  },
301
  "1475440973": {
302
+ "base_success": 1.0,
303
+ "base_utility": 2.0,
304
  "branch_car_at_8": 1.28389310836792,
305
+ "hidden_chart_oracle_success_at_8": 1.0,
306
+ "hidden_chart_oracle_utility_at_8": 2.0,
307
  "outcome_ptr_at_8": 0.0,
308
  "pairwise_causal_calibration_ece": 0.22214884767353088,
309
+ "proposal_oracle_success_at_8": 1.0,
310
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
311
+ "proposal_oracle_utility_at_8": 2.0,
312
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
313
+ "selected_success_at_8": 0.0,
314
+ "selected_success_gain_over_base_at_8": -1.0,
315
+ "selected_utility_at_8": 0.7161068916320801,
316
+ "selected_utility_gain_over_base_at_8": -1.28389310836792,
317
  "selector_regret_at_8": 1.28389310836792,
318
+ "success_selector_gap_at_8": 1.0,
319
+ "success_support_gap_at_8": 0.0,
320
+ "success_total_car_to_hidden_at_8": 1.0,
321
+ "support_gap_at_8": 0.0,
322
+ "total_car_to_hidden_at_8": 1.28389310836792
323
  },
324
  "1483645336": {
325
+ "base_success": 1.0,
326
+ "base_utility": 2.0,
327
  "branch_car_at_8": 1.9686058908700943,
328
+ "hidden_chart_oracle_success_at_8": 1.0,
329
+ "hidden_chart_oracle_utility_at_8": 2.0,
330
  "outcome_ptr_at_8": 0.0,
331
  "pairwise_causal_calibration_ece": 0.5995435922806452,
332
+ "proposal_oracle_success_at_8": 1.0,
333
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
334
+ "proposal_oracle_utility_at_8": 2.0,
335
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
336
+ "selected_success_at_8": 0.0,
337
+ "selected_success_gain_over_base_at_8": -1.0,
338
+ "selected_utility_at_8": 0.0313941091299057,
339
+ "selected_utility_gain_over_base_at_8": -1.9686058908700943,
340
  "selector_regret_at_8": 1.9686058908700943,
341
+ "success_selector_gap_at_8": 1.0,
342
+ "success_support_gap_at_8": 0.0,
343
+ "success_total_car_to_hidden_at_8": 1.0,
344
+ "support_gap_at_8": 0.0,
345
+ "total_car_to_hidden_at_8": 1.9686058908700943
346
  },
347
  "1522272686": {
348
+ "base_success": 0.0,
349
+ "base_utility": 0.19476109743118286,
350
  "branch_car_at_8": 0.017972871661186218,
351
+ "hidden_chart_oracle_success_at_8": 1.0,
352
+ "hidden_chart_oracle_utility_at_8": 2.0,
353
  "outcome_ptr_at_8": 1.0,
354
  "pairwise_causal_calibration_ece": 0.3276128524355476,
355
+ "proposal_oracle_success_at_8": 0.0,
356
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
357
+ "proposal_oracle_utility_at_8": 0.1949000507593155,
358
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0001389533281326294,
359
+ "selected_success_at_8": 0.0,
360
+ "selected_success_gain_over_base_at_8": 0.0,
361
+ "selected_utility_at_8": 0.17692717909812927,
362
+ "selected_utility_gain_over_base_at_8": -0.01783391833305359,
363
  "selector_regret_at_8": 0.017972871661186218,
364
+ "success_selector_gap_at_8": 0.0,
365
+ "success_support_gap_at_8": 1.0,
366
+ "success_total_car_to_hidden_at_8": 1.0,
367
+ "support_gap_at_8": 1.8050999492406845,
368
+ "total_car_to_hidden_at_8": 1.8230728209018707
369
  },
370
  "1528740417": {
371
+ "base_success": 1.0,
372
+ "base_utility": 2.0,
373
  "branch_car_at_8": 1.280472755432129,
374
+ "hidden_chart_oracle_success_at_8": 1.0,
375
+ "hidden_chart_oracle_utility_at_8": 2.0,
376
  "outcome_ptr_at_8": 0.0,
377
  "pairwise_causal_calibration_ece": 0.16399362968800954,
378
+ "proposal_oracle_success_at_8": 1.0,
379
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
380
+ "proposal_oracle_utility_at_8": 2.0,
381
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
382
+ "selected_success_at_8": 0.0,
383
+ "selected_success_gain_over_base_at_8": -1.0,
384
+ "selected_utility_at_8": 0.7195272445678711,
385
+ "selected_utility_gain_over_base_at_8": -1.280472755432129,
386
  "selector_regret_at_8": 1.280472755432129,
387
+ "success_selector_gap_at_8": 1.0,
388
+ "success_support_gap_at_8": 0.0,
389
+ "success_total_car_to_hidden_at_8": 1.0,
390
+ "support_gap_at_8": 0.0,
391
+ "total_car_to_hidden_at_8": 1.280472755432129
392
  },
393
  "1549324805": {
394
+ "base_success": 0.6666666666666666,
395
+ "base_utility": 1.3787474483251572,
396
  "branch_car_at_8": 0.01237945475926002,
397
+ "hidden_chart_oracle_success_at_8": 0.6666666666666666,
398
+ "hidden_chart_oracle_utility_at_8": 1.4027505218982697,
399
  "outcome_ptr_at_8": 0.0,
400
  "pairwise_causal_calibration_ece": 0.23864987006465407,
401
+ "proposal_oracle_success_at_8": 0.3333333333333333,
402
+ "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333,
403
+ "proposal_oracle_utility_at_8": 0.6833462389186025,
404
+ "proposal_oracle_utility_gain_over_base_at_8": -0.6954012094065547,
405
+ "selected_success_at_8": 0.3333333333333333,
406
+ "selected_success_gain_over_base_at_8": -0.3333333333333333,
407
+ "selected_utility_at_8": 0.6709667841593424,
408
+ "selected_utility_gain_over_base_at_8": -0.7077806641658148,
409
  "selector_regret_at_8": 0.01237945475926002,
410
+ "success_selector_gap_at_8": 0.0,
411
+ "success_support_gap_at_8": 0.3333333333333333,
412
+ "success_total_car_to_hidden_at_8": 0.3333333333333333,
413
+ "support_gap_at_8": 0.7194042829796672,
414
+ "total_car_to_hidden_at_8": 0.7317837377389272
415
  },
416
  "1607773860": {
417
+ "base_success": 1.0,
418
+ "base_utility": 2.0,
419
  "branch_car_at_8": 0.0,
420
+ "hidden_chart_oracle_success_at_8": 1.0,
421
+ "hidden_chart_oracle_utility_at_8": 2.0,
422
  "outcome_ptr_at_8": 0.0,
423
  "pairwise_causal_calibration_ece": 0.24177237819615016,
424
+ "proposal_oracle_success_at_8": 1.0,
425
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
426
+ "proposal_oracle_utility_at_8": 2.0,
427
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
428
+ "selected_success_at_8": 1.0,
429
+ "selected_success_gain_over_base_at_8": 0.0,
430
+ "selected_utility_at_8": 2.0,
431
+ "selected_utility_gain_over_base_at_8": 0.0,
432
  "selector_regret_at_8": 0.0,
433
+ "success_selector_gap_at_8": 0.0,
434
+ "success_support_gap_at_8": 0.0,
435
+ "success_total_car_to_hidden_at_8": 0.0,
436
+ "support_gap_at_8": 0.0,
437
+ "total_car_to_hidden_at_8": 0.0
438
  },
439
  "1665739511": {
440
+ "base_success": 1.0,
441
+ "base_utility": 2.0,
442
  "branch_car_at_8": 0.0,
443
+ "hidden_chart_oracle_success_at_8": 1.0,
444
+ "hidden_chart_oracle_utility_at_8": 2.0,
445
  "outcome_ptr_at_8": 0.0,
446
  "pairwise_causal_calibration_ece": 0.2508232858898205,
447
+ "proposal_oracle_success_at_8": 1.0,
448
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
449
+ "proposal_oracle_utility_at_8": 2.0,
450
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
451
+ "selected_success_at_8": 1.0,
452
+ "selected_success_gain_over_base_at_8": 0.0,
453
+ "selected_utility_at_8": 2.0,
454
+ "selected_utility_gain_over_base_at_8": 0.0,
455
  "selector_regret_at_8": 0.0,
456
+ "success_selector_gap_at_8": 0.0,
457
+ "success_support_gap_at_8": 0.0,
458
+ "success_total_car_to_hidden_at_8": 0.0,
459
+ "support_gap_at_8": 0.0,
460
+ "total_car_to_hidden_at_8": 0.0
461
  },
462
  "1747178030": {
463
+ "base_success": 0.0,
464
+ "base_utility": 0.1463065892457962,
465
  "branch_car_at_8": 0.034092940390110016,
466
+ "hidden_chart_oracle_success_at_8": 1.0,
467
+ "hidden_chart_oracle_utility_at_8": 2.0,
468
  "outcome_ptr_at_8": 0.0,
469
  "pairwise_causal_calibration_ece": 0.08595818214364334,
470
+ "proposal_oracle_success_at_8": 0.0,
471
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
472
+ "proposal_oracle_utility_at_8": 0.12480536848306656,
473
+ "proposal_oracle_utility_gain_over_base_at_8": -0.021501220762729645,
474
+ "selected_success_at_8": 0.0,
475
+ "selected_success_gain_over_base_at_8": 0.0,
476
+ "selected_utility_at_8": 0.09071242809295654,
477
+ "selected_utility_gain_over_base_at_8": -0.05559416115283966,
478
  "selector_regret_at_8": 0.034092940390110016,
479
+ "success_selector_gap_at_8": 0.0,
480
+ "success_support_gap_at_8": 1.0,
481
+ "success_total_car_to_hidden_at_8": 1.0,
482
+ "support_gap_at_8": 1.8751946315169334,
483
+ "total_car_to_hidden_at_8": 1.9092875719070435
484
  },
485
  "1881223807": {
486
+ "base_success": 0.0,
487
+ "base_utility": 0.0320269912481308,
488
  "branch_car_at_8": 0.024854734539985657,
489
+ "hidden_chart_oracle_success_at_8": 1.0,
490
+ "hidden_chart_oracle_utility_at_8": 2.0,
491
  "outcome_ptr_at_8": 1.0,
492
  "pairwise_causal_calibration_ece": 0.2146835718150399,
493
+ "proposal_oracle_success_at_8": 0.0,
494
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
495
+ "proposal_oracle_utility_at_8": 0.043482705950737,
496
+ "proposal_oracle_utility_gain_over_base_at_8": 0.011455714702606201,
497
+ "selected_success_at_8": 0.0,
498
+ "selected_success_gain_over_base_at_8": 0.0,
499
+ "selected_utility_at_8": 0.018627971410751343,
500
+ "selected_utility_gain_over_base_at_8": -0.013399019837379456,
501
  "selector_regret_at_8": 0.024854734539985657,
502
+ "success_selector_gap_at_8": 0.0,
503
+ "success_support_gap_at_8": 1.0,
504
+ "success_total_car_to_hidden_at_8": 1.0,
505
+ "support_gap_at_8": 1.956517294049263,
506
+ "total_car_to_hidden_at_8": 1.9813720285892487
507
  },
508
  "1919198188": {
509
+ "base_success": 0.0,
510
+ "base_utility": 0.03719887137413025,
511
  "branch_car_at_8": 0.018839523196220398,
512
+ "hidden_chart_oracle_success_at_8": 1.0,
513
+ "hidden_chart_oracle_utility_at_8": 2.0,
514
  "outcome_ptr_at_8": 1.0,
515
  "pairwise_causal_calibration_ece": 0.1669540555093941,
516
+ "proposal_oracle_success_at_8": 0.0,
517
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
518
+ "proposal_oracle_utility_at_8": 0.06275103986263275,
519
+ "proposal_oracle_utility_gain_over_base_at_8": 0.025552168488502502,
520
+ "selected_success_at_8": 0.0,
521
+ "selected_success_gain_over_base_at_8": 0.0,
522
+ "selected_utility_at_8": 0.043911516666412354,
523
+ "selected_utility_gain_over_base_at_8": 0.0067126452922821045,
524
  "selector_regret_at_8": 0.018839523196220398,
525
+ "success_selector_gap_at_8": 0.0,
526
+ "success_support_gap_at_8": 1.0,
527
+ "success_total_car_to_hidden_at_8": 1.0,
528
+ "support_gap_at_8": 1.9372489601373672,
529
+ "total_car_to_hidden_at_8": 1.9560884833335876
530
  },
531
  "1991414747": {
532
+ "base_success": 1.0,
533
+ "base_utility": 2.0,
534
  "branch_car_at_8": 0.0,
535
+ "hidden_chart_oracle_success_at_8": 1.0,
536
+ "hidden_chart_oracle_utility_at_8": 2.0,
537
  "outcome_ptr_at_8": 0.0,
538
  "pairwise_causal_calibration_ece": 0.11896663871153935,
539
+ "proposal_oracle_success_at_8": 1.0,
540
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
541
+ "proposal_oracle_utility_at_8": 2.0,
542
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
543
+ "selected_success_at_8": 1.0,
544
+ "selected_success_gain_over_base_at_8": 0.0,
545
+ "selected_utility_at_8": 2.0,
546
+ "selected_utility_gain_over_base_at_8": 0.0,
547
  "selector_regret_at_8": 0.0,
548
+ "success_selector_gap_at_8": 0.0,
549
+ "success_support_gap_at_8": 0.0,
550
+ "success_total_car_to_hidden_at_8": 0.0,
551
+ "support_gap_at_8": 0.0,
552
+ "total_car_to_hidden_at_8": 0.0
553
  },
554
  "1997090716": {
555
+ "base_success": 0.0,
556
+ "base_utility": 0.13927479088306427,
557
  "branch_car_at_8": 0.0,
558
+ "hidden_chart_oracle_success_at_8": 1.0,
559
+ "hidden_chart_oracle_utility_at_8": 2.0,
560
  "outcome_ptr_at_8": 1.0,
561
  "pairwise_causal_calibration_ece": 0.14617959762414967,
562
+ "proposal_oracle_success_at_8": 1.0,
563
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
564
+ "proposal_oracle_utility_at_8": 2.0,
565
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357,
566
+ "selected_success_at_8": 1.0,
567
+ "selected_success_gain_over_base_at_8": 1.0,
568
+ "selected_utility_at_8": 2.0,
569
+ "selected_utility_gain_over_base_at_8": 1.8607252091169357,
570
  "selector_regret_at_8": 0.0,
571
+ "success_selector_gap_at_8": 0.0,
572
+ "success_support_gap_at_8": 0.0,
573
+ "success_total_car_to_hidden_at_8": 0.0,
574
+ "support_gap_at_8": 0.0,
575
+ "total_car_to_hidden_at_8": 0.0
576
  },
577
  "2015956957": {
578
+ "base_success": 0.0,
579
+ "base_utility": 0.06711978986859321,
580
  "branch_car_at_8": 0.005573729798197746,
581
+ "hidden_chart_oracle_success_at_8": 0.2,
582
+ "hidden_chart_oracle_utility_at_8": 0.5298813670873642,
583
  "outcome_ptr_at_8": 0.6,
584
  "pairwise_causal_calibration_ece": 0.1431252893441326,
585
+ "proposal_oracle_success_at_8": 0.2,
586
+ "proposal_oracle_success_gain_over_base_at_8": 0.2,
587
+ "proposal_oracle_utility_at_8": 0.43981083631515505,
588
+ "proposal_oracle_utility_gain_over_base_at_8": 0.3726910464465618,
589
+ "selected_success_at_8": 0.2,
590
+ "selected_success_gain_over_base_at_8": 0.2,
591
+ "selected_utility_at_8": 0.4342371065169573,
592
+ "selected_utility_gain_over_base_at_8": 0.36711731664836406,
593
  "selector_regret_at_8": 0.005573729798197746,
594
+ "success_selector_gap_at_8": 0.0,
595
+ "success_support_gap_at_8": 0.0,
596
+ "success_total_car_to_hidden_at_8": 0.0,
597
+ "support_gap_at_8": 0.09007053077220917,
598
+ "total_car_to_hidden_at_8": 0.09564426057040691
599
  },
600
  "2054899126": {
601
+ "base_success": 1.0,
602
+ "base_utility": 2.0,
603
  "branch_car_at_8": 0.0,
604
+ "hidden_chart_oracle_success_at_8": 1.0,
605
+ "hidden_chart_oracle_utility_at_8": 2.0,
606
  "outcome_ptr_at_8": 0.0,
607
  "pairwise_causal_calibration_ece": 0.0,
608
+ "proposal_oracle_success_at_8": 1.0,
609
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
610
+ "proposal_oracle_utility_at_8": 2.0,
611
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
612
+ "selected_success_at_8": 1.0,
613
+ "selected_success_gain_over_base_at_8": 0.0,
614
+ "selected_utility_at_8": 2.0,
615
+ "selected_utility_gain_over_base_at_8": 0.0,
616
  "selector_regret_at_8": 0.0,
617
+ "success_selector_gap_at_8": 0.0,
618
+ "success_support_gap_at_8": 0.0,
619
+ "success_total_car_to_hidden_at_8": 0.0,
620
+ "support_gap_at_8": 0.0,
621
+ "total_car_to_hidden_at_8": 0.0
622
  },
623
  "211245842": {
624
+ "base_success": 0.0,
625
+ "base_utility": 0.8388344049453735,
626
  "branch_car_at_8": 0.0,
627
+ "hidden_chart_oracle_success_at_8": 1.0,
628
+ "hidden_chart_oracle_utility_at_8": 2.0,
629
  "outcome_ptr_at_8": 1.0,
630
  "pairwise_causal_calibration_ece": 0.13508468824583414,
631
+ "proposal_oracle_success_at_8": 1.0,
632
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
633
+ "proposal_oracle_utility_at_8": 2.0,
634
+ "proposal_oracle_utility_gain_over_base_at_8": 1.1611655950546265,
635
+ "selected_success_at_8": 1.0,
636
+ "selected_success_gain_over_base_at_8": 1.0,
637
+ "selected_utility_at_8": 2.0,
638
+ "selected_utility_gain_over_base_at_8": 1.1611655950546265,
639
  "selector_regret_at_8": 0.0,
640
+ "success_selector_gap_at_8": 0.0,
641
+ "success_support_gap_at_8": 0.0,
642
+ "success_total_car_to_hidden_at_8": 0.0,
643
+ "support_gap_at_8": 0.0,
644
+ "total_car_to_hidden_at_8": 0.0
645
  },
646
  "2141802816": {
647
+ "base_success": 0.0,
648
+ "base_utility": 0.07080724835395813,
649
  "branch_car_at_8": 1.861636370420456,
650
+ "hidden_chart_oracle_success_at_8": 1.0,
651
+ "hidden_chart_oracle_utility_at_8": 2.0,
652
  "outcome_ptr_at_8": 1.0,
653
  "pairwise_causal_calibration_ece": 0.09903289408743085,
654
+ "proposal_oracle_success_at_8": 1.0,
655
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
656
+ "proposal_oracle_utility_at_8": 2.0,
657
+ "proposal_oracle_utility_gain_over_base_at_8": 1.9291927516460419,
658
+ "selected_success_at_8": 0.0,
659
+ "selected_success_gain_over_base_at_8": 0.0,
660
+ "selected_utility_at_8": 0.13836362957954407,
661
+ "selected_utility_gain_over_base_at_8": 0.06755638122558594,
662
  "selector_regret_at_8": 1.861636370420456,
663
+ "success_selector_gap_at_8": 1.0,
664
+ "success_support_gap_at_8": 0.0,
665
+ "success_total_car_to_hidden_at_8": 1.0,
666
+ "support_gap_at_8": 0.0,
667
+ "total_car_to_hidden_at_8": 1.861636370420456
668
  },
669
  "256770874": {
670
+ "base_success": 0.0,
671
+ "base_utility": 0.8557480573654175,
672
  "branch_car_at_8": 0.046216949820518494,
673
+ "hidden_chart_oracle_success_at_8": 1.0,
674
+ "hidden_chart_oracle_utility_at_8": 2.0,
675
  "outcome_ptr_at_8": 0.0,
676
  "pairwise_causal_calibration_ece": 0.046630940401554466,
677
+ "proposal_oracle_success_at_8": 0.0,
678
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
679
+ "proposal_oracle_utility_at_8": 0.21061499416828156,
680
+ "proposal_oracle_utility_gain_over_base_at_8": -0.6451330631971359,
681
+ "selected_success_at_8": 0.0,
682
+ "selected_success_gain_over_base_at_8": 0.0,
683
+ "selected_utility_at_8": 0.16439804434776306,
684
+ "selected_utility_gain_over_base_at_8": -0.6913500130176544,
685
  "selector_regret_at_8": 0.046216949820518494,
686
+ "success_selector_gap_at_8": 0.0,
687
+ "success_support_gap_at_8": 1.0,
688
+ "success_total_car_to_hidden_at_8": 1.0,
689
+ "support_gap_at_8": 1.7893850058317184,
690
+ "total_car_to_hidden_at_8": 1.835601955652237
691
  },
692
  "278008524": {
693
+ "base_success": 0.0,
694
+ "base_utility": 0.10747186839580536,
695
  "branch_car_at_8": 0.020201382460072637,
696
+ "hidden_chart_oracle_success_at_8": 0.5,
697
+ "hidden_chart_oracle_utility_at_8": 1.113811194896698,
698
  "outcome_ptr_at_8": 0.5,
699
  "pairwise_causal_calibration_ece": 0.15535141304867436,
700
+ "proposal_oracle_success_at_8": 0.0,
701
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
702
+ "proposal_oracle_utility_at_8": 0.032865648390725255,
703
+ "proposal_oracle_utility_gain_over_base_at_8": -0.0746062200050801,
704
+ "selected_success_at_8": 0.0,
705
+ "selected_success_gain_over_base_at_8": 0.0,
706
+ "selected_utility_at_8": 0.012664265930652618,
707
+ "selected_utility_gain_over_base_at_8": -0.09480760246515274,
708
  "selector_regret_at_8": 0.020201382460072637,
709
+ "success_selector_gap_at_8": 0.0,
710
+ "success_support_gap_at_8": 0.5,
711
+ "success_total_car_to_hidden_at_8": 0.5,
712
+ "support_gap_at_8": 1.0809455465059727,
713
+ "total_car_to_hidden_at_8": 1.1011469289660454
714
  },
715
  "416183106": {
716
+ "base_success": 0.0,
717
+ "base_utility": 0.20032641291618347,
718
  "branch_car_at_8": 1.812951534986496,
719
+ "hidden_chart_oracle_success_at_8": 1.0,
720
+ "hidden_chart_oracle_utility_at_8": 2.0,
721
  "outcome_ptr_at_8": 1.0,
722
  "pairwise_causal_calibration_ece": 0.18072825209943494,
723
+ "proposal_oracle_success_at_8": 1.0,
724
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
725
+ "proposal_oracle_utility_at_8": 2.0,
726
+ "proposal_oracle_utility_gain_over_base_at_8": 1.7996735870838165,
727
+ "selected_success_at_8": 0.0,
728
+ "selected_success_gain_over_base_at_8": 0.0,
729
+ "selected_utility_at_8": 0.18704846501350403,
730
+ "selected_utility_gain_over_base_at_8": -0.013277947902679443,
731
  "selector_regret_at_8": 1.812951534986496,
732
+ "success_selector_gap_at_8": 1.0,
733
+ "success_support_gap_at_8": 0.0,
734
+ "success_total_car_to_hidden_at_8": 1.0,
735
+ "support_gap_at_8": 0.0,
736
+ "total_car_to_hidden_at_8": 1.812951534986496
737
  },
738
  "534717141": {
739
+ "base_success": 1.0,
740
+ "base_utility": 2.0,
741
  "branch_car_at_8": 0.0,
742
+ "hidden_chart_oracle_success_at_8": 1.0,
743
+ "hidden_chart_oracle_utility_at_8": 2.0,
744
  "outcome_ptr_at_8": 0.0,
745
  "pairwise_causal_calibration_ece": 0.2870871533378466,
746
+ "proposal_oracle_success_at_8": 1.0,
747
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
748
+ "proposal_oracle_utility_at_8": 2.0,
749
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
750
+ "selected_success_at_8": 1.0,
751
+ "selected_success_gain_over_base_at_8": 0.0,
752
+ "selected_utility_at_8": 2.0,
753
+ "selected_utility_gain_over_base_at_8": 0.0,
754
  "selector_regret_at_8": 0.0,
755
+ "success_selector_gap_at_8": 0.0,
756
+ "success_support_gap_at_8": 0.0,
757
+ "success_total_car_to_hidden_at_8": 0.0,
758
+ "support_gap_at_8": 0.0,
759
+ "total_car_to_hidden_at_8": 0.0
760
  },
761
  "551504635": {
762
+ "base_success": 0.0,
763
+ "base_utility": 0.7238867878913879,
764
  "branch_car_at_8": 1.273692786693573,
765
+ "hidden_chart_oracle_success_at_8": 1.0,
766
+ "hidden_chart_oracle_utility_at_8": 2.0,
767
  "outcome_ptr_at_8": 1.0,
768
  "pairwise_causal_calibration_ece": 0.2702494805053396,
769
+ "proposal_oracle_success_at_8": 1.0,
770
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
771
+ "proposal_oracle_utility_at_8": 2.0,
772
+ "proposal_oracle_utility_gain_over_base_at_8": 1.276113212108612,
773
+ "selected_success_at_8": 0.0,
774
+ "selected_success_gain_over_base_at_8": 0.0,
775
+ "selected_utility_at_8": 0.726307213306427,
776
+ "selected_utility_gain_over_base_at_8": 0.0024204254150390625,
777
  "selector_regret_at_8": 1.273692786693573,
778
+ "success_selector_gap_at_8": 1.0,
779
+ "success_support_gap_at_8": 0.0,
780
+ "success_total_car_to_hidden_at_8": 1.0,
781
+ "support_gap_at_8": 0.0,
782
+ "total_car_to_hidden_at_8": 1.273692786693573
783
  },
784
  "565369392": {
785
+ "base_success": 0.3333333333333333,
786
+ "base_utility": 0.7997656464576721,
787
  "branch_car_at_8": 0.04557279869914055,
788
+ "hidden_chart_oracle_success_at_8": 0.6666666666666666,
789
+ "hidden_chart_oracle_utility_at_8": 1.4266315797964733,
790
  "outcome_ptr_at_8": 0.0,
791
  "pairwise_causal_calibration_ece": 0.17390197648626904,
792
+ "proposal_oracle_success_at_8": 0.0,
793
+ "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333,
794
+ "proposal_oracle_utility_at_8": 0.1326704372962316,
795
+ "proposal_oracle_utility_gain_over_base_at_8": -0.6670952091614405,
796
+ "selected_success_at_8": 0.0,
797
+ "selected_success_gain_over_base_at_8": -0.3333333333333333,
798
+ "selected_utility_at_8": 0.08709763859709103,
799
+ "selected_utility_gain_over_base_at_8": -0.712668007860581,
800
  "selector_regret_at_8": 0.04557279869914055,
801
+ "success_selector_gap_at_8": 0.0,
802
+ "success_support_gap_at_8": 0.6666666666666666,
803
+ "success_total_car_to_hidden_at_8": 0.6666666666666666,
804
+ "support_gap_at_8": 1.2939611425002415,
805
+ "total_car_to_hidden_at_8": 1.339533941199382
806
  },
807
  "565621826": {
808
+ "base_success": 1.0,
809
+ "base_utility": 2.0,
810
  "branch_car_at_8": 0.0,
811
+ "hidden_chart_oracle_success_at_8": 1.0,
812
+ "hidden_chart_oracle_utility_at_8": 2.0,
813
  "outcome_ptr_at_8": 0.0,
814
  "pairwise_causal_calibration_ece": 0.0,
815
+ "proposal_oracle_success_at_8": 1.0,
816
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
817
+ "proposal_oracle_utility_at_8": 2.0,
818
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
819
+ "selected_success_at_8": 1.0,
820
+ "selected_success_gain_over_base_at_8": 0.0,
821
+ "selected_utility_at_8": 2.0,
822
+ "selected_utility_gain_over_base_at_8": 0.0,
823
  "selector_regret_at_8": 0.0,
824
+ "success_selector_gap_at_8": 0.0,
825
+ "success_support_gap_at_8": 0.0,
826
+ "success_total_car_to_hidden_at_8": 0.0,
827
+ "support_gap_at_8": 0.0,
828
+ "total_car_to_hidden_at_8": 0.0
829
  },
830
  "585904130": {
831
+ "base_success": 0.0,
832
+ "base_utility": 0.07529209802548091,
833
  "branch_car_at_8": 0.011032561771571636,
834
+ "hidden_chart_oracle_success_at_8": 0.0,
835
+ "hidden_chart_oracle_utility_at_8": 0.23764699200789133,
836
  "outcome_ptr_at_8": 0.6666666666666666,
837
  "pairwise_causal_calibration_ece": 0.2625659187813512,
838
+ "proposal_oracle_success_at_8": 0.0,
839
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
840
+ "proposal_oracle_utility_at_8": 0.07848061621189117,
841
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0031885181864102683,
842
+ "selected_success_at_8": 0.0,
843
+ "selected_success_gain_over_base_at_8": 0.0,
844
+ "selected_utility_at_8": 0.06744805444031954,
845
+ "selected_utility_gain_over_base_at_8": -0.007844043585161367,
846
  "selector_regret_at_8": 0.011032561771571636,
847
+ "success_selector_gap_at_8": 0.0,
848
+ "success_support_gap_at_8": 0.0,
849
+ "success_total_car_to_hidden_at_8": 0.0,
850
+ "support_gap_at_8": 0.15916637579600015,
851
+ "total_car_to_hidden_at_8": 0.1701989375675718
852
  },
853
  "626486813": {
854
+ "base_success": 0.25,
855
+ "base_utility": 0.5054400166263804,
856
  "branch_car_at_8": 0.031722720130346715,
857
+ "hidden_chart_oracle_success_at_8": 0.25,
858
+ "hidden_chart_oracle_utility_at_8": 0.6124684102833271,
859
  "outcome_ptr_at_8": 0.75,
860
  "pairwise_causal_calibration_ece": 0.23561688988950158,
861
+ "proposal_oracle_success_at_8": 0.0,
862
+ "proposal_oracle_success_gain_over_base_at_8": -0.25,
863
+ "proposal_oracle_utility_at_8": 0.07003506645560265,
864
+ "proposal_oracle_utility_gain_over_base_at_8": -0.43540495017077774,
865
+ "selected_success_at_8": 0.0,
866
+ "selected_success_gain_over_base_at_8": -0.25,
867
+ "selected_utility_at_8": 0.03831234632525593,
868
+ "selected_utility_gain_over_base_at_8": -0.46712767030112445,
869
  "selector_regret_at_8": 0.031722720130346715,
870
+ "success_selector_gap_at_8": 0.0,
871
+ "success_support_gap_at_8": 0.25,
872
+ "success_total_car_to_hidden_at_8": 0.25,
873
+ "support_gap_at_8": 0.5432130172848701,
874
+ "total_car_to_hidden_at_8": 0.5741560639580712
875
  },
876
  "636680137": {
877
+ "base_success": 1.0,
878
+ "base_utility": 2.0,
879
  "branch_car_at_8": 0.006491700187325478,
880
+ "hidden_chart_oracle_success_at_8": 1.0,
881
+ "hidden_chart_oracle_utility_at_8": 2.0,
882
  "outcome_ptr_at_8": 0.0,
883
  "pairwise_causal_calibration_ece": 0.12019361687075947,
884
+ "proposal_oracle_success_at_8": 0.0,
885
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
886
+ "proposal_oracle_utility_at_8": 0.02436021901667118,
887
+ "proposal_oracle_utility_gain_over_base_at_8": -1.9756397809833288,
888
+ "selected_success_at_8": 0.0,
889
+ "selected_success_gain_over_base_at_8": -1.0,
890
+ "selected_utility_at_8": 0.017868518829345703,
891
+ "selected_utility_gain_over_base_at_8": -1.9821314811706543,
892
  "selector_regret_at_8": 0.006491700187325478,
893
+ "success_selector_gap_at_8": 0.0,
894
+ "success_support_gap_at_8": 1.0,
895
+ "success_total_car_to_hidden_at_8": 1.0,
896
+ "support_gap_at_8": 1.9756397809833288,
897
+ "total_car_to_hidden_at_8": 1.9821314811706543
898
  },
899
  "706479600": {
900
+ "base_success": 0.0,
901
+ "base_utility": 0.8350887894630432,
902
  "branch_car_at_8": 0.0,
903
+ "hidden_chart_oracle_success_at_8": 1.0,
904
+ "hidden_chart_oracle_utility_at_8": 2.0,
905
  "outcome_ptr_at_8": 1.0,
906
  "pairwise_causal_calibration_ece": 0.22468613298280815,
907
+ "proposal_oracle_success_at_8": 1.0,
908
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
909
+ "proposal_oracle_utility_at_8": 2.0,
910
+ "proposal_oracle_utility_gain_over_base_at_8": 1.1649112105369568,
911
+ "selected_success_at_8": 1.0,
912
+ "selected_success_gain_over_base_at_8": 1.0,
913
+ "selected_utility_at_8": 2.0,
914
+ "selected_utility_gain_over_base_at_8": 1.1649112105369568,
915
  "selector_regret_at_8": 0.0,
916
+ "success_selector_gap_at_8": 0.0,
917
+ "success_support_gap_at_8": 0.0,
918
+ "success_total_car_to_hidden_at_8": 0.0,
919
+ "support_gap_at_8": 0.0,
920
+ "total_car_to_hidden_at_8": 0.0
921
  },
922
  "750448296": {
923
+ "base_success": 0.0,
924
+ "base_utility": 0.11153355985879898,
925
  "branch_car_at_8": 0.01770863775163889,
926
+ "hidden_chart_oracle_success_at_8": 0.0,
927
+ "hidden_chart_oracle_utility_at_8": 0.23580759018659592,
928
  "outcome_ptr_at_8": 0.0,
929
  "pairwise_causal_calibration_ece": 0.14149358436045273,
930
+ "proposal_oracle_success_at_8": 0.0,
931
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
932
+ "proposal_oracle_utility_at_8": 0.0669428389519453,
933
+ "proposal_oracle_utility_gain_over_base_at_8": -0.044590720906853676,
934
+ "selected_success_at_8": 0.0,
935
+ "selected_success_gain_over_base_at_8": 0.0,
936
+ "selected_utility_at_8": 0.049234201200306416,
937
+ "selected_utility_gain_over_base_at_8": -0.062299358658492565,
938
  "selector_regret_at_8": 0.01770863775163889,
939
+ "success_selector_gap_at_8": 0.0,
940
+ "success_support_gap_at_8": 0.0,
941
+ "success_total_car_to_hidden_at_8": 0.0,
942
+ "support_gap_at_8": 0.1688647512346506,
943
+ "total_car_to_hidden_at_8": 0.1865733889862895
944
  },
945
  "75070641": {
946
+ "base_success": 0.0,
947
+ "base_utility": 0.20275163650512695,
948
  "branch_car_at_8": 0.3710349500179291,
949
+ "hidden_chart_oracle_success_at_8": 1.0,
950
+ "hidden_chart_oracle_utility_at_8": 2.0,
951
  "outcome_ptr_at_8": 1.0,
952
  "pairwise_causal_calibration_ece": 0.09051828052202912,
953
+ "proposal_oracle_success_at_8": 0.0,
954
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
955
+ "proposal_oracle_utility_at_8": 0.5650836229324341,
956
+ "proposal_oracle_utility_gain_over_base_at_8": 0.36233198642730713,
957
+ "selected_success_at_8": 0.0,
958
+ "selected_success_gain_over_base_at_8": 0.0,
959
+ "selected_utility_at_8": 0.194048672914505,
960
+ "selected_utility_gain_over_base_at_8": -0.008702963590621948,
961
  "selector_regret_at_8": 0.3710349500179291,
962
+ "success_selector_gap_at_8": 0.0,
963
+ "success_support_gap_at_8": 1.0,
964
+ "success_total_car_to_hidden_at_8": 1.0,
965
+ "support_gap_at_8": 1.434916377067566,
966
+ "total_car_to_hidden_at_8": 1.805951327085495
967
  },
968
  "767747424": {
969
+ "base_success": 0.25,
970
+ "base_utility": 0.5475241877138615,
971
  "branch_car_at_8": 0.024658456211909652,
972
+ "hidden_chart_oracle_success_at_8": 0.0,
973
+ "hidden_chart_oracle_utility_at_8": 0.18863565661013126,
974
  "outcome_ptr_at_8": 0.75,
975
  "pairwise_causal_calibration_ece": 0.14269822176062383,
976
+ "proposal_oracle_success_at_8": 0.0,
977
+ "proposal_oracle_success_gain_over_base_at_8": -0.25,
978
+ "proposal_oracle_utility_at_8": 0.11239026742987335,
979
+ "proposal_oracle_utility_gain_over_base_at_8": -0.4351339202839881,
980
+ "selected_success_at_8": 0.0,
981
+ "selected_success_gain_over_base_at_8": -0.25,
982
+ "selected_utility_at_8": 0.0877318112179637,
983
+ "selected_utility_gain_over_base_at_8": -0.45979237649589777,
984
  "selector_regret_at_8": 0.024658456211909652,
985
+ "success_selector_gap_at_8": 0.0,
986
+ "success_support_gap_at_8": 0.0,
987
+ "success_total_car_to_hidden_at_8": 0.0,
988
+ "support_gap_at_8": 0.07677002181299031,
989
+ "total_car_to_hidden_at_8": 0.10090384539216757
990
  },
991
  "77147264": {
992
+ "base_success": 0.5,
993
+ "base_utility": 1.4215794801712036,
994
  "branch_car_at_8": 0.0,
995
+ "hidden_chart_oracle_success_at_8": 1.0,
996
+ "hidden_chart_oracle_utility_at_8": 2.0,
997
  "outcome_ptr_at_8": 0.5,
998
  "pairwise_causal_calibration_ece": 0.12309184879211332,
999
+ "proposal_oracle_success_at_8": 1.0,
1000
+ "proposal_oracle_success_gain_over_base_at_8": 0.5,
1001
+ "proposal_oracle_utility_at_8": 2.0,
1002
+ "proposal_oracle_utility_gain_over_base_at_8": 0.5784205198287964,
1003
+ "selected_success_at_8": 1.0,
1004
+ "selected_success_gain_over_base_at_8": 0.5,
1005
+ "selected_utility_at_8": 2.0,
1006
+ "selected_utility_gain_over_base_at_8": 0.5784205198287964,
1007
  "selector_regret_at_8": 0.0,
1008
+ "success_selector_gap_at_8": 0.0,
1009
+ "success_support_gap_at_8": 0.0,
1010
+ "success_total_car_to_hidden_at_8": 0.0,
1011
+ "support_gap_at_8": 0.0,
1012
+ "total_car_to_hidden_at_8": 0.0
1013
  },
1014
  "794181626": {
1015
+ "base_success": 0.0,
1016
+ "base_utility": 0.18095605075359344,
1017
  "branch_car_at_8": 0.0,
1018
+ "hidden_chart_oracle_success_at_8": 1.0,
1019
+ "hidden_chart_oracle_utility_at_8": 2.0,
1020
  "outcome_ptr_at_8": 1.0,
1021
  "pairwise_causal_calibration_ece": 0.0,
1022
+ "proposal_oracle_success_at_8": 1.0,
1023
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
1024
+ "proposal_oracle_utility_at_8": 2.0,
1025
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8190439492464066,
1026
+ "selected_success_at_8": 1.0,
1027
+ "selected_success_gain_over_base_at_8": 1.0,
1028
+ "selected_utility_at_8": 2.0,
1029
+ "selected_utility_gain_over_base_at_8": 1.8190439492464066,
1030
  "selector_regret_at_8": 0.0,
1031
+ "success_selector_gap_at_8": 0.0,
1032
+ "success_support_gap_at_8": 0.0,
1033
+ "success_total_car_to_hidden_at_8": 0.0,
1034
+ "support_gap_at_8": 0.0,
1035
+ "total_car_to_hidden_at_8": 0.0
1036
  },
1037
  "957101489": {
1038
+ "base_success": 0.0,
1039
+ "base_utility": 0.2039649486541748,
1040
  "branch_car_at_8": 1.432861328125,
1041
+ "hidden_chart_oracle_success_at_8": 1.0,
1042
+ "hidden_chart_oracle_utility_at_8": 2.0,
1043
  "outcome_ptr_at_8": 1.0,
1044
  "pairwise_causal_calibration_ece": 0.21219243624964876,
1045
+ "proposal_oracle_success_at_8": 1.0,
1046
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
1047
+ "proposal_oracle_utility_at_8": 2.0,
1048
+ "proposal_oracle_utility_gain_over_base_at_8": 1.7960350513458252,
1049
+ "selected_success_at_8": 0.0,
1050
+ "selected_success_gain_over_base_at_8": 0.0,
1051
+ "selected_utility_at_8": 0.567138671875,
1052
+ "selected_utility_gain_over_base_at_8": 0.3631737232208252,
1053
  "selector_regret_at_8": 1.432861328125,
1054
+ "success_selector_gap_at_8": 1.0,
1055
+ "success_support_gap_at_8": 0.0,
1056
+ "success_total_car_to_hidden_at_8": 1.0,
1057
+ "support_gap_at_8": 0.0,
1058
+ "total_car_to_hidden_at_8": 1.432861328125
1059
  },
1060
  "998761162": {
1061
+ "base_success": 0.0,
1062
+ "base_utility": 0.06781657040119171,
1063
  "branch_car_at_8": 1.939187914133072,
1064
+ "hidden_chart_oracle_success_at_8": 1.0,
1065
+ "hidden_chart_oracle_utility_at_8": 2.0,
1066
  "outcome_ptr_at_8": 1.0,
1067
  "pairwise_causal_calibration_ece": 0.11706618974004251,
1068
+ "proposal_oracle_success_at_8": 1.0,
1069
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
1070
+ "proposal_oracle_utility_at_8": 2.0,
1071
+ "proposal_oracle_utility_gain_over_base_at_8": 1.9321834295988083,
1072
+ "selected_success_at_8": 0.0,
1073
+ "selected_success_gain_over_base_at_8": 0.0,
1074
+ "selected_utility_at_8": 0.0608120858669281,
1075
+ "selected_utility_gain_over_base_at_8": -0.007004484534263611,
1076
  "selector_regret_at_8": 1.939187914133072,
1077
+ "success_selector_gap_at_8": 1.0,
1078
+ "success_support_gap_at_8": 0.0,
1079
+ "success_total_car_to_hidden_at_8": 1.0,
1080
+ "support_gap_at_8": 0.0,
1081
+ "total_car_to_hidden_at_8": 1.939187914133072
1082
  }
1083
  }
workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics_by_task.json CHANGED
@@ -1,37 +1,117 @@
1
  {
2
  "LiftPegUpright-v1": {
 
 
3
  "branch_car_at_8": 1.029554468890031,
 
 
4
  "outcome_ptr_at_8": 0.6666666666666666,
5
  "pairwise_causal_calibration_ece": 0.16081309392707618,
 
 
 
 
 
 
 
 
6
  "selector_regret_at_8": 1.029554468890031,
7
- "support_gap_at_8": 0.4282723168532054
 
 
 
 
8
  },
9
  "PickCube-v1": {
 
 
10
  "branch_car_at_8": 0.35906313359737396,
 
 
11
  "outcome_ptr_at_8": 1.0,
12
  "pairwise_causal_calibration_ece": 0.2793744975404514,
 
 
 
 
 
 
 
 
13
  "selector_regret_at_8": 0.35906313359737396,
14
- "support_gap_at_8": 1.5362297892570496
 
 
 
 
15
  },
16
  "PullCube-v1": {
 
 
17
  "branch_car_at_8": 0.017109284352045506,
 
 
18
  "outcome_ptr_at_8": 0.35294117647058826,
19
  "pairwise_causal_calibration_ece": 0.1721079818939321,
 
 
 
 
 
 
 
 
20
  "selector_regret_at_8": 0.017109284352045506,
21
- "support_gap_at_8": 0.4276017144216252
 
 
 
 
22
  },
23
  "PushCube-v1": {
 
 
24
  "branch_car_at_8": 0.4485462617415648,
 
 
25
  "outcome_ptr_at_8": 0.6153846153846154,
26
  "pairwise_causal_calibration_ece": 0.15912329792465574,
 
 
 
 
 
 
 
 
27
  "selector_regret_at_8": 0.4485462617415648,
28
- "support_gap_at_8": 0.2998153051504722
 
 
 
 
29
  },
30
  "StackCube-v1": {
 
 
31
  "branch_car_at_8": 0.5301474064588547,
 
 
32
  "outcome_ptr_at_8": 0.6666666666666666,
33
  "pairwise_causal_calibration_ece": 0.20081454850953362,
 
 
 
 
 
 
 
 
34
  "selector_regret_at_8": 0.5301474064588547,
35
- "support_gap_at_8": 0.6809894740581512
 
 
 
 
36
  }
37
  }
 
1
  {
2
  "LiftPegUpright-v1": {
3
+ "base_success": 0.3333333333333333,
4
+ "base_utility": 1.066929851969083,
5
  "branch_car_at_8": 1.029554468890031,
6
+ "hidden_chart_oracle_success_at_8": 1.0,
7
+ "hidden_chart_oracle_utility_at_8": 2.0,
8
  "outcome_ptr_at_8": 0.6666666666666666,
9
  "pairwise_causal_calibration_ece": 0.16081309392707618,
10
+ "proposal_oracle_success_at_8": 0.6666666666666666,
11
+ "proposal_oracle_success_gain_over_base_at_8": 0.3333333333333333,
12
+ "proposal_oracle_utility_at_8": 1.5717276831467946,
13
+ "proposal_oracle_utility_gain_over_base_at_8": 0.5047978311777115,
14
+ "selected_success_at_8": 0.0,
15
+ "selected_success_gain_over_base_at_8": -0.3333333333333333,
16
+ "selected_utility_at_8": 0.5421732142567635,
17
+ "selected_utility_gain_over_base_at_8": -0.5247566377123197,
18
  "selector_regret_at_8": 1.029554468890031,
19
+ "success_selector_gap_at_8": 0.6666666666666666,
20
+ "success_support_gap_at_8": 0.3333333333333333,
21
+ "success_total_car_to_hidden_at_8": 1.0,
22
+ "support_gap_at_8": 0.4282723168532054,
23
+ "total_car_to_hidden_at_8": 1.4578267857432365
24
  },
25
  "PickCube-v1": {
26
+ "base_success": 0.0,
27
+ "base_utility": 0.15342965722084045,
28
  "branch_car_at_8": 0.35906313359737396,
29
+ "hidden_chart_oracle_success_at_8": 1.0,
30
+ "hidden_chart_oracle_utility_at_8": 2.0,
31
  "outcome_ptr_at_8": 1.0,
32
  "pairwise_causal_calibration_ece": 0.2793744975404514,
33
+ "proposal_oracle_success_at_8": 0.0,
34
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
35
+ "proposal_oracle_utility_at_8": 0.46377021074295044,
36
+ "proposal_oracle_utility_gain_over_base_at_8": 0.31034055352211,
37
+ "selected_success_at_8": 0.0,
38
+ "selected_success_gain_over_base_at_8": 0.0,
39
+ "selected_utility_at_8": 0.10470707714557648,
40
+ "selected_utility_gain_over_base_at_8": -0.04872258007526398,
41
  "selector_regret_at_8": 0.35906313359737396,
42
+ "success_selector_gap_at_8": 0.0,
43
+ "success_support_gap_at_8": 1.0,
44
+ "success_total_car_to_hidden_at_8": 1.0,
45
+ "support_gap_at_8": 1.5362297892570496,
46
+ "total_car_to_hidden_at_8": 1.8952929228544235
47
  },
48
  "PullCube-v1": {
49
+ "base_success": 0.29411764705882354,
50
+ "base_utility": 0.6512299207833541,
51
  "branch_car_at_8": 0.017109284352045506,
52
+ "hidden_chart_oracle_success_at_8": 0.3235294117647059,
53
+ "hidden_chart_oracle_utility_at_8": 0.7823218367993832,
54
  "outcome_ptr_at_8": 0.35294117647058826,
55
  "pairwise_causal_calibration_ece": 0.1721079818939321,
56
+ "proposal_oracle_success_at_8": 0.14705882352941177,
57
+ "proposal_oracle_success_gain_over_base_at_8": -0.14705882352941177,
58
+ "proposal_oracle_utility_at_8": 0.3548735701530377,
59
+ "proposal_oracle_utility_gain_over_base_at_8": -0.29635635063031635,
60
+ "selected_success_at_8": 0.14705882352941177,
61
+ "selected_success_gain_over_base_at_8": -0.14705882352941177,
62
+ "selected_utility_at_8": 0.3377642858009922,
63
+ "selected_utility_gain_over_base_at_8": -0.31346563498236185,
64
  "selector_regret_at_8": 0.017109284352045506,
65
+ "success_selector_gap_at_8": 0.0,
66
+ "success_support_gap_at_8": 0.17647058823529413,
67
+ "success_total_car_to_hidden_at_8": 0.17647058823529413,
68
+ "support_gap_at_8": 0.4276017144216252,
69
+ "total_car_to_hidden_at_8": 0.44455755099839095
70
  },
71
  "PushCube-v1": {
72
+ "base_success": 0.38461538461538464,
73
+ "base_utility": 0.8217729788560134,
74
  "branch_car_at_8": 0.4485462617415648,
75
+ "hidden_chart_oracle_success_at_8": 1.0,
76
+ "hidden_chart_oracle_utility_at_8": 2.0,
77
  "outcome_ptr_at_8": 0.6153846153846154,
78
  "pairwise_causal_calibration_ece": 0.15912329792465574,
79
+ "proposal_oracle_success_at_8": 0.8461538461538461,
80
+ "proposal_oracle_success_gain_over_base_at_8": 0.46153846153846156,
81
+ "proposal_oracle_utility_at_8": 1.7001846948495278,
82
+ "proposal_oracle_utility_gain_over_base_at_8": 0.8784117159935144,
83
+ "selected_success_at_8": 0.6153846153846154,
84
+ "selected_success_gain_over_base_at_8": 0.23076923076923078,
85
+ "selected_utility_at_8": 1.251638433107963,
86
+ "selected_utility_gain_over_base_at_8": 0.4298654542519496,
87
  "selector_regret_at_8": 0.4485462617415648,
88
+ "success_selector_gap_at_8": 0.23076923076923078,
89
+ "success_support_gap_at_8": 0.15384615384615385,
90
+ "success_total_car_to_hidden_at_8": 0.38461538461538464,
91
+ "support_gap_at_8": 0.2998153051504722,
92
+ "total_car_to_hidden_at_8": 0.748361566892037
93
  },
94
  "StackCube-v1": {
95
+ "base_success": 0.2,
96
+ "base_utility": 0.8047854532798131,
97
  "branch_car_at_8": 0.5301474064588547,
98
+ "hidden_chart_oracle_success_at_8": 1.0,
99
+ "hidden_chart_oracle_utility_at_8": 2.0,
100
  "outcome_ptr_at_8": 0.6666666666666666,
101
  "pairwise_causal_calibration_ece": 0.20081454850953362,
102
+ "proposal_oracle_success_at_8": 0.6,
103
+ "proposal_oracle_success_gain_over_base_at_8": 0.4,
104
+ "proposal_oracle_utility_at_8": 1.3190105259418488,
105
+ "proposal_oracle_utility_gain_over_base_at_8": 0.5142250726620357,
106
+ "selected_success_at_8": 0.3333333333333333,
107
+ "selected_success_gain_over_base_at_8": 0.13333333333333333,
108
+ "selected_utility_at_8": 0.788863119482994,
109
+ "selected_utility_gain_over_base_at_8": -0.015922333796819052,
110
  "selector_regret_at_8": 0.5301474064588547,
111
+ "success_selector_gap_at_8": 0.26666666666666666,
112
+ "success_support_gap_at_8": 0.4,
113
+ "success_total_car_to_hidden_at_8": 0.6666666666666666,
114
+ "support_gap_at_8": 0.6809894740581512,
115
+ "total_car_to_hidden_at_8": 1.211136880517006
116
  }
117
  }
workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/report.md CHANGED
@@ -4,8 +4,24 @@ K: `8`
4
 
5
  | Metric | N | Micro mean | 95% CI | Task macro | Seed macro |
6
  | --- | ---: | ---: | ---: | ---: | ---: |
 
 
7
  | branch_car_at_8 | 69 | 0.3029 | [0.1642, 0.4617] | 0.4769 | 0.4321 |
 
 
8
  | outcome_ptr_at_8 | 69 | 0.5072 | [0.3623, 0.6087] | 0.6603 | 0.5340 |
9
  | pairwise_causal_calibration_ece | 69 | 0.1765 | [0.1491, 0.2027] | 0.1944 | 0.1718 |
 
 
 
 
 
 
 
 
10
  | selector_regret_at_8 | 69 | 0.3029 | [0.1642, 0.4617] | 0.4769 | 0.4321 |
 
 
 
11
  | support_gap_at_8 | 69 | 0.4747 | [0.3004, 0.6486] | 0.6746 | 0.5088 |
 
 
4
 
5
  | Metric | N | Micro mean | 95% CI | Task macro | Seed macro |
6
  | --- | ---: | ---: | ---: | ---: | ---: |
7
+ | base_success | 69 | 0.2899 | [0.1884, 0.4203] | 0.2424 | 0.3298 |
8
+ | base_utility | 69 | 0.7457 | [0.5696, 0.9620] | 0.6996 | 0.8569 |
9
  | branch_car_at_8 | 69 | 0.3029 | [0.1642, 0.4617] | 0.4769 | 0.4321 |
10
+ | hidden_chart_oracle_success_at_8 | 69 | 0.6667 | [0.5507, 0.7826] | 0.8647 | 0.8358 |
11
+ | hidden_chart_oracle_utility_at_8 | 69 | 1.4000 | [1.1855, 1.6062] | 1.7565 | 1.7062 |
12
  | outcome_ptr_at_8 | 69 | 0.5072 | [0.3623, 0.6087] | 0.6603 | 0.5340 |
13
  | pairwise_causal_calibration_ece | 69 | 0.1765 | [0.1491, 0.2027] | 0.1944 | 0.1718 |
14
+ | proposal_oracle_success_at_8 | 69 | 0.4203 | [0.3188, 0.5507] | 0.4520 | 0.5539 |
15
+ | proposal_oracle_success_gain_over_base_at_8 | 69 | 0.1304 | [-0.0145, 0.2609] | 0.2096 | 0.2241 |
16
+ | proposal_oracle_utility_at_8 | 69 | 0.9253 | [0.7270, 1.1546] | 1.0819 | 1.1974 |
17
+ | proposal_oracle_utility_gain_over_base_at_8 | 69 | 0.1796 | [-0.0557, 0.4002] | 0.3823 | 0.3406 |
18
+ | selected_success_at_8 | 69 | 0.2609 | [0.1739, 0.3623] | 0.2192 | 0.3199 |
19
+ | selected_success_gain_over_base_at_8 | 69 | -0.0290 | [-0.1594, 0.0870] | -0.0233 | -0.0099 |
20
+ | selected_utility_at_8 | 69 | 0.6224 | [0.4473, 0.8056] | 0.6050 | 0.7653 |
21
+ | selected_utility_gain_over_base_at_8 | 69 | -0.1233 | [-0.3535, 0.0836] | -0.0946 | -0.0915 |
22
  | selector_regret_at_8 | 69 | 0.3029 | [0.1642, 0.4617] | 0.4769 | 0.4321 |
23
+ | success_selector_gap_at_8 | 69 | 0.1594 | [0.0870, 0.2609] | 0.2328 | 0.2340 |
24
+ | success_support_gap_at_8 | 69 | 0.2464 | [0.1449, 0.3478] | 0.4127 | 0.2819 |
25
+ | success_total_car_to_hidden_at_8 | 69 | 0.4058 | [0.2754, 0.5217] | 0.6456 | 0.5160 |
26
  | support_gap_at_8 | 69 | 0.4747 | [0.3004, 0.6486] | 0.6746 | 0.5088 |
27
+ | total_car_to_hidden_at_8 | 69 | 0.7776 | [0.5436, 0.9685] | 1.1514 | 0.9409 |
workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/table.tex CHANGED
@@ -3,10 +3,26 @@
3
  \toprule
4
  Metric & N & Micro mean & CI low & CI high \\
5
  \midrule
 
 
6
  branch\_car\_at\_8 & 69 & 0.3029 & 0.1642 & 0.4617 \\
 
 
7
  outcome\_ptr\_at\_8 & 69 & 0.5072 & 0.3623 & 0.6087 \\
8
  pairwise\_causal\_calibration\_ece & 69 & 0.1765 & 0.1491 & 0.2027 \\
 
 
 
 
 
 
 
 
9
  selector\_regret\_at\_8 & 69 & 0.3029 & 0.1642 & 0.4617 \\
 
 
 
10
  support\_gap\_at\_8 & 69 & 0.4747 & 0.3004 & 0.6486 \\
 
11
  \bottomrule
12
  \end{tabular}
 
3
  \toprule
4
  Metric & N & Micro mean & CI low & CI high \\
5
  \midrule
6
+ base\_success & 69 & 0.2899 & 0.1884 & 0.4203 \\
7
+ base\_utility & 69 & 0.7457 & 0.5696 & 0.9620 \\
8
  branch\_car\_at\_8 & 69 & 0.3029 & 0.1642 & 0.4617 \\
9
+ hidden\_chart\_oracle\_success\_at\_8 & 69 & 0.6667 & 0.5507 & 0.7826 \\
10
+ hidden\_chart\_oracle\_utility\_at\_8 & 69 & 1.4000 & 1.1855 & 1.6062 \\
11
  outcome\_ptr\_at\_8 & 69 & 0.5072 & 0.3623 & 0.6087 \\
12
  pairwise\_causal\_calibration\_ece & 69 & 0.1765 & 0.1491 & 0.2027 \\
13
+ proposal\_oracle\_success\_at\_8 & 69 & 0.4203 & 0.3188 & 0.5507 \\
14
+ proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 69 & 0.1304 & -0.0145 & 0.2609 \\
15
+ proposal\_oracle\_utility\_at\_8 & 69 & 0.9253 & 0.7270 & 1.1546 \\
16
+ proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 69 & 0.1796 & -0.0557 & 0.4002 \\
17
+ selected\_success\_at\_8 & 69 & 0.2609 & 0.1739 & 0.3623 \\
18
+ selected\_success\_gain\_over\_base\_at\_8 & 69 & -0.0290 & -0.1594 & 0.0870 \\
19
+ selected\_utility\_at\_8 & 69 & 0.6224 & 0.4473 & 0.8056 \\
20
+ selected\_utility\_gain\_over\_base\_at\_8 & 69 & -0.1233 & -0.3535 & 0.0836 \\
21
  selector\_regret\_at\_8 & 69 & 0.3029 & 0.1642 & 0.4617 \\
22
+ success\_selector\_gap\_at\_8 & 69 & 0.1594 & 0.0870 & 0.2609 \\
23
+ success\_support\_gap\_at\_8 & 69 & 0.2464 & 0.1449 & 0.3478 \\
24
+ success\_total\_car\_to\_hidden\_at\_8 & 69 & 0.4058 & 0.2754 & 0.5217 \\
25
  support\_gap\_at\_8 & 69 & 0.4747 & 0.3004 & 0.6486 \\
26
+ total\_car\_to\_hidden\_at\_8 & 69 & 0.7776 & 0.5436 & 0.9685 \\
27
  \bottomrule
28
  \end{tabular}
workspace/runs/ctt_residual_rollout_val69_seed2/metrics.json ADDED
@@ -0,0 +1,1918 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": "runs/ctt_residual_full_seed2/model.pt",
3
+ "k": 8,
4
+ "measured_metric_summary": {
5
+ "base_success": {
6
+ "macro_by_seed": {
7
+ "ci": {
8
+ "confidence": 0.95,
9
+ "high": 0.4521276595744681,
10
+ "low": 0.20921985815602834,
11
+ "mean": 0.3297872340425532,
12
+ "n": 47,
13
+ "num_samples": 500
14
+ },
15
+ "mean": 0.3297872340425532,
16
+ "per_seed": {
17
+ "1025740212": 0.0,
18
+ "1055157354": 0.0,
19
+ "1055567618": 1.0,
20
+ "1113268940": 0.0,
21
+ "1113371261": 1.0,
22
+ "1176943004": 0.0,
23
+ "1189392917": 0.0,
24
+ "1231780997": 0.0,
25
+ "1271718007": 0.5,
26
+ "1273247072": 0.0,
27
+ "1358421325": 0.0,
28
+ "1375492786": 1.0,
29
+ "141984615": 0.0,
30
+ "1475440973": 1.0,
31
+ "1483645336": 1.0,
32
+ "1522272686": 0.0,
33
+ "1528740417": 1.0,
34
+ "1549324805": 0.6666666666666666,
35
+ "1607773860": 1.0,
36
+ "1665739511": 1.0,
37
+ "1747178030": 0.0,
38
+ "1881223807": 0.0,
39
+ "1919198188": 0.0,
40
+ "1991414747": 1.0,
41
+ "1997090716": 0.0,
42
+ "2015956957": 0.0,
43
+ "2054899126": 1.0,
44
+ "211245842": 0.0,
45
+ "2141802816": 0.0,
46
+ "256770874": 0.0,
47
+ "278008524": 0.0,
48
+ "416183106": 0.0,
49
+ "534717141": 1.0,
50
+ "551504635": 0.0,
51
+ "565369392": 0.3333333333333333,
52
+ "565621826": 1.0,
53
+ "585904130": 0.0,
54
+ "626486813": 0.25,
55
+ "636680137": 1.0,
56
+ "706479600": 0.0,
57
+ "750448296": 0.0,
58
+ "75070641": 0.0,
59
+ "767747424": 0.25,
60
+ "77147264": 0.5,
61
+ "794181626": 0.0,
62
+ "957101489": 0.0,
63
+ "998761162": 0.0
64
+ }
65
+ },
66
+ "macro_by_task": {
67
+ "ci": {
68
+ "confidence": 0.95,
69
+ "high": 0.33815987933634994,
70
+ "low": 0.10666666666666666,
71
+ "mean": 0.24241327300150828,
72
+ "n": 5,
73
+ "num_samples": 500
74
+ },
75
+ "mean": 0.24241327300150828,
76
+ "per_task": {
77
+ "LiftPegUpright-v1": 0.3333333333333333,
78
+ "PickCube-v1": 0.0,
79
+ "PullCube-v1": 0.29411764705882354,
80
+ "PushCube-v1": 0.38461538461538464,
81
+ "StackCube-v1": 0.2
82
+ }
83
+ },
84
+ "metric": "base_success",
85
+ "micro": {
86
+ "confidence": 0.95,
87
+ "high": 0.42028985507246375,
88
+ "low": 0.18840579710144928,
89
+ "mean": 0.2898550724637681,
90
+ "n": 69,
91
+ "num_samples": 500
92
+ },
93
+ "n": 69
94
+ },
95
+ "base_utility": {
96
+ "macro_by_seed": {
97
+ "ci": {
98
+ "confidence": 0.95,
99
+ "high": 1.0924235453427908,
100
+ "low": 0.6323911708297458,
101
+ "mean": 0.8568741302496377,
102
+ "n": 47,
103
+ "num_samples": 500
104
+ },
105
+ "mean": 0.8568741302496377,
106
+ "per_seed": {
107
+ "1025740212": 0.528373658657074,
108
+ "1055157354": 0.19648407399654388,
109
+ "1055567618": 2.0,
110
+ "1113268940": 0.42231252789497375,
111
+ "1113371261": 2.0,
112
+ "1176943004": 0.05094052851200104,
113
+ "1189392917": 0.10841864347457886,
114
+ "1231780997": 0.7270061373710632,
115
+ "1271718007": 1.415381133556366,
116
+ "1273247072": 0.15342965722084045,
117
+ "1358421325": 0.8327022790908813,
118
+ "1375492786": 2.0,
119
+ "141984615": 0.09381213908394177,
120
+ "1475440973": 2.0,
121
+ "1483645336": 2.0,
122
+ "1522272686": 0.19476109743118286,
123
+ "1528740417": 2.0,
124
+ "1549324805": 1.3787474483251572,
125
+ "1607773860": 2.0,
126
+ "1665739511": 2.0,
127
+ "1747178030": 0.1463065892457962,
128
+ "1881223807": 0.0320269912481308,
129
+ "1919198188": 0.03719887137413025,
130
+ "1991414747": 2.0,
131
+ "1997090716": 0.13927479088306427,
132
+ "2015956957": 0.06711978986859321,
133
+ "2054899126": 2.0,
134
+ "211245842": 0.8388344049453735,
135
+ "2141802816": 0.07080724835395813,
136
+ "256770874": 0.8557480573654175,
137
+ "278008524": 0.10747186839580536,
138
+ "416183106": 0.20032641291618347,
139
+ "534717141": 2.0,
140
+ "551504635": 0.7238867878913879,
141
+ "565369392": 0.7997656464576721,
142
+ "565621826": 2.0,
143
+ "585904130": 0.07529209802548091,
144
+ "626486813": 0.5054400166263804,
145
+ "636680137": 2.0,
146
+ "706479600": 0.8350887894630432,
147
+ "750448296": 0.11153355985879898,
148
+ "75070641": 0.20275163650512695,
149
+ "767747424": 0.5475241877138615,
150
+ "77147264": 1.4215794801712036,
151
+ "794181626": 0.18095605075359344,
152
+ "957101489": 0.2039649486541748,
153
+ "998761162": 0.06781657040119171
154
+ }
155
+ },
156
+ "macro_by_task": {
157
+ "ci": {
158
+ "confidence": 0.95,
159
+ "high": 0.9313609859940832,
160
+ "low": 0.4139719756444295,
161
+ "mean": 0.6996295724218208,
162
+ "n": 5,
163
+ "num_samples": 500
164
+ },
165
+ "mean": 0.6996295724218208,
166
+ "per_task": {
167
+ "LiftPegUpright-v1": 1.066929851969083,
168
+ "PickCube-v1": 0.15342965722084045,
169
+ "PullCube-v1": 0.6512299207833541,
170
+ "PushCube-v1": 0.8217729788560134,
171
+ "StackCube-v1": 0.8047854532798131
172
+ }
173
+ },
174
+ "metric": "base_utility",
175
+ "micro": {
176
+ "confidence": 0.95,
177
+ "high": 0.9619567723796312,
178
+ "low": 0.5695875146345276,
179
+ "mean": 0.7456761826086196,
180
+ "n": 69,
181
+ "num_samples": 500
182
+ },
183
+ "n": 69
184
+ },
185
+ "branch_car_at_8": {
186
+ "macro_by_seed": {
187
+ "ci": {
188
+ "confidence": 0.95,
189
+ "high": 0.6595998992103133,
190
+ "low": 0.2230010245754945,
191
+ "mean": 0.43210832736758736,
192
+ "n": 47,
193
+ "num_samples": 500
194
+ },
195
+ "mean": 0.43210832736758736,
196
+ "per_seed": {
197
+ "1025740212": 1.772506669163704,
198
+ "1055157354": 0.03480546176433563,
199
+ "1055567618": 1.9218642711639404,
200
+ "1113268940": 0.06608206033706665,
201
+ "1113371261": 0.0,
202
+ "1176943004": 0.0,
203
+ "1189392917": 0.0,
204
+ "1231780997": 0.5006794333457947,
205
+ "1271718007": 0.19734752923250198,
206
+ "1273247072": 0.35906313359737396,
207
+ "1358421325": 1.90096914768219,
208
+ "1375492786": 0.0002880295505747199,
209
+ "141984615": 0.013830550014972687,
210
+ "1475440973": 1.28389310836792,
211
+ "1483645336": 1.9686058908700943,
212
+ "1522272686": 0.017972871661186218,
213
+ "1528740417": 1.280472755432129,
214
+ "1549324805": 0.01237945475926002,
215
+ "1607773860": 0.0,
216
+ "1665739511": 0.0,
217
+ "1747178030": 0.034092940390110016,
218
+ "1881223807": 0.024854734539985657,
219
+ "1919198188": 0.018839523196220398,
220
+ "1991414747": 0.0,
221
+ "1997090716": 0.0,
222
+ "2015956957": 0.005573729798197746,
223
+ "2054899126": 0.0,
224
+ "211245842": 0.0,
225
+ "2141802816": 1.861636370420456,
226
+ "256770874": 0.046216949820518494,
227
+ "278008524": 0.020201382460072637,
228
+ "416183106": 1.812951534986496,
229
+ "534717141": 0.0,
230
+ "551504635": 1.273692786693573,
231
+ "565369392": 0.04557279869914055,
232
+ "565621826": 0.0,
233
+ "585904130": 0.011032561771571636,
234
+ "626486813": 0.031722720130346715,
235
+ "636680137": 0.006491700187325478,
236
+ "706479600": 0.0,
237
+ "750448296": 0.01770863775163889,
238
+ "75070641": 0.3710349500179291,
239
+ "767747424": 0.024658456211909652,
240
+ "77147264": 0.0,
241
+ "794181626": 0.0,
242
+ "957101489": 1.432861328125,
243
+ "998761162": 1.939187914133072
244
+ }
245
+ },
246
+ "macro_by_task": {
247
+ "ci": {
248
+ "confidence": 0.95,
249
+ "high": 0.7792545604018064,
250
+ "low": 0.18968407530785322,
251
+ "mean": 0.476884111007974,
252
+ "n": 5,
253
+ "num_samples": 500
254
+ },
255
+ "mean": 0.476884111007974,
256
+ "per_task": {
257
+ "LiftPegUpright-v1": 1.029554468890031,
258
+ "PickCube-v1": 0.35906313359737396,
259
+ "PullCube-v1": 0.017109284352045506,
260
+ "PushCube-v1": 0.4485462617415648,
261
+ "StackCube-v1": 0.5301474064588547
262
+ }
263
+ },
264
+ "metric": "branch_car_at_8",
265
+ "micro": {
266
+ "confidence": 0.95,
267
+ "high": 0.46172967990117986,
268
+ "low": 0.1641734795007126,
269
+ "mean": 0.3029191031076851,
270
+ "n": 69,
271
+ "num_samples": 500
272
+ },
273
+ "n": 69
274
+ },
275
+ "hidden_chart_oracle_success_at_8": {
276
+ "macro_by_seed": {
277
+ "ci": {
278
+ "confidence": 0.95,
279
+ "high": 0.922695035460993,
280
+ "low": 0.7351063829787235,
281
+ "mean": 0.8358156028368794,
282
+ "n": 47,
283
+ "num_samples": 500
284
+ },
285
+ "mean": 0.8358156028368794,
286
+ "per_seed": {
287
+ "1025740212": 1.0,
288
+ "1055157354": 1.0,
289
+ "1055567618": 1.0,
290
+ "1113268940": 1.0,
291
+ "1113371261": 1.0,
292
+ "1176943004": 1.0,
293
+ "1189392917": 1.0,
294
+ "1231780997": 1.0,
295
+ "1271718007": 1.0,
296
+ "1273247072": 1.0,
297
+ "1358421325": 1.0,
298
+ "1375492786": 0.0,
299
+ "141984615": 0.0,
300
+ "1475440973": 1.0,
301
+ "1483645336": 1.0,
302
+ "1522272686": 1.0,
303
+ "1528740417": 1.0,
304
+ "1549324805": 0.6666666666666666,
305
+ "1607773860": 1.0,
306
+ "1665739511": 1.0,
307
+ "1747178030": 1.0,
308
+ "1881223807": 1.0,
309
+ "1919198188": 1.0,
310
+ "1991414747": 1.0,
311
+ "1997090716": 1.0,
312
+ "2015956957": 0.2,
313
+ "2054899126": 1.0,
314
+ "211245842": 1.0,
315
+ "2141802816": 1.0,
316
+ "256770874": 1.0,
317
+ "278008524": 0.5,
318
+ "416183106": 1.0,
319
+ "534717141": 1.0,
320
+ "551504635": 1.0,
321
+ "565369392": 0.6666666666666666,
322
+ "565621826": 1.0,
323
+ "585904130": 0.0,
324
+ "626486813": 0.25,
325
+ "636680137": 1.0,
326
+ "706479600": 1.0,
327
+ "750448296": 0.0,
328
+ "75070641": 1.0,
329
+ "767747424": 0.0,
330
+ "77147264": 1.0,
331
+ "794181626": 1.0,
332
+ "957101489": 1.0,
333
+ "998761162": 1.0
334
+ }
335
+ },
336
+ "macro_by_task": {
337
+ "ci": {
338
+ "confidence": 0.95,
339
+ "high": 1.0,
340
+ "low": 0.5941176470588235,
341
+ "mean": 0.8647058823529413,
342
+ "n": 5,
343
+ "num_samples": 500
344
+ },
345
+ "mean": 0.8647058823529413,
346
+ "per_task": {
347
+ "LiftPegUpright-v1": 1.0,
348
+ "PickCube-v1": 1.0,
349
+ "PullCube-v1": 0.3235294117647059,
350
+ "PushCube-v1": 1.0,
351
+ "StackCube-v1": 1.0
352
+ }
353
+ },
354
+ "metric": "hidden_chart_oracle_success_at_8",
355
+ "micro": {
356
+ "confidence": 0.95,
357
+ "high": 0.782608695652174,
358
+ "low": 0.5507246376811594,
359
+ "mean": 0.6666666666666666,
360
+ "n": 69,
361
+ "num_samples": 500
362
+ },
363
+ "n": 69
364
+ },
365
+ "hidden_chart_oracle_utility_at_8": {
366
+ "macro_by_seed": {
367
+ "ci": {
368
+ "confidence": 0.95,
369
+ "high": 1.8615907955016442,
370
+ "low": 1.529367521088174,
371
+ "mean": 1.7062028111638965,
372
+ "n": 47,
373
+ "num_samples": 500
374
+ },
375
+ "mean": 1.7062028111638965,
376
+ "per_seed": {
377
+ "1025740212": 2.0,
378
+ "1055157354": 2.0,
379
+ "1055567618": 2.0,
380
+ "1113268940": 2.0,
381
+ "1113371261": 2.0,
382
+ "1176943004": 2.0,
383
+ "1189392917": 2.0,
384
+ "1231780997": 2.0,
385
+ "1271718007": 2.0,
386
+ "1273247072": 2.0,
387
+ "1358421325": 2.0,
388
+ "1375492786": 0.24345096945762634,
389
+ "141984615": 0.20044784247875214,
390
+ "1475440973": 2.0,
391
+ "1483645336": 2.0,
392
+ "1522272686": 2.0,
393
+ "1528740417": 2.0,
394
+ "1549324805": 1.4027505218982697,
395
+ "1607773860": 2.0,
396
+ "1665739511": 2.0,
397
+ "1747178030": 2.0,
398
+ "1881223807": 2.0,
399
+ "1919198188": 2.0,
400
+ "1991414747": 2.0,
401
+ "1997090716": 2.0,
402
+ "2015956957": 0.5298813670873642,
403
+ "2054899126": 2.0,
404
+ "211245842": 2.0,
405
+ "2141802816": 2.0,
406
+ "256770874": 2.0,
407
+ "278008524": 1.113811194896698,
408
+ "416183106": 2.0,
409
+ "534717141": 2.0,
410
+ "551504635": 2.0,
411
+ "565369392": 1.4266315797964733,
412
+ "565621826": 2.0,
413
+ "585904130": 0.23764699200789133,
414
+ "626486813": 0.6124684102833271,
415
+ "636680137": 2.0,
416
+ "706479600": 2.0,
417
+ "750448296": 0.23580759018659592,
418
+ "75070641": 2.0,
419
+ "767747424": 0.18863565661013126,
420
+ "77147264": 2.0,
421
+ "794181626": 2.0,
422
+ "957101489": 2.0,
423
+ "998761162": 2.0
424
+ }
425
+ },
426
+ "macro_by_task": {
427
+ "ci": {
428
+ "confidence": 0.95,
429
+ "high": 2.0,
430
+ "low": 1.2693931020796299,
431
+ "mean": 1.7564643673598765,
432
+ "n": 5,
433
+ "num_samples": 500
434
+ },
435
+ "mean": 1.7564643673598765,
436
+ "per_task": {
437
+ "LiftPegUpright-v1": 2.0,
438
+ "PickCube-v1": 2.0,
439
+ "PullCube-v1": 0.7823218367993832,
440
+ "PushCube-v1": 2.0,
441
+ "StackCube-v1": 2.0
442
+ }
443
+ },
444
+ "metric": "hidden_chart_oracle_utility_at_8",
445
+ "micro": {
446
+ "confidence": 0.95,
447
+ "high": 1.6062410234109215,
448
+ "low": 1.1855228276572365,
449
+ "mean": 1.3999846732054932,
450
+ "n": 69,
451
+ "num_samples": 500
452
+ },
453
+ "n": 69
454
+ },
455
+ "outcome_ptr_at_8": {
456
+ "macro_by_seed": {
457
+ "ci": {
458
+ "confidence": 0.95,
459
+ "high": 0.6567375886524823,
460
+ "low": 0.3964539007092199,
461
+ "mean": 0.5340425531914894,
462
+ "n": 47,
463
+ "num_samples": 500
464
+ },
465
+ "mean": 0.5340425531914894,
466
+ "per_seed": {
467
+ "1025740212": 1.0,
468
+ "1055157354": 1.0,
469
+ "1055567618": 0.0,
470
+ "1113268940": 1.0,
471
+ "1113371261": 0.0,
472
+ "1176943004": 1.0,
473
+ "1189392917": 1.0,
474
+ "1231780997": 1.0,
475
+ "1271718007": 0.0,
476
+ "1273247072": 1.0,
477
+ "1358421325": 1.0,
478
+ "1375492786": 0.0,
479
+ "141984615": 0.3333333333333333,
480
+ "1475440973": 0.0,
481
+ "1483645336": 0.0,
482
+ "1522272686": 1.0,
483
+ "1528740417": 0.0,
484
+ "1549324805": 0.0,
485
+ "1607773860": 0.0,
486
+ "1665739511": 0.0,
487
+ "1747178030": 0.0,
488
+ "1881223807": 1.0,
489
+ "1919198188": 1.0,
490
+ "1991414747": 0.0,
491
+ "1997090716": 1.0,
492
+ "2015956957": 0.6,
493
+ "2054899126": 0.0,
494
+ "211245842": 1.0,
495
+ "2141802816": 1.0,
496
+ "256770874": 0.0,
497
+ "278008524": 0.5,
498
+ "416183106": 1.0,
499
+ "534717141": 0.0,
500
+ "551504635": 1.0,
501
+ "565369392": 0.0,
502
+ "565621826": 0.0,
503
+ "585904130": 0.6666666666666666,
504
+ "626486813": 0.75,
505
+ "636680137": 0.0,
506
+ "706479600": 1.0,
507
+ "750448296": 0.0,
508
+ "75070641": 1.0,
509
+ "767747424": 0.75,
510
+ "77147264": 0.5,
511
+ "794181626": 1.0,
512
+ "957101489": 1.0,
513
+ "998761162": 1.0
514
+ }
515
+ },
516
+ "macro_by_task": {
517
+ "ci": {
518
+ "confidence": 0.95,
519
+ "high": 0.8564102564102564,
520
+ "low": 0.46817496229260935,
521
+ "mean": 0.6603318250377074,
522
+ "n": 5,
523
+ "num_samples": 500
524
+ },
525
+ "mean": 0.6603318250377074,
526
+ "per_task": {
527
+ "LiftPegUpright-v1": 0.6666666666666666,
528
+ "PickCube-v1": 1.0,
529
+ "PullCube-v1": 0.35294117647058826,
530
+ "PushCube-v1": 0.6153846153846154,
531
+ "StackCube-v1": 0.6666666666666666
532
+ }
533
+ },
534
+ "metric": "outcome_ptr_at_8",
535
+ "micro": {
536
+ "confidence": 0.95,
537
+ "high": 0.6086956521739131,
538
+ "low": 0.36231884057971014,
539
+ "mean": 0.5072463768115942,
540
+ "n": 69,
541
+ "num_samples": 500
542
+ },
543
+ "n": 69
544
+ },
545
+ "pairwise_causal_calibration_ece": {
546
+ "macro_by_seed": {
547
+ "ci": {
548
+ "confidence": 0.95,
549
+ "high": 0.20239102672265116,
550
+ "low": 0.14332855009235143,
551
+ "mean": 0.17181636350182927,
552
+ "n": 47,
553
+ "num_samples": 500
554
+ },
555
+ "mean": 0.17181636350182927,
556
+ "per_seed": {
557
+ "1025740212": 0.17388092747958847,
558
+ "1055157354": 0.1805709977888097,
559
+ "1055567618": 0.19289744165908876,
560
+ "1113268940": 0.032532089835823584,
561
+ "1113371261": 0.233029795067053,
562
+ "1176943004": 0.17468938592004596,
563
+ "1189392917": 0.0,
564
+ "1231780997": 0.10207358838016511,
565
+ "1271718007": 0.2350719661460603,
566
+ "1273247072": 0.2793744975404514,
567
+ "1358421325": 0.32948159974174274,
568
+ "1375492786": 0.008822272371479922,
569
+ "141984615": 0.19434281311769708,
570
+ "1475440973": 0.22214884767353088,
571
+ "1483645336": 0.5995435922806452,
572
+ "1522272686": 0.3276128524355476,
573
+ "1528740417": 0.16399362968800954,
574
+ "1549324805": 0.23864987006465407,
575
+ "1607773860": 0.24177237819615016,
576
+ "1665739511": 0.2508232858898205,
577
+ "1747178030": 0.08595818214364334,
578
+ "1881223807": 0.2146835718150399,
579
+ "1919198188": 0.1669540555093941,
580
+ "1991414747": 0.11896663871153935,
581
+ "1997090716": 0.14617959762414967,
582
+ "2015956957": 0.1431252893441326,
583
+ "2054899126": 0.0,
584
+ "211245842": 0.13508468824583414,
585
+ "2141802816": 0.09903289408743085,
586
+ "256770874": 0.046630940401554466,
587
+ "278008524": 0.15535141304867436,
588
+ "416183106": 0.18072825209943494,
589
+ "534717141": 0.2870871533378466,
590
+ "551504635": 0.2702494805053396,
591
+ "565369392": 0.17390197648626904,
592
+ "565621826": 0.0,
593
+ "585904130": 0.2625659187813512,
594
+ "626486813": 0.23561688988950158,
595
+ "636680137": 0.12019361687075947,
596
+ "706479600": 0.22468613298280815,
597
+ "750448296": 0.14149358436045273,
598
+ "75070641": 0.09051828052202912,
599
+ "767747424": 0.14269822176062383,
600
+ "77147264": 0.12309184879211332,
601
+ "794181626": 0.0,
602
+ "957101489": 0.21219243624964876,
603
+ "998761162": 0.11706618974004251
604
+ }
605
+ },
606
+ "macro_by_task": {
607
+ "ci": {
608
+ "confidence": 0.95,
609
+ "high": 0.2399502270115928,
610
+ "low": 0.16431717151236627,
611
+ "mean": 0.1944466839591298,
612
+ "n": 5,
613
+ "num_samples": 500
614
+ },
615
+ "mean": 0.1944466839591298,
616
+ "per_task": {
617
+ "LiftPegUpright-v1": 0.16081309392707618,
618
+ "PickCube-v1": 0.2793744975404514,
619
+ "PullCube-v1": 0.1721079818939321,
620
+ "PushCube-v1": 0.15912329792465574,
621
+ "StackCube-v1": 0.20081454850953362
622
+ }
623
+ },
624
+ "metric": "pairwise_causal_calibration_ece",
625
+ "micro": {
626
+ "confidence": 0.95,
627
+ "high": 0.2026857252748648,
628
+ "low": 0.1491416444266403,
629
+ "mean": 0.17647457313275544,
630
+ "n": 69,
631
+ "num_samples": 500
632
+ },
633
+ "n": 69
634
+ },
635
+ "proposal_oracle_success_at_8": {
636
+ "macro_by_seed": {
637
+ "ci": {
638
+ "confidence": 0.95,
639
+ "high": 0.6808510638297872,
640
+ "low": 0.425531914893617,
641
+ "mean": 0.5539007092198581,
642
+ "n": 47,
643
+ "num_samples": 500
644
+ },
645
+ "mean": 0.5539007092198581,
646
+ "per_seed": {
647
+ "1025740212": 1.0,
648
+ "1055157354": 0.0,
649
+ "1055567618": 1.0,
650
+ "1113268940": 0.0,
651
+ "1113371261": 1.0,
652
+ "1176943004": 1.0,
653
+ "1189392917": 1.0,
654
+ "1231780997": 0.0,
655
+ "1271718007": 0.5,
656
+ "1273247072": 0.0,
657
+ "1358421325": 1.0,
658
+ "1375492786": 0.0,
659
+ "141984615": 0.0,
660
+ "1475440973": 1.0,
661
+ "1483645336": 1.0,
662
+ "1522272686": 0.0,
663
+ "1528740417": 1.0,
664
+ "1549324805": 0.3333333333333333,
665
+ "1607773860": 1.0,
666
+ "1665739511": 1.0,
667
+ "1747178030": 0.0,
668
+ "1881223807": 0.0,
669
+ "1919198188": 0.0,
670
+ "1991414747": 1.0,
671
+ "1997090716": 1.0,
672
+ "2015956957": 0.2,
673
+ "2054899126": 1.0,
674
+ "211245842": 1.0,
675
+ "2141802816": 1.0,
676
+ "256770874": 0.0,
677
+ "278008524": 0.0,
678
+ "416183106": 1.0,
679
+ "534717141": 1.0,
680
+ "551504635": 1.0,
681
+ "565369392": 0.0,
682
+ "565621826": 1.0,
683
+ "585904130": 0.0,
684
+ "626486813": 0.0,
685
+ "636680137": 0.0,
686
+ "706479600": 1.0,
687
+ "750448296": 0.0,
688
+ "75070641": 0.0,
689
+ "767747424": 0.0,
690
+ "77147264": 1.0,
691
+ "794181626": 1.0,
692
+ "957101489": 1.0,
693
+ "998761162": 1.0
694
+ }
695
+ },
696
+ "macro_by_task": {
697
+ "ci": {
698
+ "confidence": 0.95,
699
+ "high": 0.7117948717948719,
700
+ "low": 0.14941176470588236,
701
+ "mean": 0.4519758672699849,
702
+ "n": 5,
703
+ "num_samples": 500
704
+ },
705
+ "mean": 0.4519758672699849,
706
+ "per_task": {
707
+ "LiftPegUpright-v1": 0.6666666666666666,
708
+ "PickCube-v1": 0.0,
709
+ "PullCube-v1": 0.14705882352941177,
710
+ "PushCube-v1": 0.8461538461538461,
711
+ "StackCube-v1": 0.6
712
+ }
713
+ },
714
+ "metric": "proposal_oracle_success_at_8",
715
+ "micro": {
716
+ "confidence": 0.95,
717
+ "high": 0.5507246376811594,
718
+ "low": 0.3188405797101449,
719
+ "mean": 0.42028985507246375,
720
+ "n": 69,
721
+ "num_samples": 500
722
+ },
723
+ "n": 69
724
+ },
725
+ "proposal_oracle_success_gain_over_base_at_8": {
726
+ "macro_by_seed": {
727
+ "ci": {
728
+ "confidence": 0.95,
729
+ "high": 0.35496453900709213,
730
+ "low": 0.08120567375886525,
731
+ "mean": 0.224113475177305,
732
+ "n": 47,
733
+ "num_samples": 500
734
+ },
735
+ "mean": 0.224113475177305,
736
+ "per_seed": {
737
+ "1025740212": 1.0,
738
+ "1055157354": 0.0,
739
+ "1055567618": 0.0,
740
+ "1113268940": 0.0,
741
+ "1113371261": 0.0,
742
+ "1176943004": 1.0,
743
+ "1189392917": 1.0,
744
+ "1231780997": 0.0,
745
+ "1271718007": 0.0,
746
+ "1273247072": 0.0,
747
+ "1358421325": 1.0,
748
+ "1375492786": -1.0,
749
+ "141984615": 0.0,
750
+ "1475440973": 0.0,
751
+ "1483645336": 0.0,
752
+ "1522272686": 0.0,
753
+ "1528740417": 0.0,
754
+ "1549324805": -0.3333333333333333,
755
+ "1607773860": 0.0,
756
+ "1665739511": 0.0,
757
+ "1747178030": 0.0,
758
+ "1881223807": 0.0,
759
+ "1919198188": 0.0,
760
+ "1991414747": 0.0,
761
+ "1997090716": 1.0,
762
+ "2015956957": 0.2,
763
+ "2054899126": 0.0,
764
+ "211245842": 1.0,
765
+ "2141802816": 1.0,
766
+ "256770874": 0.0,
767
+ "278008524": 0.0,
768
+ "416183106": 1.0,
769
+ "534717141": 0.0,
770
+ "551504635": 1.0,
771
+ "565369392": -0.3333333333333333,
772
+ "565621826": 0.0,
773
+ "585904130": 0.0,
774
+ "626486813": -0.25,
775
+ "636680137": -1.0,
776
+ "706479600": 1.0,
777
+ "750448296": 0.0,
778
+ "75070641": 0.0,
779
+ "767747424": -0.25,
780
+ "77147264": 0.5,
781
+ "794181626": 1.0,
782
+ "957101489": 1.0,
783
+ "998761162": 1.0
784
+ }
785
+ },
786
+ "macro_by_task": {
787
+ "ci": {
788
+ "confidence": 0.95,
789
+ "high": 0.4235897435897436,
790
+ "low": -0.0215686274509804,
791
+ "mean": 0.20956259426847662,
792
+ "n": 5,
793
+ "num_samples": 500
794
+ },
795
+ "mean": 0.20956259426847662,
796
+ "per_task": {
797
+ "LiftPegUpright-v1": 0.3333333333333333,
798
+ "PickCube-v1": 0.0,
799
+ "PullCube-v1": -0.14705882352941177,
800
+ "PushCube-v1": 0.46153846153846156,
801
+ "StackCube-v1": 0.4
802
+ }
803
+ },
804
+ "metric": "proposal_oracle_success_gain_over_base_at_8",
805
+ "micro": {
806
+ "confidence": 0.95,
807
+ "high": 0.2608695652173913,
808
+ "low": -0.014492753623188406,
809
+ "mean": 0.13043478260869565,
810
+ "n": 69,
811
+ "num_samples": 500
812
+ },
813
+ "n": 69
814
+ },
815
+ "proposal_oracle_utility_at_8": {
816
+ "macro_by_seed": {
817
+ "ci": {
818
+ "confidence": 0.95,
819
+ "high": 1.4388152989261644,
820
+ "low": 0.9575430694977415,
821
+ "mean": 1.1974476296841894,
822
+ "n": 47,
823
+ "num_samples": 500
824
+ },
825
+ "mean": 1.1974476296841894,
826
+ "per_seed": {
827
+ "1025740212": 2.0,
828
+ "1055157354": 0.22467364370822906,
829
+ "1055567618": 2.0,
830
+ "1113268940": 0.7025249004364014,
831
+ "1113371261": 2.0,
832
+ "1176943004": 2.0,
833
+ "1189392917": 2.0,
834
+ "1231780997": 0.7278411984443665,
835
+ "1271718007": 1.2635672688484192,
836
+ "1273247072": 0.46377021074295044,
837
+ "1358421325": 2.0,
838
+ "1375492786": 0.0008872548933140934,
839
+ "141984615": 0.054234166940053306,
840
+ "1475440973": 2.0,
841
+ "1483645336": 2.0,
842
+ "1522272686": 0.1949000507593155,
843
+ "1528740417": 2.0,
844
+ "1549324805": 0.6833462389186025,
845
+ "1607773860": 2.0,
846
+ "1665739511": 2.0,
847
+ "1747178030": 0.12480536848306656,
848
+ "1881223807": 0.043482705950737,
849
+ "1919198188": 0.06275103986263275,
850
+ "1991414747": 2.0,
851
+ "1997090716": 2.0,
852
+ "2015956957": 0.43981083631515505,
853
+ "2054899126": 2.0,
854
+ "211245842": 2.0,
855
+ "2141802816": 2.0,
856
+ "256770874": 0.21061499416828156,
857
+ "278008524": 0.032865648390725255,
858
+ "416183106": 2.0,
859
+ "534717141": 2.0,
860
+ "551504635": 2.0,
861
+ "565369392": 0.1326704372962316,
862
+ "565621826": 2.0,
863
+ "585904130": 0.07848061621189117,
864
+ "626486813": 0.07003506645560265,
865
+ "636680137": 0.02436021901667118,
866
+ "706479600": 2.0,
867
+ "750448296": 0.0669428389519453,
868
+ "75070641": 0.5650836229324341,
869
+ "767747424": 0.11239026742987335,
870
+ "77147264": 2.0,
871
+ "794181626": 2.0,
872
+ "957101489": 2.0,
873
+ "998761162": 2.0
874
+ }
875
+ },
876
+ "macro_by_task": {
877
+ "ci": {
878
+ "confidence": 0.95,
879
+ "high": 1.5220236249459096,
880
+ "low": 0.5982443927517891,
881
+ "mean": 1.081913336966832,
882
+ "n": 5,
883
+ "num_samples": 500
884
+ },
885
+ "mean": 1.081913336966832,
886
+ "per_task": {
887
+ "LiftPegUpright-v1": 1.5717276831467946,
888
+ "PickCube-v1": 0.46377021074295044,
889
+ "PullCube-v1": 0.3548735701530377,
890
+ "PushCube-v1": 1.7001846948495278,
891
+ "StackCube-v1": 1.3190105259418488
892
+ }
893
+ },
894
+ "metric": "proposal_oracle_utility_at_8",
895
+ "micro": {
896
+ "confidence": 0.95,
897
+ "high": 1.1545684401146676,
898
+ "low": 0.7270322287424832,
899
+ "mean": 0.9253245886521535,
900
+ "n": 69,
901
+ "num_samples": 500
902
+ },
903
+ "n": 69
904
+ },
905
+ "proposal_oracle_utility_gain_over_base_at_8": {
906
+ "macro_by_seed": {
907
+ "ci": {
908
+ "confidence": 0.95,
909
+ "high": 0.5855443862486458,
910
+ "low": 0.08830777057388746,
911
+ "mean": 0.3405734994345517,
912
+ "n": 47,
913
+ "num_samples": 500
914
+ },
915
+ "mean": 0.3405734994345517,
916
+ "per_seed": {
917
+ "1025740212": 1.471626341342926,
918
+ "1055157354": 0.02818956971168518,
919
+ "1055567618": 0.0,
920
+ "1113268940": 0.2802123725414276,
921
+ "1113371261": 0.0,
922
+ "1176943004": 1.949059471487999,
923
+ "1189392917": 1.8915813565254211,
924
+ "1231780997": 0.0008350610733032227,
925
+ "1271718007": -0.15181386470794678,
926
+ "1273247072": 0.31034055352211,
927
+ "1358421325": 1.1672977209091187,
928
+ "1375492786": -1.999112745106686,
929
+ "141984615": -0.039577972143888474,
930
+ "1475440973": 0.0,
931
+ "1483645336": 0.0,
932
+ "1522272686": 0.0001389533281326294,
933
+ "1528740417": 0.0,
934
+ "1549324805": -0.6954012094065547,
935
+ "1607773860": 0.0,
936
+ "1665739511": 0.0,
937
+ "1747178030": -0.021501220762729645,
938
+ "1881223807": 0.011455714702606201,
939
+ "1919198188": 0.025552168488502502,
940
+ "1991414747": 0.0,
941
+ "1997090716": 1.8607252091169357,
942
+ "2015956957": 0.3726910464465618,
943
+ "2054899126": 0.0,
944
+ "211245842": 1.1611655950546265,
945
+ "2141802816": 1.9291927516460419,
946
+ "256770874": -0.6451330631971359,
947
+ "278008524": -0.0746062200050801,
948
+ "416183106": 1.7996735870838165,
949
+ "534717141": 0.0,
950
+ "551504635": 1.276113212108612,
951
+ "565369392": -0.6670952091614405,
952
+ "565621826": 0.0,
953
+ "585904130": 0.0031885181864102683,
954
+ "626486813": -0.43540495017077774,
955
+ "636680137": -1.9756397809833288,
956
+ "706479600": 1.1649112105369568,
957
+ "750448296": -0.044590720906853676,
958
+ "75070641": 0.36233198642730713,
959
+ "767747424": -0.4351339202839881,
960
+ "77147264": 0.5784205198287964,
961
+ "794181626": 1.8190439492464066,
962
+ "957101489": 1.7960350513458252,
963
+ "998761162": 1.9321834295988083
964
+ }
965
+ },
966
+ "macro_by_task": {
967
+ "ci": {
968
+ "confidence": 0.95,
969
+ "high": 0.6900747065360731,
970
+ "low": -0.01290068514136069,
971
+ "mean": 0.382283764545011,
972
+ "n": 5,
973
+ "num_samples": 500
974
+ },
975
+ "mean": 0.382283764545011,
976
+ "per_task": {
977
+ "LiftPegUpright-v1": 0.5047978311777115,
978
+ "PickCube-v1": 0.31034055352211,
979
+ "PullCube-v1": -0.29635635063031635,
980
+ "PushCube-v1": 0.8784117159935144,
981
+ "StackCube-v1": 0.5142250726620357
982
+ }
983
+ },
984
+ "metric": "proposal_oracle_utility_gain_over_base_at_8",
985
+ "micro": {
986
+ "confidence": 0.95,
987
+ "high": 0.40019721312421386,
988
+ "low": -0.055687820795329586,
989
+ "mean": 0.179648406043534,
990
+ "n": 69,
991
+ "num_samples": 500
992
+ },
993
+ "n": 69
994
+ },
995
+ "selected_success_at_8": {
996
+ "macro_by_seed": {
997
+ "ci": {
998
+ "confidence": 0.95,
999
+ "high": 0.4482269503546099,
1000
+ "low": 0.19148936170212766,
1001
+ "mean": 0.3198581560283688,
1002
+ "n": 47,
1003
+ "num_samples": 500
1004
+ },
1005
+ "mean": 0.3198581560283688,
1006
+ "per_seed": {
1007
+ "1025740212": 0.0,
1008
+ "1055157354": 0.0,
1009
+ "1055567618": 0.0,
1010
+ "1113268940": 0.0,
1011
+ "1113371261": 1.0,
1012
+ "1176943004": 1.0,
1013
+ "1189392917": 1.0,
1014
+ "1231780997": 0.0,
1015
+ "1271718007": 0.5,
1016
+ "1273247072": 0.0,
1017
+ "1358421325": 0.0,
1018
+ "1375492786": 0.0,
1019
+ "141984615": 0.0,
1020
+ "1475440973": 0.0,
1021
+ "1483645336": 0.0,
1022
+ "1522272686": 0.0,
1023
+ "1528740417": 0.0,
1024
+ "1549324805": 0.3333333333333333,
1025
+ "1607773860": 1.0,
1026
+ "1665739511": 1.0,
1027
+ "1747178030": 0.0,
1028
+ "1881223807": 0.0,
1029
+ "1919198188": 0.0,
1030
+ "1991414747": 1.0,
1031
+ "1997090716": 1.0,
1032
+ "2015956957": 0.2,
1033
+ "2054899126": 1.0,
1034
+ "211245842": 1.0,
1035
+ "2141802816": 0.0,
1036
+ "256770874": 0.0,
1037
+ "278008524": 0.0,
1038
+ "416183106": 0.0,
1039
+ "534717141": 1.0,
1040
+ "551504635": 0.0,
1041
+ "565369392": 0.0,
1042
+ "565621826": 1.0,
1043
+ "585904130": 0.0,
1044
+ "626486813": 0.0,
1045
+ "636680137": 0.0,
1046
+ "706479600": 1.0,
1047
+ "750448296": 0.0,
1048
+ "75070641": 0.0,
1049
+ "767747424": 0.0,
1050
+ "77147264": 1.0,
1051
+ "794181626": 1.0,
1052
+ "957101489": 0.0,
1053
+ "998761162": 0.0
1054
+ }
1055
+ },
1056
+ "macro_by_task": {
1057
+ "ci": {
1058
+ "confidence": 0.95,
1059
+ "high": 0.43589743589743596,
1060
+ "low": 0.058823529411764705,
1061
+ "mean": 0.21915535444947212,
1062
+ "n": 5,
1063
+ "num_samples": 500
1064
+ },
1065
+ "mean": 0.21915535444947212,
1066
+ "per_task": {
1067
+ "LiftPegUpright-v1": 0.0,
1068
+ "PickCube-v1": 0.0,
1069
+ "PullCube-v1": 0.14705882352941177,
1070
+ "PushCube-v1": 0.6153846153846154,
1071
+ "StackCube-v1": 0.3333333333333333
1072
+ }
1073
+ },
1074
+ "metric": "selected_success_at_8",
1075
+ "micro": {
1076
+ "confidence": 0.95,
1077
+ "high": 0.36231884057971014,
1078
+ "low": 0.17391304347826086,
1079
+ "mean": 0.2608695652173913,
1080
+ "n": 69,
1081
+ "num_samples": 500
1082
+ },
1083
+ "n": 69
1084
+ },
1085
+ "selected_success_gain_over_base_at_8": {
1086
+ "macro_by_seed": {
1087
+ "ci": {
1088
+ "confidence": 0.95,
1089
+ "high": 0.11950354609929079,
1090
+ "low": -0.16241134751773048,
1091
+ "mean": -0.0099290780141844,
1092
+ "n": 47,
1093
+ "num_samples": 500
1094
+ },
1095
+ "mean": -0.0099290780141844,
1096
+ "per_seed": {
1097
+ "1025740212": 0.0,
1098
+ "1055157354": 0.0,
1099
+ "1055567618": -1.0,
1100
+ "1113268940": 0.0,
1101
+ "1113371261": 0.0,
1102
+ "1176943004": 1.0,
1103
+ "1189392917": 1.0,
1104
+ "1231780997": 0.0,
1105
+ "1271718007": 0.0,
1106
+ "1273247072": 0.0,
1107
+ "1358421325": 0.0,
1108
+ "1375492786": -1.0,
1109
+ "141984615": 0.0,
1110
+ "1475440973": -1.0,
1111
+ "1483645336": -1.0,
1112
+ "1522272686": 0.0,
1113
+ "1528740417": -1.0,
1114
+ "1549324805": -0.3333333333333333,
1115
+ "1607773860": 0.0,
1116
+ "1665739511": 0.0,
1117
+ "1747178030": 0.0,
1118
+ "1881223807": 0.0,
1119
+ "1919198188": 0.0,
1120
+ "1991414747": 0.0,
1121
+ "1997090716": 1.0,
1122
+ "2015956957": 0.2,
1123
+ "2054899126": 0.0,
1124
+ "211245842": 1.0,
1125
+ "2141802816": 0.0,
1126
+ "256770874": 0.0,
1127
+ "278008524": 0.0,
1128
+ "416183106": 0.0,
1129
+ "534717141": 0.0,
1130
+ "551504635": 0.0,
1131
+ "565369392": -0.3333333333333333,
1132
+ "565621826": 0.0,
1133
+ "585904130": 0.0,
1134
+ "626486813": -0.25,
1135
+ "636680137": -1.0,
1136
+ "706479600": 1.0,
1137
+ "750448296": 0.0,
1138
+ "75070641": 0.0,
1139
+ "767747424": -0.25,
1140
+ "77147264": 0.5,
1141
+ "794181626": 1.0,
1142
+ "957101489": 0.0,
1143
+ "998761162": 0.0
1144
+ }
1145
+ },
1146
+ "macro_by_task": {
1147
+ "ci": {
1148
+ "confidence": 0.95,
1149
+ "high": 0.14564102564102566,
1150
+ "low": -0.20274509803921567,
1151
+ "mean": -0.02325791855203619,
1152
+ "n": 5,
1153
+ "num_samples": 500
1154
+ },
1155
+ "mean": -0.02325791855203619,
1156
+ "per_task": {
1157
+ "LiftPegUpright-v1": -0.3333333333333333,
1158
+ "PickCube-v1": 0.0,
1159
+ "PullCube-v1": -0.14705882352941177,
1160
+ "PushCube-v1": 0.23076923076923078,
1161
+ "StackCube-v1": 0.13333333333333333
1162
+ }
1163
+ },
1164
+ "metric": "selected_success_gain_over_base_at_8",
1165
+ "micro": {
1166
+ "confidence": 0.95,
1167
+ "high": 0.08695652173913043,
1168
+ "low": -0.15942028985507245,
1169
+ "mean": -0.028985507246376812,
1170
+ "n": 69,
1171
+ "num_samples": 500
1172
+ },
1173
+ "n": 69
1174
+ },
1175
+ "selected_utility_at_8": {
1176
+ "macro_by_seed": {
1177
+ "ci": {
1178
+ "confidence": 0.95,
1179
+ "high": 0.9970987782217497,
1180
+ "low": 0.5411203128707609,
1181
+ "mean": 0.7653393023166019,
1182
+ "n": 47,
1183
+ "num_samples": 500
1184
+ },
1185
+ "mean": 0.7653393023166019,
1186
+ "per_seed": {
1187
+ "1025740212": 0.22749333083629608,
1188
+ "1055157354": 0.18986818194389343,
1189
+ "1055567618": 0.07813572883605957,
1190
+ "1113268940": 0.6364428400993347,
1191
+ "1113371261": 2.0,
1192
+ "1176943004": 2.0,
1193
+ "1189392917": 2.0,
1194
+ "1231780997": 0.22716176509857178,
1195
+ "1271718007": 1.0662197396159172,
1196
+ "1273247072": 0.10470707714557648,
1197
+ "1358421325": 0.09903085231781006,
1198
+ "1375492786": 0.0005992253427393734,
1199
+ "141984615": 0.04040361692508062,
1200
+ "1475440973": 0.7161068916320801,
1201
+ "1483645336": 0.0313941091299057,
1202
+ "1522272686": 0.17692717909812927,
1203
+ "1528740417": 0.7195272445678711,
1204
+ "1549324805": 0.6709667841593424,
1205
+ "1607773860": 2.0,
1206
+ "1665739511": 2.0,
1207
+ "1747178030": 0.09071242809295654,
1208
+ "1881223807": 0.018627971410751343,
1209
+ "1919198188": 0.043911516666412354,
1210
+ "1991414747": 2.0,
1211
+ "1997090716": 2.0,
1212
+ "2015956957": 0.4342371065169573,
1213
+ "2054899126": 2.0,
1214
+ "211245842": 2.0,
1215
+ "2141802816": 0.13836362957954407,
1216
+ "256770874": 0.16439804434776306,
1217
+ "278008524": 0.012664265930652618,
1218
+ "416183106": 0.18704846501350403,
1219
+ "534717141": 2.0,
1220
+ "551504635": 0.726307213306427,
1221
+ "565369392": 0.08709763859709103,
1222
+ "565621826": 2.0,
1223
+ "585904130": 0.06744805444031954,
1224
+ "626486813": 0.03831234632525593,
1225
+ "636680137": 0.017868518829345703,
1226
+ "706479600": 2.0,
1227
+ "750448296": 0.049234201200306416,
1228
+ "75070641": 0.194048672914505,
1229
+ "767747424": 0.0877318112179637,
1230
+ "77147264": 2.0,
1231
+ "794181626": 2.0,
1232
+ "957101489": 0.567138671875,
1233
+ "998761162": 0.0608120858669281
1234
+ }
1235
+ },
1236
+ "macro_by_task": {
1237
+ "ci": {
1238
+ "confidence": 0.95,
1239
+ "high": 0.9739732449329817,
1240
+ "low": 0.2881497273441431,
1241
+ "mean": 0.6050292259588579,
1242
+ "n": 5,
1243
+ "num_samples": 500
1244
+ },
1245
+ "mean": 0.6050292259588579,
1246
+ "per_task": {
1247
+ "LiftPegUpright-v1": 0.5421732142567635,
1248
+ "PickCube-v1": 0.10470707714557648,
1249
+ "PullCube-v1": 0.3377642858009922,
1250
+ "PushCube-v1": 1.251638433107963,
1251
+ "StackCube-v1": 0.788863119482994
1252
+ }
1253
+ },
1254
+ "metric": "selected_utility_at_8",
1255
+ "micro": {
1256
+ "confidence": 0.95,
1257
+ "high": 0.8055619765385189,
1258
+ "low": 0.44728328000781115,
1259
+ "mean": 0.6224054855444685,
1260
+ "n": 69,
1261
+ "num_samples": 500
1262
+ },
1263
+ "n": 69
1264
+ },
1265
+ "selected_utility_gain_over_base_at_8": {
1266
+ "macro_by_seed": {
1267
+ "ci": {
1268
+ "confidence": 0.95,
1269
+ "high": 0.13649843921744537,
1270
+ "low": -0.349154780383126,
1271
+ "mean": -0.09153482793303577,
1272
+ "n": 47,
1273
+ "num_samples": 500
1274
+ },
1275
+ "mean": -0.09153482793303577,
1276
+ "per_seed": {
1277
+ "1025740212": -0.3008803278207779,
1278
+ "1055157354": -0.006615892052650452,
1279
+ "1055567618": -1.9218642711639404,
1280
+ "1113268940": 0.21413031220436096,
1281
+ "1113371261": 0.0,
1282
+ "1176943004": 1.949059471487999,
1283
+ "1189392917": 1.8915813565254211,
1284
+ "1231780997": -0.49984437227249146,
1285
+ "1271718007": -0.34916139394044876,
1286
+ "1273247072": -0.04872258007526398,
1287
+ "1358421325": -0.7336714267730713,
1288
+ "1375492786": -1.9994007746572606,
1289
+ "141984615": -0.05340852215886116,
1290
+ "1475440973": -1.28389310836792,
1291
+ "1483645336": -1.9686058908700943,
1292
+ "1522272686": -0.01783391833305359,
1293
+ "1528740417": -1.280472755432129,
1294
+ "1549324805": -0.7077806641658148,
1295
+ "1607773860": 0.0,
1296
+ "1665739511": 0.0,
1297
+ "1747178030": -0.05559416115283966,
1298
+ "1881223807": -0.013399019837379456,
1299
+ "1919198188": 0.0067126452922821045,
1300
+ "1991414747": 0.0,
1301
+ "1997090716": 1.8607252091169357,
1302
+ "2015956957": 0.36711731664836406,
1303
+ "2054899126": 0.0,
1304
+ "211245842": 1.1611655950546265,
1305
+ "2141802816": 0.06755638122558594,
1306
+ "256770874": -0.6913500130176544,
1307
+ "278008524": -0.09480760246515274,
1308
+ "416183106": -0.013277947902679443,
1309
+ "534717141": 0.0,
1310
+ "551504635": 0.0024204254150390625,
1311
+ "565369392": -0.712668007860581,
1312
+ "565621826": 0.0,
1313
+ "585904130": -0.007844043585161367,
1314
+ "626486813": -0.46712767030112445,
1315
+ "636680137": -1.9821314811706543,
1316
+ "706479600": 1.1649112105369568,
1317
+ "750448296": -0.062299358658492565,
1318
+ "75070641": -0.008702963590621948,
1319
+ "767747424": -0.45979237649589777,
1320
+ "77147264": 0.5784205198287964,
1321
+ "794181626": 1.8190439492464066,
1322
+ "957101489": 0.3631737232208252,
1323
+ "998761162": -0.007004484534263611
1324
+ }
1325
+ },
1326
+ "macro_by_task": {
1327
+ "ci": {
1328
+ "confidence": 0.95,
1329
+ "high": 0.1623927814226884,
1330
+ "low": -0.3557238355283534,
1331
+ "mean": -0.09460034646296299,
1332
+ "n": 5,
1333
+ "num_samples": 500
1334
+ },
1335
+ "mean": -0.09460034646296299,
1336
+ "per_task": {
1337
+ "LiftPegUpright-v1": -0.5247566377123197,
1338
+ "PickCube-v1": -0.04872258007526398,
1339
+ "PullCube-v1": -0.31346563498236185,
1340
+ "PushCube-v1": 0.4298654542519496,
1341
+ "StackCube-v1": -0.015922333796819052
1342
+ }
1343
+ },
1344
+ "metric": "selected_utility_gain_over_base_at_8",
1345
+ "micro": {
1346
+ "confidence": 0.95,
1347
+ "high": 0.08361495469791302,
1348
+ "low": -0.3535496463047583,
1349
+ "mean": -0.1232706970641511,
1350
+ "n": 69,
1351
+ "num_samples": 500
1352
+ },
1353
+ "n": 69
1354
+ },
1355
+ "selector_regret_at_8": {
1356
+ "macro_by_seed": {
1357
+ "ci": {
1358
+ "confidence": 0.95,
1359
+ "high": 0.6595998992103133,
1360
+ "low": 0.2230010245754945,
1361
+ "mean": 0.43210832736758736,
1362
+ "n": 47,
1363
+ "num_samples": 500
1364
+ },
1365
+ "mean": 0.43210832736758736,
1366
+ "per_seed": {
1367
+ "1025740212": 1.772506669163704,
1368
+ "1055157354": 0.03480546176433563,
1369
+ "1055567618": 1.9218642711639404,
1370
+ "1113268940": 0.06608206033706665,
1371
+ "1113371261": 0.0,
1372
+ "1176943004": 0.0,
1373
+ "1189392917": 0.0,
1374
+ "1231780997": 0.5006794333457947,
1375
+ "1271718007": 0.19734752923250198,
1376
+ "1273247072": 0.35906313359737396,
1377
+ "1358421325": 1.90096914768219,
1378
+ "1375492786": 0.0002880295505747199,
1379
+ "141984615": 0.013830550014972687,
1380
+ "1475440973": 1.28389310836792,
1381
+ "1483645336": 1.9686058908700943,
1382
+ "1522272686": 0.017972871661186218,
1383
+ "1528740417": 1.280472755432129,
1384
+ "1549324805": 0.01237945475926002,
1385
+ "1607773860": 0.0,
1386
+ "1665739511": 0.0,
1387
+ "1747178030": 0.034092940390110016,
1388
+ "1881223807": 0.024854734539985657,
1389
+ "1919198188": 0.018839523196220398,
1390
+ "1991414747": 0.0,
1391
+ "1997090716": 0.0,
1392
+ "2015956957": 0.005573729798197746,
1393
+ "2054899126": 0.0,
1394
+ "211245842": 0.0,
1395
+ "2141802816": 1.861636370420456,
1396
+ "256770874": 0.046216949820518494,
1397
+ "278008524": 0.020201382460072637,
1398
+ "416183106": 1.812951534986496,
1399
+ "534717141": 0.0,
1400
+ "551504635": 1.273692786693573,
1401
+ "565369392": 0.04557279869914055,
1402
+ "565621826": 0.0,
1403
+ "585904130": 0.011032561771571636,
1404
+ "626486813": 0.031722720130346715,
1405
+ "636680137": 0.006491700187325478,
1406
+ "706479600": 0.0,
1407
+ "750448296": 0.01770863775163889,
1408
+ "75070641": 0.3710349500179291,
1409
+ "767747424": 0.024658456211909652,
1410
+ "77147264": 0.0,
1411
+ "794181626": 0.0,
1412
+ "957101489": 1.432861328125,
1413
+ "998761162": 1.939187914133072
1414
+ }
1415
+ },
1416
+ "macro_by_task": {
1417
+ "ci": {
1418
+ "confidence": 0.95,
1419
+ "high": 0.7792545604018064,
1420
+ "low": 0.18968407530785322,
1421
+ "mean": 0.476884111007974,
1422
+ "n": 5,
1423
+ "num_samples": 500
1424
+ },
1425
+ "mean": 0.476884111007974,
1426
+ "per_task": {
1427
+ "LiftPegUpright-v1": 1.029554468890031,
1428
+ "PickCube-v1": 0.35906313359737396,
1429
+ "PullCube-v1": 0.017109284352045506,
1430
+ "PushCube-v1": 0.4485462617415648,
1431
+ "StackCube-v1": 0.5301474064588547
1432
+ }
1433
+ },
1434
+ "metric": "selector_regret_at_8",
1435
+ "micro": {
1436
+ "confidence": 0.95,
1437
+ "high": 0.46172967990117986,
1438
+ "low": 0.1641734795007126,
1439
+ "mean": 0.3029191031076851,
1440
+ "n": 69,
1441
+ "num_samples": 500
1442
+ },
1443
+ "n": 69
1444
+ },
1445
+ "success_selector_gap_at_8": {
1446
+ "macro_by_seed": {
1447
+ "ci": {
1448
+ "confidence": 0.95,
1449
+ "high": 0.3617021276595745,
1450
+ "low": 0.10638297872340426,
1451
+ "mean": 0.23404255319148937,
1452
+ "n": 47,
1453
+ "num_samples": 500
1454
+ },
1455
+ "mean": 0.23404255319148937,
1456
+ "per_seed": {
1457
+ "1025740212": 1.0,
1458
+ "1055157354": 0.0,
1459
+ "1055567618": 1.0,
1460
+ "1113268940": 0.0,
1461
+ "1113371261": 0.0,
1462
+ "1176943004": 0.0,
1463
+ "1189392917": 0.0,
1464
+ "1231780997": 0.0,
1465
+ "1271718007": 0.0,
1466
+ "1273247072": 0.0,
1467
+ "1358421325": 1.0,
1468
+ "1375492786": 0.0,
1469
+ "141984615": 0.0,
1470
+ "1475440973": 1.0,
1471
+ "1483645336": 1.0,
1472
+ "1522272686": 0.0,
1473
+ "1528740417": 1.0,
1474
+ "1549324805": 0.0,
1475
+ "1607773860": 0.0,
1476
+ "1665739511": 0.0,
1477
+ "1747178030": 0.0,
1478
+ "1881223807": 0.0,
1479
+ "1919198188": 0.0,
1480
+ "1991414747": 0.0,
1481
+ "1997090716": 0.0,
1482
+ "2015956957": 0.0,
1483
+ "2054899126": 0.0,
1484
+ "211245842": 0.0,
1485
+ "2141802816": 1.0,
1486
+ "256770874": 0.0,
1487
+ "278008524": 0.0,
1488
+ "416183106": 1.0,
1489
+ "534717141": 0.0,
1490
+ "551504635": 1.0,
1491
+ "565369392": 0.0,
1492
+ "565621826": 0.0,
1493
+ "585904130": 0.0,
1494
+ "626486813": 0.0,
1495
+ "636680137": 0.0,
1496
+ "706479600": 0.0,
1497
+ "750448296": 0.0,
1498
+ "75070641": 0.0,
1499
+ "767747424": 0.0,
1500
+ "77147264": 0.0,
1501
+ "794181626": 0.0,
1502
+ "957101489": 1.0,
1503
+ "998761162": 1.0
1504
+ }
1505
+ },
1506
+ "macro_by_task": {
1507
+ "ci": {
1508
+ "confidence": 0.95,
1509
+ "high": 0.4461538461538462,
1510
+ "low": 0.046153846153846156,
1511
+ "mean": 0.2328205128205128,
1512
+ "n": 5,
1513
+ "num_samples": 500
1514
+ },
1515
+ "mean": 0.2328205128205128,
1516
+ "per_task": {
1517
+ "LiftPegUpright-v1": 0.6666666666666666,
1518
+ "PickCube-v1": 0.0,
1519
+ "PullCube-v1": 0.0,
1520
+ "PushCube-v1": 0.23076923076923078,
1521
+ "StackCube-v1": 0.26666666666666666
1522
+ }
1523
+ },
1524
+ "metric": "success_selector_gap_at_8",
1525
+ "micro": {
1526
+ "confidence": 0.95,
1527
+ "high": 0.2608695652173913,
1528
+ "low": 0.08695652173913043,
1529
+ "mean": 0.15942028985507245,
1530
+ "n": 69,
1531
+ "num_samples": 500
1532
+ },
1533
+ "n": 69
1534
+ },
1535
+ "success_support_gap_at_8": {
1536
+ "macro_by_seed": {
1537
+ "ci": {
1538
+ "confidence": 0.95,
1539
+ "high": 0.4148936170212766,
1540
+ "low": 0.1595744680851064,
1541
+ "mean": 0.2819148936170212,
1542
+ "n": 47,
1543
+ "num_samples": 500
1544
+ },
1545
+ "mean": 0.2819148936170212,
1546
+ "per_seed": {
1547
+ "1025740212": 0.0,
1548
+ "1055157354": 1.0,
1549
+ "1055567618": 0.0,
1550
+ "1113268940": 1.0,
1551
+ "1113371261": 0.0,
1552
+ "1176943004": 0.0,
1553
+ "1189392917": 0.0,
1554
+ "1231780997": 1.0,
1555
+ "1271718007": 0.5,
1556
+ "1273247072": 1.0,
1557
+ "1358421325": 0.0,
1558
+ "1375492786": 0.0,
1559
+ "141984615": 0.0,
1560
+ "1475440973": 0.0,
1561
+ "1483645336": 0.0,
1562
+ "1522272686": 1.0,
1563
+ "1528740417": 0.0,
1564
+ "1549324805": 0.3333333333333333,
1565
+ "1607773860": 0.0,
1566
+ "1665739511": 0.0,
1567
+ "1747178030": 1.0,
1568
+ "1881223807": 1.0,
1569
+ "1919198188": 1.0,
1570
+ "1991414747": 0.0,
1571
+ "1997090716": 0.0,
1572
+ "2015956957": 0.0,
1573
+ "2054899126": 0.0,
1574
+ "211245842": 0.0,
1575
+ "2141802816": 0.0,
1576
+ "256770874": 1.0,
1577
+ "278008524": 0.5,
1578
+ "416183106": 0.0,
1579
+ "534717141": 0.0,
1580
+ "551504635": 0.0,
1581
+ "565369392": 0.6666666666666666,
1582
+ "565621826": 0.0,
1583
+ "585904130": 0.0,
1584
+ "626486813": 0.25,
1585
+ "636680137": 1.0,
1586
+ "706479600": 0.0,
1587
+ "750448296": 0.0,
1588
+ "75070641": 1.0,
1589
+ "767747424": 0.0,
1590
+ "77147264": 0.0,
1591
+ "794181626": 0.0,
1592
+ "957101489": 0.0,
1593
+ "998761162": 0.0
1594
+ }
1595
+ },
1596
+ "macro_by_task": {
1597
+ "ci": {
1598
+ "confidence": 0.95,
1599
+ "high": 0.7152941176470589,
1600
+ "low": 0.19879336349924587,
1601
+ "mean": 0.4127300150829562,
1602
+ "n": 5,
1603
+ "num_samples": 500
1604
+ },
1605
+ "mean": 0.4127300150829562,
1606
+ "per_task": {
1607
+ "LiftPegUpright-v1": 0.3333333333333333,
1608
+ "PickCube-v1": 1.0,
1609
+ "PullCube-v1": 0.17647058823529413,
1610
+ "PushCube-v1": 0.15384615384615385,
1611
+ "StackCube-v1": 0.4
1612
+ }
1613
+ },
1614
+ "metric": "success_support_gap_at_8",
1615
+ "micro": {
1616
+ "confidence": 0.95,
1617
+ "high": 0.34782608695652173,
1618
+ "low": 0.14492753623188406,
1619
+ "mean": 0.2463768115942029,
1620
+ "n": 69,
1621
+ "num_samples": 500
1622
+ },
1623
+ "n": 69
1624
+ },
1625
+ "success_total_car_to_hidden_at_8": {
1626
+ "macro_by_seed": {
1627
+ "ci": {
1628
+ "confidence": 0.95,
1629
+ "high": 0.6524822695035459,
1630
+ "low": 0.3776595744680851,
1631
+ "mean": 0.5159574468085107,
1632
+ "n": 47,
1633
+ "num_samples": 500
1634
+ },
1635
+ "mean": 0.5159574468085107,
1636
+ "per_seed": {
1637
+ "1025740212": 1.0,
1638
+ "1055157354": 1.0,
1639
+ "1055567618": 1.0,
1640
+ "1113268940": 1.0,
1641
+ "1113371261": 0.0,
1642
+ "1176943004": 0.0,
1643
+ "1189392917": 0.0,
1644
+ "1231780997": 1.0,
1645
+ "1271718007": 0.5,
1646
+ "1273247072": 1.0,
1647
+ "1358421325": 1.0,
1648
+ "1375492786": 0.0,
1649
+ "141984615": 0.0,
1650
+ "1475440973": 1.0,
1651
+ "1483645336": 1.0,
1652
+ "1522272686": 1.0,
1653
+ "1528740417": 1.0,
1654
+ "1549324805": 0.3333333333333333,
1655
+ "1607773860": 0.0,
1656
+ "1665739511": 0.0,
1657
+ "1747178030": 1.0,
1658
+ "1881223807": 1.0,
1659
+ "1919198188": 1.0,
1660
+ "1991414747": 0.0,
1661
+ "1997090716": 0.0,
1662
+ "2015956957": 0.0,
1663
+ "2054899126": 0.0,
1664
+ "211245842": 0.0,
1665
+ "2141802816": 1.0,
1666
+ "256770874": 1.0,
1667
+ "278008524": 0.5,
1668
+ "416183106": 1.0,
1669
+ "534717141": 0.0,
1670
+ "551504635": 1.0,
1671
+ "565369392": 0.6666666666666666,
1672
+ "565621826": 0.0,
1673
+ "585904130": 0.0,
1674
+ "626486813": 0.25,
1675
+ "636680137": 1.0,
1676
+ "706479600": 0.0,
1677
+ "750448296": 0.0,
1678
+ "75070641": 1.0,
1679
+ "767747424": 0.0,
1680
+ "77147264": 0.0,
1681
+ "794181626": 0.0,
1682
+ "957101489": 1.0,
1683
+ "998761162": 1.0
1684
+ }
1685
+ },
1686
+ "macro_by_task": {
1687
+ "ci": {
1688
+ "confidence": 0.95,
1689
+ "high": 0.9333333333333332,
1690
+ "low": 0.3161387631975867,
1691
+ "mean": 0.645550527903469,
1692
+ "n": 5,
1693
+ "num_samples": 500
1694
+ },
1695
+ "mean": 0.645550527903469,
1696
+ "per_task": {
1697
+ "LiftPegUpright-v1": 1.0,
1698
+ "PickCube-v1": 1.0,
1699
+ "PullCube-v1": 0.17647058823529413,
1700
+ "PushCube-v1": 0.38461538461538464,
1701
+ "StackCube-v1": 0.6666666666666666
1702
+ }
1703
+ },
1704
+ "metric": "success_total_car_to_hidden_at_8",
1705
+ "micro": {
1706
+ "confidence": 0.95,
1707
+ "high": 0.5217391304347826,
1708
+ "low": 0.2753623188405797,
1709
+ "mean": 0.4057971014492754,
1710
+ "n": 69,
1711
+ "num_samples": 500
1712
+ },
1713
+ "n": 69
1714
+ },
1715
+ "support_gap_at_8": {
1716
+ "macro_by_seed": {
1717
+ "ci": {
1718
+ "confidence": 0.95,
1719
+ "high": 0.7286775349375151,
1720
+ "low": 0.30572481987938444,
1721
+ "mean": 0.5087829326731087,
1722
+ "n": 47,
1723
+ "num_samples": 500
1724
+ },
1725
+ "mean": 0.5087829326731087,
1726
+ "per_seed": {
1727
+ "1025740212": 0.0,
1728
+ "1055157354": 1.775326356291771,
1729
+ "1055567618": 0.0,
1730
+ "1113268940": 1.2974750995635986,
1731
+ "1113371261": 0.0,
1732
+ "1176943004": 0.0,
1733
+ "1189392917": 0.0,
1734
+ "1231780997": 1.2721588015556335,
1735
+ "1271718007": 0.7364327311515808,
1736
+ "1273247072": 1.5362297892570496,
1737
+ "1358421325": 0.0,
1738
+ "1375492786": 0.24256371456431225,
1739
+ "141984615": 0.14621367553869882,
1740
+ "1475440973": 0.0,
1741
+ "1483645336": 0.0,
1742
+ "1522272686": 1.8050999492406845,
1743
+ "1528740417": 0.0,
1744
+ "1549324805": 0.7194042829796672,
1745
+ "1607773860": 0.0,
1746
+ "1665739511": 0.0,
1747
+ "1747178030": 1.8751946315169334,
1748
+ "1881223807": 1.956517294049263,
1749
+ "1919198188": 1.9372489601373672,
1750
+ "1991414747": 0.0,
1751
+ "1997090716": 0.0,
1752
+ "2015956957": 0.09007053077220917,
1753
+ "2054899126": 0.0,
1754
+ "211245842": 0.0,
1755
+ "2141802816": 0.0,
1756
+ "256770874": 1.7893850058317184,
1757
+ "278008524": 1.0809455465059727,
1758
+ "416183106": 0.0,
1759
+ "534717141": 0.0,
1760
+ "551504635": 0.0,
1761
+ "565369392": 1.2939611425002415,
1762
+ "565621826": 0.0,
1763
+ "585904130": 0.15916637579600015,
1764
+ "626486813": 0.5432130172848701,
1765
+ "636680137": 1.9756397809833288,
1766
+ "706479600": 0.0,
1767
+ "750448296": 0.1688647512346506,
1768
+ "75070641": 1.434916377067566,
1769
+ "767747424": 0.07677002181299031,
1770
+ "77147264": 0.0,
1771
+ "794181626": 0.0,
1772
+ "957101489": 0.0,
1773
+ "998761162": 0.0
1774
+ }
1775
+ },
1776
+ "macro_by_task": {
1777
+ "ci": {
1778
+ "confidence": 0.95,
1779
+ "high": 1.143456111250185,
1780
+ "low": 0.37662127119948,
1781
+ "mean": 0.6745817199481008,
1782
+ "n": 5,
1783
+ "num_samples": 500
1784
+ },
1785
+ "mean": 0.6745817199481008,
1786
+ "per_task": {
1787
+ "LiftPegUpright-v1": 0.4282723168532054,
1788
+ "PickCube-v1": 1.5362297892570496,
1789
+ "PullCube-v1": 0.4276017144216252,
1790
+ "PushCube-v1": 0.2998153051504722,
1791
+ "StackCube-v1": 0.6809894740581512
1792
+ }
1793
+ },
1794
+ "metric": "support_gap_at_8",
1795
+ "micro": {
1796
+ "confidence": 0.95,
1797
+ "high": 0.648598097173659,
1798
+ "low": 0.30040934245672374,
1799
+ "mean": 0.4747356965005789,
1800
+ "n": 69,
1801
+ "num_samples": 500
1802
+ },
1803
+ "n": 69
1804
+ },
1805
+ "total_car_to_hidden_at_8": {
1806
+ "macro_by_seed": {
1807
+ "ci": {
1808
+ "confidence": 0.95,
1809
+ "high": 1.1787349130094698,
1810
+ "low": 0.7077262575365768,
1811
+ "mean": 0.9408635088472943,
1812
+ "n": 47,
1813
+ "num_samples": 500
1814
+ },
1815
+ "mean": 0.9408635088472943,
1816
+ "per_seed": {
1817
+ "1025740212": 1.772506669163704,
1818
+ "1055157354": 1.8101318180561066,
1819
+ "1055567618": 1.9218642711639404,
1820
+ "1113268940": 1.3635571599006653,
1821
+ "1113371261": 0.0,
1822
+ "1176943004": 0.0,
1823
+ "1189392917": 0.0,
1824
+ "1231780997": 1.7728382349014282,
1825
+ "1271718007": 0.9337802603840828,
1826
+ "1273247072": 1.8952929228544235,
1827
+ "1358421325": 1.90096914768219,
1828
+ "1375492786": 0.24285174411488697,
1829
+ "141984615": 0.1600442255536715,
1830
+ "1475440973": 1.28389310836792,
1831
+ "1483645336": 1.9686058908700943,
1832
+ "1522272686": 1.8230728209018707,
1833
+ "1528740417": 1.280472755432129,
1834
+ "1549324805": 0.7317837377389272,
1835
+ "1607773860": 0.0,
1836
+ "1665739511": 0.0,
1837
+ "1747178030": 1.9092875719070435,
1838
+ "1881223807": 1.9813720285892487,
1839
+ "1919198188": 1.9560884833335876,
1840
+ "1991414747": 0.0,
1841
+ "1997090716": 0.0,
1842
+ "2015956957": 0.09564426057040691,
1843
+ "2054899126": 0.0,
1844
+ "211245842": 0.0,
1845
+ "2141802816": 1.861636370420456,
1846
+ "256770874": 1.835601955652237,
1847
+ "278008524": 1.1011469289660454,
1848
+ "416183106": 1.812951534986496,
1849
+ "534717141": 0.0,
1850
+ "551504635": 1.273692786693573,
1851
+ "565369392": 1.339533941199382,
1852
+ "565621826": 0.0,
1853
+ "585904130": 0.1701989375675718,
1854
+ "626486813": 0.5741560639580712,
1855
+ "636680137": 1.9821314811706543,
1856
+ "706479600": 0.0,
1857
+ "750448296": 0.1865733889862895,
1858
+ "75070641": 1.805951327085495,
1859
+ "767747424": 0.10090384539216757,
1860
+ "77147264": 0.0,
1861
+ "794181626": 0.0,
1862
+ "957101489": 1.432861328125,
1863
+ "998761162": 1.939187914133072
1864
+ }
1865
+ },
1866
+ "macro_by_task": {
1867
+ "ci": {
1868
+ "confidence": 0.95,
1869
+ "high": 1.5834752595424653,
1870
+ "low": 0.6586342200808432,
1871
+ "mean": 1.1514351414010187,
1872
+ "n": 5,
1873
+ "num_samples": 500
1874
+ },
1875
+ "mean": 1.1514351414010187,
1876
+ "per_task": {
1877
+ "LiftPegUpright-v1": 1.4578267857432365,
1878
+ "PickCube-v1": 1.8952929228544235,
1879
+ "PullCube-v1": 0.44455755099839095,
1880
+ "PushCube-v1": 0.748361566892037,
1881
+ "StackCube-v1": 1.211136880517006
1882
+ }
1883
+ },
1884
+ "metric": "total_car_to_hidden_at_8",
1885
+ "micro": {
1886
+ "confidence": 0.95,
1887
+ "high": 0.9684723974298348,
1888
+ "low": 0.5435695670675014,
1889
+ "mean": 0.7775791876610247,
1890
+ "n": 69,
1891
+ "num_samples": 500
1892
+ },
1893
+ "n": 69
1894
+ }
1895
+ },
1896
+ "num_rows": 69,
1897
+ "report_type": "ctt_generated_measured_rollout",
1898
+ "schema_version": 1,
1899
+ "source_content_hash": "90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957",
1900
+ "source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372",
1901
+ "success_summary": {
1902
+ "base_success_rate": 0.2898550724637681,
1903
+ "base_utility_mean": 0.7456761826086196,
1904
+ "hidden_chart_oracle_success_rate": 0.6666666666666666,
1905
+ "hidden_chart_oracle_utility_mean": 1.3999846732054932,
1906
+ "max_restore_error": 1.4901161193847656e-08,
1907
+ "proposal_oracle_success_gain_over_base": 0.13043478260869565,
1908
+ "proposal_oracle_success_rate": 0.42028985507246375,
1909
+ "proposal_oracle_utility_mean": 0.9253245886521535,
1910
+ "selected_success_gain_over_base": -0.028985507246376812,
1911
+ "selected_success_rate": 0.2608695652173913,
1912
+ "selected_utility_mean": 0.6224054855444685,
1913
+ "success_selector_gap": 0.15942028985507245,
1914
+ "success_support_gap": 0.2463768115942029
1915
+ },
1916
+ "target_content_hash": "fa0fe0e3881ee70ce91492445871614d01b3a47fc7c576659c53ff17ddee4e42",
1917
+ "target_split_hash": "9d1dc1c10868f76fa080895df0f2730c173f12b6db14932599d55c2b60859381"
1918
+ }
workspace/runs/ctt_residual_rollout_val69_seed2/metrics_by_seed.json ADDED
@@ -0,0 +1,1083 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "1025740212": {
3
+ "base_success": 0.0,
4
+ "base_utility": 0.528373658657074,
5
+ "branch_car_at_8": 1.772506669163704,
6
+ "hidden_chart_oracle_success_at_8": 1.0,
7
+ "hidden_chart_oracle_utility_at_8": 2.0,
8
+ "outcome_ptr_at_8": 1.0,
9
+ "pairwise_causal_calibration_ece": 0.17388092747958847,
10
+ "proposal_oracle_success_at_8": 1.0,
11
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
12
+ "proposal_oracle_utility_at_8": 2.0,
13
+ "proposal_oracle_utility_gain_over_base_at_8": 1.471626341342926,
14
+ "selected_success_at_8": 0.0,
15
+ "selected_success_gain_over_base_at_8": 0.0,
16
+ "selected_utility_at_8": 0.22749333083629608,
17
+ "selected_utility_gain_over_base_at_8": -0.3008803278207779,
18
+ "selector_regret_at_8": 1.772506669163704,
19
+ "success_selector_gap_at_8": 1.0,
20
+ "success_support_gap_at_8": 0.0,
21
+ "success_total_car_to_hidden_at_8": 1.0,
22
+ "support_gap_at_8": 0.0,
23
+ "total_car_to_hidden_at_8": 1.772506669163704
24
+ },
25
+ "1055157354": {
26
+ "base_success": 0.0,
27
+ "base_utility": 0.19648407399654388,
28
+ "branch_car_at_8": 0.03480546176433563,
29
+ "hidden_chart_oracle_success_at_8": 1.0,
30
+ "hidden_chart_oracle_utility_at_8": 2.0,
31
+ "outcome_ptr_at_8": 1.0,
32
+ "pairwise_causal_calibration_ece": 0.1805709977888097,
33
+ "proposal_oracle_success_at_8": 0.0,
34
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
35
+ "proposal_oracle_utility_at_8": 0.22467364370822906,
36
+ "proposal_oracle_utility_gain_over_base_at_8": 0.02818956971168518,
37
+ "selected_success_at_8": 0.0,
38
+ "selected_success_gain_over_base_at_8": 0.0,
39
+ "selected_utility_at_8": 0.18986818194389343,
40
+ "selected_utility_gain_over_base_at_8": -0.006615892052650452,
41
+ "selector_regret_at_8": 0.03480546176433563,
42
+ "success_selector_gap_at_8": 0.0,
43
+ "success_support_gap_at_8": 1.0,
44
+ "success_total_car_to_hidden_at_8": 1.0,
45
+ "support_gap_at_8": 1.775326356291771,
46
+ "total_car_to_hidden_at_8": 1.8101318180561066
47
+ },
48
+ "1055567618": {
49
+ "base_success": 1.0,
50
+ "base_utility": 2.0,
51
+ "branch_car_at_8": 1.9218642711639404,
52
+ "hidden_chart_oracle_success_at_8": 1.0,
53
+ "hidden_chart_oracle_utility_at_8": 2.0,
54
+ "outcome_ptr_at_8": 0.0,
55
+ "pairwise_causal_calibration_ece": 0.19289744165908876,
56
+ "proposal_oracle_success_at_8": 1.0,
57
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
58
+ "proposal_oracle_utility_at_8": 2.0,
59
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
60
+ "selected_success_at_8": 0.0,
61
+ "selected_success_gain_over_base_at_8": -1.0,
62
+ "selected_utility_at_8": 0.07813572883605957,
63
+ "selected_utility_gain_over_base_at_8": -1.9218642711639404,
64
+ "selector_regret_at_8": 1.9218642711639404,
65
+ "success_selector_gap_at_8": 1.0,
66
+ "success_support_gap_at_8": 0.0,
67
+ "success_total_car_to_hidden_at_8": 1.0,
68
+ "support_gap_at_8": 0.0,
69
+ "total_car_to_hidden_at_8": 1.9218642711639404
70
+ },
71
+ "1113268940": {
72
+ "base_success": 0.0,
73
+ "base_utility": 0.42231252789497375,
74
+ "branch_car_at_8": 0.06608206033706665,
75
+ "hidden_chart_oracle_success_at_8": 1.0,
76
+ "hidden_chart_oracle_utility_at_8": 2.0,
77
+ "outcome_ptr_at_8": 1.0,
78
+ "pairwise_causal_calibration_ece": 0.032532089835823584,
79
+ "proposal_oracle_success_at_8": 0.0,
80
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
81
+ "proposal_oracle_utility_at_8": 0.7025249004364014,
82
+ "proposal_oracle_utility_gain_over_base_at_8": 0.2802123725414276,
83
+ "selected_success_at_8": 0.0,
84
+ "selected_success_gain_over_base_at_8": 0.0,
85
+ "selected_utility_at_8": 0.6364428400993347,
86
+ "selected_utility_gain_over_base_at_8": 0.21413031220436096,
87
+ "selector_regret_at_8": 0.06608206033706665,
88
+ "success_selector_gap_at_8": 0.0,
89
+ "success_support_gap_at_8": 1.0,
90
+ "success_total_car_to_hidden_at_8": 1.0,
91
+ "support_gap_at_8": 1.2974750995635986,
92
+ "total_car_to_hidden_at_8": 1.3635571599006653
93
+ },
94
+ "1113371261": {
95
+ "base_success": 1.0,
96
+ "base_utility": 2.0,
97
+ "branch_car_at_8": 0.0,
98
+ "hidden_chart_oracle_success_at_8": 1.0,
99
+ "hidden_chart_oracle_utility_at_8": 2.0,
100
+ "outcome_ptr_at_8": 0.0,
101
+ "pairwise_causal_calibration_ece": 0.233029795067053,
102
+ "proposal_oracle_success_at_8": 1.0,
103
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
104
+ "proposal_oracle_utility_at_8": 2.0,
105
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
106
+ "selected_success_at_8": 1.0,
107
+ "selected_success_gain_over_base_at_8": 0.0,
108
+ "selected_utility_at_8": 2.0,
109
+ "selected_utility_gain_over_base_at_8": 0.0,
110
+ "selector_regret_at_8": 0.0,
111
+ "success_selector_gap_at_8": 0.0,
112
+ "success_support_gap_at_8": 0.0,
113
+ "success_total_car_to_hidden_at_8": 0.0,
114
+ "support_gap_at_8": 0.0,
115
+ "total_car_to_hidden_at_8": 0.0
116
+ },
117
+ "1176943004": {
118
+ "base_success": 0.0,
119
+ "base_utility": 0.05094052851200104,
120
+ "branch_car_at_8": 0.0,
121
+ "hidden_chart_oracle_success_at_8": 1.0,
122
+ "hidden_chart_oracle_utility_at_8": 2.0,
123
+ "outcome_ptr_at_8": 1.0,
124
+ "pairwise_causal_calibration_ece": 0.17468938592004596,
125
+ "proposal_oracle_success_at_8": 1.0,
126
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
127
+ "proposal_oracle_utility_at_8": 2.0,
128
+ "proposal_oracle_utility_gain_over_base_at_8": 1.949059471487999,
129
+ "selected_success_at_8": 1.0,
130
+ "selected_success_gain_over_base_at_8": 1.0,
131
+ "selected_utility_at_8": 2.0,
132
+ "selected_utility_gain_over_base_at_8": 1.949059471487999,
133
+ "selector_regret_at_8": 0.0,
134
+ "success_selector_gap_at_8": 0.0,
135
+ "success_support_gap_at_8": 0.0,
136
+ "success_total_car_to_hidden_at_8": 0.0,
137
+ "support_gap_at_8": 0.0,
138
+ "total_car_to_hidden_at_8": 0.0
139
+ },
140
+ "1189392917": {
141
+ "base_success": 0.0,
142
+ "base_utility": 0.10841864347457886,
143
+ "branch_car_at_8": 0.0,
144
+ "hidden_chart_oracle_success_at_8": 1.0,
145
+ "hidden_chart_oracle_utility_at_8": 2.0,
146
+ "outcome_ptr_at_8": 1.0,
147
+ "pairwise_causal_calibration_ece": 0.0,
148
+ "proposal_oracle_success_at_8": 1.0,
149
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
150
+ "proposal_oracle_utility_at_8": 2.0,
151
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211,
152
+ "selected_success_at_8": 1.0,
153
+ "selected_success_gain_over_base_at_8": 1.0,
154
+ "selected_utility_at_8": 2.0,
155
+ "selected_utility_gain_over_base_at_8": 1.8915813565254211,
156
+ "selector_regret_at_8": 0.0,
157
+ "success_selector_gap_at_8": 0.0,
158
+ "success_support_gap_at_8": 0.0,
159
+ "success_total_car_to_hidden_at_8": 0.0,
160
+ "support_gap_at_8": 0.0,
161
+ "total_car_to_hidden_at_8": 0.0
162
+ },
163
+ "1231780997": {
164
+ "base_success": 0.0,
165
+ "base_utility": 0.7270061373710632,
166
+ "branch_car_at_8": 0.5006794333457947,
167
+ "hidden_chart_oracle_success_at_8": 1.0,
168
+ "hidden_chart_oracle_utility_at_8": 2.0,
169
+ "outcome_ptr_at_8": 1.0,
170
+ "pairwise_causal_calibration_ece": 0.10207358838016511,
171
+ "proposal_oracle_success_at_8": 0.0,
172
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
173
+ "proposal_oracle_utility_at_8": 0.7278411984443665,
174
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0008350610733032227,
175
+ "selected_success_at_8": 0.0,
176
+ "selected_success_gain_over_base_at_8": 0.0,
177
+ "selected_utility_at_8": 0.22716176509857178,
178
+ "selected_utility_gain_over_base_at_8": -0.49984437227249146,
179
+ "selector_regret_at_8": 0.5006794333457947,
180
+ "success_selector_gap_at_8": 0.0,
181
+ "success_support_gap_at_8": 1.0,
182
+ "success_total_car_to_hidden_at_8": 1.0,
183
+ "support_gap_at_8": 1.2721588015556335,
184
+ "total_car_to_hidden_at_8": 1.7728382349014282
185
+ },
186
+ "1271718007": {
187
+ "base_success": 0.5,
188
+ "base_utility": 1.415381133556366,
189
+ "branch_car_at_8": 0.19734752923250198,
190
+ "hidden_chart_oracle_success_at_8": 1.0,
191
+ "hidden_chart_oracle_utility_at_8": 2.0,
192
+ "outcome_ptr_at_8": 0.0,
193
+ "pairwise_causal_calibration_ece": 0.2350719661460603,
194
+ "proposal_oracle_success_at_8": 0.5,
195
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
196
+ "proposal_oracle_utility_at_8": 1.2635672688484192,
197
+ "proposal_oracle_utility_gain_over_base_at_8": -0.15181386470794678,
198
+ "selected_success_at_8": 0.5,
199
+ "selected_success_gain_over_base_at_8": 0.0,
200
+ "selected_utility_at_8": 1.0662197396159172,
201
+ "selected_utility_gain_over_base_at_8": -0.34916139394044876,
202
+ "selector_regret_at_8": 0.19734752923250198,
203
+ "success_selector_gap_at_8": 0.0,
204
+ "success_support_gap_at_8": 0.5,
205
+ "success_total_car_to_hidden_at_8": 0.5,
206
+ "support_gap_at_8": 0.7364327311515808,
207
+ "total_car_to_hidden_at_8": 0.9337802603840828
208
+ },
209
+ "1273247072": {
210
+ "base_success": 0.0,
211
+ "base_utility": 0.15342965722084045,
212
+ "branch_car_at_8": 0.35906313359737396,
213
+ "hidden_chart_oracle_success_at_8": 1.0,
214
+ "hidden_chart_oracle_utility_at_8": 2.0,
215
+ "outcome_ptr_at_8": 1.0,
216
+ "pairwise_causal_calibration_ece": 0.2793744975404514,
217
+ "proposal_oracle_success_at_8": 0.0,
218
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
219
+ "proposal_oracle_utility_at_8": 0.46377021074295044,
220
+ "proposal_oracle_utility_gain_over_base_at_8": 0.31034055352211,
221
+ "selected_success_at_8": 0.0,
222
+ "selected_success_gain_over_base_at_8": 0.0,
223
+ "selected_utility_at_8": 0.10470707714557648,
224
+ "selected_utility_gain_over_base_at_8": -0.04872258007526398,
225
+ "selector_regret_at_8": 0.35906313359737396,
226
+ "success_selector_gap_at_8": 0.0,
227
+ "success_support_gap_at_8": 1.0,
228
+ "success_total_car_to_hidden_at_8": 1.0,
229
+ "support_gap_at_8": 1.5362297892570496,
230
+ "total_car_to_hidden_at_8": 1.8952929228544235
231
+ },
232
+ "1358421325": {
233
+ "base_success": 0.0,
234
+ "base_utility": 0.8327022790908813,
235
+ "branch_car_at_8": 1.90096914768219,
236
+ "hidden_chart_oracle_success_at_8": 1.0,
237
+ "hidden_chart_oracle_utility_at_8": 2.0,
238
+ "outcome_ptr_at_8": 1.0,
239
+ "pairwise_causal_calibration_ece": 0.32948159974174274,
240
+ "proposal_oracle_success_at_8": 1.0,
241
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
242
+ "proposal_oracle_utility_at_8": 2.0,
243
+ "proposal_oracle_utility_gain_over_base_at_8": 1.1672977209091187,
244
+ "selected_success_at_8": 0.0,
245
+ "selected_success_gain_over_base_at_8": 0.0,
246
+ "selected_utility_at_8": 0.09903085231781006,
247
+ "selected_utility_gain_over_base_at_8": -0.7336714267730713,
248
+ "selector_regret_at_8": 1.90096914768219,
249
+ "success_selector_gap_at_8": 1.0,
250
+ "success_support_gap_at_8": 0.0,
251
+ "success_total_car_to_hidden_at_8": 1.0,
252
+ "support_gap_at_8": 0.0,
253
+ "total_car_to_hidden_at_8": 1.90096914768219
254
+ },
255
+ "1375492786": {
256
+ "base_success": 1.0,
257
+ "base_utility": 2.0,
258
+ "branch_car_at_8": 0.0002880295505747199,
259
+ "hidden_chart_oracle_success_at_8": 0.0,
260
+ "hidden_chart_oracle_utility_at_8": 0.24345096945762634,
261
+ "outcome_ptr_at_8": 0.0,
262
+ "pairwise_causal_calibration_ece": 0.008822272371479922,
263
+ "proposal_oracle_success_at_8": 0.0,
264
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
265
+ "proposal_oracle_utility_at_8": 0.0008872548933140934,
266
+ "proposal_oracle_utility_gain_over_base_at_8": -1.999112745106686,
267
+ "selected_success_at_8": 0.0,
268
+ "selected_success_gain_over_base_at_8": -1.0,
269
+ "selected_utility_at_8": 0.0005992253427393734,
270
+ "selected_utility_gain_over_base_at_8": -1.9994007746572606,
271
+ "selector_regret_at_8": 0.0002880295505747199,
272
+ "success_selector_gap_at_8": 0.0,
273
+ "success_support_gap_at_8": 0.0,
274
+ "success_total_car_to_hidden_at_8": 0.0,
275
+ "support_gap_at_8": 0.24256371456431225,
276
+ "total_car_to_hidden_at_8": 0.24285174411488697
277
+ },
278
+ "141984615": {
279
+ "base_success": 0.0,
280
+ "base_utility": 0.09381213908394177,
281
+ "branch_car_at_8": 0.013830550014972687,
282
+ "hidden_chart_oracle_success_at_8": 0.0,
283
+ "hidden_chart_oracle_utility_at_8": 0.20044784247875214,
284
+ "outcome_ptr_at_8": 0.3333333333333333,
285
+ "pairwise_causal_calibration_ece": 0.19434281311769708,
286
+ "proposal_oracle_success_at_8": 0.0,
287
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
288
+ "proposal_oracle_utility_at_8": 0.054234166940053306,
289
+ "proposal_oracle_utility_gain_over_base_at_8": -0.039577972143888474,
290
+ "selected_success_at_8": 0.0,
291
+ "selected_success_gain_over_base_at_8": 0.0,
292
+ "selected_utility_at_8": 0.04040361692508062,
293
+ "selected_utility_gain_over_base_at_8": -0.05340852215886116,
294
+ "selector_regret_at_8": 0.013830550014972687,
295
+ "success_selector_gap_at_8": 0.0,
296
+ "success_support_gap_at_8": 0.0,
297
+ "success_total_car_to_hidden_at_8": 0.0,
298
+ "support_gap_at_8": 0.14621367553869882,
299
+ "total_car_to_hidden_at_8": 0.1600442255536715
300
+ },
301
+ "1475440973": {
302
+ "base_success": 1.0,
303
+ "base_utility": 2.0,
304
+ "branch_car_at_8": 1.28389310836792,
305
+ "hidden_chart_oracle_success_at_8": 1.0,
306
+ "hidden_chart_oracle_utility_at_8": 2.0,
307
+ "outcome_ptr_at_8": 0.0,
308
+ "pairwise_causal_calibration_ece": 0.22214884767353088,
309
+ "proposal_oracle_success_at_8": 1.0,
310
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
311
+ "proposal_oracle_utility_at_8": 2.0,
312
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
313
+ "selected_success_at_8": 0.0,
314
+ "selected_success_gain_over_base_at_8": -1.0,
315
+ "selected_utility_at_8": 0.7161068916320801,
316
+ "selected_utility_gain_over_base_at_8": -1.28389310836792,
317
+ "selector_regret_at_8": 1.28389310836792,
318
+ "success_selector_gap_at_8": 1.0,
319
+ "success_support_gap_at_8": 0.0,
320
+ "success_total_car_to_hidden_at_8": 1.0,
321
+ "support_gap_at_8": 0.0,
322
+ "total_car_to_hidden_at_8": 1.28389310836792
323
+ },
324
+ "1483645336": {
325
+ "base_success": 1.0,
326
+ "base_utility": 2.0,
327
+ "branch_car_at_8": 1.9686058908700943,
328
+ "hidden_chart_oracle_success_at_8": 1.0,
329
+ "hidden_chart_oracle_utility_at_8": 2.0,
330
+ "outcome_ptr_at_8": 0.0,
331
+ "pairwise_causal_calibration_ece": 0.5995435922806452,
332
+ "proposal_oracle_success_at_8": 1.0,
333
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
334
+ "proposal_oracle_utility_at_8": 2.0,
335
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
336
+ "selected_success_at_8": 0.0,
337
+ "selected_success_gain_over_base_at_8": -1.0,
338
+ "selected_utility_at_8": 0.0313941091299057,
339
+ "selected_utility_gain_over_base_at_8": -1.9686058908700943,
340
+ "selector_regret_at_8": 1.9686058908700943,
341
+ "success_selector_gap_at_8": 1.0,
342
+ "success_support_gap_at_8": 0.0,
343
+ "success_total_car_to_hidden_at_8": 1.0,
344
+ "support_gap_at_8": 0.0,
345
+ "total_car_to_hidden_at_8": 1.9686058908700943
346
+ },
347
+ "1522272686": {
348
+ "base_success": 0.0,
349
+ "base_utility": 0.19476109743118286,
350
+ "branch_car_at_8": 0.017972871661186218,
351
+ "hidden_chart_oracle_success_at_8": 1.0,
352
+ "hidden_chart_oracle_utility_at_8": 2.0,
353
+ "outcome_ptr_at_8": 1.0,
354
+ "pairwise_causal_calibration_ece": 0.3276128524355476,
355
+ "proposal_oracle_success_at_8": 0.0,
356
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
357
+ "proposal_oracle_utility_at_8": 0.1949000507593155,
358
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0001389533281326294,
359
+ "selected_success_at_8": 0.0,
360
+ "selected_success_gain_over_base_at_8": 0.0,
361
+ "selected_utility_at_8": 0.17692717909812927,
362
+ "selected_utility_gain_over_base_at_8": -0.01783391833305359,
363
+ "selector_regret_at_8": 0.017972871661186218,
364
+ "success_selector_gap_at_8": 0.0,
365
+ "success_support_gap_at_8": 1.0,
366
+ "success_total_car_to_hidden_at_8": 1.0,
367
+ "support_gap_at_8": 1.8050999492406845,
368
+ "total_car_to_hidden_at_8": 1.8230728209018707
369
+ },
370
+ "1528740417": {
371
+ "base_success": 1.0,
372
+ "base_utility": 2.0,
373
+ "branch_car_at_8": 1.280472755432129,
374
+ "hidden_chart_oracle_success_at_8": 1.0,
375
+ "hidden_chart_oracle_utility_at_8": 2.0,
376
+ "outcome_ptr_at_8": 0.0,
377
+ "pairwise_causal_calibration_ece": 0.16399362968800954,
378
+ "proposal_oracle_success_at_8": 1.0,
379
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
380
+ "proposal_oracle_utility_at_8": 2.0,
381
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
382
+ "selected_success_at_8": 0.0,
383
+ "selected_success_gain_over_base_at_8": -1.0,
384
+ "selected_utility_at_8": 0.7195272445678711,
385
+ "selected_utility_gain_over_base_at_8": -1.280472755432129,
386
+ "selector_regret_at_8": 1.280472755432129,
387
+ "success_selector_gap_at_8": 1.0,
388
+ "success_support_gap_at_8": 0.0,
389
+ "success_total_car_to_hidden_at_8": 1.0,
390
+ "support_gap_at_8": 0.0,
391
+ "total_car_to_hidden_at_8": 1.280472755432129
392
+ },
393
+ "1549324805": {
394
+ "base_success": 0.6666666666666666,
395
+ "base_utility": 1.3787474483251572,
396
+ "branch_car_at_8": 0.01237945475926002,
397
+ "hidden_chart_oracle_success_at_8": 0.6666666666666666,
398
+ "hidden_chart_oracle_utility_at_8": 1.4027505218982697,
399
+ "outcome_ptr_at_8": 0.0,
400
+ "pairwise_causal_calibration_ece": 0.23864987006465407,
401
+ "proposal_oracle_success_at_8": 0.3333333333333333,
402
+ "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333,
403
+ "proposal_oracle_utility_at_8": 0.6833462389186025,
404
+ "proposal_oracle_utility_gain_over_base_at_8": -0.6954012094065547,
405
+ "selected_success_at_8": 0.3333333333333333,
406
+ "selected_success_gain_over_base_at_8": -0.3333333333333333,
407
+ "selected_utility_at_8": 0.6709667841593424,
408
+ "selected_utility_gain_over_base_at_8": -0.7077806641658148,
409
+ "selector_regret_at_8": 0.01237945475926002,
410
+ "success_selector_gap_at_8": 0.0,
411
+ "success_support_gap_at_8": 0.3333333333333333,
412
+ "success_total_car_to_hidden_at_8": 0.3333333333333333,
413
+ "support_gap_at_8": 0.7194042829796672,
414
+ "total_car_to_hidden_at_8": 0.7317837377389272
415
+ },
416
+ "1607773860": {
417
+ "base_success": 1.0,
418
+ "base_utility": 2.0,
419
+ "branch_car_at_8": 0.0,
420
+ "hidden_chart_oracle_success_at_8": 1.0,
421
+ "hidden_chart_oracle_utility_at_8": 2.0,
422
+ "outcome_ptr_at_8": 0.0,
423
+ "pairwise_causal_calibration_ece": 0.24177237819615016,
424
+ "proposal_oracle_success_at_8": 1.0,
425
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
426
+ "proposal_oracle_utility_at_8": 2.0,
427
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
428
+ "selected_success_at_8": 1.0,
429
+ "selected_success_gain_over_base_at_8": 0.0,
430
+ "selected_utility_at_8": 2.0,
431
+ "selected_utility_gain_over_base_at_8": 0.0,
432
+ "selector_regret_at_8": 0.0,
433
+ "success_selector_gap_at_8": 0.0,
434
+ "success_support_gap_at_8": 0.0,
435
+ "success_total_car_to_hidden_at_8": 0.0,
436
+ "support_gap_at_8": 0.0,
437
+ "total_car_to_hidden_at_8": 0.0
438
+ },
439
+ "1665739511": {
440
+ "base_success": 1.0,
441
+ "base_utility": 2.0,
442
+ "branch_car_at_8": 0.0,
443
+ "hidden_chart_oracle_success_at_8": 1.0,
444
+ "hidden_chart_oracle_utility_at_8": 2.0,
445
+ "outcome_ptr_at_8": 0.0,
446
+ "pairwise_causal_calibration_ece": 0.2508232858898205,
447
+ "proposal_oracle_success_at_8": 1.0,
448
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
449
+ "proposal_oracle_utility_at_8": 2.0,
450
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
451
+ "selected_success_at_8": 1.0,
452
+ "selected_success_gain_over_base_at_8": 0.0,
453
+ "selected_utility_at_8": 2.0,
454
+ "selected_utility_gain_over_base_at_8": 0.0,
455
+ "selector_regret_at_8": 0.0,
456
+ "success_selector_gap_at_8": 0.0,
457
+ "success_support_gap_at_8": 0.0,
458
+ "success_total_car_to_hidden_at_8": 0.0,
459
+ "support_gap_at_8": 0.0,
460
+ "total_car_to_hidden_at_8": 0.0
461
+ },
462
+ "1747178030": {
463
+ "base_success": 0.0,
464
+ "base_utility": 0.1463065892457962,
465
+ "branch_car_at_8": 0.034092940390110016,
466
+ "hidden_chart_oracle_success_at_8": 1.0,
467
+ "hidden_chart_oracle_utility_at_8": 2.0,
468
+ "outcome_ptr_at_8": 0.0,
469
+ "pairwise_causal_calibration_ece": 0.08595818214364334,
470
+ "proposal_oracle_success_at_8": 0.0,
471
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
472
+ "proposal_oracle_utility_at_8": 0.12480536848306656,
473
+ "proposal_oracle_utility_gain_over_base_at_8": -0.021501220762729645,
474
+ "selected_success_at_8": 0.0,
475
+ "selected_success_gain_over_base_at_8": 0.0,
476
+ "selected_utility_at_8": 0.09071242809295654,
477
+ "selected_utility_gain_over_base_at_8": -0.05559416115283966,
478
+ "selector_regret_at_8": 0.034092940390110016,
479
+ "success_selector_gap_at_8": 0.0,
480
+ "success_support_gap_at_8": 1.0,
481
+ "success_total_car_to_hidden_at_8": 1.0,
482
+ "support_gap_at_8": 1.8751946315169334,
483
+ "total_car_to_hidden_at_8": 1.9092875719070435
484
+ },
485
+ "1881223807": {
486
+ "base_success": 0.0,
487
+ "base_utility": 0.0320269912481308,
488
+ "branch_car_at_8": 0.024854734539985657,
489
+ "hidden_chart_oracle_success_at_8": 1.0,
490
+ "hidden_chart_oracle_utility_at_8": 2.0,
491
+ "outcome_ptr_at_8": 1.0,
492
+ "pairwise_causal_calibration_ece": 0.2146835718150399,
493
+ "proposal_oracle_success_at_8": 0.0,
494
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
495
+ "proposal_oracle_utility_at_8": 0.043482705950737,
496
+ "proposal_oracle_utility_gain_over_base_at_8": 0.011455714702606201,
497
+ "selected_success_at_8": 0.0,
498
+ "selected_success_gain_over_base_at_8": 0.0,
499
+ "selected_utility_at_8": 0.018627971410751343,
500
+ "selected_utility_gain_over_base_at_8": -0.013399019837379456,
501
+ "selector_regret_at_8": 0.024854734539985657,
502
+ "success_selector_gap_at_8": 0.0,
503
+ "success_support_gap_at_8": 1.0,
504
+ "success_total_car_to_hidden_at_8": 1.0,
505
+ "support_gap_at_8": 1.956517294049263,
506
+ "total_car_to_hidden_at_8": 1.9813720285892487
507
+ },
508
+ "1919198188": {
509
+ "base_success": 0.0,
510
+ "base_utility": 0.03719887137413025,
511
+ "branch_car_at_8": 0.018839523196220398,
512
+ "hidden_chart_oracle_success_at_8": 1.0,
513
+ "hidden_chart_oracle_utility_at_8": 2.0,
514
+ "outcome_ptr_at_8": 1.0,
515
+ "pairwise_causal_calibration_ece": 0.1669540555093941,
516
+ "proposal_oracle_success_at_8": 0.0,
517
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
518
+ "proposal_oracle_utility_at_8": 0.06275103986263275,
519
+ "proposal_oracle_utility_gain_over_base_at_8": 0.025552168488502502,
520
+ "selected_success_at_8": 0.0,
521
+ "selected_success_gain_over_base_at_8": 0.0,
522
+ "selected_utility_at_8": 0.043911516666412354,
523
+ "selected_utility_gain_over_base_at_8": 0.0067126452922821045,
524
+ "selector_regret_at_8": 0.018839523196220398,
525
+ "success_selector_gap_at_8": 0.0,
526
+ "success_support_gap_at_8": 1.0,
527
+ "success_total_car_to_hidden_at_8": 1.0,
528
+ "support_gap_at_8": 1.9372489601373672,
529
+ "total_car_to_hidden_at_8": 1.9560884833335876
530
+ },
531
+ "1991414747": {
532
+ "base_success": 1.0,
533
+ "base_utility": 2.0,
534
+ "branch_car_at_8": 0.0,
535
+ "hidden_chart_oracle_success_at_8": 1.0,
536
+ "hidden_chart_oracle_utility_at_8": 2.0,
537
+ "outcome_ptr_at_8": 0.0,
538
+ "pairwise_causal_calibration_ece": 0.11896663871153935,
539
+ "proposal_oracle_success_at_8": 1.0,
540
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
541
+ "proposal_oracle_utility_at_8": 2.0,
542
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
543
+ "selected_success_at_8": 1.0,
544
+ "selected_success_gain_over_base_at_8": 0.0,
545
+ "selected_utility_at_8": 2.0,
546
+ "selected_utility_gain_over_base_at_8": 0.0,
547
+ "selector_regret_at_8": 0.0,
548
+ "success_selector_gap_at_8": 0.0,
549
+ "success_support_gap_at_8": 0.0,
550
+ "success_total_car_to_hidden_at_8": 0.0,
551
+ "support_gap_at_8": 0.0,
552
+ "total_car_to_hidden_at_8": 0.0
553
+ },
554
+ "1997090716": {
555
+ "base_success": 0.0,
556
+ "base_utility": 0.13927479088306427,
557
+ "branch_car_at_8": 0.0,
558
+ "hidden_chart_oracle_success_at_8": 1.0,
559
+ "hidden_chart_oracle_utility_at_8": 2.0,
560
+ "outcome_ptr_at_8": 1.0,
561
+ "pairwise_causal_calibration_ece": 0.14617959762414967,
562
+ "proposal_oracle_success_at_8": 1.0,
563
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
564
+ "proposal_oracle_utility_at_8": 2.0,
565
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357,
566
+ "selected_success_at_8": 1.0,
567
+ "selected_success_gain_over_base_at_8": 1.0,
568
+ "selected_utility_at_8": 2.0,
569
+ "selected_utility_gain_over_base_at_8": 1.8607252091169357,
570
+ "selector_regret_at_8": 0.0,
571
+ "success_selector_gap_at_8": 0.0,
572
+ "success_support_gap_at_8": 0.0,
573
+ "success_total_car_to_hidden_at_8": 0.0,
574
+ "support_gap_at_8": 0.0,
575
+ "total_car_to_hidden_at_8": 0.0
576
+ },
577
+ "2015956957": {
578
+ "base_success": 0.0,
579
+ "base_utility": 0.06711978986859321,
580
+ "branch_car_at_8": 0.005573729798197746,
581
+ "hidden_chart_oracle_success_at_8": 0.2,
582
+ "hidden_chart_oracle_utility_at_8": 0.5298813670873642,
583
+ "outcome_ptr_at_8": 0.6,
584
+ "pairwise_causal_calibration_ece": 0.1431252893441326,
585
+ "proposal_oracle_success_at_8": 0.2,
586
+ "proposal_oracle_success_gain_over_base_at_8": 0.2,
587
+ "proposal_oracle_utility_at_8": 0.43981083631515505,
588
+ "proposal_oracle_utility_gain_over_base_at_8": 0.3726910464465618,
589
+ "selected_success_at_8": 0.2,
590
+ "selected_success_gain_over_base_at_8": 0.2,
591
+ "selected_utility_at_8": 0.4342371065169573,
592
+ "selected_utility_gain_over_base_at_8": 0.36711731664836406,
593
+ "selector_regret_at_8": 0.005573729798197746,
594
+ "success_selector_gap_at_8": 0.0,
595
+ "success_support_gap_at_8": 0.0,
596
+ "success_total_car_to_hidden_at_8": 0.0,
597
+ "support_gap_at_8": 0.09007053077220917,
598
+ "total_car_to_hidden_at_8": 0.09564426057040691
599
+ },
600
+ "2054899126": {
601
+ "base_success": 1.0,
602
+ "base_utility": 2.0,
603
+ "branch_car_at_8": 0.0,
604
+ "hidden_chart_oracle_success_at_8": 1.0,
605
+ "hidden_chart_oracle_utility_at_8": 2.0,
606
+ "outcome_ptr_at_8": 0.0,
607
+ "pairwise_causal_calibration_ece": 0.0,
608
+ "proposal_oracle_success_at_8": 1.0,
609
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
610
+ "proposal_oracle_utility_at_8": 2.0,
611
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
612
+ "selected_success_at_8": 1.0,
613
+ "selected_success_gain_over_base_at_8": 0.0,
614
+ "selected_utility_at_8": 2.0,
615
+ "selected_utility_gain_over_base_at_8": 0.0,
616
+ "selector_regret_at_8": 0.0,
617
+ "success_selector_gap_at_8": 0.0,
618
+ "success_support_gap_at_8": 0.0,
619
+ "success_total_car_to_hidden_at_8": 0.0,
620
+ "support_gap_at_8": 0.0,
621
+ "total_car_to_hidden_at_8": 0.0
622
+ },
623
+ "211245842": {
624
+ "base_success": 0.0,
625
+ "base_utility": 0.8388344049453735,
626
+ "branch_car_at_8": 0.0,
627
+ "hidden_chart_oracle_success_at_8": 1.0,
628
+ "hidden_chart_oracle_utility_at_8": 2.0,
629
+ "outcome_ptr_at_8": 1.0,
630
+ "pairwise_causal_calibration_ece": 0.13508468824583414,
631
+ "proposal_oracle_success_at_8": 1.0,
632
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
633
+ "proposal_oracle_utility_at_8": 2.0,
634
+ "proposal_oracle_utility_gain_over_base_at_8": 1.1611655950546265,
635
+ "selected_success_at_8": 1.0,
636
+ "selected_success_gain_over_base_at_8": 1.0,
637
+ "selected_utility_at_8": 2.0,
638
+ "selected_utility_gain_over_base_at_8": 1.1611655950546265,
639
+ "selector_regret_at_8": 0.0,
640
+ "success_selector_gap_at_8": 0.0,
641
+ "success_support_gap_at_8": 0.0,
642
+ "success_total_car_to_hidden_at_8": 0.0,
643
+ "support_gap_at_8": 0.0,
644
+ "total_car_to_hidden_at_8": 0.0
645
+ },
646
+ "2141802816": {
647
+ "base_success": 0.0,
648
+ "base_utility": 0.07080724835395813,
649
+ "branch_car_at_8": 1.861636370420456,
650
+ "hidden_chart_oracle_success_at_8": 1.0,
651
+ "hidden_chart_oracle_utility_at_8": 2.0,
652
+ "outcome_ptr_at_8": 1.0,
653
+ "pairwise_causal_calibration_ece": 0.09903289408743085,
654
+ "proposal_oracle_success_at_8": 1.0,
655
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
656
+ "proposal_oracle_utility_at_8": 2.0,
657
+ "proposal_oracle_utility_gain_over_base_at_8": 1.9291927516460419,
658
+ "selected_success_at_8": 0.0,
659
+ "selected_success_gain_over_base_at_8": 0.0,
660
+ "selected_utility_at_8": 0.13836362957954407,
661
+ "selected_utility_gain_over_base_at_8": 0.06755638122558594,
662
+ "selector_regret_at_8": 1.861636370420456,
663
+ "success_selector_gap_at_8": 1.0,
664
+ "success_support_gap_at_8": 0.0,
665
+ "success_total_car_to_hidden_at_8": 1.0,
666
+ "support_gap_at_8": 0.0,
667
+ "total_car_to_hidden_at_8": 1.861636370420456
668
+ },
669
+ "256770874": {
670
+ "base_success": 0.0,
671
+ "base_utility": 0.8557480573654175,
672
+ "branch_car_at_8": 0.046216949820518494,
673
+ "hidden_chart_oracle_success_at_8": 1.0,
674
+ "hidden_chart_oracle_utility_at_8": 2.0,
675
+ "outcome_ptr_at_8": 0.0,
676
+ "pairwise_causal_calibration_ece": 0.046630940401554466,
677
+ "proposal_oracle_success_at_8": 0.0,
678
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
679
+ "proposal_oracle_utility_at_8": 0.21061499416828156,
680
+ "proposal_oracle_utility_gain_over_base_at_8": -0.6451330631971359,
681
+ "selected_success_at_8": 0.0,
682
+ "selected_success_gain_over_base_at_8": 0.0,
683
+ "selected_utility_at_8": 0.16439804434776306,
684
+ "selected_utility_gain_over_base_at_8": -0.6913500130176544,
685
+ "selector_regret_at_8": 0.046216949820518494,
686
+ "success_selector_gap_at_8": 0.0,
687
+ "success_support_gap_at_8": 1.0,
688
+ "success_total_car_to_hidden_at_8": 1.0,
689
+ "support_gap_at_8": 1.7893850058317184,
690
+ "total_car_to_hidden_at_8": 1.835601955652237
691
+ },
692
+ "278008524": {
693
+ "base_success": 0.0,
694
+ "base_utility": 0.10747186839580536,
695
+ "branch_car_at_8": 0.020201382460072637,
696
+ "hidden_chart_oracle_success_at_8": 0.5,
697
+ "hidden_chart_oracle_utility_at_8": 1.113811194896698,
698
+ "outcome_ptr_at_8": 0.5,
699
+ "pairwise_causal_calibration_ece": 0.15535141304867436,
700
+ "proposal_oracle_success_at_8": 0.0,
701
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
702
+ "proposal_oracle_utility_at_8": 0.032865648390725255,
703
+ "proposal_oracle_utility_gain_over_base_at_8": -0.0746062200050801,
704
+ "selected_success_at_8": 0.0,
705
+ "selected_success_gain_over_base_at_8": 0.0,
706
+ "selected_utility_at_8": 0.012664265930652618,
707
+ "selected_utility_gain_over_base_at_8": -0.09480760246515274,
708
+ "selector_regret_at_8": 0.020201382460072637,
709
+ "success_selector_gap_at_8": 0.0,
710
+ "success_support_gap_at_8": 0.5,
711
+ "success_total_car_to_hidden_at_8": 0.5,
712
+ "support_gap_at_8": 1.0809455465059727,
713
+ "total_car_to_hidden_at_8": 1.1011469289660454
714
+ },
715
+ "416183106": {
716
+ "base_success": 0.0,
717
+ "base_utility": 0.20032641291618347,
718
+ "branch_car_at_8": 1.812951534986496,
719
+ "hidden_chart_oracle_success_at_8": 1.0,
720
+ "hidden_chart_oracle_utility_at_8": 2.0,
721
+ "outcome_ptr_at_8": 1.0,
722
+ "pairwise_causal_calibration_ece": 0.18072825209943494,
723
+ "proposal_oracle_success_at_8": 1.0,
724
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
725
+ "proposal_oracle_utility_at_8": 2.0,
726
+ "proposal_oracle_utility_gain_over_base_at_8": 1.7996735870838165,
727
+ "selected_success_at_8": 0.0,
728
+ "selected_success_gain_over_base_at_8": 0.0,
729
+ "selected_utility_at_8": 0.18704846501350403,
730
+ "selected_utility_gain_over_base_at_8": -0.013277947902679443,
731
+ "selector_regret_at_8": 1.812951534986496,
732
+ "success_selector_gap_at_8": 1.0,
733
+ "success_support_gap_at_8": 0.0,
734
+ "success_total_car_to_hidden_at_8": 1.0,
735
+ "support_gap_at_8": 0.0,
736
+ "total_car_to_hidden_at_8": 1.812951534986496
737
+ },
738
+ "534717141": {
739
+ "base_success": 1.0,
740
+ "base_utility": 2.0,
741
+ "branch_car_at_8": 0.0,
742
+ "hidden_chart_oracle_success_at_8": 1.0,
743
+ "hidden_chart_oracle_utility_at_8": 2.0,
744
+ "outcome_ptr_at_8": 0.0,
745
+ "pairwise_causal_calibration_ece": 0.2870871533378466,
746
+ "proposal_oracle_success_at_8": 1.0,
747
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
748
+ "proposal_oracle_utility_at_8": 2.0,
749
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
750
+ "selected_success_at_8": 1.0,
751
+ "selected_success_gain_over_base_at_8": 0.0,
752
+ "selected_utility_at_8": 2.0,
753
+ "selected_utility_gain_over_base_at_8": 0.0,
754
+ "selector_regret_at_8": 0.0,
755
+ "success_selector_gap_at_8": 0.0,
756
+ "success_support_gap_at_8": 0.0,
757
+ "success_total_car_to_hidden_at_8": 0.0,
758
+ "support_gap_at_8": 0.0,
759
+ "total_car_to_hidden_at_8": 0.0
760
+ },
761
+ "551504635": {
762
+ "base_success": 0.0,
763
+ "base_utility": 0.7238867878913879,
764
+ "branch_car_at_8": 1.273692786693573,
765
+ "hidden_chart_oracle_success_at_8": 1.0,
766
+ "hidden_chart_oracle_utility_at_8": 2.0,
767
+ "outcome_ptr_at_8": 1.0,
768
+ "pairwise_causal_calibration_ece": 0.2702494805053396,
769
+ "proposal_oracle_success_at_8": 1.0,
770
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
771
+ "proposal_oracle_utility_at_8": 2.0,
772
+ "proposal_oracle_utility_gain_over_base_at_8": 1.276113212108612,
773
+ "selected_success_at_8": 0.0,
774
+ "selected_success_gain_over_base_at_8": 0.0,
775
+ "selected_utility_at_8": 0.726307213306427,
776
+ "selected_utility_gain_over_base_at_8": 0.0024204254150390625,
777
+ "selector_regret_at_8": 1.273692786693573,
778
+ "success_selector_gap_at_8": 1.0,
779
+ "success_support_gap_at_8": 0.0,
780
+ "success_total_car_to_hidden_at_8": 1.0,
781
+ "support_gap_at_8": 0.0,
782
+ "total_car_to_hidden_at_8": 1.273692786693573
783
+ },
784
+ "565369392": {
785
+ "base_success": 0.3333333333333333,
786
+ "base_utility": 0.7997656464576721,
787
+ "branch_car_at_8": 0.04557279869914055,
788
+ "hidden_chart_oracle_success_at_8": 0.6666666666666666,
789
+ "hidden_chart_oracle_utility_at_8": 1.4266315797964733,
790
+ "outcome_ptr_at_8": 0.0,
791
+ "pairwise_causal_calibration_ece": 0.17390197648626904,
792
+ "proposal_oracle_success_at_8": 0.0,
793
+ "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333,
794
+ "proposal_oracle_utility_at_8": 0.1326704372962316,
795
+ "proposal_oracle_utility_gain_over_base_at_8": -0.6670952091614405,
796
+ "selected_success_at_8": 0.0,
797
+ "selected_success_gain_over_base_at_8": -0.3333333333333333,
798
+ "selected_utility_at_8": 0.08709763859709103,
799
+ "selected_utility_gain_over_base_at_8": -0.712668007860581,
800
+ "selector_regret_at_8": 0.04557279869914055,
801
+ "success_selector_gap_at_8": 0.0,
802
+ "success_support_gap_at_8": 0.6666666666666666,
803
+ "success_total_car_to_hidden_at_8": 0.6666666666666666,
804
+ "support_gap_at_8": 1.2939611425002415,
805
+ "total_car_to_hidden_at_8": 1.339533941199382
806
+ },
807
+ "565621826": {
808
+ "base_success": 1.0,
809
+ "base_utility": 2.0,
810
+ "branch_car_at_8": 0.0,
811
+ "hidden_chart_oracle_success_at_8": 1.0,
812
+ "hidden_chart_oracle_utility_at_8": 2.0,
813
+ "outcome_ptr_at_8": 0.0,
814
+ "pairwise_causal_calibration_ece": 0.0,
815
+ "proposal_oracle_success_at_8": 1.0,
816
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
817
+ "proposal_oracle_utility_at_8": 2.0,
818
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0,
819
+ "selected_success_at_8": 1.0,
820
+ "selected_success_gain_over_base_at_8": 0.0,
821
+ "selected_utility_at_8": 2.0,
822
+ "selected_utility_gain_over_base_at_8": 0.0,
823
+ "selector_regret_at_8": 0.0,
824
+ "success_selector_gap_at_8": 0.0,
825
+ "success_support_gap_at_8": 0.0,
826
+ "success_total_car_to_hidden_at_8": 0.0,
827
+ "support_gap_at_8": 0.0,
828
+ "total_car_to_hidden_at_8": 0.0
829
+ },
830
+ "585904130": {
831
+ "base_success": 0.0,
832
+ "base_utility": 0.07529209802548091,
833
+ "branch_car_at_8": 0.011032561771571636,
834
+ "hidden_chart_oracle_success_at_8": 0.0,
835
+ "hidden_chart_oracle_utility_at_8": 0.23764699200789133,
836
+ "outcome_ptr_at_8": 0.6666666666666666,
837
+ "pairwise_causal_calibration_ece": 0.2625659187813512,
838
+ "proposal_oracle_success_at_8": 0.0,
839
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
840
+ "proposal_oracle_utility_at_8": 0.07848061621189117,
841
+ "proposal_oracle_utility_gain_over_base_at_8": 0.0031885181864102683,
842
+ "selected_success_at_8": 0.0,
843
+ "selected_success_gain_over_base_at_8": 0.0,
844
+ "selected_utility_at_8": 0.06744805444031954,
845
+ "selected_utility_gain_over_base_at_8": -0.007844043585161367,
846
+ "selector_regret_at_8": 0.011032561771571636,
847
+ "success_selector_gap_at_8": 0.0,
848
+ "success_support_gap_at_8": 0.0,
849
+ "success_total_car_to_hidden_at_8": 0.0,
850
+ "support_gap_at_8": 0.15916637579600015,
851
+ "total_car_to_hidden_at_8": 0.1701989375675718
852
+ },
853
+ "626486813": {
854
+ "base_success": 0.25,
855
+ "base_utility": 0.5054400166263804,
856
+ "branch_car_at_8": 0.031722720130346715,
857
+ "hidden_chart_oracle_success_at_8": 0.25,
858
+ "hidden_chart_oracle_utility_at_8": 0.6124684102833271,
859
+ "outcome_ptr_at_8": 0.75,
860
+ "pairwise_causal_calibration_ece": 0.23561688988950158,
861
+ "proposal_oracle_success_at_8": 0.0,
862
+ "proposal_oracle_success_gain_over_base_at_8": -0.25,
863
+ "proposal_oracle_utility_at_8": 0.07003506645560265,
864
+ "proposal_oracle_utility_gain_over_base_at_8": -0.43540495017077774,
865
+ "selected_success_at_8": 0.0,
866
+ "selected_success_gain_over_base_at_8": -0.25,
867
+ "selected_utility_at_8": 0.03831234632525593,
868
+ "selected_utility_gain_over_base_at_8": -0.46712767030112445,
869
+ "selector_regret_at_8": 0.031722720130346715,
870
+ "success_selector_gap_at_8": 0.0,
871
+ "success_support_gap_at_8": 0.25,
872
+ "success_total_car_to_hidden_at_8": 0.25,
873
+ "support_gap_at_8": 0.5432130172848701,
874
+ "total_car_to_hidden_at_8": 0.5741560639580712
875
+ },
876
+ "636680137": {
877
+ "base_success": 1.0,
878
+ "base_utility": 2.0,
879
+ "branch_car_at_8": 0.006491700187325478,
880
+ "hidden_chart_oracle_success_at_8": 1.0,
881
+ "hidden_chart_oracle_utility_at_8": 2.0,
882
+ "outcome_ptr_at_8": 0.0,
883
+ "pairwise_causal_calibration_ece": 0.12019361687075947,
884
+ "proposal_oracle_success_at_8": 0.0,
885
+ "proposal_oracle_success_gain_over_base_at_8": -1.0,
886
+ "proposal_oracle_utility_at_8": 0.02436021901667118,
887
+ "proposal_oracle_utility_gain_over_base_at_8": -1.9756397809833288,
888
+ "selected_success_at_8": 0.0,
889
+ "selected_success_gain_over_base_at_8": -1.0,
890
+ "selected_utility_at_8": 0.017868518829345703,
891
+ "selected_utility_gain_over_base_at_8": -1.9821314811706543,
892
+ "selector_regret_at_8": 0.006491700187325478,
893
+ "success_selector_gap_at_8": 0.0,
894
+ "success_support_gap_at_8": 1.0,
895
+ "success_total_car_to_hidden_at_8": 1.0,
896
+ "support_gap_at_8": 1.9756397809833288,
897
+ "total_car_to_hidden_at_8": 1.9821314811706543
898
+ },
899
+ "706479600": {
900
+ "base_success": 0.0,
901
+ "base_utility": 0.8350887894630432,
902
+ "branch_car_at_8": 0.0,
903
+ "hidden_chart_oracle_success_at_8": 1.0,
904
+ "hidden_chart_oracle_utility_at_8": 2.0,
905
+ "outcome_ptr_at_8": 1.0,
906
+ "pairwise_causal_calibration_ece": 0.22468613298280815,
907
+ "proposal_oracle_success_at_8": 1.0,
908
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
909
+ "proposal_oracle_utility_at_8": 2.0,
910
+ "proposal_oracle_utility_gain_over_base_at_8": 1.1649112105369568,
911
+ "selected_success_at_8": 1.0,
912
+ "selected_success_gain_over_base_at_8": 1.0,
913
+ "selected_utility_at_8": 2.0,
914
+ "selected_utility_gain_over_base_at_8": 1.1649112105369568,
915
+ "selector_regret_at_8": 0.0,
916
+ "success_selector_gap_at_8": 0.0,
917
+ "success_support_gap_at_8": 0.0,
918
+ "success_total_car_to_hidden_at_8": 0.0,
919
+ "support_gap_at_8": 0.0,
920
+ "total_car_to_hidden_at_8": 0.0
921
+ },
922
+ "750448296": {
923
+ "base_success": 0.0,
924
+ "base_utility": 0.11153355985879898,
925
+ "branch_car_at_8": 0.01770863775163889,
926
+ "hidden_chart_oracle_success_at_8": 0.0,
927
+ "hidden_chart_oracle_utility_at_8": 0.23580759018659592,
928
+ "outcome_ptr_at_8": 0.0,
929
+ "pairwise_causal_calibration_ece": 0.14149358436045273,
930
+ "proposal_oracle_success_at_8": 0.0,
931
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
932
+ "proposal_oracle_utility_at_8": 0.0669428389519453,
933
+ "proposal_oracle_utility_gain_over_base_at_8": -0.044590720906853676,
934
+ "selected_success_at_8": 0.0,
935
+ "selected_success_gain_over_base_at_8": 0.0,
936
+ "selected_utility_at_8": 0.049234201200306416,
937
+ "selected_utility_gain_over_base_at_8": -0.062299358658492565,
938
+ "selector_regret_at_8": 0.01770863775163889,
939
+ "success_selector_gap_at_8": 0.0,
940
+ "success_support_gap_at_8": 0.0,
941
+ "success_total_car_to_hidden_at_8": 0.0,
942
+ "support_gap_at_8": 0.1688647512346506,
943
+ "total_car_to_hidden_at_8": 0.1865733889862895
944
+ },
945
+ "75070641": {
946
+ "base_success": 0.0,
947
+ "base_utility": 0.20275163650512695,
948
+ "branch_car_at_8": 0.3710349500179291,
949
+ "hidden_chart_oracle_success_at_8": 1.0,
950
+ "hidden_chart_oracle_utility_at_8": 2.0,
951
+ "outcome_ptr_at_8": 1.0,
952
+ "pairwise_causal_calibration_ece": 0.09051828052202912,
953
+ "proposal_oracle_success_at_8": 0.0,
954
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
955
+ "proposal_oracle_utility_at_8": 0.5650836229324341,
956
+ "proposal_oracle_utility_gain_over_base_at_8": 0.36233198642730713,
957
+ "selected_success_at_8": 0.0,
958
+ "selected_success_gain_over_base_at_8": 0.0,
959
+ "selected_utility_at_8": 0.194048672914505,
960
+ "selected_utility_gain_over_base_at_8": -0.008702963590621948,
961
+ "selector_regret_at_8": 0.3710349500179291,
962
+ "success_selector_gap_at_8": 0.0,
963
+ "success_support_gap_at_8": 1.0,
964
+ "success_total_car_to_hidden_at_8": 1.0,
965
+ "support_gap_at_8": 1.434916377067566,
966
+ "total_car_to_hidden_at_8": 1.805951327085495
967
+ },
968
+ "767747424": {
969
+ "base_success": 0.25,
970
+ "base_utility": 0.5475241877138615,
971
+ "branch_car_at_8": 0.024658456211909652,
972
+ "hidden_chart_oracle_success_at_8": 0.0,
973
+ "hidden_chart_oracle_utility_at_8": 0.18863565661013126,
974
+ "outcome_ptr_at_8": 0.75,
975
+ "pairwise_causal_calibration_ece": 0.14269822176062383,
976
+ "proposal_oracle_success_at_8": 0.0,
977
+ "proposal_oracle_success_gain_over_base_at_8": -0.25,
978
+ "proposal_oracle_utility_at_8": 0.11239026742987335,
979
+ "proposal_oracle_utility_gain_over_base_at_8": -0.4351339202839881,
980
+ "selected_success_at_8": 0.0,
981
+ "selected_success_gain_over_base_at_8": -0.25,
982
+ "selected_utility_at_8": 0.0877318112179637,
983
+ "selected_utility_gain_over_base_at_8": -0.45979237649589777,
984
+ "selector_regret_at_8": 0.024658456211909652,
985
+ "success_selector_gap_at_8": 0.0,
986
+ "success_support_gap_at_8": 0.0,
987
+ "success_total_car_to_hidden_at_8": 0.0,
988
+ "support_gap_at_8": 0.07677002181299031,
989
+ "total_car_to_hidden_at_8": 0.10090384539216757
990
+ },
991
+ "77147264": {
992
+ "base_success": 0.5,
993
+ "base_utility": 1.4215794801712036,
994
+ "branch_car_at_8": 0.0,
995
+ "hidden_chart_oracle_success_at_8": 1.0,
996
+ "hidden_chart_oracle_utility_at_8": 2.0,
997
+ "outcome_ptr_at_8": 0.5,
998
+ "pairwise_causal_calibration_ece": 0.12309184879211332,
999
+ "proposal_oracle_success_at_8": 1.0,
1000
+ "proposal_oracle_success_gain_over_base_at_8": 0.5,
1001
+ "proposal_oracle_utility_at_8": 2.0,
1002
+ "proposal_oracle_utility_gain_over_base_at_8": 0.5784205198287964,
1003
+ "selected_success_at_8": 1.0,
1004
+ "selected_success_gain_over_base_at_8": 0.5,
1005
+ "selected_utility_at_8": 2.0,
1006
+ "selected_utility_gain_over_base_at_8": 0.5784205198287964,
1007
+ "selector_regret_at_8": 0.0,
1008
+ "success_selector_gap_at_8": 0.0,
1009
+ "success_support_gap_at_8": 0.0,
1010
+ "success_total_car_to_hidden_at_8": 0.0,
1011
+ "support_gap_at_8": 0.0,
1012
+ "total_car_to_hidden_at_8": 0.0
1013
+ },
1014
+ "794181626": {
1015
+ "base_success": 0.0,
1016
+ "base_utility": 0.18095605075359344,
1017
+ "branch_car_at_8": 0.0,
1018
+ "hidden_chart_oracle_success_at_8": 1.0,
1019
+ "hidden_chart_oracle_utility_at_8": 2.0,
1020
+ "outcome_ptr_at_8": 1.0,
1021
+ "pairwise_causal_calibration_ece": 0.0,
1022
+ "proposal_oracle_success_at_8": 1.0,
1023
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
1024
+ "proposal_oracle_utility_at_8": 2.0,
1025
+ "proposal_oracle_utility_gain_over_base_at_8": 1.8190439492464066,
1026
+ "selected_success_at_8": 1.0,
1027
+ "selected_success_gain_over_base_at_8": 1.0,
1028
+ "selected_utility_at_8": 2.0,
1029
+ "selected_utility_gain_over_base_at_8": 1.8190439492464066,
1030
+ "selector_regret_at_8": 0.0,
1031
+ "success_selector_gap_at_8": 0.0,
1032
+ "success_support_gap_at_8": 0.0,
1033
+ "success_total_car_to_hidden_at_8": 0.0,
1034
+ "support_gap_at_8": 0.0,
1035
+ "total_car_to_hidden_at_8": 0.0
1036
+ },
1037
+ "957101489": {
1038
+ "base_success": 0.0,
1039
+ "base_utility": 0.2039649486541748,
1040
+ "branch_car_at_8": 1.432861328125,
1041
+ "hidden_chart_oracle_success_at_8": 1.0,
1042
+ "hidden_chart_oracle_utility_at_8": 2.0,
1043
+ "outcome_ptr_at_8": 1.0,
1044
+ "pairwise_causal_calibration_ece": 0.21219243624964876,
1045
+ "proposal_oracle_success_at_8": 1.0,
1046
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
1047
+ "proposal_oracle_utility_at_8": 2.0,
1048
+ "proposal_oracle_utility_gain_over_base_at_8": 1.7960350513458252,
1049
+ "selected_success_at_8": 0.0,
1050
+ "selected_success_gain_over_base_at_8": 0.0,
1051
+ "selected_utility_at_8": 0.567138671875,
1052
+ "selected_utility_gain_over_base_at_8": 0.3631737232208252,
1053
+ "selector_regret_at_8": 1.432861328125,
1054
+ "success_selector_gap_at_8": 1.0,
1055
+ "success_support_gap_at_8": 0.0,
1056
+ "success_total_car_to_hidden_at_8": 1.0,
1057
+ "support_gap_at_8": 0.0,
1058
+ "total_car_to_hidden_at_8": 1.432861328125
1059
+ },
1060
+ "998761162": {
1061
+ "base_success": 0.0,
1062
+ "base_utility": 0.06781657040119171,
1063
+ "branch_car_at_8": 1.939187914133072,
1064
+ "hidden_chart_oracle_success_at_8": 1.0,
1065
+ "hidden_chart_oracle_utility_at_8": 2.0,
1066
+ "outcome_ptr_at_8": 1.0,
1067
+ "pairwise_causal_calibration_ece": 0.11706618974004251,
1068
+ "proposal_oracle_success_at_8": 1.0,
1069
+ "proposal_oracle_success_gain_over_base_at_8": 1.0,
1070
+ "proposal_oracle_utility_at_8": 2.0,
1071
+ "proposal_oracle_utility_gain_over_base_at_8": 1.9321834295988083,
1072
+ "selected_success_at_8": 0.0,
1073
+ "selected_success_gain_over_base_at_8": 0.0,
1074
+ "selected_utility_at_8": 0.0608120858669281,
1075
+ "selected_utility_gain_over_base_at_8": -0.007004484534263611,
1076
+ "selector_regret_at_8": 1.939187914133072,
1077
+ "success_selector_gap_at_8": 1.0,
1078
+ "success_support_gap_at_8": 0.0,
1079
+ "success_total_car_to_hidden_at_8": 1.0,
1080
+ "support_gap_at_8": 0.0,
1081
+ "total_car_to_hidden_at_8": 1.939187914133072
1082
+ }
1083
+ }
workspace/runs/ctt_residual_rollout_val69_seed2/metrics_by_task.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "LiftPegUpright-v1": {
3
+ "base_success": 0.3333333333333333,
4
+ "base_utility": 1.066929851969083,
5
+ "branch_car_at_8": 1.029554468890031,
6
+ "hidden_chart_oracle_success_at_8": 1.0,
7
+ "hidden_chart_oracle_utility_at_8": 2.0,
8
+ "outcome_ptr_at_8": 0.6666666666666666,
9
+ "pairwise_causal_calibration_ece": 0.16081309392707618,
10
+ "proposal_oracle_success_at_8": 0.6666666666666666,
11
+ "proposal_oracle_success_gain_over_base_at_8": 0.3333333333333333,
12
+ "proposal_oracle_utility_at_8": 1.5717276831467946,
13
+ "proposal_oracle_utility_gain_over_base_at_8": 0.5047978311777115,
14
+ "selected_success_at_8": 0.0,
15
+ "selected_success_gain_over_base_at_8": -0.3333333333333333,
16
+ "selected_utility_at_8": 0.5421732142567635,
17
+ "selected_utility_gain_over_base_at_8": -0.5247566377123197,
18
+ "selector_regret_at_8": 1.029554468890031,
19
+ "success_selector_gap_at_8": 0.6666666666666666,
20
+ "success_support_gap_at_8": 0.3333333333333333,
21
+ "success_total_car_to_hidden_at_8": 1.0,
22
+ "support_gap_at_8": 0.4282723168532054,
23
+ "total_car_to_hidden_at_8": 1.4578267857432365
24
+ },
25
+ "PickCube-v1": {
26
+ "base_success": 0.0,
27
+ "base_utility": 0.15342965722084045,
28
+ "branch_car_at_8": 0.35906313359737396,
29
+ "hidden_chart_oracle_success_at_8": 1.0,
30
+ "hidden_chart_oracle_utility_at_8": 2.0,
31
+ "outcome_ptr_at_8": 1.0,
32
+ "pairwise_causal_calibration_ece": 0.2793744975404514,
33
+ "proposal_oracle_success_at_8": 0.0,
34
+ "proposal_oracle_success_gain_over_base_at_8": 0.0,
35
+ "proposal_oracle_utility_at_8": 0.46377021074295044,
36
+ "proposal_oracle_utility_gain_over_base_at_8": 0.31034055352211,
37
+ "selected_success_at_8": 0.0,
38
+ "selected_success_gain_over_base_at_8": 0.0,
39
+ "selected_utility_at_8": 0.10470707714557648,
40
+ "selected_utility_gain_over_base_at_8": -0.04872258007526398,
41
+ "selector_regret_at_8": 0.35906313359737396,
42
+ "success_selector_gap_at_8": 0.0,
43
+ "success_support_gap_at_8": 1.0,
44
+ "success_total_car_to_hidden_at_8": 1.0,
45
+ "support_gap_at_8": 1.5362297892570496,
46
+ "total_car_to_hidden_at_8": 1.8952929228544235
47
+ },
48
+ "PullCube-v1": {
49
+ "base_success": 0.29411764705882354,
50
+ "base_utility": 0.6512299207833541,
51
+ "branch_car_at_8": 0.017109284352045506,
52
+ "hidden_chart_oracle_success_at_8": 0.3235294117647059,
53
+ "hidden_chart_oracle_utility_at_8": 0.7823218367993832,
54
+ "outcome_ptr_at_8": 0.35294117647058826,
55
+ "pairwise_causal_calibration_ece": 0.1721079818939321,
56
+ "proposal_oracle_success_at_8": 0.14705882352941177,
57
+ "proposal_oracle_success_gain_over_base_at_8": -0.14705882352941177,
58
+ "proposal_oracle_utility_at_8": 0.3548735701530377,
59
+ "proposal_oracle_utility_gain_over_base_at_8": -0.29635635063031635,
60
+ "selected_success_at_8": 0.14705882352941177,
61
+ "selected_success_gain_over_base_at_8": -0.14705882352941177,
62
+ "selected_utility_at_8": 0.3377642858009922,
63
+ "selected_utility_gain_over_base_at_8": -0.31346563498236185,
64
+ "selector_regret_at_8": 0.017109284352045506,
65
+ "success_selector_gap_at_8": 0.0,
66
+ "success_support_gap_at_8": 0.17647058823529413,
67
+ "success_total_car_to_hidden_at_8": 0.17647058823529413,
68
+ "support_gap_at_8": 0.4276017144216252,
69
+ "total_car_to_hidden_at_8": 0.44455755099839095
70
+ },
71
+ "PushCube-v1": {
72
+ "base_success": 0.38461538461538464,
73
+ "base_utility": 0.8217729788560134,
74
+ "branch_car_at_8": 0.4485462617415648,
75
+ "hidden_chart_oracle_success_at_8": 1.0,
76
+ "hidden_chart_oracle_utility_at_8": 2.0,
77
+ "outcome_ptr_at_8": 0.6153846153846154,
78
+ "pairwise_causal_calibration_ece": 0.15912329792465574,
79
+ "proposal_oracle_success_at_8": 0.8461538461538461,
80
+ "proposal_oracle_success_gain_over_base_at_8": 0.46153846153846156,
81
+ "proposal_oracle_utility_at_8": 1.7001846948495278,
82
+ "proposal_oracle_utility_gain_over_base_at_8": 0.8784117159935144,
83
+ "selected_success_at_8": 0.6153846153846154,
84
+ "selected_success_gain_over_base_at_8": 0.23076923076923078,
85
+ "selected_utility_at_8": 1.251638433107963,
86
+ "selected_utility_gain_over_base_at_8": 0.4298654542519496,
87
+ "selector_regret_at_8": 0.4485462617415648,
88
+ "success_selector_gap_at_8": 0.23076923076923078,
89
+ "success_support_gap_at_8": 0.15384615384615385,
90
+ "success_total_car_to_hidden_at_8": 0.38461538461538464,
91
+ "support_gap_at_8": 0.2998153051504722,
92
+ "total_car_to_hidden_at_8": 0.748361566892037
93
+ },
94
+ "StackCube-v1": {
95
+ "base_success": 0.2,
96
+ "base_utility": 0.8047854532798131,
97
+ "branch_car_at_8": 0.5301474064588547,
98
+ "hidden_chart_oracle_success_at_8": 1.0,
99
+ "hidden_chart_oracle_utility_at_8": 2.0,
100
+ "outcome_ptr_at_8": 0.6666666666666666,
101
+ "pairwise_causal_calibration_ece": 0.20081454850953362,
102
+ "proposal_oracle_success_at_8": 0.6,
103
+ "proposal_oracle_success_gain_over_base_at_8": 0.4,
104
+ "proposal_oracle_utility_at_8": 1.3190105259418488,
105
+ "proposal_oracle_utility_gain_over_base_at_8": 0.5142250726620357,
106
+ "selected_success_at_8": 0.3333333333333333,
107
+ "selected_success_gain_over_base_at_8": 0.13333333333333333,
108
+ "selected_utility_at_8": 0.788863119482994,
109
+ "selected_utility_gain_over_base_at_8": -0.015922333796819052,
110
+ "selector_regret_at_8": 0.5301474064588547,
111
+ "success_selector_gap_at_8": 0.26666666666666666,
112
+ "success_support_gap_at_8": 0.4,
113
+ "success_total_car_to_hidden_at_8": 0.6666666666666666,
114
+ "support_gap_at_8": 0.6809894740581512,
115
+ "total_car_to_hidden_at_8": 1.211136880517006
116
+ }
117
+ }