Upload per-dim trainmax test aggregate
Browse files- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json +0 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/command.txt +1 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/data_hash.txt +1 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/eval.log +3 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/git_hash.txt +1 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/measured_metrics/metrics.json +0 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/measured_metrics/metrics_by_seed.json +0 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/measured_metrics/metrics_by_task.json +179 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/measured_metrics/report.md +40 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/measured_metrics/table.tex +41 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/metrics.json +0 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/metrics_by_seed.json +0 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/metrics_by_task.json +179 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/report.md +77 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/source_data_hash.txt +1 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/split_hash.txt +1 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/table.tex +41 -0
- workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/train.log +1 -0
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/command.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
python scripts/build_ctt_rollout_comparison.py --run-glob runs/ctt_residual_base_context_obs_rollout_test_perdim_trainmax_seed* --out-dir runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison --k 8 --bootstrap-samples 1000
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/data_hash.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/eval.log
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
runs/ctt_residual_base_context_obs_rollout_test_perdim_trainmax_seed0: measured_candidates.json
|
| 2 |
+
runs/ctt_residual_base_context_obs_rollout_test_perdim_trainmax_seed1: measured_candidates.json
|
| 3 |
+
runs/ctt_residual_base_context_obs_rollout_test_perdim_trainmax_seed2: measured_candidates.json
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/git_hash.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
14e81292e2004925f81e0bb4810f6f144bfd87b7
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/measured_metrics/metrics.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/measured_metrics/metrics_by_seed.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/measured_metrics/metrics_by_task.json
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"LiftPegUpright-v1": {
|
| 3 |
+
"any_generated_unsafe_known_at_8": 0.0,
|
| 4 |
+
"base_safety_label_known": 1.0,
|
| 5 |
+
"base_success": 0.0,
|
| 6 |
+
"base_unsafe_known": 0.0,
|
| 7 |
+
"base_utility": 0.4796089559793472,
|
| 8 |
+
"branch_car_at_8": 0.18885904252529145,
|
| 9 |
+
"generated_safety_label_coverage_at_8": 1.0,
|
| 10 |
+
"generated_unsafe_rate_known_at_8": 0.0,
|
| 11 |
+
"hidden_chart_oracle_success_at_8": 1.0,
|
| 12 |
+
"hidden_chart_oracle_utility_at_8": 2.0,
|
| 13 |
+
"ncar_to_hidden_chart_oracle_at_8": 1.0056207874968426,
|
| 14 |
+
"ncar_to_proposal_oracle_at_8": 0.6789057634923168,
|
| 15 |
+
"outcome_ptr_at_8": 0.3333333333333333,
|
| 16 |
+
"pairwise_causal_calibration_ece": 0.17461037477790295,
|
| 17 |
+
"proposal_oracle_safety_label_known_at_8": 1.0,
|
| 18 |
+
"proposal_oracle_success_at_8": 0.13333333333333333,
|
| 19 |
+
"proposal_oracle_success_gain_over_base_at_8": 0.13333333333333333,
|
| 20 |
+
"proposal_oracle_unsafe_known_at_8": 0.0,
|
| 21 |
+
"proposal_oracle_utility_at_8": 0.6601470321416855,
|
| 22 |
+
"proposal_oracle_utility_gain_over_base_at_8": 0.18053807616233825,
|
| 23 |
+
"selected_safety_label_known_at_8": 1.0,
|
| 24 |
+
"selected_success_at_8": 0.0,
|
| 25 |
+
"selected_success_gain_over_base_at_8": 0.0,
|
| 26 |
+
"selected_unsafe_known_at_8": 0.0,
|
| 27 |
+
"selected_utility_at_8": 0.47128798961639407,
|
| 28 |
+
"selected_utility_gain_over_base_at_8": -0.008320966362953186,
|
| 29 |
+
"selector_gap_fraction_to_hidden_at_8": 0.14638959816683447,
|
| 30 |
+
"selector_regret_at_8": 0.18885904252529145,
|
| 31 |
+
"success_selector_gap_at_8": 0.13333333333333333,
|
| 32 |
+
"success_support_gap_at_8": 0.8666666666666667,
|
| 33 |
+
"success_total_car_to_hidden_at_8": 1.0,
|
| 34 |
+
"support_gap_at_8": 1.3398529678583144,
|
| 35 |
+
"support_gap_fraction_to_hidden_at_8": 0.8592311961008738,
|
| 36 |
+
"total_car_to_hidden_at_8": 1.528712010383606
|
| 37 |
+
},
|
| 38 |
+
"PickCube-v1": {
|
| 39 |
+
"any_generated_unsafe_known_at_8": 0.0,
|
| 40 |
+
"base_safety_label_known": 1.0,
|
| 41 |
+
"base_success": 1.0,
|
| 42 |
+
"base_unsafe_known": 0.0,
|
| 43 |
+
"base_utility": 2.0,
|
| 44 |
+
"branch_car_at_8": 0.7539208928743998,
|
| 45 |
+
"generated_safety_label_coverage_at_8": 1.0,
|
| 46 |
+
"generated_unsafe_rate_known_at_8": 0.0,
|
| 47 |
+
"hidden_chart_oracle_success_at_8": 1.0,
|
| 48 |
+
"hidden_chart_oracle_utility_at_8": 2.0,
|
| 49 |
+
"ncar_to_proposal_oracle_at_8": 0.27456935655376624,
|
| 50 |
+
"outcome_ptr_at_8": 0.0,
|
| 51 |
+
"pairwise_causal_calibration_ece": 0.4028758622177973,
|
| 52 |
+
"proposal_oracle_safety_label_known_at_8": 1.0,
|
| 53 |
+
"proposal_oracle_success_at_8": 0.3333333333333333,
|
| 54 |
+
"proposal_oracle_success_gain_over_base_at_8": -0.6666666666666666,
|
| 55 |
+
"proposal_oracle_unsafe_known_at_8": 0.0,
|
| 56 |
+
"proposal_oracle_utility_at_8": 1.021850009759267,
|
| 57 |
+
"proposal_oracle_utility_gain_over_base_at_8": -0.9781499902407328,
|
| 58 |
+
"selected_safety_label_known_at_8": 1.0,
|
| 59 |
+
"selected_success_at_8": 0.0,
|
| 60 |
+
"selected_success_gain_over_base_at_8": -1.0,
|
| 61 |
+
"selected_unsafe_known_at_8": 0.0,
|
| 62 |
+
"selected_utility_at_8": 0.26792911688486737,
|
| 63 |
+
"selected_utility_gain_over_base_at_8": -1.7320708831151326,
|
| 64 |
+
"selector_regret_at_8": 0.7539208928743998,
|
| 65 |
+
"success_selector_gap_at_8": 0.3333333333333333,
|
| 66 |
+
"success_support_gap_at_8": 0.6666666666666666,
|
| 67 |
+
"success_total_car_to_hidden_at_8": 1.0,
|
| 68 |
+
"support_gap_at_8": 0.9781499902407328,
|
| 69 |
+
"total_car_to_hidden_at_8": 1.7320708831151326
|
| 70 |
+
},
|
| 71 |
+
"PullCube-v1": {
|
| 72 |
+
"any_generated_unsafe_known_at_8": 0.7066666666666667,
|
| 73 |
+
"base_safety_label_known": 1.0,
|
| 74 |
+
"base_success": 0.36,
|
| 75 |
+
"base_unsafe_known": 0.0,
|
| 76 |
+
"base_utility": 0.8032506120204925,
|
| 77 |
+
"branch_car_at_8": 0.06862825746337572,
|
| 78 |
+
"generated_safety_label_coverage_at_8": 1.0,
|
| 79 |
+
"generated_unsafe_rate_known_at_8": 0.5816666666666667,
|
| 80 |
+
"hidden_chart_oracle_success_at_8": 0.52,
|
| 81 |
+
"hidden_chart_oracle_utility_at_8": 1.1467878860235214,
|
| 82 |
+
"ncar_to_hidden_chart_oracle_at_8": 1.3075100566723548,
|
| 83 |
+
"ncar_to_proposal_oracle_at_8": 0.5092805557938541,
|
| 84 |
+
"outcome_ptr_at_8": 0.32,
|
| 85 |
+
"pairwise_causal_calibration_ece": 0.19854171059884482,
|
| 86 |
+
"proposal_oracle_safety_label_known_at_8": 1.0,
|
| 87 |
+
"proposal_oracle_success_at_8": 0.17333333333333334,
|
| 88 |
+
"proposal_oracle_success_gain_over_base_at_8": -0.18666666666666668,
|
| 89 |
+
"proposal_oracle_unsafe_known_at_8": 0.5466666666666666,
|
| 90 |
+
"proposal_oracle_utility_at_8": 0.46170220340291657,
|
| 91 |
+
"proposal_oracle_utility_gain_over_base_at_8": -0.34154840861757596,
|
| 92 |
+
"selected_safety_label_known_at_8": 1.0,
|
| 93 |
+
"selected_success_at_8": 0.14666666666666667,
|
| 94 |
+
"selected_success_gain_over_base_at_8": -0.21333333333333335,
|
| 95 |
+
"selected_unsafe_known_at_8": 0.5333333333333333,
|
| 96 |
+
"selected_utility_at_8": 0.39307394593954087,
|
| 97 |
+
"selected_utility_gain_over_base_at_8": -0.4101766660809517,
|
| 98 |
+
"selector_gap_fraction_to_hidden_at_8": 0.3721528711155151,
|
| 99 |
+
"selector_regret_at_8": 0.06862825746337572,
|
| 100 |
+
"success_selector_gap_at_8": 0.02666666666666667,
|
| 101 |
+
"success_support_gap_at_8": 0.3466666666666667,
|
| 102 |
+
"success_total_car_to_hidden_at_8": 0.37333333333333335,
|
| 103 |
+
"support_gap_at_8": 0.6881442584097386,
|
| 104 |
+
"support_gap_fraction_to_hidden_at_8": 1.0971174661984022,
|
| 105 |
+
"total_car_to_hidden_at_8": 0.7538664453228314
|
| 106 |
+
},
|
| 107 |
+
"PushCube-v1": {
|
| 108 |
+
"any_generated_unsafe_known_at_8": 0.0,
|
| 109 |
+
"base_safety_label_known": 1.0,
|
| 110 |
+
"base_success": 0.8333333333333334,
|
| 111 |
+
"base_unsafe_known": 0.0,
|
| 112 |
+
"base_utility": 1.6734557673335075,
|
| 113 |
+
"branch_car_at_8": 0.41695255951748955,
|
| 114 |
+
"generated_safety_label_coverage_at_8": 1.0,
|
| 115 |
+
"generated_unsafe_rate_known_at_8": 0.0,
|
| 116 |
+
"hidden_chart_oracle_success_at_8": 0.8333333333333334,
|
| 117 |
+
"hidden_chart_oracle_utility_at_8": 1.7015800600250561,
|
| 118 |
+
"ncar_to_hidden_chart_oracle_at_8": 0.3255141393427919,
|
| 119 |
+
"ncar_to_proposal_oracle_at_8": 0.17843067932762913,
|
| 120 |
+
"outcome_ptr_at_8": 0.16666666666666666,
|
| 121 |
+
"pairwise_causal_calibration_ece": 0.25042093435733487,
|
| 122 |
+
"proposal_oracle_safety_label_known_at_8": 1.0,
|
| 123 |
+
"proposal_oracle_success_at_8": 0.5555555555555556,
|
| 124 |
+
"proposal_oracle_success_gain_over_base_at_8": -0.2777777777777778,
|
| 125 |
+
"proposal_oracle_unsafe_known_at_8": 0.0,
|
| 126 |
+
"proposal_oracle_utility_at_8": 1.1790093589160178,
|
| 127 |
+
"proposal_oracle_utility_gain_over_base_at_8": -0.49444640841748977,
|
| 128 |
+
"selected_safety_label_known_at_8": 1.0,
|
| 129 |
+
"selected_success_at_8": 0.3333333333333333,
|
| 130 |
+
"selected_success_gain_over_base_at_8": -0.5,
|
| 131 |
+
"selected_unsafe_known_at_8": 0.0,
|
| 132 |
+
"selected_utility_at_8": 0.7620567993985282,
|
| 133 |
+
"selected_utility_gain_over_base_at_8": -0.9113989679349793,
|
| 134 |
+
"selector_gap_fraction_to_hidden_at_8": 0.48464542735161803,
|
| 135 |
+
"selector_regret_at_8": 0.41695255951748955,
|
| 136 |
+
"success_selector_gap_at_8": 0.2222222222222222,
|
| 137 |
+
"success_support_gap_at_8": 0.3333333333333333,
|
| 138 |
+
"success_total_car_to_hidden_at_8": 0.5,
|
| 139 |
+
"support_gap_at_8": 0.6220440144340197,
|
| 140 |
+
"support_gap_fraction_to_hidden_at_8": 0.007535380326794263,
|
| 141 |
+
"total_car_to_hidden_at_8": 0.939523260626528
|
| 142 |
+
},
|
| 143 |
+
"StackCube-v1": {
|
| 144 |
+
"any_generated_unsafe_known_at_8": 0.0,
|
| 145 |
+
"base_safety_label_known": 1.0,
|
| 146 |
+
"base_success": 0.09090909090909091,
|
| 147 |
+
"base_unsafe_known": 0.0,
|
| 148 |
+
"base_utility": 0.5003613992170854,
|
| 149 |
+
"branch_car_at_8": 0.3588122549382123,
|
| 150 |
+
"generated_safety_label_coverage_at_8": 1.0,
|
| 151 |
+
"generated_unsafe_rate_known_at_8": 0.0,
|
| 152 |
+
"hidden_chart_oracle_success_at_8": 1.0,
|
| 153 |
+
"hidden_chart_oracle_utility_at_8": 2.0,
|
| 154 |
+
"ncar_to_hidden_chart_oracle_at_8": 1.0530175208494181,
|
| 155 |
+
"ncar_to_proposal_oracle_at_8": 3.7287207392774975,
|
| 156 |
+
"outcome_ptr_at_8": 0.6060606060606061,
|
| 157 |
+
"pairwise_causal_calibration_ece": 0.1913742697792418,
|
| 158 |
+
"proposal_oracle_safety_label_known_at_8": 1.0,
|
| 159 |
+
"proposal_oracle_success_at_8": 0.18181818181818182,
|
| 160 |
+
"proposal_oracle_success_gain_over_base_at_8": 0.09090909090909091,
|
| 161 |
+
"proposal_oracle_unsafe_known_at_8": 0.0,
|
| 162 |
+
"proposal_oracle_utility_at_8": 0.634040683056369,
|
| 163 |
+
"proposal_oracle_utility_gain_over_base_at_8": 0.13367928383928357,
|
| 164 |
+
"selected_safety_label_known_at_8": 1.0,
|
| 165 |
+
"selected_success_at_8": 0.0,
|
| 166 |
+
"selected_success_gain_over_base_at_8": -0.09090909090909091,
|
| 167 |
+
"selected_unsafe_known_at_8": 0.0,
|
| 168 |
+
"selected_utility_at_8": 0.2752284281181567,
|
| 169 |
+
"selected_utility_gain_over_base_at_8": -0.22513297109892874,
|
| 170 |
+
"selector_gap_fraction_to_hidden_at_8": 0.2000703650638142,
|
| 171 |
+
"selector_regret_at_8": 0.3588122549382123,
|
| 172 |
+
"success_selector_gap_at_8": 0.18181818181818182,
|
| 173 |
+
"success_support_gap_at_8": 0.8181818181818182,
|
| 174 |
+
"success_total_car_to_hidden_at_8": 1.0,
|
| 175 |
+
"support_gap_at_8": 1.365959316943631,
|
| 176 |
+
"support_gap_fraction_to_hidden_at_8": 0.8529471625482842,
|
| 177 |
+
"total_car_to_hidden_at_8": 1.7247715718818433
|
| 178 |
+
}
|
| 179 |
+
}
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/measured_metrics/report.md
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Metric Evaluation (measured)
|
| 2 |
+
|
| 3 |
+
K: `8`
|
| 4 |
+
|
| 5 |
+
| Metric | N | Micro mean | 95% CI | Task macro | Seed macro |
|
| 6 |
+
| --- | ---: | ---: | ---: | ---: | ---: |
|
| 7 |
+
| any_generated_unsafe_known_at_8 | 144 | 0.3681 | [0.2847, 0.4514] | 0.1413 | 0.2917 |
|
| 8 |
+
| base_safety_label_known | 144 | 1.0000 | [1.0000, 1.0000] | 1.0000 | 1.0000 |
|
| 9 |
+
| base_success | 144 | 0.3333 | [0.2569, 0.4097] | 0.4568 | 0.3083 |
|
| 10 |
+
| base_unsafe_known | 144 | 0.0000 | [0.0000, 0.0000] | 0.0000 | 0.0000 |
|
| 11 |
+
| base_utility | 144 | 0.8338 | [0.6953, 0.9699] | 1.0913 | 0.8028 |
|
| 12 |
+
| branch_car_at_8 | 144 | 0.2055 | [0.1335, 0.2890] | 0.3574 | 0.2204 |
|
| 13 |
+
| generated_safety_label_coverage_at_8 | 144 | 1.0000 | [1.0000, 1.0000] | 1.0000 | 1.0000 |
|
| 14 |
+
| generated_unsafe_rate_known_at_8 | 144 | 0.3030 | [0.2318, 0.3707] | 0.1163 | 0.2372 |
|
| 15 |
+
| hidden_chart_oracle_success_at_8 | 144 | 0.7292 | [0.6597, 0.7986] | 0.8707 | 0.7854 |
|
| 16 |
+
| hidden_chart_oracle_utility_at_8 | 144 | 1.5183 | [1.3941, 1.6438] | 1.7697 | 1.6191 |
|
| 17 |
+
| ncar_to_hidden_chart_oracle_at_8 | 105 | 1.1356 | [0.9502, 1.3391] | 0.9229 | 1.0992 |
|
| 18 |
+
| ncar_to_proposal_oracle_at_8 | 117 | 1.2924 | [0.5646, 2.4405] | 1.0740 | 1.4299 |
|
| 19 |
+
| outcome_ptr_at_8 | 144 | 0.3611 | [0.2847, 0.4444] | 0.2852 | 0.3632 |
|
| 20 |
+
| pairwise_causal_calibration_ece | 144 | 0.2051 | [0.1835, 0.2261] | 0.2436 | 0.2105 |
|
| 21 |
+
| proposal_oracle_safety_label_known_at_8 | 144 | 1.0000 | [1.0000, 1.0000] | 1.0000 | 1.0000 |
|
| 22 |
+
| proposal_oracle_success_at_8 | 144 | 0.2222 | [0.1597, 0.2917] | 0.2755 | 0.2229 |
|
| 23 |
+
| proposal_oracle_success_gain_over_base_at_8 | 144 | -0.1111 | [-0.1875, -0.0347] | -0.1814 | -0.0854 |
|
| 24 |
+
| proposal_oracle_unsafe_known_at_8 | 144 | 0.2847 | [0.2083, 0.3611] | 0.1093 | 0.2229 |
|
| 25 |
+
| proposal_oracle_utility_at_8 | 144 | 0.6232 | [0.5049, 0.7533] | 0.7913 | 0.6365 |
|
| 26 |
+
| proposal_oracle_utility_gain_over_base_at_8 | 144 | -0.2106 | [-0.3489, -0.0733] | -0.3000 | -0.1663 |
|
| 27 |
+
| selected_safety_label_known_at_8 | 144 | 1.0000 | [1.0000, 1.0000] | 1.0000 | 1.0000 |
|
| 28 |
+
| selected_success_at_8 | 144 | 0.1181 | [0.0694, 0.1736] | 0.0960 | 0.1104 |
|
| 29 |
+
| selected_success_gain_over_base_at_8 | 144 | -0.2153 | [-0.2847, -0.1458] | -0.3608 | -0.1979 |
|
| 30 |
+
| selected_unsafe_known_at_8 | 144 | 0.2778 | [0.2083, 0.3542] | 0.1067 | 0.2056 |
|
| 31 |
+
| selected_utility_at_8 | 144 | 0.4177 | [0.3282, 0.5208] | 0.4339 | 0.4161 |
|
| 32 |
+
| selected_utility_gain_over_base_at_8 | 144 | -0.4161 | [-0.5485, -0.2891] | -0.6574 | -0.3867 |
|
| 33 |
+
| selector_gap_fraction_to_hidden_at_8 | 105 | 0.2972 | [0.1927, 0.4117] | 0.3008 | 0.2764 |
|
| 34 |
+
| selector_regret_at_8 | 144 | 0.2055 | [0.1335, 0.2890] | 0.3574 | 0.2204 |
|
| 35 |
+
| success_selector_gap_at_8 | 144 | 0.1042 | [0.0556, 0.1597] | 0.1795 | 0.1125 |
|
| 36 |
+
| success_support_gap_at_8 | 144 | 0.5139 | [0.4375, 0.5972] | 0.6063 | 0.5708 |
|
| 37 |
+
| success_total_car_to_hidden_at_8 | 144 | 0.6111 | [0.5347, 0.6875] | 0.7747 | 0.6750 |
|
| 38 |
+
| support_gap_at_8 | 144 | 0.9091 | [0.7773, 1.0456] | 0.9988 | 0.9983 |
|
| 39 |
+
| support_gap_fraction_to_hidden_at_8 | 105 | 0.9311 | [0.7587, 1.1355] | 0.7042 | 0.9021 |
|
| 40 |
+
| total_car_to_hidden_at_8 | 144 | 1.1007 | [0.9704, 1.2309] | 1.3358 | 1.2031 |
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/measured_metrics/table.tex
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
% Auto-generated by scripts/eval_metrics.py
|
| 2 |
+
\begin{tabular}{lrrrr}
|
| 3 |
+
\toprule
|
| 4 |
+
Metric & N & Micro mean & CI low & CI high \\
|
| 5 |
+
\midrule
|
| 6 |
+
any\_generated\_unsafe\_known\_at\_8 & 144 & 0.3681 & 0.2847 & 0.4514 \\
|
| 7 |
+
base\_safety\_label\_known & 144 & 1.0000 & 1.0000 & 1.0000 \\
|
| 8 |
+
base\_success & 144 & 0.3333 & 0.2569 & 0.4097 \\
|
| 9 |
+
base\_unsafe\_known & 144 & 0.0000 & 0.0000 & 0.0000 \\
|
| 10 |
+
base\_utility & 144 & 0.8338 & 0.6953 & 0.9699 \\
|
| 11 |
+
branch\_car\_at\_8 & 144 & 0.2055 & 0.1335 & 0.2890 \\
|
| 12 |
+
generated\_safety\_label\_coverage\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
|
| 13 |
+
generated\_unsafe\_rate\_known\_at\_8 & 144 & 0.3030 & 0.2318 & 0.3707 \\
|
| 14 |
+
hidden\_chart\_oracle\_success\_at\_8 & 144 & 0.7292 & 0.6597 & 0.7986 \\
|
| 15 |
+
hidden\_chart\_oracle\_utility\_at\_8 & 144 & 1.5183 & 1.3941 & 1.6438 \\
|
| 16 |
+
ncar\_to\_hidden\_chart\_oracle\_at\_8 & 105 & 1.1356 & 0.9502 & 1.3391 \\
|
| 17 |
+
ncar\_to\_proposal\_oracle\_at\_8 & 117 & 1.2924 & 0.5646 & 2.4405 \\
|
| 18 |
+
outcome\_ptr\_at\_8 & 144 & 0.3611 & 0.2847 & 0.4444 \\
|
| 19 |
+
pairwise\_causal\_calibration\_ece & 144 & 0.2051 & 0.1835 & 0.2261 \\
|
| 20 |
+
proposal\_oracle\_safety\_label\_known\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
|
| 21 |
+
proposal\_oracle\_success\_at\_8 & 144 & 0.2222 & 0.1597 & 0.2917 \\
|
| 22 |
+
proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 144 & -0.1111 & -0.1875 & -0.0347 \\
|
| 23 |
+
proposal\_oracle\_unsafe\_known\_at\_8 & 144 & 0.2847 & 0.2083 & 0.3611 \\
|
| 24 |
+
proposal\_oracle\_utility\_at\_8 & 144 & 0.6232 & 0.5049 & 0.7533 \\
|
| 25 |
+
proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 144 & -0.2106 & -0.3489 & -0.0733 \\
|
| 26 |
+
selected\_safety\_label\_known\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
|
| 27 |
+
selected\_success\_at\_8 & 144 & 0.1181 & 0.0694 & 0.1736 \\
|
| 28 |
+
selected\_success\_gain\_over\_base\_at\_8 & 144 & -0.2153 & -0.2847 & -0.1458 \\
|
| 29 |
+
selected\_unsafe\_known\_at\_8 & 144 & 0.2778 & 0.2083 & 0.3542 \\
|
| 30 |
+
selected\_utility\_at\_8 & 144 & 0.4177 & 0.3282 & 0.5208 \\
|
| 31 |
+
selected\_utility\_gain\_over\_base\_at\_8 & 144 & -0.4161 & -0.5485 & -0.2891 \\
|
| 32 |
+
selector\_gap\_fraction\_to\_hidden\_at\_8 & 105 & 0.2972 & 0.1927 & 0.4117 \\
|
| 33 |
+
selector\_regret\_at\_8 & 144 & 0.2055 & 0.1335 & 0.2890 \\
|
| 34 |
+
success\_selector\_gap\_at\_8 & 144 & 0.1042 & 0.0556 & 0.1597 \\
|
| 35 |
+
success\_support\_gap\_at\_8 & 144 & 0.5139 & 0.4375 & 0.5972 \\
|
| 36 |
+
success\_total\_car\_to\_hidden\_at\_8 & 144 & 0.6111 & 0.5347 & 0.6875 \\
|
| 37 |
+
support\_gap\_at\_8 & 144 & 0.9091 & 0.7773 & 1.0456 \\
|
| 38 |
+
support\_gap\_fraction\_to\_hidden\_at\_8 & 105 & 0.9311 & 0.7587 & 1.1355 \\
|
| 39 |
+
total\_car\_to\_hidden\_at\_8 & 144 & 1.1007 & 0.9704 & 1.2309 \\
|
| 40 |
+
\bottomrule
|
| 41 |
+
\end{tabular}
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/metrics.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/metrics_by_seed.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/metrics_by_task.json
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"LiftPegUpright-v1": {
|
| 3 |
+
"any_generated_unsafe_known_at_8": 0.0,
|
| 4 |
+
"base_safety_label_known": 1.0,
|
| 5 |
+
"base_success": 0.0,
|
| 6 |
+
"base_unsafe_known": 0.0,
|
| 7 |
+
"base_utility": 0.4796089559793472,
|
| 8 |
+
"branch_car_at_8": 0.18885904252529145,
|
| 9 |
+
"generated_safety_label_coverage_at_8": 1.0,
|
| 10 |
+
"generated_unsafe_rate_known_at_8": 0.0,
|
| 11 |
+
"hidden_chart_oracle_success_at_8": 1.0,
|
| 12 |
+
"hidden_chart_oracle_utility_at_8": 2.0,
|
| 13 |
+
"ncar_to_hidden_chart_oracle_at_8": 1.0056207874968426,
|
| 14 |
+
"ncar_to_proposal_oracle_at_8": 0.6789057634923168,
|
| 15 |
+
"outcome_ptr_at_8": 0.3333333333333333,
|
| 16 |
+
"pairwise_causal_calibration_ece": 0.17461037477790295,
|
| 17 |
+
"proposal_oracle_safety_label_known_at_8": 1.0,
|
| 18 |
+
"proposal_oracle_success_at_8": 0.13333333333333333,
|
| 19 |
+
"proposal_oracle_success_gain_over_base_at_8": 0.13333333333333333,
|
| 20 |
+
"proposal_oracle_unsafe_known_at_8": 0.0,
|
| 21 |
+
"proposal_oracle_utility_at_8": 0.6601470321416855,
|
| 22 |
+
"proposal_oracle_utility_gain_over_base_at_8": 0.18053807616233825,
|
| 23 |
+
"selected_safety_label_known_at_8": 1.0,
|
| 24 |
+
"selected_success_at_8": 0.0,
|
| 25 |
+
"selected_success_gain_over_base_at_8": 0.0,
|
| 26 |
+
"selected_unsafe_known_at_8": 0.0,
|
| 27 |
+
"selected_utility_at_8": 0.47128798961639407,
|
| 28 |
+
"selected_utility_gain_over_base_at_8": -0.008320966362953186,
|
| 29 |
+
"selector_gap_fraction_to_hidden_at_8": 0.14638959816683447,
|
| 30 |
+
"selector_regret_at_8": 0.18885904252529145,
|
| 31 |
+
"success_selector_gap_at_8": 0.13333333333333333,
|
| 32 |
+
"success_support_gap_at_8": 0.8666666666666667,
|
| 33 |
+
"success_total_car_to_hidden_at_8": 1.0,
|
| 34 |
+
"support_gap_at_8": 1.3398529678583144,
|
| 35 |
+
"support_gap_fraction_to_hidden_at_8": 0.8592311961008738,
|
| 36 |
+
"total_car_to_hidden_at_8": 1.528712010383606
|
| 37 |
+
},
|
| 38 |
+
"PickCube-v1": {
|
| 39 |
+
"any_generated_unsafe_known_at_8": 0.0,
|
| 40 |
+
"base_safety_label_known": 1.0,
|
| 41 |
+
"base_success": 1.0,
|
| 42 |
+
"base_unsafe_known": 0.0,
|
| 43 |
+
"base_utility": 2.0,
|
| 44 |
+
"branch_car_at_8": 0.7539208928743998,
|
| 45 |
+
"generated_safety_label_coverage_at_8": 1.0,
|
| 46 |
+
"generated_unsafe_rate_known_at_8": 0.0,
|
| 47 |
+
"hidden_chart_oracle_success_at_8": 1.0,
|
| 48 |
+
"hidden_chart_oracle_utility_at_8": 2.0,
|
| 49 |
+
"ncar_to_proposal_oracle_at_8": 0.27456935655376624,
|
| 50 |
+
"outcome_ptr_at_8": 0.0,
|
| 51 |
+
"pairwise_causal_calibration_ece": 0.4028758622177973,
|
| 52 |
+
"proposal_oracle_safety_label_known_at_8": 1.0,
|
| 53 |
+
"proposal_oracle_success_at_8": 0.3333333333333333,
|
| 54 |
+
"proposal_oracle_success_gain_over_base_at_8": -0.6666666666666666,
|
| 55 |
+
"proposal_oracle_unsafe_known_at_8": 0.0,
|
| 56 |
+
"proposal_oracle_utility_at_8": 1.021850009759267,
|
| 57 |
+
"proposal_oracle_utility_gain_over_base_at_8": -0.9781499902407328,
|
| 58 |
+
"selected_safety_label_known_at_8": 1.0,
|
| 59 |
+
"selected_success_at_8": 0.0,
|
| 60 |
+
"selected_success_gain_over_base_at_8": -1.0,
|
| 61 |
+
"selected_unsafe_known_at_8": 0.0,
|
| 62 |
+
"selected_utility_at_8": 0.26792911688486737,
|
| 63 |
+
"selected_utility_gain_over_base_at_8": -1.7320708831151326,
|
| 64 |
+
"selector_regret_at_8": 0.7539208928743998,
|
| 65 |
+
"success_selector_gap_at_8": 0.3333333333333333,
|
| 66 |
+
"success_support_gap_at_8": 0.6666666666666666,
|
| 67 |
+
"success_total_car_to_hidden_at_8": 1.0,
|
| 68 |
+
"support_gap_at_8": 0.9781499902407328,
|
| 69 |
+
"total_car_to_hidden_at_8": 1.7320708831151326
|
| 70 |
+
},
|
| 71 |
+
"PullCube-v1": {
|
| 72 |
+
"any_generated_unsafe_known_at_8": 0.7066666666666667,
|
| 73 |
+
"base_safety_label_known": 1.0,
|
| 74 |
+
"base_success": 0.36,
|
| 75 |
+
"base_unsafe_known": 0.0,
|
| 76 |
+
"base_utility": 0.8032506120204925,
|
| 77 |
+
"branch_car_at_8": 0.06862825746337572,
|
| 78 |
+
"generated_safety_label_coverage_at_8": 1.0,
|
| 79 |
+
"generated_unsafe_rate_known_at_8": 0.5816666666666667,
|
| 80 |
+
"hidden_chart_oracle_success_at_8": 0.52,
|
| 81 |
+
"hidden_chart_oracle_utility_at_8": 1.1467878860235214,
|
| 82 |
+
"ncar_to_hidden_chart_oracle_at_8": 1.3075100566723548,
|
| 83 |
+
"ncar_to_proposal_oracle_at_8": 0.5092805557938541,
|
| 84 |
+
"outcome_ptr_at_8": 0.32,
|
| 85 |
+
"pairwise_causal_calibration_ece": 0.19854171059884482,
|
| 86 |
+
"proposal_oracle_safety_label_known_at_8": 1.0,
|
| 87 |
+
"proposal_oracle_success_at_8": 0.17333333333333334,
|
| 88 |
+
"proposal_oracle_success_gain_over_base_at_8": -0.18666666666666668,
|
| 89 |
+
"proposal_oracle_unsafe_known_at_8": 0.5466666666666666,
|
| 90 |
+
"proposal_oracle_utility_at_8": 0.46170220340291657,
|
| 91 |
+
"proposal_oracle_utility_gain_over_base_at_8": -0.34154840861757596,
|
| 92 |
+
"selected_safety_label_known_at_8": 1.0,
|
| 93 |
+
"selected_success_at_8": 0.14666666666666667,
|
| 94 |
+
"selected_success_gain_over_base_at_8": -0.21333333333333335,
|
| 95 |
+
"selected_unsafe_known_at_8": 0.5333333333333333,
|
| 96 |
+
"selected_utility_at_8": 0.39307394593954087,
|
| 97 |
+
"selected_utility_gain_over_base_at_8": -0.4101766660809517,
|
| 98 |
+
"selector_gap_fraction_to_hidden_at_8": 0.3721528711155151,
|
| 99 |
+
"selector_regret_at_8": 0.06862825746337572,
|
| 100 |
+
"success_selector_gap_at_8": 0.02666666666666667,
|
| 101 |
+
"success_support_gap_at_8": 0.3466666666666667,
|
| 102 |
+
"success_total_car_to_hidden_at_8": 0.37333333333333335,
|
| 103 |
+
"support_gap_at_8": 0.6881442584097386,
|
| 104 |
+
"support_gap_fraction_to_hidden_at_8": 1.0971174661984022,
|
| 105 |
+
"total_car_to_hidden_at_8": 0.7538664453228314
|
| 106 |
+
},
|
| 107 |
+
"PushCube-v1": {
|
| 108 |
+
"any_generated_unsafe_known_at_8": 0.0,
|
| 109 |
+
"base_safety_label_known": 1.0,
|
| 110 |
+
"base_success": 0.8333333333333334,
|
| 111 |
+
"base_unsafe_known": 0.0,
|
| 112 |
+
"base_utility": 1.6734557673335075,
|
| 113 |
+
"branch_car_at_8": 0.41695255951748955,
|
| 114 |
+
"generated_safety_label_coverage_at_8": 1.0,
|
| 115 |
+
"generated_unsafe_rate_known_at_8": 0.0,
|
| 116 |
+
"hidden_chart_oracle_success_at_8": 0.8333333333333334,
|
| 117 |
+
"hidden_chart_oracle_utility_at_8": 1.7015800600250561,
|
| 118 |
+
"ncar_to_hidden_chart_oracle_at_8": 0.3255141393427919,
|
| 119 |
+
"ncar_to_proposal_oracle_at_8": 0.17843067932762913,
|
| 120 |
+
"outcome_ptr_at_8": 0.16666666666666666,
|
| 121 |
+
"pairwise_causal_calibration_ece": 0.25042093435733487,
|
| 122 |
+
"proposal_oracle_safety_label_known_at_8": 1.0,
|
| 123 |
+
"proposal_oracle_success_at_8": 0.5555555555555556,
|
| 124 |
+
"proposal_oracle_success_gain_over_base_at_8": -0.2777777777777778,
|
| 125 |
+
"proposal_oracle_unsafe_known_at_8": 0.0,
|
| 126 |
+
"proposal_oracle_utility_at_8": 1.1790093589160178,
|
| 127 |
+
"proposal_oracle_utility_gain_over_base_at_8": -0.49444640841748977,
|
| 128 |
+
"selected_safety_label_known_at_8": 1.0,
|
| 129 |
+
"selected_success_at_8": 0.3333333333333333,
|
| 130 |
+
"selected_success_gain_over_base_at_8": -0.5,
|
| 131 |
+
"selected_unsafe_known_at_8": 0.0,
|
| 132 |
+
"selected_utility_at_8": 0.7620567993985282,
|
| 133 |
+
"selected_utility_gain_over_base_at_8": -0.9113989679349793,
|
| 134 |
+
"selector_gap_fraction_to_hidden_at_8": 0.48464542735161803,
|
| 135 |
+
"selector_regret_at_8": 0.41695255951748955,
|
| 136 |
+
"success_selector_gap_at_8": 0.2222222222222222,
|
| 137 |
+
"success_support_gap_at_8": 0.3333333333333333,
|
| 138 |
+
"success_total_car_to_hidden_at_8": 0.5,
|
| 139 |
+
"support_gap_at_8": 0.6220440144340197,
|
| 140 |
+
"support_gap_fraction_to_hidden_at_8": 0.007535380326794263,
|
| 141 |
+
"total_car_to_hidden_at_8": 0.939523260626528
|
| 142 |
+
},
|
| 143 |
+
"StackCube-v1": {
|
| 144 |
+
"any_generated_unsafe_known_at_8": 0.0,
|
| 145 |
+
"base_safety_label_known": 1.0,
|
| 146 |
+
"base_success": 0.09090909090909091,
|
| 147 |
+
"base_unsafe_known": 0.0,
|
| 148 |
+
"base_utility": 0.5003613992170854,
|
| 149 |
+
"branch_car_at_8": 0.3588122549382123,
|
| 150 |
+
"generated_safety_label_coverage_at_8": 1.0,
|
| 151 |
+
"generated_unsafe_rate_known_at_8": 0.0,
|
| 152 |
+
"hidden_chart_oracle_success_at_8": 1.0,
|
| 153 |
+
"hidden_chart_oracle_utility_at_8": 2.0,
|
| 154 |
+
"ncar_to_hidden_chart_oracle_at_8": 1.0530175208494181,
|
| 155 |
+
"ncar_to_proposal_oracle_at_8": 3.7287207392774975,
|
| 156 |
+
"outcome_ptr_at_8": 0.6060606060606061,
|
| 157 |
+
"pairwise_causal_calibration_ece": 0.1913742697792418,
|
| 158 |
+
"proposal_oracle_safety_label_known_at_8": 1.0,
|
| 159 |
+
"proposal_oracle_success_at_8": 0.18181818181818182,
|
| 160 |
+
"proposal_oracle_success_gain_over_base_at_8": 0.09090909090909091,
|
| 161 |
+
"proposal_oracle_unsafe_known_at_8": 0.0,
|
| 162 |
+
"proposal_oracle_utility_at_8": 0.634040683056369,
|
| 163 |
+
"proposal_oracle_utility_gain_over_base_at_8": 0.13367928383928357,
|
| 164 |
+
"selected_safety_label_known_at_8": 1.0,
|
| 165 |
+
"selected_success_at_8": 0.0,
|
| 166 |
+
"selected_success_gain_over_base_at_8": -0.09090909090909091,
|
| 167 |
+
"selected_unsafe_known_at_8": 0.0,
|
| 168 |
+
"selected_utility_at_8": 0.2752284281181567,
|
| 169 |
+
"selected_utility_gain_over_base_at_8": -0.22513297109892874,
|
| 170 |
+
"selector_gap_fraction_to_hidden_at_8": 0.2000703650638142,
|
| 171 |
+
"selector_regret_at_8": 0.3588122549382123,
|
| 172 |
+
"success_selector_gap_at_8": 0.18181818181818182,
|
| 173 |
+
"success_support_gap_at_8": 0.8181818181818182,
|
| 174 |
+
"success_total_car_to_hidden_at_8": 1.0,
|
| 175 |
+
"support_gap_at_8": 1.365959316943631,
|
| 176 |
+
"support_gap_fraction_to_hidden_at_8": 0.8529471625482842,
|
| 177 |
+
"total_car_to_hidden_at_8": 1.7247715718818433
|
| 178 |
+
}
|
| 179 |
+
}
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/report.md
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CTT Test Measured Rollout Comparison
|
| 2 |
+
|
| 3 |
+
Runs: `3`
|
| 4 |
+
Rows: `144`
|
| 5 |
+
K: `8`
|
| 6 |
+
|
| 7 |
+
| Metric | N | Micro mean | 95% CI |
|
| 8 |
+
| --- | ---: | ---: | ---: |
|
| 9 |
+
| any_generated_unsafe_known_at_8 | 144 | 0.3681 | [0.2847, 0.4514] |
|
| 10 |
+
| base_safety_label_known | 144 | 1.0000 | [1.0000, 1.0000] |
|
| 11 |
+
| base_success | 144 | 0.3333 | [0.2569, 0.4097] |
|
| 12 |
+
| base_unsafe_known | 144 | 0.0000 | [0.0000, 0.0000] |
|
| 13 |
+
| base_utility | 144 | 0.8338 | [0.6953, 0.9699] |
|
| 14 |
+
| branch_car_at_8 | 144 | 0.2055 | [0.1335, 0.2890] |
|
| 15 |
+
| generated_safety_label_coverage_at_8 | 144 | 1.0000 | [1.0000, 1.0000] |
|
| 16 |
+
| generated_unsafe_rate_known_at_8 | 144 | 0.3030 | [0.2318, 0.3707] |
|
| 17 |
+
| hidden_chart_oracle_success_at_8 | 144 | 0.7292 | [0.6597, 0.7986] |
|
| 18 |
+
| hidden_chart_oracle_utility_at_8 | 144 | 1.5183 | [1.3941, 1.6438] |
|
| 19 |
+
| ncar_to_hidden_chart_oracle_at_8 | 105 | 1.1356 | [0.9502, 1.3391] |
|
| 20 |
+
| ncar_to_proposal_oracle_at_8 | 117 | 1.2924 | [0.5646, 2.4405] |
|
| 21 |
+
| outcome_ptr_at_8 | 144 | 0.3611 | [0.2847, 0.4444] |
|
| 22 |
+
| pairwise_causal_calibration_ece | 144 | 0.2051 | [0.1835, 0.2261] |
|
| 23 |
+
| proposal_oracle_safety_label_known_at_8 | 144 | 1.0000 | [1.0000, 1.0000] |
|
| 24 |
+
| proposal_oracle_success_at_8 | 144 | 0.2222 | [0.1597, 0.2917] |
|
| 25 |
+
| proposal_oracle_success_gain_over_base_at_8 | 144 | -0.1111 | [-0.1875, -0.0347] |
|
| 26 |
+
| proposal_oracle_unsafe_known_at_8 | 144 | 0.2847 | [0.2083, 0.3611] |
|
| 27 |
+
| proposal_oracle_utility_at_8 | 144 | 0.6232 | [0.5049, 0.7533] |
|
| 28 |
+
| proposal_oracle_utility_gain_over_base_at_8 | 144 | -0.2106 | [-0.3489, -0.0733] |
|
| 29 |
+
| selected_safety_label_known_at_8 | 144 | 1.0000 | [1.0000, 1.0000] |
|
| 30 |
+
| selected_success_at_8 | 144 | 0.1181 | [0.0694, 0.1736] |
|
| 31 |
+
| selected_success_gain_over_base_at_8 | 144 | -0.2153 | [-0.2847, -0.1458] |
|
| 32 |
+
| selected_unsafe_known_at_8 | 144 | 0.2778 | [0.2083, 0.3542] |
|
| 33 |
+
| selected_utility_at_8 | 144 | 0.4177 | [0.3282, 0.5208] |
|
| 34 |
+
| selected_utility_gain_over_base_at_8 | 144 | -0.4161 | [-0.5485, -0.2891] |
|
| 35 |
+
| selector_gap_fraction_to_hidden_at_8 | 105 | 0.2972 | [0.1927, 0.4117] |
|
| 36 |
+
| selector_regret_at_8 | 144 | 0.2055 | [0.1335, 0.2890] |
|
| 37 |
+
| success_selector_gap_at_8 | 144 | 0.1042 | [0.0556, 0.1597] |
|
| 38 |
+
| success_support_gap_at_8 | 144 | 0.5139 | [0.4375, 0.5972] |
|
| 39 |
+
| success_total_car_to_hidden_at_8 | 144 | 0.6111 | [0.5347, 0.6875] |
|
| 40 |
+
| support_gap_at_8 | 144 | 0.9091 | [0.7773, 1.0456] |
|
| 41 |
+
| support_gap_fraction_to_hidden_at_8 | 105 | 0.9311 | [0.7587, 1.1355] |
|
| 42 |
+
| total_car_to_hidden_at_8 | 144 | 1.1007 | [0.9704, 1.2309] |
|
| 43 |
+
|
| 44 |
+
| Success/Utility | Mean |
|
| 45 |
+
| --- | ---: |
|
| 46 |
+
| base_success_rate | 0.3333 |
|
| 47 |
+
| selected_success_rate | 0.1181 |
|
| 48 |
+
| proposal_oracle_success_rate | 0.2222 |
|
| 49 |
+
| hidden_chart_oracle_success_rate | 0.7292 |
|
| 50 |
+
| selected_success_gain_over_base | -0.2153 |
|
| 51 |
+
| proposal_oracle_success_gain_over_base | -0.1111 |
|
| 52 |
+
| success_support_gap | 0.5139 |
|
| 53 |
+
| success_selector_gap | 0.1042 |
|
| 54 |
+
| base_utility_mean | 0.8338 |
|
| 55 |
+
| selected_utility_mean | 0.4177 |
|
| 56 |
+
| proposal_oracle_utility_mean | 0.6232 |
|
| 57 |
+
| hidden_chart_oracle_utility_mean | 1.5183 |
|
| 58 |
+
| ncar_to_proposal_oracle | 1.2924 |
|
| 59 |
+
| ncar_to_hidden_chart_oracle | 1.1356 |
|
| 60 |
+
| utility_support_gap_fraction_to_hidden | 0.9311 |
|
| 61 |
+
| utility_selector_gap_fraction_to_hidden | 0.2972 |
|
| 62 |
+
| generated_safety_label_coverage | 1.0000 |
|
| 63 |
+
| generated_unsafe_rate_known | 0.3030 |
|
| 64 |
+
| any_generated_unsafe_known | 0.3681 |
|
| 65 |
+
| selected_safety_label_known_rate | 1.0000 |
|
| 66 |
+
| selected_unsafe_rate_known | 0.2778 |
|
| 67 |
+
| proposal_oracle_safety_label_known_rate | 1.0000 |
|
| 68 |
+
| proposal_oracle_unsafe_rate_known | 0.2847 |
|
| 69 |
+
| base_safety_label_known_rate | 1.0000 |
|
| 70 |
+
| base_unsafe_rate_known | 0.0000 |
|
| 71 |
+
|
| 72 |
+
These are measured generated-candidate rollouts, not PPTC proxies.
|
| 73 |
+
|
| 74 |
+
Run dirs:
|
| 75 |
+
- `runs/ctt_residual_base_context_obs_rollout_test_perdim_trainmax_seed0`
|
| 76 |
+
- `runs/ctt_residual_base_context_obs_rollout_test_perdim_trainmax_seed1`
|
| 77 |
+
- `runs/ctt_residual_base_context_obs_rollout_test_perdim_trainmax_seed2`
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/source_data_hash.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/split_hash.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/table.tex
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
% Auto-generated by scripts/eval_metrics.py
|
| 2 |
+
\begin{tabular}{lrrrr}
|
| 3 |
+
\toprule
|
| 4 |
+
Metric & N & Micro mean & CI low & CI high \\
|
| 5 |
+
\midrule
|
| 6 |
+
any\_generated\_unsafe\_known\_at\_8 & 144 & 0.3681 & 0.2847 & 0.4514 \\
|
| 7 |
+
base\_safety\_label\_known & 144 & 1.0000 & 1.0000 & 1.0000 \\
|
| 8 |
+
base\_success & 144 & 0.3333 & 0.2569 & 0.4097 \\
|
| 9 |
+
base\_unsafe\_known & 144 & 0.0000 & 0.0000 & 0.0000 \\
|
| 10 |
+
base\_utility & 144 & 0.8338 & 0.6953 & 0.9699 \\
|
| 11 |
+
branch\_car\_at\_8 & 144 & 0.2055 & 0.1335 & 0.2890 \\
|
| 12 |
+
generated\_safety\_label\_coverage\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
|
| 13 |
+
generated\_unsafe\_rate\_known\_at\_8 & 144 & 0.3030 & 0.2318 & 0.3707 \\
|
| 14 |
+
hidden\_chart\_oracle\_success\_at\_8 & 144 & 0.7292 & 0.6597 & 0.7986 \\
|
| 15 |
+
hidden\_chart\_oracle\_utility\_at\_8 & 144 & 1.5183 & 1.3941 & 1.6438 \\
|
| 16 |
+
ncar\_to\_hidden\_chart\_oracle\_at\_8 & 105 & 1.1356 & 0.9502 & 1.3391 \\
|
| 17 |
+
ncar\_to\_proposal\_oracle\_at\_8 & 117 & 1.2924 & 0.5646 & 2.4405 \\
|
| 18 |
+
outcome\_ptr\_at\_8 & 144 & 0.3611 & 0.2847 & 0.4444 \\
|
| 19 |
+
pairwise\_causal\_calibration\_ece & 144 & 0.2051 & 0.1835 & 0.2261 \\
|
| 20 |
+
proposal\_oracle\_safety\_label\_known\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
|
| 21 |
+
proposal\_oracle\_success\_at\_8 & 144 & 0.2222 & 0.1597 & 0.2917 \\
|
| 22 |
+
proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 144 & -0.1111 & -0.1875 & -0.0347 \\
|
| 23 |
+
proposal\_oracle\_unsafe\_known\_at\_8 & 144 & 0.2847 & 0.2083 & 0.3611 \\
|
| 24 |
+
proposal\_oracle\_utility\_at\_8 & 144 & 0.6232 & 0.5049 & 0.7533 \\
|
| 25 |
+
proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 144 & -0.2106 & -0.3489 & -0.0733 \\
|
| 26 |
+
selected\_safety\_label\_known\_at\_8 & 144 & 1.0000 & 1.0000 & 1.0000 \\
|
| 27 |
+
selected\_success\_at\_8 & 144 & 0.1181 & 0.0694 & 0.1736 \\
|
| 28 |
+
selected\_success\_gain\_over\_base\_at\_8 & 144 & -0.2153 & -0.2847 & -0.1458 \\
|
| 29 |
+
selected\_unsafe\_known\_at\_8 & 144 & 0.2778 & 0.2083 & 0.3542 \\
|
| 30 |
+
selected\_utility\_at\_8 & 144 & 0.4177 & 0.3282 & 0.5208 \\
|
| 31 |
+
selected\_utility\_gain\_over\_base\_at\_8 & 144 & -0.4161 & -0.5485 & -0.2891 \\
|
| 32 |
+
selector\_gap\_fraction\_to\_hidden\_at\_8 & 105 & 0.2972 & 0.1927 & 0.4117 \\
|
| 33 |
+
selector\_regret\_at\_8 & 144 & 0.2055 & 0.1335 & 0.2890 \\
|
| 34 |
+
success\_selector\_gap\_at\_8 & 144 & 0.1042 & 0.0556 & 0.1597 \\
|
| 35 |
+
success\_support\_gap\_at\_8 & 144 & 0.5139 & 0.4375 & 0.5972 \\
|
| 36 |
+
success\_total\_car\_to\_hidden\_at\_8 & 144 & 0.6111 & 0.5347 & 0.6875 \\
|
| 37 |
+
support\_gap\_at\_8 & 144 & 0.9091 & 0.7773 & 1.0456 \\
|
| 38 |
+
support\_gap\_fraction\_to\_hidden\_at\_8 & 105 & 0.9311 & 0.7587 & 1.1355 \\
|
| 39 |
+
total\_car\_to\_hidden\_at\_8 & 144 & 1.1007 & 0.9704 & 1.2309 \\
|
| 40 |
+
\bottomrule
|
| 41 |
+
\end{tabular}
|
workspace/runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/train.log
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
comparison artifact; source runs trained/evaluated separately
|