auto-sync 2026-07-02T17:27:17Z workspace (part 3)
Browse files
workspace/results/paper_analysis.json
CHANGED
|
@@ -1,6 +1,29 @@
|
|
| 1 |
{
|
|
|
|
| 2 |
"best_clean_key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005",
|
| 3 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
"mechanism_gap": {
|
| 5 |
"best_clean_vs_direct_same_ckpt": 0.1060869565217391,
|
| 6 |
"best_clean_vs_h16": 0.09159420289855075,
|
|
@@ -3724,6 +3747,8 @@
|
|
| 3724 |
},
|
| 3725 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8": {
|
| 3726 |
"candidate_oracle_rollouts": 8,
|
|
|
|
|
|
|
| 3727 |
"candidate_oracle_type_counts": {
|
| 3728 |
"retrieval_residual_policy_residual": 867,
|
| 3729 |
"retrieval_residual_residual_near_miss": 201,
|
|
@@ -3745,6 +3770,8 @@
|
|
| 3745 |
"mean_candidate_oracle_improvement_rate": 0.4602898550724637,
|
| 3746 |
"mean_candidate_oracle_progress": 0.6443418554713328,
|
| 3747 |
"mean_candidate_oracle_score_gain_over_selected": 0.16034898876064066,
|
|
|
|
|
|
|
| 3748 |
"mean_candidate_oracle_success_rate": 0.4307246376811594,
|
| 3749 |
"mean_candidate_oracle_unique_count": NaN,
|
| 3750 |
"mean_progress": 0.571575072284851,
|
|
@@ -3847,6 +3874,7 @@
|
|
| 3847 |
"std_success": 0.012173913043478276
|
| 3848 |
},
|
| 3849 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace": {
|
|
|
|
| 3850 |
"candidate_oracle_best_branch_rank_counts": {
|
| 3851 |
"1": 931,
|
| 3852 |
"2": 162,
|
|
@@ -3858,6 +3886,8 @@
|
|
| 3858 |
"8": 149
|
| 3859 |
},
|
| 3860 |
"candidate_oracle_rollouts": 8,
|
|
|
|
|
|
|
| 3861 |
"candidate_oracle_type_counts": {
|
| 3862 |
"retrieval_residual_policy_residual": 867,
|
| 3863 |
"retrieval_residual_residual_near_miss": 201,
|
|
@@ -3907,9 +3937,15 @@
|
|
| 3907 |
0.2336231884057971,
|
| 3908 |
0.25043478260869567
|
| 3909 |
],
|
|
|
|
| 3910 |
"mean_candidate_oracle_improvement_rate": 0.4602898550724637,
|
|
|
|
| 3911 |
"mean_candidate_oracle_progress": 0.6443418554713328,
|
|
|
|
| 3912 |
"mean_candidate_oracle_score_gain_over_selected": 0.16034898876064066,
|
|
|
|
|
|
|
|
|
|
| 3913 |
"mean_candidate_oracle_success_rate": 0.4307246376811594,
|
| 3914 |
"mean_candidate_oracle_unique_count": 8.0,
|
| 3915 |
"mean_progress": 0.571575072284851,
|
|
@@ -11093,6 +11129,7 @@
|
|
| 11093 |
"std_success": 0.017506862458598824
|
| 11094 |
},
|
| 11095 |
"transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_oraclek8": {
|
|
|
|
| 11096 |
"candidate_oracle_best_branch_rank_counts": {
|
| 11097 |
"1": 985,
|
| 11098 |
"2": 154,
|
|
@@ -11104,6 +11141,8 @@
|
|
| 11104 |
"8": 78
|
| 11105 |
},
|
| 11106 |
"candidate_oracle_rollouts": 8,
|
|
|
|
|
|
|
| 11107 |
"candidate_oracle_type_counts": {
|
| 11108 |
"retrieval_residual_policy_residual": 468,
|
| 11109 |
"retrieval_residual_residual_near_miss": 256,
|
|
@@ -11152,9 +11191,15 @@
|
|
| 11152 |
0.2533333333333333,
|
| 11153 |
0.23099631368984497
|
| 11154 |
],
|
|
|
|
| 11155 |
"mean_candidate_oracle_improvement_rate": 0.4289855072463768,
|
|
|
|
| 11156 |
"mean_candidate_oracle_progress": 0.6426025666080523,
|
|
|
|
| 11157 |
"mean_candidate_oracle_score_gain_over_selected": 0.10811787949955981,
|
|
|
|
|
|
|
|
|
|
| 11158 |
"mean_candidate_oracle_success_rate": 0.44347826086956516,
|
| 11159 |
"mean_candidate_oracle_unique_count": 7.998840579710145,
|
| 11160 |
"mean_progress": 0.5992048837379486,
|
|
@@ -11923,6 +11968,7 @@
|
|
| 11923 |
"std_success": 0.012215250727945203
|
| 11924 |
},
|
| 11925 |
"transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8": {
|
|
|
|
| 11926 |
"candidate_oracle_best_branch_rank_counts": {
|
| 11927 |
"1": 984,
|
| 11928 |
"2": 156,
|
|
@@ -11934,6 +11980,8 @@
|
|
| 11934 |
"8": 79
|
| 11935 |
},
|
| 11936 |
"candidate_oracle_rollouts": 8,
|
|
|
|
|
|
|
| 11937 |
"candidate_oracle_type_counts": {
|
| 11938 |
"retrieval_residual_policy_residual": 445,
|
| 11939 |
"retrieval_residual_residual_near_miss": 273,
|
|
@@ -11982,9 +12030,15 @@
|
|
| 11982 |
0.2539130434782609,
|
| 11983 |
0.23215674392768773
|
| 11984 |
],
|
|
|
|
| 11985 |
"mean_candidate_oracle_improvement_rate": 0.42956521739130443,
|
|
|
|
| 11986 |
"mean_candidate_oracle_progress": 0.6425710423981797,
|
|
|
|
| 11987 |
"mean_candidate_oracle_score_gain_over_selected": 0.10723679356600928,
|
|
|
|
|
|
|
|
|
|
| 11988 |
"mean_candidate_oracle_success_rate": 0.44347826086956516,
|
| 11989 |
"mean_candidate_oracle_unique_count": 7.998840579710145,
|
| 11990 |
"mean_progress": 0.5994939735267257,
|
|
@@ -14013,6 +14067,7 @@
|
|
| 14013 |
"std_success": 0.017650246200160362
|
| 14014 |
},
|
| 14015 |
"transport_field_reground_fieldonly_k6matched_b12_clean_k6_oraclek8": {
|
|
|
|
| 14016 |
"candidate_oracle_best_branch_rank_counts": {
|
| 14017 |
"1": 937,
|
| 14018 |
"2": 150,
|
|
@@ -14024,6 +14079,8 @@
|
|
| 14024 |
"8": 86
|
| 14025 |
},
|
| 14026 |
"candidate_oracle_rollouts": 8,
|
|
|
|
|
|
|
| 14027 |
"candidate_oracle_type_counts": {
|
| 14028 |
"retrieval_residual_policy_residual": 440,
|
| 14029 |
"retrieval_residual_residual_near_miss": 256,
|
|
@@ -14073,9 +14130,15 @@
|
|
| 14073 |
0.2515942028985507,
|
| 14074 |
0.2289855072463768
|
| 14075 |
],
|
|
|
|
| 14076 |
"mean_candidate_oracle_improvement_rate": 0.4568115942028985,
|
|
|
|
| 14077 |
"mean_candidate_oracle_progress": 0.6483499062207082,
|
|
|
|
| 14078 |
"mean_candidate_oracle_score_gain_over_selected": 0.11794837779432965,
|
|
|
|
|
|
|
|
|
|
| 14079 |
"mean_candidate_oracle_success_rate": 0.4428985507246377,
|
| 14080 |
"mean_candidate_oracle_unique_count": 8.0,
|
| 14081 |
"mean_progress": 0.5973945180108495,
|
|
|
|
| 1 |
{
|
| 2 |
+
"best_candidate_oracle_key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
|
| 3 |
"best_clean_key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005",
|
| 4 |
+
"causal_action_decomposition": {
|
| 5 |
+
"base": 0.29739130434782607,
|
| 6 |
+
"clean_gain": 0.09159420289855075,
|
| 7 |
+
"closed_fraction_of_noexpert_gap": 0.3361702127659574,
|
| 8 |
+
"full_oracle": 0.6933333333333334,
|
| 9 |
+
"full_oracle_gap": 0.30434782608695654,
|
| 10 |
+
"proposal_oracle": 0.44347826086956516,
|
| 11 |
+
"same_state_gain_over_base": 0.27246376811594214,
|
| 12 |
+
"same_state_oracle": 0.5698550724637682,
|
| 13 |
+
"selected": 0.3889855072463768,
|
| 14 |
+
"selector_gap": 0.05449275362318834,
|
| 15 |
+
"support_gap": 0.12637681159420305,
|
| 16 |
+
"total_gap_to_same_state": 0.1808695652173914
|
| 17 |
+
},
|
| 18 |
+
"causal_action_targets": {
|
| 19 |
+
"best_paper_target_range": [
|
| 20 |
+
0.47,
|
| 21 |
+
0.52
|
| 22 |
+
],
|
| 23 |
+
"selected_success_for_65pct_gap_closure": 0.47449275362318843,
|
| 24 |
+
"selected_success_for_75pct_gap_closure": 0.5017391304347827
|
| 25 |
+
},
|
| 26 |
+
"generated_utc": "2026-07-02T17:31:38+00:00",
|
| 27 |
"mechanism_gap": {
|
| 28 |
"best_clean_vs_direct_same_ckpt": 0.1060869565217391,
|
| 29 |
"best_clean_vs_h16": 0.09159420289855075,
|
|
|
|
| 3747 |
},
|
| 3748 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8": {
|
| 3749 |
"candidate_oracle_rollouts": 8,
|
| 3750 |
+
"candidate_oracle_selector_gap_progress": 0.07803014863367896,
|
| 3751 |
+
"candidate_oracle_selector_gap_success": 0.08231884057971006,
|
| 3752 |
"candidate_oracle_type_counts": {
|
| 3753 |
"retrieval_residual_policy_residual": 867,
|
| 3754 |
"retrieval_residual_residual_near_miss": 201,
|
|
|
|
| 3770 |
"mean_candidate_oracle_improvement_rate": 0.4602898550724637,
|
| 3771 |
"mean_candidate_oracle_progress": 0.6443418554713328,
|
| 3772 |
"mean_candidate_oracle_score_gain_over_selected": 0.16034898876064066,
|
| 3773 |
+
"mean_candidate_oracle_selected_branch_progress": 0.5663117068376539,
|
| 3774 |
+
"mean_candidate_oracle_selected_branch_success_rate": 0.3484057971014493,
|
| 3775 |
"mean_candidate_oracle_success_rate": 0.4307246376811594,
|
| 3776 |
"mean_candidate_oracle_unique_count": NaN,
|
| 3777 |
"mean_progress": 0.571575072284851,
|
|
|
|
| 3874 |
"std_success": 0.012173913043478276
|
| 3875 |
},
|
| 3876 |
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace": {
|
| 3877 |
+
"candidate_oracle_base_trace_coverage": 0.9768115942028985,
|
| 3878 |
"candidate_oracle_best_branch_rank_counts": {
|
| 3879 |
"1": 931,
|
| 3880 |
"2": 162,
|
|
|
|
| 3886 |
"8": 149
|
| 3887 |
},
|
| 3888 |
"candidate_oracle_rollouts": 8,
|
| 3889 |
+
"candidate_oracle_selector_gap_progress": 0.07803014863367896,
|
| 3890 |
+
"candidate_oracle_selector_gap_success": 0.08231884057971006,
|
| 3891 |
"candidate_oracle_type_counts": {
|
| 3892 |
"retrieval_residual_policy_residual": 867,
|
| 3893 |
"retrieval_residual_residual_near_miss": 201,
|
|
|
|
| 3937 |
0.2336231884057971,
|
| 3938 |
0.25043478260869567
|
| 3939 |
],
|
| 3940 |
+
"mean_candidate_oracle_car_to_proposal_oracle": 0.16034898921338922,
|
| 3941 |
"mean_candidate_oracle_improvement_rate": 0.4602898550724637,
|
| 3942 |
+
"mean_candidate_oracle_ncar_to_proposal_oracle": 0.9816454565106858,
|
| 3943 |
"mean_candidate_oracle_progress": 0.6443418554713328,
|
| 3944 |
+
"mean_candidate_oracle_ptr_at_k": 0.4712166172106825,
|
| 3945 |
"mean_candidate_oracle_score_gain_over_selected": 0.16034898876064066,
|
| 3946 |
+
"mean_candidate_oracle_selected_branch_progress": 0.5663117068376539,
|
| 3947 |
+
"mean_candidate_oracle_selected_branch_success_rate": 0.3484057971014493,
|
| 3948 |
+
"mean_candidate_oracle_selector_regret_at_k": 0.16034898921338922,
|
| 3949 |
"mean_candidate_oracle_success_rate": 0.4307246376811594,
|
| 3950 |
"mean_candidate_oracle_unique_count": 8.0,
|
| 3951 |
"mean_progress": 0.571575072284851,
|
|
|
|
| 11129 |
"std_success": 0.017506862458598824
|
| 11130 |
},
|
| 11131 |
"transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_oraclek8": {
|
| 11132 |
+
"candidate_oracle_base_trace_coverage": 0.8515942028985507,
|
| 11133 |
"candidate_oracle_best_branch_rank_counts": {
|
| 11134 |
"1": 985,
|
| 11135 |
"2": 154,
|
|
|
|
| 11141 |
"8": 78
|
| 11142 |
},
|
| 11143 |
"candidate_oracle_rollouts": 8,
|
| 11144 |
+
"candidate_oracle_selector_gap_progress": 0.04840773510598173,
|
| 11145 |
+
"candidate_oracle_selector_gap_success": 0.05971014492753618,
|
| 11146 |
"candidate_oracle_type_counts": {
|
| 11147 |
"retrieval_residual_policy_residual": 468,
|
| 11148 |
"retrieval_residual_residual_near_miss": 256,
|
|
|
|
| 11191 |
0.2533333333333333,
|
| 11192 |
0.23099631368984497
|
| 11193 |
],
|
| 11194 |
+
"mean_candidate_oracle_car_to_proposal_oracle": 0.10811788003351809,
|
| 11195 |
"mean_candidate_oracle_improvement_rate": 0.4289855072463768,
|
| 11196 |
+
"mean_candidate_oracle_ncar_to_proposal_oracle": 1.2157991567169883,
|
| 11197 |
"mean_candidate_oracle_progress": 0.6426025666080523,
|
| 11198 |
+
"mean_candidate_oracle_ptr_at_k": 0.4758339006126617,
|
| 11199 |
"mean_candidate_oracle_score_gain_over_selected": 0.10811787949955981,
|
| 11200 |
+
"mean_candidate_oracle_selected_branch_progress": 0.5941948315020705,
|
| 11201 |
+
"mean_candidate_oracle_selected_branch_success_rate": 0.383768115942029,
|
| 11202 |
+
"mean_candidate_oracle_selector_regret_at_k": 0.10811788003351809,
|
| 11203 |
"mean_candidate_oracle_success_rate": 0.44347826086956516,
|
| 11204 |
"mean_candidate_oracle_unique_count": 7.998840579710145,
|
| 11205 |
"mean_progress": 0.5992048837379486,
|
|
|
|
| 11968 |
"std_success": 0.012215250727945203
|
| 11969 |
},
|
| 11970 |
"transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8": {
|
| 11971 |
+
"candidate_oracle_base_trace_coverage": 0.8121739130434783,
|
| 11972 |
"candidate_oracle_best_branch_rank_counts": {
|
| 11973 |
"1": 984,
|
| 11974 |
"2": 156,
|
|
|
|
| 11980 |
"8": 79
|
| 11981 |
},
|
| 11982 |
"candidate_oracle_rollouts": 8,
|
| 11983 |
+
"candidate_oracle_selector_gap_progress": 0.04810635927092766,
|
| 11984 |
+
"candidate_oracle_selector_gap_success": 0.05913043478260871,
|
| 11985 |
"candidate_oracle_type_counts": {
|
| 11986 |
"retrieval_residual_policy_residual": 445,
|
| 11987 |
"retrieval_residual_residual_near_miss": 273,
|
|
|
|
| 12030 |
0.2539130434782609,
|
| 12031 |
0.23215674392768773
|
| 12032 |
],
|
| 12033 |
+
"mean_candidate_oracle_car_to_proposal_oracle": 0.1072367940535364,
|
| 12034 |
"mean_candidate_oracle_improvement_rate": 0.42956521739130443,
|
| 12035 |
+
"mean_candidate_oracle_ncar_to_proposal_oracle": 1.2488320610772363,
|
| 12036 |
"mean_candidate_oracle_progress": 0.6425710423981797,
|
| 12037 |
+
"mean_candidate_oracle_ptr_at_k": 0.4732334047109208,
|
| 12038 |
"mean_candidate_oracle_score_gain_over_selected": 0.10723679356600928,
|
| 12039 |
+
"mean_candidate_oracle_selected_branch_progress": 0.5944646831272521,
|
| 12040 |
+
"mean_candidate_oracle_selected_branch_success_rate": 0.38434782608695645,
|
| 12041 |
+
"mean_candidate_oracle_selector_regret_at_k": 0.1072367940535364,
|
| 12042 |
"mean_candidate_oracle_success_rate": 0.44347826086956516,
|
| 12043 |
"mean_candidate_oracle_unique_count": 7.998840579710145,
|
| 12044 |
"mean_progress": 0.5994939735267257,
|
|
|
|
| 14067 |
"std_success": 0.017650246200160362
|
| 14068 |
},
|
| 14069 |
"transport_field_reground_fieldonly_k6matched_b12_clean_k6_oraclek8": {
|
| 14070 |
+
"candidate_oracle_base_trace_coverage": 0.8023188405797101,
|
| 14071 |
"candidate_oracle_best_branch_rank_counts": {
|
| 14072 |
"1": 937,
|
| 14073 |
"2": 150,
|
|
|
|
| 14079 |
"8": 86
|
| 14080 |
},
|
| 14081 |
"candidate_oracle_rollouts": 8,
|
| 14082 |
+
"candidate_oracle_selector_gap_progress": 0.05475997270408861,
|
| 14083 |
+
"candidate_oracle_selector_gap_success": 0.0631884057971015,
|
| 14084 |
"candidate_oracle_type_counts": {
|
| 14085 |
"retrieval_residual_policy_residual": 440,
|
| 14086 |
"retrieval_residual_residual_near_miss": 256,
|
|
|
|
| 14130 |
0.2515942028985507,
|
| 14131 |
0.2289855072463768
|
| 14132 |
],
|
| 14133 |
+
"mean_candidate_oracle_car_to_proposal_oracle": 0.11794837850119001,
|
| 14134 |
"mean_candidate_oracle_improvement_rate": 0.4568115942028985,
|
| 14135 |
+
"mean_candidate_oracle_ncar_to_proposal_oracle": 1.4397874961800154,
|
| 14136 |
"mean_candidate_oracle_progress": 0.6483499062207082,
|
| 14137 |
+
"mean_candidate_oracle_ptr_at_k": 0.4602601156069364,
|
| 14138 |
"mean_candidate_oracle_score_gain_over_selected": 0.11794837779432965,
|
| 14139 |
+
"mean_candidate_oracle_selected_branch_progress": 0.5935899335166196,
|
| 14140 |
+
"mean_candidate_oracle_selected_branch_success_rate": 0.3797101449275362,
|
| 14141 |
+
"mean_candidate_oracle_selector_regret_at_k": 0.11794837850119001,
|
| 14142 |
"mean_candidate_oracle_success_rate": 0.4428985507246377,
|
| 14143 |
"mean_candidate_oracle_unique_count": 8.0,
|
| 14144 |
"mean_progress": 0.5973945180108495,
|
workspace/results/paper_analysis.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
# Paper Analysis
|
| 2 |
|
| 3 |
-
Generated: `2026-07-
|
| 4 |
|
| 5 |
## Main Seed Statistics
|
| 6 |
|
|
@@ -201,10 +201,20 @@ Generated: `2026-07-02T16:52:46+00:00`
|
|
| 201 |
- Remaining clean-to-same-state proposal gap is +18.09 pp.
|
| 202 |
- Full lattice adds expert proposals and reaches 69.33%, a +12.35 pp gain over no-expert.
|
| 203 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 204 |
## Candidate-Oracle Diagnostic
|
| 205 |
|
| 206 |
- Best diagnostic prefix (`transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8`) reaches 44.35% with mean progress 64.26%; this is diagnostic-only because it uses measured rollout outcomes after generating candidates.
|
| 207 |
- Mean oracle-prefix score gain over the selected branch is +0.107, which isolates ranking/abstention headroom inside the clean proposal set.
|
|
|
|
| 208 |
- Mean unique candidates in the prefix: 8.00.
|
| 209 |
- Candidate-oracle best type counts: {'retrieval_residual_policy_residual': 445, 'retrieval_residual_residual_near_miss': 273, 'retrieval_residual_residual_near_miss+residual_wrong_gripper': 262, 'retrieval_residual_residual_no_op': 285, 'retrieval_residual_residual_wrong_gripper': 460}.
|
| 210 |
- Mean best branch rank in the field-ordered prefix: 2.48; rank histogram {'1': 984, '2': 156, '3': 129, '4': 141, '5': 101, '6': 56, '7': 79, '8': 79}.
|
|
|
|
| 1 |
# Paper Analysis
|
| 2 |
|
| 3 |
+
Generated: `2026-07-02T17:31:38+00:00`
|
| 4 |
|
| 5 |
## Main Seed Statistics
|
| 6 |
|
|
|
|
| 201 |
- Remaining clean-to-same-state proposal gap is +18.09 pp.
|
| 202 |
- Full lattice adds expert proposals and reaches 69.33%, a +12.35 pp gain over no-expert.
|
| 203 |
|
| 204 |
+
## Causal Action Regret Decomposition
|
| 205 |
+
|
| 206 |
+
| base | selected clean | proposal oracle | same-state no-expert oracle | support gap | selector gap | gap closed |
|
| 207 |
+
|---:|---:|---:|---:|---:|---:|---:|
|
| 208 |
+
| 29.74% | 38.90% | 44.35% | 56.99% | +12.64 pp | +5.45 pp | 33.6% |
|
| 209 |
+
|
| 210 |
+
- Current clean policy closes 33.6% of the h16-to-same-state-no-expert gap.
|
| 211 |
+
- Closing 65--75% of that gap implies selected success targets of 47.45%--50.17%.
|
| 212 |
+
|
| 213 |
## Candidate-Oracle Diagnostic
|
| 214 |
|
| 215 |
- Best diagnostic prefix (`transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8`) reaches 44.35% with mean progress 64.26%; this is diagnostic-only because it uses measured rollout outcomes after generating candidates.
|
| 216 |
- Mean oracle-prefix score gain over the selected branch is +0.107, which isolates ranking/abstention headroom inside the clean proposal set.
|
| 217 |
+
- CAR-to-proposal-oracle from raw prefix traces is +0.107; PTR@K is 47.32% over rows with base trace coverage 81.22%.
|
| 218 |
- Mean unique candidates in the prefix: 8.00.
|
| 219 |
- Candidate-oracle best type counts: {'retrieval_residual_policy_residual': 445, 'retrieval_residual_residual_near_miss': 273, 'retrieval_residual_residual_near_miss+residual_wrong_gripper': 262, 'retrieval_residual_residual_no_op': 285, 'retrieval_residual_residual_wrong_gripper': 460}.
|
| 220 |
- Mean best branch rank in the field-ordered prefix: 2.48; rank histogram {'1': 984, '2': 156, '3': 129, '4': 141, '5': 101, '6': 56, '7': 79, '8': 79}.
|
workspace/scripts/build_paper_analysis.py
CHANGED
|
@@ -3,16 +3,29 @@ from __future__ import annotations
|
|
| 3 |
|
| 4 |
import json
|
| 5 |
import math
|
|
|
|
| 6 |
from collections import Counter
|
| 7 |
from dataclasses import dataclass
|
| 8 |
from datetime import datetime, timezone
|
| 9 |
from pathlib import Path
|
| 10 |
from typing import Any
|
| 11 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
RESULTS_DIR = Path("results")
|
| 14 |
OUT_JSON = RESULTS_DIR / "paper_analysis.json"
|
| 15 |
OUT_MD = RESULTS_DIR / "paper_analysis.md"
|
|
|
|
|
|
|
| 16 |
CANONICAL_H16_ROLLOUT = Path("/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs")
|
| 17 |
FALLBACK_BEST_CLEAN_KEY = "residual_k4_consensus_grid035040045_noopbonus003"
|
| 18 |
NON_DEPLOYMENT_KEYS = {
|
|
@@ -1480,13 +1493,33 @@ def _normalize_summary(data: dict[str, Any], rows: list[dict[str, Any]], *, sour
|
|
| 1480 |
if row.get("candidate_oracle_success_rate") is not None
|
| 1481 |
]
|
| 1482 |
if candidate_oracle_success:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1483 |
output.update(
|
| 1484 |
{
|
| 1485 |
"candidate_oracle_rollouts": int(data.get("candidate_oracle_rollouts") or 0),
|
| 1486 |
"candidate_oracle_unique_tolerance": data.get(
|
| 1487 |
"candidate_oracle_unique_tolerance"
|
| 1488 |
),
|
| 1489 |
-
"mean_candidate_oracle_success_rate":
|
| 1490 |
"std_candidate_oracle_success_rate": _sample_std(
|
| 1491 |
candidate_oracle_success
|
| 1492 |
),
|
|
@@ -1496,12 +1529,18 @@ def _normalize_summary(data: dict[str, Any], rows: list[dict[str, Any]], *, sour
|
|
| 1496 |
for index, row in enumerate(rows)
|
| 1497 |
if row.get("candidate_oracle_success_rate") is not None
|
| 1498 |
},
|
| 1499 |
-
"mean_candidate_oracle_progress":
|
| 1500 |
-
|
| 1501 |
-
|
| 1502 |
-
|
| 1503 |
-
|
| 1504 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1505 |
),
|
| 1506 |
"mean_candidate_oracle_score_gain_over_selected": _mean(
|
| 1507 |
[
|
|
@@ -1530,6 +1569,8 @@ def _normalize_summary(data: dict[str, Any], rows: list[dict[str, Any]], *, sour
|
|
| 1530 |
),
|
| 1531 |
}
|
| 1532 |
)
|
|
|
|
|
|
|
| 1533 |
for key in (
|
| 1534 |
"mean_candidate_oracle_best_branch_rank",
|
| 1535 |
"candidate_oracle_best_branch_rank_counts",
|
|
@@ -1542,6 +1583,53 @@ def _normalize_summary(data: dict[str, Any], rows: list[dict[str, Any]], *, sour
|
|
| 1542 |
return output
|
| 1543 |
|
| 1544 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1545 |
def _per_task(rows: list[dict[str, Any]]) -> dict[str, dict[str, float]]:
|
| 1546 |
task_values: dict[str, list[float]] = {}
|
| 1547 |
task_counts: dict[str, list[int]] = {}
|
|
@@ -1826,6 +1914,36 @@ def _render_markdown(report: dict[str, Any]) -> str:
|
|
| 1826 |
f"- Full lattice adds expert proposals and reaches {_pct(methods['same_state_full']['mean_success'])}, a {_pp(gap['same_state_full_vs_no_expert'])} gain over no-expert.",
|
| 1827 |
]
|
| 1828 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1829 |
oracle_key, oracle = _best_candidate_oracle(methods)
|
| 1830 |
if oracle and oracle.get("num_completed"):
|
| 1831 |
branch_success = oracle.get("mean_candidate_oracle_branch_success_rates") or []
|
|
@@ -1850,6 +1968,12 @@ def _render_markdown(report: dict[str, Any]) -> str:
|
|
| 1850 |
f"{oracle.get('mean_candidate_oracle_score_gain_over_selected', 0.0):+.3f}, "
|
| 1851 |
"which isolates ranking/abstention headroom inside the clean proposal set."
|
| 1852 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1853 |
(
|
| 1854 |
"- Mean unique candidates in the prefix: "
|
| 1855 |
f"{oracle.get('mean_candidate_oracle_unique_count', 0.0):.2f}."
|
|
@@ -1950,6 +2074,13 @@ def _render_markdown(report: dict[str, Any]) -> str:
|
|
| 1950 |
def build_report() -> dict[str, Any]:
|
| 1951 |
methods = _load_methods()
|
| 1952 |
best_clean_key = _best_clean_key(methods)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1953 |
paired_deltas = {
|
| 1954 |
"best_clean - canonical_h16": _paired_delta(methods, best_clean_key, "h16_policy_canonical"),
|
| 1955 |
"best_clean - direct_same_ckpt": _paired_delta(methods, best_clean_key, "near_miss_policy_bc5"),
|
|
@@ -1973,6 +2104,13 @@ def build_report() -> dict[str, Any]:
|
|
| 1973 |
"same_state_full_vs_no_expert": methods["same_state_full"]["mean_success"]
|
| 1974 |
- methods["same_state_no_expert"]["mean_success"],
|
| 1975 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1976 |
return {
|
| 1977 |
"generated_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
| 1978 |
"methods": methods,
|
|
@@ -1982,17 +2120,83 @@ def build_report() -> dict[str, Any]:
|
|
| 1982 |
"no_expert_vs_best_clean": _per_task_delta(methods, "same_state_no_expert", best_clean_key),
|
| 1983 |
},
|
| 1984 |
"mechanism_gap": mechanism_gap,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1985 |
"best_clean_key": best_clean_key,
|
| 1986 |
}
|
| 1987 |
|
| 1988 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1989 |
def main() -> int:
|
| 1990 |
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
| 1991 |
report = build_report()
|
| 1992 |
OUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
|
| 1993 |
OUT_MD.write_text(_render_markdown(report), encoding="utf-8")
|
|
|
|
|
|
|
| 1994 |
print(f"Wrote {OUT_JSON}")
|
| 1995 |
print(f"Wrote {OUT_MD}")
|
|
|
|
| 1996 |
return 0
|
| 1997 |
|
| 1998 |
|
|
|
|
| 3 |
|
| 4 |
import json
|
| 5 |
import math
|
| 6 |
+
import sys
|
| 7 |
from collections import Counter
|
| 8 |
from dataclasses import dataclass
|
| 9 |
from datetime import datetime, timezone
|
| 10 |
from pathlib import Path
|
| 11 |
from typing import Any
|
| 12 |
|
| 13 |
+
ROOT_DIR = Path(__file__).resolve().parents[1]
|
| 14 |
+
if str(ROOT_DIR) not in sys.path:
|
| 15 |
+
sys.path.insert(0, str(ROOT_DIR))
|
| 16 |
+
|
| 17 |
+
from dovla_cil.eval.metrics import (
|
| 18 |
+
candidate_prefix_causal_metrics,
|
| 19 |
+
causal_action_decomposition,
|
| 20 |
+
finite_mean,
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
|
| 24 |
RESULTS_DIR = Path("results")
|
| 25 |
OUT_JSON = RESULTS_DIR / "paper_analysis.json"
|
| 26 |
OUT_MD = RESULTS_DIR / "paper_analysis.md"
|
| 27 |
+
LATEX_TABLES_DIR = Path("latex") / "tables"
|
| 28 |
+
OUT_CAR_TABLE = LATEX_TABLES_DIR / "car_decomposition.tex"
|
| 29 |
CANONICAL_H16_ROLLOUT = Path("/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs")
|
| 30 |
FALLBACK_BEST_CLEAN_KEY = "residual_k4_consensus_grid035040045_noopbonus003"
|
| 31 |
NON_DEPLOYMENT_KEYS = {
|
|
|
|
| 1493 |
if row.get("candidate_oracle_success_rate") is not None
|
| 1494 |
]
|
| 1495 |
if candidate_oracle_success:
|
| 1496 |
+
raw_prefix_metrics = _candidate_prefix_metrics_from_raw(rows)
|
| 1497 |
+
selected_branch_success = [
|
| 1498 |
+
float(row["candidate_oracle_selected_branch_success_rate"])
|
| 1499 |
+
for row in rows
|
| 1500 |
+
if row.get("candidate_oracle_selected_branch_success_rate") is not None
|
| 1501 |
+
]
|
| 1502 |
+
selected_branch_progress = [
|
| 1503 |
+
float(row["candidate_oracle_selected_branch_progress"])
|
| 1504 |
+
for row in rows
|
| 1505 |
+
if row.get("candidate_oracle_selected_branch_progress") is not None
|
| 1506 |
+
]
|
| 1507 |
+
oracle_progress = [
|
| 1508 |
+
float(row["candidate_oracle_progress"])
|
| 1509 |
+
for row in rows
|
| 1510 |
+
if row.get("candidate_oracle_progress") is not None
|
| 1511 |
+
]
|
| 1512 |
+
selected_success_mean = _mean(selected_branch_success)
|
| 1513 |
+
oracle_success_mean = _mean(candidate_oracle_success)
|
| 1514 |
+
selected_progress_mean = _mean(selected_branch_progress)
|
| 1515 |
+
oracle_progress_mean = _mean(oracle_progress)
|
| 1516 |
output.update(
|
| 1517 |
{
|
| 1518 |
"candidate_oracle_rollouts": int(data.get("candidate_oracle_rollouts") or 0),
|
| 1519 |
"candidate_oracle_unique_tolerance": data.get(
|
| 1520 |
"candidate_oracle_unique_tolerance"
|
| 1521 |
),
|
| 1522 |
+
"mean_candidate_oracle_success_rate": oracle_success_mean,
|
| 1523 |
"std_candidate_oracle_success_rate": _sample_std(
|
| 1524 |
candidate_oracle_success
|
| 1525 |
),
|
|
|
|
| 1529 |
for index, row in enumerate(rows)
|
| 1530 |
if row.get("candidate_oracle_success_rate") is not None
|
| 1531 |
},
|
| 1532 |
+
"mean_candidate_oracle_progress": oracle_progress_mean,
|
| 1533 |
+
"mean_candidate_oracle_selected_branch_success_rate": (
|
| 1534 |
+
selected_success_mean
|
| 1535 |
+
),
|
| 1536 |
+
"mean_candidate_oracle_selected_branch_progress": (
|
| 1537 |
+
selected_progress_mean
|
| 1538 |
+
),
|
| 1539 |
+
"candidate_oracle_selector_gap_success": (
|
| 1540 |
+
oracle_success_mean - selected_success_mean
|
| 1541 |
+
),
|
| 1542 |
+
"candidate_oracle_selector_gap_progress": (
|
| 1543 |
+
oracle_progress_mean - selected_progress_mean
|
| 1544 |
),
|
| 1545 |
"mean_candidate_oracle_score_gain_over_selected": _mean(
|
| 1546 |
[
|
|
|
|
| 1569 |
),
|
| 1570 |
}
|
| 1571 |
)
|
| 1572 |
+
if raw_prefix_metrics:
|
| 1573 |
+
output.update(raw_prefix_metrics)
|
| 1574 |
for key in (
|
| 1575 |
"mean_candidate_oracle_best_branch_rank",
|
| 1576 |
"candidate_oracle_best_branch_rank_counts",
|
|
|
|
| 1583 |
return output
|
| 1584 |
|
| 1585 |
|
| 1586 |
+
def _candidate_prefix_metrics_from_raw(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
| 1587 |
+
prefix_metrics: list[dict[str, Any]] = []
|
| 1588 |
+
for row in rows:
|
| 1589 |
+
path = row.get("path")
|
| 1590 |
+
if not path:
|
| 1591 |
+
continue
|
| 1592 |
+
raw_path = Path(str(path))
|
| 1593 |
+
if not raw_path.exists():
|
| 1594 |
+
continue
|
| 1595 |
+
raw = _load_json(raw_path)
|
| 1596 |
+
for item in raw.get("rows", []):
|
| 1597 |
+
scores = item.get("candidate_oracle_branch_scores") or []
|
| 1598 |
+
if not scores:
|
| 1599 |
+
continue
|
| 1600 |
+
metrics = candidate_prefix_causal_metrics(
|
| 1601 |
+
branch_scores=[float(value) for value in scores],
|
| 1602 |
+
selected_score=(
|
| 1603 |
+
float(item["candidate_oracle_selected_branch_score"])
|
| 1604 |
+
if item.get("candidate_oracle_selected_branch_score") is not None
|
| 1605 |
+
else None
|
| 1606 |
+
),
|
| 1607 |
+
branch_types=[str(value) for value in item.get("candidate_oracle_types", [])],
|
| 1608 |
+
valid_mask=[bool(value) for value in item.get("candidate_oracle_valid_mask", [])]
|
| 1609 |
+
or None,
|
| 1610 |
+
)
|
| 1611 |
+
if metrics:
|
| 1612 |
+
prefix_metrics.append(metrics)
|
| 1613 |
+
if not prefix_metrics:
|
| 1614 |
+
return {}
|
| 1615 |
+
ptr = finite_mean([item.get("ptr_at_k") for item in prefix_metrics])
|
| 1616 |
+
ncar = finite_mean([item.get("ncar_to_proposal_oracle") for item in prefix_metrics])
|
| 1617 |
+
return {
|
| 1618 |
+
"mean_candidate_oracle_car_to_proposal_oracle": _mean(
|
| 1619 |
+
[float(item["car_to_proposal_oracle"]) for item in prefix_metrics]
|
| 1620 |
+
),
|
| 1621 |
+
"mean_candidate_oracle_selector_regret_at_k": _mean(
|
| 1622 |
+
[float(item["selector_regret_at_k"]) for item in prefix_metrics]
|
| 1623 |
+
),
|
| 1624 |
+
"mean_candidate_oracle_ptr_at_k": ptr,
|
| 1625 |
+
"mean_candidate_oracle_ncar_to_proposal_oracle": ncar,
|
| 1626 |
+
"candidate_oracle_base_trace_coverage": (
|
| 1627 |
+
sum(1 for item in prefix_metrics if item.get("base_utility") is not None)
|
| 1628 |
+
/ len(prefix_metrics)
|
| 1629 |
+
),
|
| 1630 |
+
}
|
| 1631 |
+
|
| 1632 |
+
|
| 1633 |
def _per_task(rows: list[dict[str, Any]]) -> dict[str, dict[str, float]]:
|
| 1634 |
task_values: dict[str, list[float]] = {}
|
| 1635 |
task_counts: dict[str, list[int]] = {}
|
|
|
|
| 1914 |
f"- Full lattice adds expert proposals and reaches {_pct(methods['same_state_full']['mean_success'])}, a {_pp(gap['same_state_full_vs_no_expert'])} gain over no-expert.",
|
| 1915 |
]
|
| 1916 |
)
|
| 1917 |
+
decomposition = report["causal_action_decomposition"]
|
| 1918 |
+
targets = report["causal_action_targets"]
|
| 1919 |
+
lines.extend(
|
| 1920 |
+
[
|
| 1921 |
+
"",
|
| 1922 |
+
"## Causal Action Regret Decomposition",
|
| 1923 |
+
"",
|
| 1924 |
+
"| base | selected clean | proposal oracle | same-state no-expert oracle | support gap | selector gap | gap closed |",
|
| 1925 |
+
"|---:|---:|---:|---:|---:|---:|---:|",
|
| 1926 |
+
(
|
| 1927 |
+
f"| {_pct(decomposition['base'])} | {_pct(decomposition['selected'])} | "
|
| 1928 |
+
f"{_pct(decomposition['proposal_oracle'])} | "
|
| 1929 |
+
f"{_pct(decomposition['same_state_oracle'])} | "
|
| 1930 |
+
f"{_pp(decomposition['support_gap'])} | "
|
| 1931 |
+
f"{_pp(decomposition['selector_gap'])} | "
|
| 1932 |
+
f"{decomposition['closed_fraction_of_noexpert_gap'] * 100:.1f}% |"
|
| 1933 |
+
),
|
| 1934 |
+
"",
|
| 1935 |
+
(
|
| 1936 |
+
"- Current clean policy closes "
|
| 1937 |
+
f"{decomposition['closed_fraction_of_noexpert_gap'] * 100:.1f}% "
|
| 1938 |
+
"of the h16-to-same-state-no-expert gap."
|
| 1939 |
+
),
|
| 1940 |
+
(
|
| 1941 |
+
"- Closing 65--75% of that gap implies selected success targets of "
|
| 1942 |
+
f"{_pct(targets['selected_success_for_65pct_gap_closure'])}--"
|
| 1943 |
+
f"{_pct(targets['selected_success_for_75pct_gap_closure'])}."
|
| 1944 |
+
),
|
| 1945 |
+
]
|
| 1946 |
+
)
|
| 1947 |
oracle_key, oracle = _best_candidate_oracle(methods)
|
| 1948 |
if oracle and oracle.get("num_completed"):
|
| 1949 |
branch_success = oracle.get("mean_candidate_oracle_branch_success_rates") or []
|
|
|
|
| 1968 |
f"{oracle.get('mean_candidate_oracle_score_gain_over_selected', 0.0):+.3f}, "
|
| 1969 |
"which isolates ranking/abstention headroom inside the clean proposal set."
|
| 1970 |
),
|
| 1971 |
+
(
|
| 1972 |
+
"- CAR-to-proposal-oracle from raw prefix traces is "
|
| 1973 |
+
f"{oracle.get('mean_candidate_oracle_car_to_proposal_oracle', 0.0):+.3f}; "
|
| 1974 |
+
f"PTR@K is {_pct(oracle.get('mean_candidate_oracle_ptr_at_k'))} "
|
| 1975 |
+
f"over rows with base trace coverage {_pct(oracle.get('candidate_oracle_base_trace_coverage'))}."
|
| 1976 |
+
),
|
| 1977 |
(
|
| 1978 |
"- Mean unique candidates in the prefix: "
|
| 1979 |
f"{oracle.get('mean_candidate_oracle_unique_count', 0.0):.2f}."
|
|
|
|
| 2074 |
def build_report() -> dict[str, Any]:
|
| 2075 |
methods = _load_methods()
|
| 2076 |
best_clean_key = _best_clean_key(methods)
|
| 2077 |
+
oracle_key, oracle_method = _best_candidate_oracle(methods)
|
| 2078 |
+
proposal_oracle_success = (
|
| 2079 |
+
float(oracle_method["mean_candidate_oracle_success_rate"])
|
| 2080 |
+
if oracle_method is not None
|
| 2081 |
+
and oracle_method.get("mean_candidate_oracle_success_rate") is not None
|
| 2082 |
+
else float("nan")
|
| 2083 |
+
)
|
| 2084 |
paired_deltas = {
|
| 2085 |
"best_clean - canonical_h16": _paired_delta(methods, best_clean_key, "h16_policy_canonical"),
|
| 2086 |
"best_clean - direct_same_ckpt": _paired_delta(methods, best_clean_key, "near_miss_policy_bc5"),
|
|
|
|
| 2104 |
"same_state_full_vs_no_expert": methods["same_state_full"]["mean_success"]
|
| 2105 |
- methods["same_state_no_expert"]["mean_success"],
|
| 2106 |
}
|
| 2107 |
+
decomposition = causal_action_decomposition(
|
| 2108 |
+
base=methods["h16_policy_canonical"]["mean_success"],
|
| 2109 |
+
selected=methods[best_clean_key]["mean_success"],
|
| 2110 |
+
proposal_oracle=proposal_oracle_success,
|
| 2111 |
+
same_state_oracle=methods["same_state_no_expert"]["mean_success"],
|
| 2112 |
+
full_oracle=methods["same_state_full"]["mean_success"],
|
| 2113 |
+
)
|
| 2114 |
return {
|
| 2115 |
"generated_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
| 2116 |
"methods": methods,
|
|
|
|
| 2120 |
"no_expert_vs_best_clean": _per_task_delta(methods, "same_state_no_expert", best_clean_key),
|
| 2121 |
},
|
| 2122 |
"mechanism_gap": mechanism_gap,
|
| 2123 |
+
"causal_action_decomposition": decomposition.to_dict(),
|
| 2124 |
+
"causal_action_targets": {
|
| 2125 |
+
"selected_success_for_65pct_gap_closure": decomposition.target_for_gap_closure(0.65),
|
| 2126 |
+
"selected_success_for_75pct_gap_closure": decomposition.target_for_gap_closure(0.75),
|
| 2127 |
+
"best_paper_target_range": [0.47, 0.52],
|
| 2128 |
+
},
|
| 2129 |
+
"best_candidate_oracle_key": oracle_key,
|
| 2130 |
"best_clean_key": best_clean_key,
|
| 2131 |
}
|
| 2132 |
|
| 2133 |
|
| 2134 |
+
def _latex_pct(value: float | None) -> str:
|
| 2135 |
+
if value is None:
|
| 2136 |
+
return "--"
|
| 2137 |
+
value = float(value)
|
| 2138 |
+
if math.isnan(value):
|
| 2139 |
+
return "--"
|
| 2140 |
+
return f"{value * 100:.2f}"
|
| 2141 |
+
|
| 2142 |
+
|
| 2143 |
+
def _latex_pp(value: float | None) -> str:
|
| 2144 |
+
if value is None:
|
| 2145 |
+
return "--"
|
| 2146 |
+
value = float(value)
|
| 2147 |
+
if math.isnan(value):
|
| 2148 |
+
return "--"
|
| 2149 |
+
return f"{value * 100:.2f}"
|
| 2150 |
+
|
| 2151 |
+
|
| 2152 |
+
def _render_car_decomposition_table(report: dict[str, Any]) -> str:
|
| 2153 |
+
decomposition = report["causal_action_decomposition"]
|
| 2154 |
+
targets = report["causal_action_targets"]
|
| 2155 |
+
closed = decomposition["closed_fraction_of_noexpert_gap"]
|
| 2156 |
+
closed_text = "--" if math.isnan(float(closed)) else f"{float(closed) * 100:.1f}"
|
| 2157 |
+
return (
|
| 2158 |
+
"\\begin{table}[t]\n"
|
| 2159 |
+
"\\centering\n"
|
| 2160 |
+
"\\caption{Causal Action Regret decomposition on the current six-task "
|
| 2161 |
+
"diagnostic. Support is the proposal-generation gap to the hidden "
|
| 2162 |
+
"same-state no-expert oracle; selector is the clean proposal-oracle "
|
| 2163 |
+
"headroom left by the deployed selector.}\n"
|
| 2164 |
+
"\\label{tab:car-decomposition}\n"
|
| 2165 |
+
"\\small\n"
|
| 2166 |
+
"\\begin{tabular}{@{}lrrrrrr@{}}\n"
|
| 2167 |
+
"\\toprule\n"
|
| 2168 |
+
"Method & Base & Prop. oracle & Selected & State oracle & "
|
| 2169 |
+
"Support & Selector \\\\\n"
|
| 2170 |
+
"\\midrule\n"
|
| 2171 |
+
"Current CIL-Atlas V0 & "
|
| 2172 |
+
f"{_latex_pct(decomposition['base'])} & "
|
| 2173 |
+
f"{_latex_pct(decomposition['proposal_oracle'])} & "
|
| 2174 |
+
f"{_latex_pct(decomposition['selected'])} & "
|
| 2175 |
+
f"{_latex_pct(decomposition['same_state_oracle'])} & "
|
| 2176 |
+
f"{_latex_pp(decomposition['support_gap'])} & "
|
| 2177 |
+
f"{_latex_pp(decomposition['selector_gap'])} \\\\\n"
|
| 2178 |
+
"\\bottomrule\n"
|
| 2179 |
+
"\\end{tabular}\n"
|
| 2180 |
+
"\\vspace{2pt}\n"
|
| 2181 |
+
"\\footnotesize Clean gain is "
|
| 2182 |
+
f"{_latex_pp(decomposition['clean_gain'])} points; gap closed is "
|
| 2183 |
+
f"{closed_text}\\%; 65--75\\% closure targets are "
|
| 2184 |
+
f"{_latex_pct(targets['selected_success_for_65pct_gap_closure'])}--"
|
| 2185 |
+
f"{_latex_pct(targets['selected_success_for_75pct_gap_closure'])}.\n"
|
| 2186 |
+
"\\end{table}\n"
|
| 2187 |
+
)
|
| 2188 |
+
|
| 2189 |
+
|
| 2190 |
def main() -> int:
|
| 2191 |
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
|
| 2192 |
report = build_report()
|
| 2193 |
OUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
|
| 2194 |
OUT_MD.write_text(_render_markdown(report), encoding="utf-8")
|
| 2195 |
+
LATEX_TABLES_DIR.mkdir(parents=True, exist_ok=True)
|
| 2196 |
+
OUT_CAR_TABLE.write_text(_render_car_decomposition_table(report), encoding="utf-8")
|
| 2197 |
print(f"Wrote {OUT_JSON}")
|
| 2198 |
print(f"Wrote {OUT_MD}")
|
| 2199 |
+
print(f"Wrote {OUT_CAR_TABLE}")
|
| 2200 |
return 0
|
| 2201 |
|
| 2202 |
|