anhtld commited on
Commit
dda08d5
·
verified ·
1 Parent(s): 666f655

auto-sync 2026-07-02T17:27:17Z workspace (part 3)

Browse files
workspace/results/paper_analysis.json CHANGED
@@ -1,6 +1,29 @@
1
  {
 
2
  "best_clean_key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005",
3
- "generated_utc": "2026-07-02T16:52:46+00:00",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4
  "mechanism_gap": {
5
  "best_clean_vs_direct_same_ckpt": 0.1060869565217391,
6
  "best_clean_vs_h16": 0.09159420289855075,
@@ -3724,6 +3747,8 @@
3724
  },
3725
  "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8": {
3726
  "candidate_oracle_rollouts": 8,
 
 
3727
  "candidate_oracle_type_counts": {
3728
  "retrieval_residual_policy_residual": 867,
3729
  "retrieval_residual_residual_near_miss": 201,
@@ -3745,6 +3770,8 @@
3745
  "mean_candidate_oracle_improvement_rate": 0.4602898550724637,
3746
  "mean_candidate_oracle_progress": 0.6443418554713328,
3747
  "mean_candidate_oracle_score_gain_over_selected": 0.16034898876064066,
 
 
3748
  "mean_candidate_oracle_success_rate": 0.4307246376811594,
3749
  "mean_candidate_oracle_unique_count": NaN,
3750
  "mean_progress": 0.571575072284851,
@@ -3847,6 +3874,7 @@
3847
  "std_success": 0.012173913043478276
3848
  },
3849
  "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace": {
 
3850
  "candidate_oracle_best_branch_rank_counts": {
3851
  "1": 931,
3852
  "2": 162,
@@ -3858,6 +3886,8 @@
3858
  "8": 149
3859
  },
3860
  "candidate_oracle_rollouts": 8,
 
 
3861
  "candidate_oracle_type_counts": {
3862
  "retrieval_residual_policy_residual": 867,
3863
  "retrieval_residual_residual_near_miss": 201,
@@ -3907,9 +3937,15 @@
3907
  0.2336231884057971,
3908
  0.25043478260869567
3909
  ],
 
3910
  "mean_candidate_oracle_improvement_rate": 0.4602898550724637,
 
3911
  "mean_candidate_oracle_progress": 0.6443418554713328,
 
3912
  "mean_candidate_oracle_score_gain_over_selected": 0.16034898876064066,
 
 
 
3913
  "mean_candidate_oracle_success_rate": 0.4307246376811594,
3914
  "mean_candidate_oracle_unique_count": 8.0,
3915
  "mean_progress": 0.571575072284851,
@@ -11093,6 +11129,7 @@
11093
  "std_success": 0.017506862458598824
11094
  },
11095
  "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_oraclek8": {
 
11096
  "candidate_oracle_best_branch_rank_counts": {
11097
  "1": 985,
11098
  "2": 154,
@@ -11104,6 +11141,8 @@
11104
  "8": 78
11105
  },
11106
  "candidate_oracle_rollouts": 8,
 
 
11107
  "candidate_oracle_type_counts": {
11108
  "retrieval_residual_policy_residual": 468,
11109
  "retrieval_residual_residual_near_miss": 256,
@@ -11152,9 +11191,15 @@
11152
  0.2533333333333333,
11153
  0.23099631368984497
11154
  ],
 
11155
  "mean_candidate_oracle_improvement_rate": 0.4289855072463768,
 
11156
  "mean_candidate_oracle_progress": 0.6426025666080523,
 
11157
  "mean_candidate_oracle_score_gain_over_selected": 0.10811787949955981,
 
 
 
11158
  "mean_candidate_oracle_success_rate": 0.44347826086956516,
11159
  "mean_candidate_oracle_unique_count": 7.998840579710145,
11160
  "mean_progress": 0.5992048837379486,
@@ -11923,6 +11968,7 @@
11923
  "std_success": 0.012215250727945203
11924
  },
11925
  "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8": {
 
11926
  "candidate_oracle_best_branch_rank_counts": {
11927
  "1": 984,
11928
  "2": 156,
@@ -11934,6 +11980,8 @@
11934
  "8": 79
11935
  },
11936
  "candidate_oracle_rollouts": 8,
 
 
11937
  "candidate_oracle_type_counts": {
11938
  "retrieval_residual_policy_residual": 445,
11939
  "retrieval_residual_residual_near_miss": 273,
@@ -11982,9 +12030,15 @@
11982
  0.2539130434782609,
11983
  0.23215674392768773
11984
  ],
 
11985
  "mean_candidate_oracle_improvement_rate": 0.42956521739130443,
 
11986
  "mean_candidate_oracle_progress": 0.6425710423981797,
 
11987
  "mean_candidate_oracle_score_gain_over_selected": 0.10723679356600928,
 
 
 
11988
  "mean_candidate_oracle_success_rate": 0.44347826086956516,
11989
  "mean_candidate_oracle_unique_count": 7.998840579710145,
11990
  "mean_progress": 0.5994939735267257,
@@ -14013,6 +14067,7 @@
14013
  "std_success": 0.017650246200160362
14014
  },
14015
  "transport_field_reground_fieldonly_k6matched_b12_clean_k6_oraclek8": {
 
14016
  "candidate_oracle_best_branch_rank_counts": {
14017
  "1": 937,
14018
  "2": 150,
@@ -14024,6 +14079,8 @@
14024
  "8": 86
14025
  },
14026
  "candidate_oracle_rollouts": 8,
 
 
14027
  "candidate_oracle_type_counts": {
14028
  "retrieval_residual_policy_residual": 440,
14029
  "retrieval_residual_residual_near_miss": 256,
@@ -14073,9 +14130,15 @@
14073
  0.2515942028985507,
14074
  0.2289855072463768
14075
  ],
 
14076
  "mean_candidate_oracle_improvement_rate": 0.4568115942028985,
 
14077
  "mean_candidate_oracle_progress": 0.6483499062207082,
 
14078
  "mean_candidate_oracle_score_gain_over_selected": 0.11794837779432965,
 
 
 
14079
  "mean_candidate_oracle_success_rate": 0.4428985507246377,
14080
  "mean_candidate_oracle_unique_count": 8.0,
14081
  "mean_progress": 0.5973945180108495,
 
1
  {
2
+ "best_candidate_oracle_key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
3
  "best_clean_key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005",
4
+ "causal_action_decomposition": {
5
+ "base": 0.29739130434782607,
6
+ "clean_gain": 0.09159420289855075,
7
+ "closed_fraction_of_noexpert_gap": 0.3361702127659574,
8
+ "full_oracle": 0.6933333333333334,
9
+ "full_oracle_gap": 0.30434782608695654,
10
+ "proposal_oracle": 0.44347826086956516,
11
+ "same_state_gain_over_base": 0.27246376811594214,
12
+ "same_state_oracle": 0.5698550724637682,
13
+ "selected": 0.3889855072463768,
14
+ "selector_gap": 0.05449275362318834,
15
+ "support_gap": 0.12637681159420305,
16
+ "total_gap_to_same_state": 0.1808695652173914
17
+ },
18
+ "causal_action_targets": {
19
+ "best_paper_target_range": [
20
+ 0.47,
21
+ 0.52
22
+ ],
23
+ "selected_success_for_65pct_gap_closure": 0.47449275362318843,
24
+ "selected_success_for_75pct_gap_closure": 0.5017391304347827
25
+ },
26
+ "generated_utc": "2026-07-02T17:31:38+00:00",
27
  "mechanism_gap": {
28
  "best_clean_vs_direct_same_ckpt": 0.1060869565217391,
29
  "best_clean_vs_h16": 0.09159420289855075,
 
3747
  },
3748
  "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8": {
3749
  "candidate_oracle_rollouts": 8,
3750
+ "candidate_oracle_selector_gap_progress": 0.07803014863367896,
3751
+ "candidate_oracle_selector_gap_success": 0.08231884057971006,
3752
  "candidate_oracle_type_counts": {
3753
  "retrieval_residual_policy_residual": 867,
3754
  "retrieval_residual_residual_near_miss": 201,
 
3770
  "mean_candidate_oracle_improvement_rate": 0.4602898550724637,
3771
  "mean_candidate_oracle_progress": 0.6443418554713328,
3772
  "mean_candidate_oracle_score_gain_over_selected": 0.16034898876064066,
3773
+ "mean_candidate_oracle_selected_branch_progress": 0.5663117068376539,
3774
+ "mean_candidate_oracle_selected_branch_success_rate": 0.3484057971014493,
3775
  "mean_candidate_oracle_success_rate": 0.4307246376811594,
3776
  "mean_candidate_oracle_unique_count": NaN,
3777
  "mean_progress": 0.571575072284851,
 
3874
  "std_success": 0.012173913043478276
3875
  },
3876
  "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace": {
3877
+ "candidate_oracle_base_trace_coverage": 0.9768115942028985,
3878
  "candidate_oracle_best_branch_rank_counts": {
3879
  "1": 931,
3880
  "2": 162,
 
3886
  "8": 149
3887
  },
3888
  "candidate_oracle_rollouts": 8,
3889
+ "candidate_oracle_selector_gap_progress": 0.07803014863367896,
3890
+ "candidate_oracle_selector_gap_success": 0.08231884057971006,
3891
  "candidate_oracle_type_counts": {
3892
  "retrieval_residual_policy_residual": 867,
3893
  "retrieval_residual_residual_near_miss": 201,
 
3937
  0.2336231884057971,
3938
  0.25043478260869567
3939
  ],
3940
+ "mean_candidate_oracle_car_to_proposal_oracle": 0.16034898921338922,
3941
  "mean_candidate_oracle_improvement_rate": 0.4602898550724637,
3942
+ "mean_candidate_oracle_ncar_to_proposal_oracle": 0.9816454565106858,
3943
  "mean_candidate_oracle_progress": 0.6443418554713328,
3944
+ "mean_candidate_oracle_ptr_at_k": 0.4712166172106825,
3945
  "mean_candidate_oracle_score_gain_over_selected": 0.16034898876064066,
3946
+ "mean_candidate_oracle_selected_branch_progress": 0.5663117068376539,
3947
+ "mean_candidate_oracle_selected_branch_success_rate": 0.3484057971014493,
3948
+ "mean_candidate_oracle_selector_regret_at_k": 0.16034898921338922,
3949
  "mean_candidate_oracle_success_rate": 0.4307246376811594,
3950
  "mean_candidate_oracle_unique_count": 8.0,
3951
  "mean_progress": 0.571575072284851,
 
11129
  "std_success": 0.017506862458598824
11130
  },
11131
  "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_oraclek8": {
11132
+ "candidate_oracle_base_trace_coverage": 0.8515942028985507,
11133
  "candidate_oracle_best_branch_rank_counts": {
11134
  "1": 985,
11135
  "2": 154,
 
11141
  "8": 78
11142
  },
11143
  "candidate_oracle_rollouts": 8,
11144
+ "candidate_oracle_selector_gap_progress": 0.04840773510598173,
11145
+ "candidate_oracle_selector_gap_success": 0.05971014492753618,
11146
  "candidate_oracle_type_counts": {
11147
  "retrieval_residual_policy_residual": 468,
11148
  "retrieval_residual_residual_near_miss": 256,
 
11191
  0.2533333333333333,
11192
  0.23099631368984497
11193
  ],
11194
+ "mean_candidate_oracle_car_to_proposal_oracle": 0.10811788003351809,
11195
  "mean_candidate_oracle_improvement_rate": 0.4289855072463768,
11196
+ "mean_candidate_oracle_ncar_to_proposal_oracle": 1.2157991567169883,
11197
  "mean_candidate_oracle_progress": 0.6426025666080523,
11198
+ "mean_candidate_oracle_ptr_at_k": 0.4758339006126617,
11199
  "mean_candidate_oracle_score_gain_over_selected": 0.10811787949955981,
11200
+ "mean_candidate_oracle_selected_branch_progress": 0.5941948315020705,
11201
+ "mean_candidate_oracle_selected_branch_success_rate": 0.383768115942029,
11202
+ "mean_candidate_oracle_selector_regret_at_k": 0.10811788003351809,
11203
  "mean_candidate_oracle_success_rate": 0.44347826086956516,
11204
  "mean_candidate_oracle_unique_count": 7.998840579710145,
11205
  "mean_progress": 0.5992048837379486,
 
11968
  "std_success": 0.012215250727945203
11969
  },
11970
  "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8": {
11971
+ "candidate_oracle_base_trace_coverage": 0.8121739130434783,
11972
  "candidate_oracle_best_branch_rank_counts": {
11973
  "1": 984,
11974
  "2": 156,
 
11980
  "8": 79
11981
  },
11982
  "candidate_oracle_rollouts": 8,
11983
+ "candidate_oracle_selector_gap_progress": 0.04810635927092766,
11984
+ "candidate_oracle_selector_gap_success": 0.05913043478260871,
11985
  "candidate_oracle_type_counts": {
11986
  "retrieval_residual_policy_residual": 445,
11987
  "retrieval_residual_residual_near_miss": 273,
 
12030
  0.2539130434782609,
12031
  0.23215674392768773
12032
  ],
12033
+ "mean_candidate_oracle_car_to_proposal_oracle": 0.1072367940535364,
12034
  "mean_candidate_oracle_improvement_rate": 0.42956521739130443,
12035
+ "mean_candidate_oracle_ncar_to_proposal_oracle": 1.2488320610772363,
12036
  "mean_candidate_oracle_progress": 0.6425710423981797,
12037
+ "mean_candidate_oracle_ptr_at_k": 0.4732334047109208,
12038
  "mean_candidate_oracle_score_gain_over_selected": 0.10723679356600928,
12039
+ "mean_candidate_oracle_selected_branch_progress": 0.5944646831272521,
12040
+ "mean_candidate_oracle_selected_branch_success_rate": 0.38434782608695645,
12041
+ "mean_candidate_oracle_selector_regret_at_k": 0.1072367940535364,
12042
  "mean_candidate_oracle_success_rate": 0.44347826086956516,
12043
  "mean_candidate_oracle_unique_count": 7.998840579710145,
12044
  "mean_progress": 0.5994939735267257,
 
14067
  "std_success": 0.017650246200160362
14068
  },
14069
  "transport_field_reground_fieldonly_k6matched_b12_clean_k6_oraclek8": {
14070
+ "candidate_oracle_base_trace_coverage": 0.8023188405797101,
14071
  "candidate_oracle_best_branch_rank_counts": {
14072
  "1": 937,
14073
  "2": 150,
 
14079
  "8": 86
14080
  },
14081
  "candidate_oracle_rollouts": 8,
14082
+ "candidate_oracle_selector_gap_progress": 0.05475997270408861,
14083
+ "candidate_oracle_selector_gap_success": 0.0631884057971015,
14084
  "candidate_oracle_type_counts": {
14085
  "retrieval_residual_policy_residual": 440,
14086
  "retrieval_residual_residual_near_miss": 256,
 
14130
  0.2515942028985507,
14131
  0.2289855072463768
14132
  ],
14133
+ "mean_candidate_oracle_car_to_proposal_oracle": 0.11794837850119001,
14134
  "mean_candidate_oracle_improvement_rate": 0.4568115942028985,
14135
+ "mean_candidate_oracle_ncar_to_proposal_oracle": 1.4397874961800154,
14136
  "mean_candidate_oracle_progress": 0.6483499062207082,
14137
+ "mean_candidate_oracle_ptr_at_k": 0.4602601156069364,
14138
  "mean_candidate_oracle_score_gain_over_selected": 0.11794837779432965,
14139
+ "mean_candidate_oracle_selected_branch_progress": 0.5935899335166196,
14140
+ "mean_candidate_oracle_selected_branch_success_rate": 0.3797101449275362,
14141
+ "mean_candidate_oracle_selector_regret_at_k": 0.11794837850119001,
14142
  "mean_candidate_oracle_success_rate": 0.4428985507246377,
14143
  "mean_candidate_oracle_unique_count": 8.0,
14144
  "mean_progress": 0.5973945180108495,
workspace/results/paper_analysis.md CHANGED
@@ -1,6 +1,6 @@
1
  # Paper Analysis
2
 
3
- Generated: `2026-07-02T16:52:46+00:00`
4
 
5
  ## Main Seed Statistics
6
 
@@ -201,10 +201,20 @@ Generated: `2026-07-02T16:52:46+00:00`
201
  - Remaining clean-to-same-state proposal gap is +18.09 pp.
202
  - Full lattice adds expert proposals and reaches 69.33%, a +12.35 pp gain over no-expert.
203
 
 
 
 
 
 
 
 
 
 
204
  ## Candidate-Oracle Diagnostic
205
 
206
  - Best diagnostic prefix (`transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8`) reaches 44.35% with mean progress 64.26%; this is diagnostic-only because it uses measured rollout outcomes after generating candidates.
207
  - Mean oracle-prefix score gain over the selected branch is +0.107, which isolates ranking/abstention headroom inside the clean proposal set.
 
208
  - Mean unique candidates in the prefix: 8.00.
209
  - Candidate-oracle best type counts: {'retrieval_residual_policy_residual': 445, 'retrieval_residual_residual_near_miss': 273, 'retrieval_residual_residual_near_miss+residual_wrong_gripper': 262, 'retrieval_residual_residual_no_op': 285, 'retrieval_residual_residual_wrong_gripper': 460}.
210
  - Mean best branch rank in the field-ordered prefix: 2.48; rank histogram {'1': 984, '2': 156, '3': 129, '4': 141, '5': 101, '6': 56, '7': 79, '8': 79}.
 
1
  # Paper Analysis
2
 
3
+ Generated: `2026-07-02T17:31:38+00:00`
4
 
5
  ## Main Seed Statistics
6
 
 
201
  - Remaining clean-to-same-state proposal gap is +18.09 pp.
202
  - Full lattice adds expert proposals and reaches 69.33%, a +12.35 pp gain over no-expert.
203
 
204
+ ## Causal Action Regret Decomposition
205
+
206
+ | base | selected clean | proposal oracle | same-state no-expert oracle | support gap | selector gap | gap closed |
207
+ |---:|---:|---:|---:|---:|---:|---:|
208
+ | 29.74% | 38.90% | 44.35% | 56.99% | +12.64 pp | +5.45 pp | 33.6% |
209
+
210
+ - Current clean policy closes 33.6% of the h16-to-same-state-no-expert gap.
211
+ - Closing 65--75% of that gap implies selected success targets of 47.45%--50.17%.
212
+
213
  ## Candidate-Oracle Diagnostic
214
 
215
  - Best diagnostic prefix (`transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8`) reaches 44.35% with mean progress 64.26%; this is diagnostic-only because it uses measured rollout outcomes after generating candidates.
216
  - Mean oracle-prefix score gain over the selected branch is +0.107, which isolates ranking/abstention headroom inside the clean proposal set.
217
+ - CAR-to-proposal-oracle from raw prefix traces is +0.107; PTR@K is 47.32% over rows with base trace coverage 81.22%.
218
  - Mean unique candidates in the prefix: 8.00.
219
  - Candidate-oracle best type counts: {'retrieval_residual_policy_residual': 445, 'retrieval_residual_residual_near_miss': 273, 'retrieval_residual_residual_near_miss+residual_wrong_gripper': 262, 'retrieval_residual_residual_no_op': 285, 'retrieval_residual_residual_wrong_gripper': 460}.
220
  - Mean best branch rank in the field-ordered prefix: 2.48; rank histogram {'1': 984, '2': 156, '3': 129, '4': 141, '5': 101, '6': 56, '7': 79, '8': 79}.
workspace/scripts/build_paper_analysis.py CHANGED
@@ -3,16 +3,29 @@ from __future__ import annotations
3
 
4
  import json
5
  import math
 
6
  from collections import Counter
7
  from dataclasses import dataclass
8
  from datetime import datetime, timezone
9
  from pathlib import Path
10
  from typing import Any
11
 
 
 
 
 
 
 
 
 
 
 
12
 
13
  RESULTS_DIR = Path("results")
14
  OUT_JSON = RESULTS_DIR / "paper_analysis.json"
15
  OUT_MD = RESULTS_DIR / "paper_analysis.md"
 
 
16
  CANONICAL_H16_ROLLOUT = Path("/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs")
17
  FALLBACK_BEST_CLEAN_KEY = "residual_k4_consensus_grid035040045_noopbonus003"
18
  NON_DEPLOYMENT_KEYS = {
@@ -1480,13 +1493,33 @@ def _normalize_summary(data: dict[str, Any], rows: list[dict[str, Any]], *, sour
1480
  if row.get("candidate_oracle_success_rate") is not None
1481
  ]
1482
  if candidate_oracle_success:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1483
  output.update(
1484
  {
1485
  "candidate_oracle_rollouts": int(data.get("candidate_oracle_rollouts") or 0),
1486
  "candidate_oracle_unique_tolerance": data.get(
1487
  "candidate_oracle_unique_tolerance"
1488
  ),
1489
- "mean_candidate_oracle_success_rate": _mean(candidate_oracle_success),
1490
  "std_candidate_oracle_success_rate": _sample_std(
1491
  candidate_oracle_success
1492
  ),
@@ -1496,12 +1529,18 @@ def _normalize_summary(data: dict[str, Any], rows: list[dict[str, Any]], *, sour
1496
  for index, row in enumerate(rows)
1497
  if row.get("candidate_oracle_success_rate") is not None
1498
  },
1499
- "mean_candidate_oracle_progress": _mean(
1500
- [
1501
- float(row["candidate_oracle_progress"])
1502
- for row in rows
1503
- if row.get("candidate_oracle_progress") is not None
1504
- ]
 
 
 
 
 
 
1505
  ),
1506
  "mean_candidate_oracle_score_gain_over_selected": _mean(
1507
  [
@@ -1530,6 +1569,8 @@ def _normalize_summary(data: dict[str, Any], rows: list[dict[str, Any]], *, sour
1530
  ),
1531
  }
1532
  )
 
 
1533
  for key in (
1534
  "mean_candidate_oracle_best_branch_rank",
1535
  "candidate_oracle_best_branch_rank_counts",
@@ -1542,6 +1583,53 @@ def _normalize_summary(data: dict[str, Any], rows: list[dict[str, Any]], *, sour
1542
  return output
1543
 
1544
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1545
  def _per_task(rows: list[dict[str, Any]]) -> dict[str, dict[str, float]]:
1546
  task_values: dict[str, list[float]] = {}
1547
  task_counts: dict[str, list[int]] = {}
@@ -1826,6 +1914,36 @@ def _render_markdown(report: dict[str, Any]) -> str:
1826
  f"- Full lattice adds expert proposals and reaches {_pct(methods['same_state_full']['mean_success'])}, a {_pp(gap['same_state_full_vs_no_expert'])} gain over no-expert.",
1827
  ]
1828
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1829
  oracle_key, oracle = _best_candidate_oracle(methods)
1830
  if oracle and oracle.get("num_completed"):
1831
  branch_success = oracle.get("mean_candidate_oracle_branch_success_rates") or []
@@ -1850,6 +1968,12 @@ def _render_markdown(report: dict[str, Any]) -> str:
1850
  f"{oracle.get('mean_candidate_oracle_score_gain_over_selected', 0.0):+.3f}, "
1851
  "which isolates ranking/abstention headroom inside the clean proposal set."
1852
  ),
 
 
 
 
 
 
1853
  (
1854
  "- Mean unique candidates in the prefix: "
1855
  f"{oracle.get('mean_candidate_oracle_unique_count', 0.0):.2f}."
@@ -1950,6 +2074,13 @@ def _render_markdown(report: dict[str, Any]) -> str:
1950
  def build_report() -> dict[str, Any]:
1951
  methods = _load_methods()
1952
  best_clean_key = _best_clean_key(methods)
 
 
 
 
 
 
 
1953
  paired_deltas = {
1954
  "best_clean - canonical_h16": _paired_delta(methods, best_clean_key, "h16_policy_canonical"),
1955
  "best_clean - direct_same_ckpt": _paired_delta(methods, best_clean_key, "near_miss_policy_bc5"),
@@ -1973,6 +2104,13 @@ def build_report() -> dict[str, Any]:
1973
  "same_state_full_vs_no_expert": methods["same_state_full"]["mean_success"]
1974
  - methods["same_state_no_expert"]["mean_success"],
1975
  }
 
 
 
 
 
 
 
1976
  return {
1977
  "generated_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
1978
  "methods": methods,
@@ -1982,17 +2120,83 @@ def build_report() -> dict[str, Any]:
1982
  "no_expert_vs_best_clean": _per_task_delta(methods, "same_state_no_expert", best_clean_key),
1983
  },
1984
  "mechanism_gap": mechanism_gap,
 
 
 
 
 
 
 
1985
  "best_clean_key": best_clean_key,
1986
  }
1987
 
1988
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1989
  def main() -> int:
1990
  RESULTS_DIR.mkdir(parents=True, exist_ok=True)
1991
  report = build_report()
1992
  OUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
1993
  OUT_MD.write_text(_render_markdown(report), encoding="utf-8")
 
 
1994
  print(f"Wrote {OUT_JSON}")
1995
  print(f"Wrote {OUT_MD}")
 
1996
  return 0
1997
 
1998
 
 
3
 
4
  import json
5
  import math
6
+ import sys
7
  from collections import Counter
8
  from dataclasses import dataclass
9
  from datetime import datetime, timezone
10
  from pathlib import Path
11
  from typing import Any
12
 
13
+ ROOT_DIR = Path(__file__).resolve().parents[1]
14
+ if str(ROOT_DIR) not in sys.path:
15
+ sys.path.insert(0, str(ROOT_DIR))
16
+
17
+ from dovla_cil.eval.metrics import (
18
+ candidate_prefix_causal_metrics,
19
+ causal_action_decomposition,
20
+ finite_mean,
21
+ )
22
+
23
 
24
  RESULTS_DIR = Path("results")
25
  OUT_JSON = RESULTS_DIR / "paper_analysis.json"
26
  OUT_MD = RESULTS_DIR / "paper_analysis.md"
27
+ LATEX_TABLES_DIR = Path("latex") / "tables"
28
+ OUT_CAR_TABLE = LATEX_TABLES_DIR / "car_decomposition.tex"
29
  CANONICAL_H16_ROLLOUT = Path("/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs")
30
  FALLBACK_BEST_CLEAN_KEY = "residual_k4_consensus_grid035040045_noopbonus003"
31
  NON_DEPLOYMENT_KEYS = {
 
1493
  if row.get("candidate_oracle_success_rate") is not None
1494
  ]
1495
  if candidate_oracle_success:
1496
+ raw_prefix_metrics = _candidate_prefix_metrics_from_raw(rows)
1497
+ selected_branch_success = [
1498
+ float(row["candidate_oracle_selected_branch_success_rate"])
1499
+ for row in rows
1500
+ if row.get("candidate_oracle_selected_branch_success_rate") is not None
1501
+ ]
1502
+ selected_branch_progress = [
1503
+ float(row["candidate_oracle_selected_branch_progress"])
1504
+ for row in rows
1505
+ if row.get("candidate_oracle_selected_branch_progress") is not None
1506
+ ]
1507
+ oracle_progress = [
1508
+ float(row["candidate_oracle_progress"])
1509
+ for row in rows
1510
+ if row.get("candidate_oracle_progress") is not None
1511
+ ]
1512
+ selected_success_mean = _mean(selected_branch_success)
1513
+ oracle_success_mean = _mean(candidate_oracle_success)
1514
+ selected_progress_mean = _mean(selected_branch_progress)
1515
+ oracle_progress_mean = _mean(oracle_progress)
1516
  output.update(
1517
  {
1518
  "candidate_oracle_rollouts": int(data.get("candidate_oracle_rollouts") or 0),
1519
  "candidate_oracle_unique_tolerance": data.get(
1520
  "candidate_oracle_unique_tolerance"
1521
  ),
1522
+ "mean_candidate_oracle_success_rate": oracle_success_mean,
1523
  "std_candidate_oracle_success_rate": _sample_std(
1524
  candidate_oracle_success
1525
  ),
 
1529
  for index, row in enumerate(rows)
1530
  if row.get("candidate_oracle_success_rate") is not None
1531
  },
1532
+ "mean_candidate_oracle_progress": oracle_progress_mean,
1533
+ "mean_candidate_oracle_selected_branch_success_rate": (
1534
+ selected_success_mean
1535
+ ),
1536
+ "mean_candidate_oracle_selected_branch_progress": (
1537
+ selected_progress_mean
1538
+ ),
1539
+ "candidate_oracle_selector_gap_success": (
1540
+ oracle_success_mean - selected_success_mean
1541
+ ),
1542
+ "candidate_oracle_selector_gap_progress": (
1543
+ oracle_progress_mean - selected_progress_mean
1544
  ),
1545
  "mean_candidate_oracle_score_gain_over_selected": _mean(
1546
  [
 
1569
  ),
1570
  }
1571
  )
1572
+ if raw_prefix_metrics:
1573
+ output.update(raw_prefix_metrics)
1574
  for key in (
1575
  "mean_candidate_oracle_best_branch_rank",
1576
  "candidate_oracle_best_branch_rank_counts",
 
1583
  return output
1584
 
1585
 
1586
+ def _candidate_prefix_metrics_from_raw(rows: list[dict[str, Any]]) -> dict[str, Any]:
1587
+ prefix_metrics: list[dict[str, Any]] = []
1588
+ for row in rows:
1589
+ path = row.get("path")
1590
+ if not path:
1591
+ continue
1592
+ raw_path = Path(str(path))
1593
+ if not raw_path.exists():
1594
+ continue
1595
+ raw = _load_json(raw_path)
1596
+ for item in raw.get("rows", []):
1597
+ scores = item.get("candidate_oracle_branch_scores") or []
1598
+ if not scores:
1599
+ continue
1600
+ metrics = candidate_prefix_causal_metrics(
1601
+ branch_scores=[float(value) for value in scores],
1602
+ selected_score=(
1603
+ float(item["candidate_oracle_selected_branch_score"])
1604
+ if item.get("candidate_oracle_selected_branch_score") is not None
1605
+ else None
1606
+ ),
1607
+ branch_types=[str(value) for value in item.get("candidate_oracle_types", [])],
1608
+ valid_mask=[bool(value) for value in item.get("candidate_oracle_valid_mask", [])]
1609
+ or None,
1610
+ )
1611
+ if metrics:
1612
+ prefix_metrics.append(metrics)
1613
+ if not prefix_metrics:
1614
+ return {}
1615
+ ptr = finite_mean([item.get("ptr_at_k") for item in prefix_metrics])
1616
+ ncar = finite_mean([item.get("ncar_to_proposal_oracle") for item in prefix_metrics])
1617
+ return {
1618
+ "mean_candidate_oracle_car_to_proposal_oracle": _mean(
1619
+ [float(item["car_to_proposal_oracle"]) for item in prefix_metrics]
1620
+ ),
1621
+ "mean_candidate_oracle_selector_regret_at_k": _mean(
1622
+ [float(item["selector_regret_at_k"]) for item in prefix_metrics]
1623
+ ),
1624
+ "mean_candidate_oracle_ptr_at_k": ptr,
1625
+ "mean_candidate_oracle_ncar_to_proposal_oracle": ncar,
1626
+ "candidate_oracle_base_trace_coverage": (
1627
+ sum(1 for item in prefix_metrics if item.get("base_utility") is not None)
1628
+ / len(prefix_metrics)
1629
+ ),
1630
+ }
1631
+
1632
+
1633
  def _per_task(rows: list[dict[str, Any]]) -> dict[str, dict[str, float]]:
1634
  task_values: dict[str, list[float]] = {}
1635
  task_counts: dict[str, list[int]] = {}
 
1914
  f"- Full lattice adds expert proposals and reaches {_pct(methods['same_state_full']['mean_success'])}, a {_pp(gap['same_state_full_vs_no_expert'])} gain over no-expert.",
1915
  ]
1916
  )
1917
+ decomposition = report["causal_action_decomposition"]
1918
+ targets = report["causal_action_targets"]
1919
+ lines.extend(
1920
+ [
1921
+ "",
1922
+ "## Causal Action Regret Decomposition",
1923
+ "",
1924
+ "| base | selected clean | proposal oracle | same-state no-expert oracle | support gap | selector gap | gap closed |",
1925
+ "|---:|---:|---:|---:|---:|---:|---:|",
1926
+ (
1927
+ f"| {_pct(decomposition['base'])} | {_pct(decomposition['selected'])} | "
1928
+ f"{_pct(decomposition['proposal_oracle'])} | "
1929
+ f"{_pct(decomposition['same_state_oracle'])} | "
1930
+ f"{_pp(decomposition['support_gap'])} | "
1931
+ f"{_pp(decomposition['selector_gap'])} | "
1932
+ f"{decomposition['closed_fraction_of_noexpert_gap'] * 100:.1f}% |"
1933
+ ),
1934
+ "",
1935
+ (
1936
+ "- Current clean policy closes "
1937
+ f"{decomposition['closed_fraction_of_noexpert_gap'] * 100:.1f}% "
1938
+ "of the h16-to-same-state-no-expert gap."
1939
+ ),
1940
+ (
1941
+ "- Closing 65--75% of that gap implies selected success targets of "
1942
+ f"{_pct(targets['selected_success_for_65pct_gap_closure'])}--"
1943
+ f"{_pct(targets['selected_success_for_75pct_gap_closure'])}."
1944
+ ),
1945
+ ]
1946
+ )
1947
  oracle_key, oracle = _best_candidate_oracle(methods)
1948
  if oracle and oracle.get("num_completed"):
1949
  branch_success = oracle.get("mean_candidate_oracle_branch_success_rates") or []
 
1968
  f"{oracle.get('mean_candidate_oracle_score_gain_over_selected', 0.0):+.3f}, "
1969
  "which isolates ranking/abstention headroom inside the clean proposal set."
1970
  ),
1971
+ (
1972
+ "- CAR-to-proposal-oracle from raw prefix traces is "
1973
+ f"{oracle.get('mean_candidate_oracle_car_to_proposal_oracle', 0.0):+.3f}; "
1974
+ f"PTR@K is {_pct(oracle.get('mean_candidate_oracle_ptr_at_k'))} "
1975
+ f"over rows with base trace coverage {_pct(oracle.get('candidate_oracle_base_trace_coverage'))}."
1976
+ ),
1977
  (
1978
  "- Mean unique candidates in the prefix: "
1979
  f"{oracle.get('mean_candidate_oracle_unique_count', 0.0):.2f}."
 
2074
  def build_report() -> dict[str, Any]:
2075
  methods = _load_methods()
2076
  best_clean_key = _best_clean_key(methods)
2077
+ oracle_key, oracle_method = _best_candidate_oracle(methods)
2078
+ proposal_oracle_success = (
2079
+ float(oracle_method["mean_candidate_oracle_success_rate"])
2080
+ if oracle_method is not None
2081
+ and oracle_method.get("mean_candidate_oracle_success_rate") is not None
2082
+ else float("nan")
2083
+ )
2084
  paired_deltas = {
2085
  "best_clean - canonical_h16": _paired_delta(methods, best_clean_key, "h16_policy_canonical"),
2086
  "best_clean - direct_same_ckpt": _paired_delta(methods, best_clean_key, "near_miss_policy_bc5"),
 
2104
  "same_state_full_vs_no_expert": methods["same_state_full"]["mean_success"]
2105
  - methods["same_state_no_expert"]["mean_success"],
2106
  }
2107
+ decomposition = causal_action_decomposition(
2108
+ base=methods["h16_policy_canonical"]["mean_success"],
2109
+ selected=methods[best_clean_key]["mean_success"],
2110
+ proposal_oracle=proposal_oracle_success,
2111
+ same_state_oracle=methods["same_state_no_expert"]["mean_success"],
2112
+ full_oracle=methods["same_state_full"]["mean_success"],
2113
+ )
2114
  return {
2115
  "generated_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
2116
  "methods": methods,
 
2120
  "no_expert_vs_best_clean": _per_task_delta(methods, "same_state_no_expert", best_clean_key),
2121
  },
2122
  "mechanism_gap": mechanism_gap,
2123
+ "causal_action_decomposition": decomposition.to_dict(),
2124
+ "causal_action_targets": {
2125
+ "selected_success_for_65pct_gap_closure": decomposition.target_for_gap_closure(0.65),
2126
+ "selected_success_for_75pct_gap_closure": decomposition.target_for_gap_closure(0.75),
2127
+ "best_paper_target_range": [0.47, 0.52],
2128
+ },
2129
+ "best_candidate_oracle_key": oracle_key,
2130
  "best_clean_key": best_clean_key,
2131
  }
2132
 
2133
 
2134
+ def _latex_pct(value: float | None) -> str:
2135
+ if value is None:
2136
+ return "--"
2137
+ value = float(value)
2138
+ if math.isnan(value):
2139
+ return "--"
2140
+ return f"{value * 100:.2f}"
2141
+
2142
+
2143
+ def _latex_pp(value: float | None) -> str:
2144
+ if value is None:
2145
+ return "--"
2146
+ value = float(value)
2147
+ if math.isnan(value):
2148
+ return "--"
2149
+ return f"{value * 100:.2f}"
2150
+
2151
+
2152
+ def _render_car_decomposition_table(report: dict[str, Any]) -> str:
2153
+ decomposition = report["causal_action_decomposition"]
2154
+ targets = report["causal_action_targets"]
2155
+ closed = decomposition["closed_fraction_of_noexpert_gap"]
2156
+ closed_text = "--" if math.isnan(float(closed)) else f"{float(closed) * 100:.1f}"
2157
+ return (
2158
+ "\\begin{table}[t]\n"
2159
+ "\\centering\n"
2160
+ "\\caption{Causal Action Regret decomposition on the current six-task "
2161
+ "diagnostic. Support is the proposal-generation gap to the hidden "
2162
+ "same-state no-expert oracle; selector is the clean proposal-oracle "
2163
+ "headroom left by the deployed selector.}\n"
2164
+ "\\label{tab:car-decomposition}\n"
2165
+ "\\small\n"
2166
+ "\\begin{tabular}{@{}lrrrrrr@{}}\n"
2167
+ "\\toprule\n"
2168
+ "Method & Base & Prop. oracle & Selected & State oracle & "
2169
+ "Support & Selector \\\\\n"
2170
+ "\\midrule\n"
2171
+ "Current CIL-Atlas V0 & "
2172
+ f"{_latex_pct(decomposition['base'])} & "
2173
+ f"{_latex_pct(decomposition['proposal_oracle'])} & "
2174
+ f"{_latex_pct(decomposition['selected'])} & "
2175
+ f"{_latex_pct(decomposition['same_state_oracle'])} & "
2176
+ f"{_latex_pp(decomposition['support_gap'])} & "
2177
+ f"{_latex_pp(decomposition['selector_gap'])} \\\\\n"
2178
+ "\\bottomrule\n"
2179
+ "\\end{tabular}\n"
2180
+ "\\vspace{2pt}\n"
2181
+ "\\footnotesize Clean gain is "
2182
+ f"{_latex_pp(decomposition['clean_gain'])} points; gap closed is "
2183
+ f"{closed_text}\\%; 65--75\\% closure targets are "
2184
+ f"{_latex_pct(targets['selected_success_for_65pct_gap_closure'])}--"
2185
+ f"{_latex_pct(targets['selected_success_for_75pct_gap_closure'])}.\n"
2186
+ "\\end{table}\n"
2187
+ )
2188
+
2189
+
2190
  def main() -> int:
2191
  RESULTS_DIR.mkdir(parents=True, exist_ok=True)
2192
  report = build_report()
2193
  OUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
2194
  OUT_MD.write_text(_render_markdown(report), encoding="utf-8")
2195
+ LATEX_TABLES_DIR.mkdir(parents=True, exist_ok=True)
2196
+ OUT_CAR_TABLE.write_text(_render_car_decomposition_table(report), encoding="utf-8")
2197
  print(f"Wrote {OUT_JSON}")
2198
  print(f"Wrote {OUT_MD}")
2199
+ print(f"Wrote {OUT_CAR_TABLE}")
2200
  return 0
2201
 
2202