Auto-sync: 2026-06-28 13:38:23 (part 2)

results/paper_analysis.json

@@ -0,0 +1,791 @@
+{
+  "generated_utc": "2026-06-28T17:38:38+00:00",
+  "mechanism_gap": {
+    "best_clean_vs_direct_same_ckpt": 0.06724637681159418,
+    "best_clean_vs_h16": 0.05275362318840582,
+    "same_state_full_vs_no_expert": 0.12347826086956515,
+    "same_state_no_expert_vs_best_clean": 0.21971014492753632,
+    "same_state_no_expert_vs_h16": 0.27246376811594214
+  },
+  "methods": {
+    "best_clean_residual_k2": {
+      "ci95_success": 0.040145061606012736,
+      "label": "K2 residual transport, safe + margin 0.20",
+      "mean_action_mse_to_best": 0.3984417730505052,
+      "mean_progress": 0.5670320824102237,
+      "mean_success": 0.3501449275362319,
+      "num_completed": 3,
+      "per_task_success": {
+        "LiftPegUpright-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.26340865087329823,
+          "std_success": 0.06057881385012102
+        },
+        "PickCube-v1": {
+          "mean_num_groups": 196.66666666666666,
+          "mean_success": 0.3225911568302873,
+          "std_success": 0.029955807435728066
+        },
+        "PullCube-v1": {
+          "mean_num_groups": 81.0,
+          "mean_success": 0.21334270018480547,
+          "std_success": 0.03862357075869046
+        },
+        "PushCube-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.7478514959028968,
+          "std_success": 0.07707721123855243
+        },
+        "StackCube-v1": {
+          "mean_num_groups": 93.33333333333333,
+          "mean_success": 0.18057498057498056,
+          "std_success": 0.044823450143885134
+        }
+      },
+      "seed_action_mse_to_best": {
+        "0": 0.38253434254990326,
+        "1": 0.3923974754207808,
+        "2": 0.4203935011808315
+      },
+      "seed_progress": {
+        "0": 0.5547752405812397,
+        "1": 0.5637077170092126,
+        "2": 0.5826132896402186
+      },
+      "seed_success": {
+        "0": 0.3426086956521739,
+        "1": 0.3391304347826087,
+        "2": 0.36869565217391304
+      },
+      "selected_candidate_type_counts": {
+        "retrieval_residual_policy_residual": 1610,
+        "retrieval_residual_residual_no_op": 84,
+        "retrieval_residual_residual_wrong_gripper": 31
+      },
+      "selected_residual_scale_counts": {},
+      "selected_type_outcomes": {
+        "retrieval_residual_policy_residual": {
+          "count": 1610.0,
+          "mean_progress": 0.5620848575190935,
+          "success_rate": 0.3453416149068323
+        },
+        "retrieval_residual_residual_no_op": {
+          "count": 84.0,
+          "mean_progress": 0.6583140224129671,
+          "success_rate": 0.4166666666666667
+        },
+        "retrieval_residual_residual_wrong_gripper": {
+          "count": 31.0,
+          "mean_progress": 0.5766239893292228,
+          "success_rate": 0.41935483870967744
+        }
+      },
+      "source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_knn2_scale0p40_safe_types_margin0p20_summary.json",
+      "std_success": 0.016159257814221867
+    },
+    "gaussian_field": {
+      "best_config": "k32_sigma0.35",
+      "ci95_success": 0.03242840579710147,
+      "label": "Gaussian field search",
+      "mean_action_mse_to_best": 0.41566191710828654,
+      "mean_progress": 0.5343561867265489,
+      "mean_success": 0.2910144927536232,
+      "num_completed": 3,
+      "per_task_success": {
+        "LiftPegUpright-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.2242410592099261,
+          "std_success": 0.021337673409952483
+        },
+        "PickCube-v1": {
+          "mean_num_groups": 196.66666666666666,
+          "mean_success": 0.205857347161695,
+          "std_success": 0.02323219995097751
+        },
+        "PullCube-v1": {
+          "mean_num_groups": 81.0,
+          "mean_success": 0.18976481608060555,
+          "std_success": 0.056985676016897836
+        },
+        "PushCube-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.7240118778985432,
+          "std_success": 0.08854264670468585
+        },
+        "StackCube-v1": {
+          "mean_num_groups": 93.33333333333333,
+          "mean_success": 0.1442927442927443,
+          "std_success": 0.06254861803217816
+        }
+      },
+      "seed_action_mse_to_best": {
+        "0": 0.3982355571473422,
+        "1": 0.4134289959891011,
+        "2": 0.43532119818841636
+      },
+      "seed_progress": {
+        "0": 0.523629953644198,
+        "1": 0.5489098613255697,
+        "2": 0.5305287452098792
+      },
+      "seed_success": {
+        "0": 0.2782608695652174,
+        "1": 0.30434782608695654,
+        "2": 0.29043478260869565
+      },
+      "selected_candidate_type_counts": {
+        "field_selected": 1725
+      },
+      "selected_residual_scale_counts": {},
+      "selected_type_outcomes": {
+        "field_selected": {
+          "count": 1725.0,
+          "mean_progress": 0.5343561867265489,
+          "success_rate": 0.2910144927536232
+        }
+      },
+      "source": "results/h16_field_sweep_summary.json",
+      "std_success": 0.01305313652080894
+    },
+    "h16_policy_canonical": {
+      "ci95_success": 0.032619725485313895,
+      "label": "Direct h=16 policy, canonical rollout",
+      "mean_action_mse_to_best": 0.39868307013577525,
+      "mean_progress": 0.5443993710955061,
+      "mean_success": 0.29739130434782607,
+      "num_completed": 3,
+      "per_task_success": {
+        "LiftPegUpright-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.23062733783413925,
+          "std_success": 0.022553378573476553
+        },
+        "PickCube-v1": {
+          "mean_num_groups": 196.66666666666666,
+          "mean_success": 0.205857347161695,
+          "std_success": 0.02323219995097751
+        },
+        "PullCube-v1": {
+          "mean_num_groups": 81.0,
+          "mean_success": 0.19785448469658995,
+          "std_success": 0.060076299188250165
+        },
+        "PushCube-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.7505571718758995,
+          "std_success": 0.08230105593270941
+        },
+        "StackCube-v1": {
+          "mean_num_groups": 93.33333333333333,
+          "mean_success": 0.14096644096644095,
+          "std_success": 0.05389485806900829
+        }
+      },
+      "seed_action_mse_to_best": {
+        "0": 0.38102538562825194,
+        "1": 0.3963353596538629,
+        "2": 0.4186884651252109
+      },
+      "seed_progress": {
+        "0": 0.5311609336215517,
+        "1": 0.5577803822104698,
+        "2": 0.5442567974544964
+      },
+      "seed_success": {
+        "0": 0.28347826086956524,
+        "1": 0.3095652173913043,
+        "2": 0.2991304347826087
+      },
+      "selected_candidate_type_counts": {
+        "policy_continuous": 1725
+      },
+      "selected_residual_scale_counts": {},
+      "selected_type_outcomes": {
+        "policy_continuous": {
+          "count": 1725.0,
+          "mean_progress": 0.544399371095506,
+          "success_rate": 0.29739130434782607
+        }
+      },
+      "source": "/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs/seed_*/online_rollout.json",
+      "std_success": 0.01313014684394911
+    },
+    "near_miss_policy_bc5": {
+      "ci95_success": 0.019955942028985537,
+      "label": "Near-miss proposal policy, direct",
+      "mean_action_mse_to_best": 0.3943580119071555,
+      "mean_progress": 0.5199238422948863,
+      "mean_success": 0.2828985507246377,
+      "num_completed": 3,
+      "per_task_success": {
+        "LiftPegUpright-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.246534113371651,
+          "std_success": 0.03109492306125915
+        },
+        "PickCube-v1": {
+          "mean_num_groups": 196.66666666666666,
+          "mean_success": 0.18615010191097148,
+          "std_success": 0.04054046058455721
+        },
+        "PullCube-v1": {
+          "mean_num_groups": 81.0,
+          "mean_success": 0.14173818384344702,
+          "std_success": 0.07763680512318745
+        },
+        "PushCube-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.6353647728200077,
+          "std_success": 0.0750496464623429
+        },
+        "StackCube-v1": {
+          "mean_num_groups": 93.33333333333333,
+          "mean_success": 0.2624042624042624,
+          "std_success": 0.05357661572836266
+        }
+      },
+      "seed_action_mse_to_best": {
+        "0": 0.3831931037115662,
+        "1": 0.3916490852298296,
+        "2": 0.4082318467800708
+      },
+      "seed_progress": {
+        "0": 0.5299277274051438,
+        "1": 0.5173212753750208,
+        "2": 0.5125225241044942
+      },
+      "seed_success": {
+        "0": 0.2782608695652174,
+        "1": 0.2921739130434783,
+        "2": 0.2782608695652174
+      },
+      "selected_candidate_type_counts": {
+        "policy_continuous": 1725
+      },
+      "selected_residual_scale_counts": {},
+      "selected_type_outcomes": {
+        "policy_continuous": {
+          "count": 1725.0,
+          "mean_progress": 0.5199238422948862,
+          "success_rate": 0.28289855072463765
+        }
+      },
+      "source": "results/h16_policy_ckpt_near_miss_policy_bc5_summary.json",
+      "std_success": 0.008032699397420892
+    },
+    "residual_k4_consensus": {
+      "ci95_success": 0.04490086956521744,
+      "label": "K4 mean-by-type tangent consensus",
+      "mean_action_mse_to_best": 0.3951832763526751,
+      "mean_progress": 0.5664916701836668,
+      "mean_success": 0.34956521739130436,
+      "num_completed": 3,
+      "per_task_success": {
+        "LiftPegUpright-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.26340865087329823,
+          "std_success": 0.06057881385012102
+        },
+        "PickCube-v1": {
+          "mean_num_groups": 196.66666666666666,
+          "mean_success": 0.32271924934968416,
+          "std_success": 0.02563840631968968
+        },
+        "PullCube-v1": {
+          "mean_num_groups": 81.0,
+          "mean_success": 0.21710336447178555,
+          "std_success": 0.028486855778825676
+        },
+        "PushCube-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.7382993906061106,
+          "std_success": 0.07739498258812846
+        },
+        "StackCube-v1": {
+          "mean_num_groups": 93.33333333333333,
+          "mean_success": 0.1842786842786843,
+          "std_success": 0.0420986975433789
+        }
+      },
+      "seed_action_mse_to_best": {
+        "0": 0.3816084327896976,
+        "1": 0.38771247025741185,
+        "2": 0.41622892601091577
+      },
+      "seed_progress": {
+        "0": 0.550848600073596,
+        "1": 0.5633281786707433,
+        "2": 0.5852982318066611
+      },
+      "seed_success": {
+        "0": 0.3391304347826087,
+        "1": 0.3391304347826087,
+        "2": 0.37043478260869567
+      },
+      "selected_candidate_type_counts": {
+        "retrieval_residual_policy_residual": 1666,
+        "retrieval_residual_residual_no_op": 35,
+        "retrieval_residual_residual_wrong_gripper": 24
+      },
+      "selected_residual_scale_counts": {},
+      "selected_type_outcomes": {
+        "retrieval_residual_policy_residual": {
+          "count": 1666.0,
+          "mean_progress": 0.5619538651180497,
+          "success_rate": 0.3427370948379352
+        },
+        "retrieval_residual_residual_no_op": {
+          "count": 35.0,
+          "mean_progress": 0.7849920756050519,
+          "success_rate": 0.6285714285714286
+        },
+        "retrieval_residual_residual_wrong_gripper": {
+          "count": 24.0,
+          "mean_progress": 0.5628445472490663,
+          "success_rate": 0.4166666666666667
+        }
+      },
+      "source": "results/h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_k4s040_safe_margin0p20_mean_by_type_summary.json",
+      "std_success": 0.018073573644196997
+    },
+    "same_state_full": {
+      "ci95_success": 0.0885808391099616,
+      "label": "Same-state lattice, full",
+      "mean_action_mse_to_best": 0.4375445205177369,
+      "mean_progress": 0.8108624100247802,
+      "mean_success": 0.6933333333333334,
+      "num_completed": 3,
+      "per_task_success": {
+        "LiftPegUpright-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.7319153640685664,
+          "std_success": 0.009244868116419832
+        },
+        "PickCube-v1": {
+          "mean_num_groups": 196.66666666666666,
+          "mean_success": 0.8418873799308582,
+          "std_success": 0.06029163337598174
+        },
+        "PullCube-v1": {
+          "mean_num_groups": 81.0,
+          "mean_success": 0.2241133135869978,
+          "std_success": 0.05014441905467882
+        },
+        "PushCube-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.8192156765992872,
+          "std_success": 0.0284217046535236
+        },
+        "StackCube-v1": {
+          "mean_num_groups": 93.33333333333333,
+          "mean_success": 0.6082510082510082,
+          "std_success": 0.06356803112780139
+        }
+      },
+      "seed_action_mse_to_best": {
+        "0": 0.414993303792207,
+        "1": 0.41892785357189893,
+        "2": 0.47871240418910493
+      },
+      "seed_progress": {
+        "0": 0.792444871255885,
+        "1": 0.8205086112848442,
+        "2": 0.8196337475336116
+      },
+      "seed_success": {
+        "0": 0.6521739130434783,
+        "1": 0.7147826086956521,
+        "2": 0.7130434782608696
+      },
+      "selected_candidate_type_counts": {
+        "lattice_expert": 977,
+        "lattice_near_miss": 348,
+        "lattice_no_op": 177,
+        "lattice_random_negative": 138,
+        "lattice_wrong_direction": 30,
+        "lattice_wrong_gripper": 55
+      },
+      "selected_residual_scale_counts": {},
+      "selected_type_outcomes": {
+        "lattice_expert": {
+          "count": 977.0,
+          "mean_progress": 0.9339633623143789,
+          "success_rate": 0.8444216990788127
+        },
+        "lattice_near_miss": {
+          "count": 348.0,
+          "mean_progress": 0.8058800027416698,
+          "success_rate": 0.6436781609195402
+        },
+        "lattice_no_op": {
+          "count": 177.0,
+          "mean_progress": 0.6444597694665025,
+          "success_rate": 0.4745762711864407
+        },
+        "lattice_random_negative": {
+          "count": 138.0,
+          "mean_progress": 0.29152617198617564,
+          "success_rate": 0.14492753623188406
+        },
+        "lattice_wrong_direction": {
+          "count": 30.0,
+          "mean_progress": 0.5927432135989269,
+          "success_rate": 0.4666666666666667
+        },
+        "lattice_wrong_gripper": {
+          "count": 55.0,
+          "mean_progress": 0.6132168003611944,
+          "success_rate": 0.5272727272727272
+        }
+      },
+      "source": "results/h16_lattice_summary.json",
+      "std_success": 0.035655708555783816
+    },
+    "same_state_near_miss": {
+      "ci95_success": 0.08178741440600439,
+      "label": "Same-state lattice, near-miss only",
+      "mean_action_mse_to_best": 0.34738104417867804,
+      "mean_progress": 0.7515200069436928,
+      "mean_success": 0.5594202898550724,
+      "num_completed": 3,
+      "per_task_success": {
+        "LiftPegUpright-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.6289923744792142,
+          "std_success": 0.0538077190869994
+        },
+        "PickCube-v1": {
+          "mean_num_groups": 196.66666666666666,
+          "mean_success": 0.5813155242503069,
+          "std_success": 0.05906115334855372
+        },
+        "PullCube-v1": {
+          "mean_num_groups": 81.0,
+          "mean_success": 0.150226576542366,
+          "std_success": 0.043012136508133576
+        },
+        "PushCube-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.8253741417845063,
+          "std_success": 0.04067409265225504
+        },
+        "StackCube-v1": {
+          "mean_num_groups": 93.33333333333333,
+          "mean_success": 0.504107004107004,
+          "std_success": 0.021606778206277356
+        }
+      },
+      "seed_action_mse_to_best": {
+        "0": 0.33686841549711183,
+        "1": 0.338796136389365,
+        "2": 0.36647858064955746
+      },
+      "seed_progress": {
+        "0": 0.7402807480075242,
+        "1": 0.7557949085549815,
+        "2": 0.7584843642685725
+      },
+      "seed_success": {
+        "0": 0.5217391304347826,
+        "1": 0.5826086956521739,
+        "2": 0.5739130434782609
+      },
+      "selected_candidate_type_counts": {
+        "lattice_near_miss": 1725
+      },
+      "selected_residual_scale_counts": {},
+      "selected_type_outcomes": {
+        "lattice_near_miss": {
+          "count": 1725.0,
+          "mean_progress": 0.7515200069436927,
+          "success_rate": 0.5594202898550724
+        }
+      },
+      "source": "results/h16_lattice_near_miss_only_v2_summary.json",
+      "std_success": 0.032921207801740716
+    },
+    "same_state_no_expert": {
+      "ci95_success": 0.11474666666666679,
+      "label": "Same-state lattice, no expert",
+      "mean_action_mse_to_best": 0.45901029001275234,
+      "mean_progress": 0.7501174603224449,
+      "mean_success": 0.5698550724637682,
+      "num_completed": 3,
+      "per_task_success": {
+        "LiftPegUpright-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.617157168344332,
+          "std_success": 0.03559446921051549
+        },
+        "PickCube-v1": {
+          "mean_num_groups": 196.66666666666666,
+          "mean_success": 0.6214788633266894,
+          "std_success": 0.10345152045609288
+        },
+        "PullCube-v1": {
+          "mean_num_groups": 81.0,
+          "mean_success": 0.1984797853218906,
+          "std_success": 0.049441545131695956
+        },
+        "PushCube-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.8010429852645989,
+          "std_success": 0.018497036403234836
+        },
+        "StackCube-v1": {
+          "mean_num_groups": 93.33333333333333,
+          "mean_success": 0.4825174825174825,
+          "std_success": 0.04949074107006737
+        }
+      },
+      "seed_action_mse_to_best": {
+        "0": 0.4291479373977864,
+        "1": 0.44271026729079693,
+        "2": 0.5051726653496735
+      },
+      "seed_progress": {
+        "0": 0.7308024795159049,
+        "1": 0.756582982868928,
+        "2": 0.7629669185825017
+      },
+      "seed_success": {
+        "0": 0.5165217391304348,
+        "1": 0.5965217391304348,
+        "2": 0.5965217391304348
+      },
+      "selected_candidate_type_counts": {
+        "lattice_near_miss": 1263,
+        "lattice_no_op": 222,
+        "lattice_random_negative": 144,
+        "lattice_wrong_direction": 34,
+        "lattice_wrong_gripper": 62
+      },
+      "selected_residual_scale_counts": {},
+      "selected_type_outcomes": {
+        "lattice_near_miss": {
+          "count": 1263.0,
+          "mean_progress": 0.824350294011836,
+          "success_rate": 0.6318289786223278
+        },
+        "lattice_no_op": {
+          "count": 222.0,
+          "mean_progress": 0.6904902144729554,
+          "success_rate": 0.5225225225225225
+        },
+        "lattice_random_negative": {
+          "count": 144.0,
+          "mean_progress": 0.28417760477815235,
+          "success_rate": 0.1388888888888889
+        },
+        "lattice_wrong_direction": {
+          "count": 34.0,
+          "mean_progress": 0.5852376738672748,
+          "success_rate": 0.47058823529411764
+        },
+        "lattice_wrong_gripper": {
+          "count": 62.0,
+          "mean_progress": 0.6240276468827599,
+          "success_rate": 0.532258064516129
+        }
+      },
+      "source": "results/h16_lattice_no_expert_summary.json",
+      "std_success": 0.046188021535170105
+    },
+    "same_state_policy_baseline": {
+      "ci95_success": 0.12189879370089796,
+      "label": "Same-state no-expert + policy candidate",
+      "mean_action_mse_to_best": 0.43814348461106417,
+      "mean_progress": 0.631166610757726,
+      "mean_success": 0.40695652173913044,
+      "num_completed": 3,
+      "per_task_success": {
+        "LiftPegUpright-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.3079640315664629,
+          "std_success": 0.04002483097272376
+        },
+        "PickCube-v1": {
+          "mean_num_groups": 196.66666666666666,
+          "mean_success": 0.3750668614799049,
+          "std_success": 0.05362858173802174
+        },
+        "PullCube-v1": {
+          "mean_num_groups": 81.0,
+          "mean_success": 0.21089845826687928,
+          "std_success": 0.048311495828015846
+        },
+        "PushCube-v1": {
+          "mean_num_groups": 102.0,
+          "mean_success": 0.749962517815899,
+          "std_success": 0.08915824474967014
+        },
+        "StackCube-v1": {
+          "mean_num_groups": 93.33333333333333,
+          "mean_success": 0.37118067118067116,
+          "std_success": 0.04773148081461797
+        }
+      },
+      "seed_action_mse_to_best": {
+        "0": 0.4125794651391714,
+        "1": 0.42664123004302384,
+        "2": 0.4752097586509974
+      },
+      "seed_progress": {
+        "0": 0.6054906688468612,
+        "1": 0.6635976886036603,
+        "2": 0.624411474822656
+      },
+      "seed_success": {
+        "0": 0.3617391304347826,
+        "1": 0.4591304347826087,
+        "2": 0.4
+      },
+      "selected_candidate_type_counts": {
+        "lattice_near_miss": 448,
+        "lattice_no_op": 119,
+        "lattice_random_negative": 75,
+        "lattice_wrong_direction": 16,
+        "lattice_wrong_gripper": 45,
+        "policy_continuous": 1022
+      },
+      "selected_residual_scale_counts": {},
+      "selected_type_outcomes": {
+        "lattice_near_miss": {
+          "count": 448.0,
+          "mean_progress": 0.8320851847196796,
+          "success_rate": 0.640625
+        },
+        "lattice_no_op": {
+          "count": 119.0,
+          "mean_progress": 0.7418963548516025,
+          "success_rate": 0.5966386554621849
+        },
+        "lattice_random_negative": {
+          "count": 75.0,
+          "mean_progress": 0.3766690089305242,
+          "success_rate": 0.25333333333333335
+        },
+        "lattice_wrong_direction": {
+          "count": 16.0,
+          "mean_progress": 0.6197442151606083,
+          "success_rate": 0.5
+        },
+        "lattice_wrong_gripper": {
+          "count": 45.0,
+          "mean_progress": 0.6137187815995679,
+          "success_rate": 0.5111111111111111
+        },
+        "policy_continuous": {
+          "count": 1022.0,
+          "mean_progress": 0.5498230394236597,
+          "success_rate": 0.2876712328767123
+        }
+      },
+      "source": "results/h16_lattice_no_expert_policy_baseline_margin000_summary.json",
+      "std_success": 0.04906690775535958
+    }
+  },
+  "paired_deltas": {
+    "best_clean - canonical_h16": {
+      "ci95_delta": 0.051546144179697446,
+      "left": "best_clean_residual_k2",
+      "mean_delta": 0.052753623188405784,
+      "right": "h16_policy_canonical",
+      "seed_deltas": {
+        "0": 0.059130434782608654,
+        "1": 0.029565217391304355,
+        "2": 0.06956521739130433
+      },
+      "seeds": [
+        0,
+        1,
+        2
+      ],
+      "std_delta": 0.020748440774693643
+    },
+    "best_clean - direct_same_ckpt": {
+      "ci95_delta": 0.054366209142189946,
+      "left": "best_clean_residual_k2",
+      "mean_delta": 0.06724637681159418,
+      "right": "near_miss_policy_bc5",
+      "seed_deltas": {
+        "0": 0.0643478260869565,
+        "1": 0.0469565217391304,
+        "2": 0.09043478260869564
+      },
+      "seeds": [
+        0,
+        1,
+        2
+      ],
+      "std_delta": 0.021883578073248564
+    },
+    "full_lattice - no_expert_lattice": {
+      "ci95_delta": 0.02628111194117426,
+      "left": "same_state_full",
+      "mean_delta": 0.12347826086956519,
+      "right": "same_state_no_expert",
+      "seed_deltas": {
+        "0": 0.13565217391304352,
+        "1": 0.11826086956521731,
+        "2": 0.11652173913043473
+      },
+      "seeds": [
+        0,
+        1,
+        2
+      ],
+      "std_delta": 0.010578717443996964
+    },
+    "no_expert_lattice - canonical_h16": {
+      "ci95_delta": 0.08579749715341586,
+      "left": "same_state_no_expert",
+      "mean_delta": 0.27246376811594203,
+      "right": "h16_policy_canonical",
+      "seed_deltas": {
+        "0": 0.23304347826086952,
+        "1": 0.2869565217391305,
+        "2": 0.2973913043478261
+      },
+      "seeds": [
+        0,
+        1,
+        2
+      ],
+      "std_delta": 0.034535353063435366
+    },
+    "policy_candidate_lattice - no_expert_lattice": {
+      "ci95_delta": 0.07549705023149442,
+      "left": "same_state_policy_baseline",
+      "mean_delta": -0.1628985507246377,
+      "right": "same_state_no_expert",
+      "seed_deltas": {
+        "0": -0.15478260869565214,
+        "1": -0.13739130434782615,
+        "2": -0.1965217391304348
+      },
+      "seeds": [
+        0,
+        1,
+        2
+      ],
+      "std_delta": 0.030389199819318615
+    }
+  },
+  "per_task_deltas": {
+    "best_clean_vs_h16": {
+      "LiftPegUpright-v1": 0.03278131303915899,
+      "PickCube-v1": 0.11673380966859229,
+      "PullCube-v1": 0.015488215488215523,
+      "PushCube-v1": -0.0027056759730027524,
+      "StackCube-v1": 0.039608539608539606
+    },
+    "no_expert_vs_best_clean": {
+      "LiftPegUpright-v1": 0.35374851747103375,
+      "PickCube-v1": 0.2988877064964022,
+      "PullCube-v1": -0.014862914862914883,
+      "PushCube-v1": 0.05319148936170215,
+      "StackCube-v1": 0.30194250194250194
+    }
+  }
+}

results/paper_analysis.md

@@ -0,0 +1,76 @@
+# Paper Analysis
+
+Generated: `2026-06-28T17:38:38+00:00`
+
+## Main Seed Statistics
+
+| key | method | n | success | 95% CI | progress | action MSE | gain vs canonical h16 |
+|---|---|---:|---:|---:|---:|---:|---:|
+| h16_policy_canonical | Direct h=16 policy, canonical rollout | 3 | 29.74% +/- 1.31 | +/- 3.26 | 54.44% | 0.399 | +0.00 pp |
+| gaussian_field | Gaussian field search | 3 | 29.10% +/- 1.31 | +/- 3.24 | 53.44% | 0.416 | -0.64 pp |
+| near_miss_policy_bc5 | Near-miss proposal policy, direct | 3 | 28.29% +/- 0.80 | +/- 2.00 | 51.99% | 0.394 | -1.45 pp |
+| best_clean_residual_k2 | K2 residual transport, safe + margin 0.20 | 3 | 35.01% +/- 1.62 | +/- 4.01 | 56.70% | 0.398 | +5.28 pp |
+| residual_k4_consensus | K4 mean-by-type tangent consensus | 3 | 34.96% +/- 1.81 | +/- 4.49 | 56.65% | 0.395 | +5.22 pp |
+| same_state_near_miss | Same-state lattice, near-miss only | 3 | 55.94% +/- 3.29 | +/- 8.18 | 75.15% | 0.347 | +26.20 pp |
+| same_state_no_expert | Same-state lattice, no expert | 3 | 56.99% +/- 4.62 | +/- 11.47 | 75.01% | 0.459 | +27.25 pp |
+| same_state_policy_baseline | Same-state no-expert + policy candidate | 3 | 40.70% +/- 4.91 | +/- 12.19 | 63.12% | 0.438 | +10.96 pp |
+| same_state_full | Same-state lattice, full | 3 | 69.33% +/- 3.57 | +/- 8.86 | 81.09% | 0.438 | +39.59 pp |
+
+## Paired Seed Deltas
+
+| comparison | seeds | mean delta | 95% CI | seed deltas |
+|---|---:|---:|---:|---|
+| best_clean - canonical_h16 | 3 | +5.28 pp | +/- 5.15 | 0:+5.91, 1:+2.96, 2:+6.96 |
+| best_clean - direct_same_ckpt | 3 | +6.72 pp | +/- 5.44 | 0:+6.43, 1:+4.70, 2:+9.04 |
+| no_expert_lattice - canonical_h16 | 3 | +27.25 pp | +/- 8.58 | 0:+23.30, 1:+28.70, 2:+29.74 |
+| full_lattice - no_expert_lattice | 3 | +12.35 pp | +/- 2.63 | 0:+13.57, 1:+11.83, 2:+11.65 |
+| policy_candidate_lattice - no_expert_lattice | 3 | -16.29 pp | +/- 7.55 | 0:-15.48, 1:-13.74, 2:-19.65 |
+
+## Per-Task Mean Success
+
+| task | h16 policy | best clean | near-miss lattice | no-expert lattice | full lattice | clean-h16 delta | noexpert-clean gap |
+|---|---:|---:|---:|---:|---:|---:|---:|
+| LiftPegUpright-v1 | 23.06% | 26.34% | 62.90% | 61.72% | 73.19% | +3.28 pp | +35.37 pp |
+| PickCube-v1 | 20.59% | 32.26% | 58.13% | 62.15% | 84.19% | +11.67 pp | +29.89 pp |
+| PullCube-v1 | 19.79% | 21.33% | 15.02% | 19.85% | 22.41% | +1.55 pp | -1.49 pp |
+| PushCube-v1 | 75.06% | 74.79% | 82.54% | 80.10% | 81.92% | -0.27 pp | +5.32 pp |
+| StackCube-v1 | 14.10% | 18.06% | 50.41% | 48.25% | 60.83% | +3.96 pp | +30.19 pp |
+
+## Mechanism Gap
+
+- Best clean residual transport improves over canonical h16 by +5.28 pp.
+- Same-state no-expert lattice improves over canonical h16 by +27.25 pp.
+- Remaining clean-to-same-state proposal gap is +21.97 pp.
+- Full lattice adds expert proposals and reaches 69.33%, a +12.35 pp gain over no-expert.
+
+## Selection Histograms
+
+- `same_state_near_miss`: lattice_near_miss=1725 (100.0%)
+- `same_state_no_expert`: lattice_near_miss=1263 (73.2%), lattice_no_op=222 (12.9%), lattice_random_negative=144 (8.3%), lattice_wrong_gripper=62 (3.6%), lattice_wrong_direction=34 (2.0%)
+- `same_state_policy_baseline`: policy_continuous=1022 (59.2%), lattice_near_miss=448 (26.0%), lattice_no_op=119 (6.9%), lattice_random_negative=75 (4.3%), lattice_wrong_gripper=45 (2.6%), lattice_wrong_direction=16 (0.9%)
+- `same_state_full`: lattice_expert=977 (56.6%), lattice_near_miss=348 (20.2%), lattice_no_op=177 (10.3%), lattice_random_negative=138 (8.0%), lattice_wrong_gripper=55 (3.2%), lattice_wrong_direction=30 (1.7%)
+- `best_clean_residual_k2`: retrieval_residual_policy_residual=1610 (93.3%), retrieval_residual_residual_no_op=84 (4.9%), retrieval_residual_residual_wrong_gripper=31 (1.8%)
+
+## Selected-Type Outcomes
+
+These rows are measured from raw rollout rows. In residual retrieval, `policy_residual` is the zero-residual action, i.e. abstaining to the current policy mean.
+
+| method | selected type | count | success | progress |
+|---|---|---:|---:|---:|
+| best_clean_residual_k2 | retrieval_residual_policy_residual | 1610 | 34.53% | 56.21% |
+| best_clean_residual_k2 | retrieval_residual_residual_no_op | 84 | 41.67% | 65.83% |
+| best_clean_residual_k2 | retrieval_residual_residual_wrong_gripper | 31 | 41.94% | 57.66% |
+| residual_k4_consensus | retrieval_residual_policy_residual | 1666 | 34.27% | 56.20% |
+| residual_k4_consensus | retrieval_residual_residual_no_op | 35 | 62.86% | 78.50% |
+| residual_k4_consensus | retrieval_residual_residual_wrong_gripper | 24 | 41.67% | 56.28% |
+| same_state_no_expert | lattice_near_miss | 1263 | 63.18% | 82.44% |
+| same_state_no_expert | lattice_no_op | 222 | 52.25% | 69.05% |
+| same_state_no_expert | lattice_random_negative | 144 | 13.89% | 28.42% |
+| same_state_no_expert | lattice_wrong_gripper | 62 | 53.23% | 62.40% |
+| same_state_no_expert | lattice_wrong_direction | 34 | 47.06% | 58.52% |
+| same_state_policy_baseline | policy_continuous | 1022 | 28.77% | 54.98% |
+| same_state_policy_baseline | lattice_near_miss | 448 | 64.06% | 83.21% |
+| same_state_policy_baseline | lattice_no_op | 119 | 59.66% | 74.19% |
+| same_state_policy_baseline | lattice_random_negative | 75 | 25.33% | 37.67% |
+| same_state_policy_baseline | lattice_wrong_gripper | 45 | 51.11% | 61.37% |
+| same_state_policy_baseline | lattice_wrong_direction | 16 | 50.00% | 61.97% |

results/paper_core_results.md

@@ -3,6 +3,12 @@
 All rows use 3 seeds and 575 validation groups per seed unless noted. The direct policy
 baseline is the h=16 rank-checkpoint online rollout (`29.74%`).
 
+For paired seed deltas, per-task gaps, and selection histograms, regenerate and
+read `paper_analysis.md` with `python3 scripts/build_paper_analysis.py`. Current
+paired analysis: best clean K2 residual transport is `+5.28 pp` over canonical
+h=16, same-state no-expert lattice is `+27.25 pp`, and the remaining
+clean-to-same-state proposal gap is `+21.97 pp`.
+
 | Method | Uses same-state proposals | Uses expert proposal | Success | Gain vs policy | Interpretation |
 |---|---:|---:|---:|---:|---|
 | Direct h=16 policy | No | No | 29.74% | -- | BC policy cannot exploit high oracle ceiling |

results/paper_story_memo.md

@@ -26,7 +26,9 @@ when queried on proposal geometry that matches those local counterfactuals.
 | All-split field-teacher distillation does not fix checkpointing/coverage | allmap direct is 28.00%; field-guided best is 26.49% despite 100% target coverage | Negative diagnostic |
 | Residual family consistency improves clean transport | policy/no-op/wrong-gripper typed residuals reach 33.74%, above raw 33.33% | Supported as diagnostic |
 | Counterfactual advantage abstention improves clean transport | requiring field advantage over the zero-residual policy raises typed residual transport to 34.84%, and K2 retrieval reaches 35.01% | Current best clean result |
+| Clean residual transport behaves like sparse intervention | `paper_analysis.md` shows K2 residual retrieval abstains to zero-residual policy on 93.3% of states, while selected nonzero no-op/wrong-gripper residuals succeed at ~41.7-41.9% vs 34.5% for abstention | Stronger clean-mechanism framing |
 | Tangent consensus is close but does not beat raw K2 residuals | K4 mean-by-type residual consensus reaches 34.96%, just below the 35.01% K2 raw residual row | Near-tie diagnostic |
+| The proposal gap is now quantified | `paper_analysis.md` reports best clean +5.28 pp over canonical h16, same-state no-expert +27.25 pp, leaving a +21.97 pp clean-to-same-state gap | Core paper tension |
 | Policy fallback is not the same-state mechanism | adding a policy baseline candidate to the no-expert same-state lattice drops 56.99% to 40.70% even with margin 0.00 | Negative diagnostic |
 | Z-score retrieval metric does not help | z-score rows reach 32.23-32.81%, below raw retrieval | Negative diagnostic |
 | Train-split residual family reliability does not recover the typed mask | after fixing threshold pass-through, scale-0.35 thresholds 0.10/0.25 reach 33.33%/33.28%, below typed safe residuals | Negative diagnostic |
@@ -89,7 +91,7 @@ test-time search. The cleaner novelty is:
 
 ## Job Status
 
-Last checked: `2026-06-28 17:28 UTC`. Ray-search jobs are queued, pending on
+Last checked: `2026-06-28 17:36 UTC`. Ray-search jobs are queued, pending on
 GPU priority.
 
 - `14858328`-`14858333`: completed train-split `field_selected_noexpert_bc5`;
@@ -144,7 +146,9 @@ GPU priority.
 - `14868993`/`14868995`/`14868997`/`14868999`: pending counterfactual tangent
   ray-search rollouts for K1/K2/K4 safe residual retrieval with scale grids.
   Summary jobs `14868994`/`14868996`/`14868998`/`14869000` and table rebuild
-  `14869605` are dependency-gated on those rollouts.
+  `14869605` are dependency-gated on those rollouts. Updated rebuild job
+  `14869860` is also dependency-gated and will regenerate both paper table and
+  paired analysis outputs with the current script.
 - `14869627`: completed CPU Apptainer smoke for the new residual scale-grid
   selector. It selected index `3` on a two-residual/two-scale toy case and
   returned the expected action `0.20`, validating the candidate expansion and
@@ -166,6 +170,8 @@ GPU priority.
   result.
 - Use K2 typed safe residual transport with advantage abstention (35.01%) only as the current best clean
   deployment diagnostic, not as a SOTA claim.
+- Use `results/paper_analysis.md` for paired seed deltas, per-task gaps, and
+  selection histograms when writing reviewer-facing tables.
 - Treat z-score retrieval, repaired train-family reliability priors, Gaussian hybrids,
   field optimization, field-teacher/tangent distillation, policy-relative anchoring, tangent consensus,
   and same-state policy-baseline fallback as negative or near-tie diagnostics

scripts/build_paper_analysis.py

@@ -0,0 +1,488 @@
+#!/usr/bin/env python
+from __future__ import annotations
+
+import json
+import math
+from collections import Counter
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+RESULTS_DIR = Path("results")
+OUT_JSON = RESULTS_DIR / "paper_analysis.json"
+OUT_MD = RESULTS_DIR / "paper_analysis.md"
+CANONICAL_H16_ROLLOUT = Path("/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs")
+
+
+@dataclass(frozen=True)
+class MethodSpec:
+    key: str
+    label: str
+    summary_path: str | None = None
+    raw_rollout_glob: str | None = None
+    summary_mode: str = "standard"
+
+
+METHODS = [
+    MethodSpec(
+        key="h16_policy_canonical",
+        label="Direct h=16 policy, canonical rollout",
+        raw_rollout_glob=str(CANONICAL_H16_ROLLOUT / "seed_*/online_rollout.json"),
+    ),
+    MethodSpec(
+        key="gaussian_field",
+        label="Gaussian field search",
+        summary_path="h16_field_sweep_summary.json",
+        summary_mode="field_sweep_best",
+    ),
+    MethodSpec(
+        key="near_miss_policy_bc5",
+        label="Near-miss proposal policy, direct",
+        summary_path="h16_policy_ckpt_near_miss_policy_bc5_summary.json",
+    ),
+    MethodSpec(
+        key="best_clean_residual_k2",
+        label="K2 residual transport, safe + margin 0.20",
+        summary_path=(
+            "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
+            "knn2_scale0p40_safe_types_margin0p20_summary.json"
+        ),
+    ),
+    MethodSpec(
+        key="residual_k4_consensus",
+        label="K4 mean-by-type tangent consensus",
+        summary_path=(
+            "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
+            "k4s040_safe_margin0p20_mean_by_type_summary.json"
+        ),
+    ),
+    MethodSpec(
+        key="same_state_near_miss",
+        label="Same-state lattice, near-miss only",
+        summary_path="h16_lattice_near_miss_only_v2_summary.json",
+    ),
+    MethodSpec(
+        key="same_state_no_expert",
+        label="Same-state lattice, no expert",
+        summary_path="h16_lattice_no_expert_summary.json",
+    ),
+    MethodSpec(
+        key="same_state_policy_baseline",
+        label="Same-state no-expert + policy candidate",
+        summary_path="h16_lattice_no_expert_policy_baseline_margin000_summary.json",
+    ),
+    MethodSpec(
+        key="same_state_full",
+        label="Same-state lattice, full",
+        summary_path="h16_lattice_summary.json",
+    ),
+]
+
+
+def _load_json(path: Path) -> dict[str, Any]:
+    with path.open("r", encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def _mean(values: list[float]) -> float:
+    return sum(values) / len(values) if values else float("nan")
+
+
+def _sample_std(values: list[float]) -> float:
+    if len(values) <= 1:
+        return 0.0
+    mean = _mean(values)
+    return math.sqrt(sum((value - mean) ** 2 for value in values) / (len(values) - 1))
+
+
+def _ci95(values: list[float]) -> float:
+    if len(values) <= 1:
+        return 0.0
+    t_crit = {
+        1: 12.706,
+        2: 4.303,
+        3: 3.182,
+        4: 2.776,
+        5: 2.571,
+        6: 2.447,
+        7: 2.365,
+        8: 2.306,
+        9: 2.262,
+        10: 2.228,
+    }.get(len(values) - 1, 1.96)
+    return t_crit * _sample_std(values) / math.sqrt(len(values))
+
+
+def _success(row: dict[str, Any]) -> float:
+    return float(row["policy_rollout_success_rate"])
+
+
+def _progress(row: dict[str, Any]) -> float:
+    return float(row.get("policy_rollout_progress", float("nan")))
+
+
+def _action_mse(row: dict[str, Any]) -> float:
+    return float(row.get("action_mse_to_best", float("nan")))
+
+
+def _seed(row: dict[str, Any], fallback: int) -> int:
+    return int(row.get("seed", fallback))
+
+
+def _standard_summary(path: Path) -> dict[str, Any]:
+    data = _load_json(path)
+    rows = list(data.get("rows", []))
+    return _normalize_summary(data, rows, source=str(path))
+
+
+def _field_sweep_best(path: Path) -> dict[str, Any]:
+    data = _load_json(path)
+    best_config = data.get("best", {}).get("config")
+    rows = [row for row in data.get("rows", []) if row.get("config") == best_config]
+    normalized = _normalize_summary(data.get("best", data), rows, source=str(path))
+    normalized["best_config"] = best_config
+    return normalized
+
+
+def _raw_rollout_summary(pattern: str) -> dict[str, Any]:
+    rows = []
+    for index, path in enumerate(sorted(Path().glob(pattern) if not pattern.startswith("/") else Path("/").glob(pattern[1:]))):
+        row = _load_json(path)
+        row = dict(row)
+        row["path"] = str(path)
+        row["seed"] = _seed(row, index)
+        rows.append(row)
+    return _normalize_summary({}, rows, source=pattern)
+
+
+def _normalize_summary(data: dict[str, Any], rows: list[dict[str, Any]], *, source: str) -> dict[str, Any]:
+    successes = [_success(row) for row in rows]
+    progress = [_progress(row) for row in rows]
+    action_mse = [_action_mse(row) for row in rows]
+    selected_counts = Counter()
+    selected_counts.update(data.get("selected_candidate_type_counts", {}))
+    selected_scale_counts = Counter()
+    selected_scale_counts.update(data.get("selected_residual_scale_counts", {}))
+    expected_count = sum(int(row.get("num_groups", 0)) for row in rows)
+    has_top_level_selected_counts = (
+        bool(selected_counts) and sum(int(value) for value in selected_counts.values()) == expected_count
+    )
+    has_top_level_scale_counts = (
+        bool(selected_scale_counts)
+        and sum(int(value) for value in selected_scale_counts.values()) == expected_count
+    )
+    if not has_top_level_selected_counts:
+        selected_counts = Counter()
+    if not has_top_level_scale_counts:
+        selected_scale_counts = Counter()
+    if not has_top_level_selected_counts or not has_top_level_scale_counts:
+        for row in rows:
+            path = row.get("path")
+            if not path:
+                continue
+            raw_path = Path(str(path))
+            if not raw_path.exists():
+                continue
+            raw = _load_json(raw_path)
+            if not has_top_level_selected_counts:
+                selected_counts.update(raw.get("selected_candidate_type_counts", {}))
+            if not has_top_level_scale_counts:
+                selected_scale_counts.update(raw.get("selected_residual_scale_counts", {}))
+    return {
+        "source": source,
+        "num_completed": len(rows),
+        "mean_success": _mean(successes),
+        "std_success": _sample_std(successes),
+        "ci95_success": _ci95(successes),
+        "mean_progress": _mean(progress),
+        "mean_action_mse_to_best": _mean(action_mse),
+        "seed_success": {_seed(row, index): _success(row) for index, row in enumerate(rows)},
+        "seed_progress": {_seed(row, index): _progress(row) for index, row in enumerate(rows)},
+        "seed_action_mse_to_best": {_seed(row, index): _action_mse(row) for index, row in enumerate(rows)},
+        "per_task_success": _per_task(rows),
+        "selected_candidate_type_counts": dict(selected_counts),
+        "selected_residual_scale_counts": dict(selected_scale_counts),
+        "selected_type_outcomes": _selected_type_outcomes(rows),
+    }
+
+
+def _per_task(rows: list[dict[str, Any]]) -> dict[str, dict[str, float]]:
+    task_values: dict[str, list[float]] = {}
+    task_counts: dict[str, list[int]] = {}
+    for row in rows:
+        for task, metrics in row.get("per_task", {}).items():
+            task_values.setdefault(task, []).append(float(metrics["policy_rollout_success_rate"]))
+            task_counts.setdefault(task, []).append(int(metrics.get("num_groups", 0)))
+    return {
+        task: {
+            "mean_success": _mean(values),
+            "std_success": _sample_std(values),
+            "mean_num_groups": _mean([float(value) for value in task_counts.get(task, [])]),
+        }
+        for task, values in sorted(task_values.items())
+    }
+
+
+def _selected_type_outcomes(rows: list[dict[str, Any]]) -> dict[str, dict[str, float]]:
+    grouped: dict[str, dict[str, float]] = {}
+    for row in rows:
+        path = row.get("path")
+        if not path:
+            continue
+        raw_path = Path(str(path))
+        if not raw_path.exists():
+            continue
+        raw = _load_json(raw_path)
+        for item in raw.get("rows", []):
+            candidate_type = str(item.get("nearest_candidate_type") or "unknown")
+            stats = grouped.setdefault(
+                candidate_type,
+                {"count": 0.0, "success_sum": 0.0, "progress_sum": 0.0},
+            )
+            stats["count"] += 1.0
+            stats["success_sum"] += 1.0 if item.get("success") else 0.0
+            stats["progress_sum"] += float(item.get("progress", 0.0))
+    return {
+        candidate_type: {
+            "count": values["count"],
+            "success_rate": values["success_sum"] / values["count"] if values["count"] else float("nan"),
+            "mean_progress": values["progress_sum"] / values["count"] if values["count"] else float("nan"),
+        }
+        for candidate_type, values in sorted(
+            grouped.items(),
+            key=lambda item: (-item[1]["count"], item[0]),
+        )
+    }
+
+
+def _load_methods() -> dict[str, dict[str, Any]]:
+    methods: dict[str, dict[str, Any]] = {}
+    for spec in METHODS:
+        if spec.summary_path:
+            path = RESULTS_DIR / spec.summary_path
+            if not path.exists():
+                methods[spec.key] = {"missing": True, "source": str(path), "label": spec.label}
+                continue
+            if spec.summary_mode == "field_sweep_best":
+                method = _field_sweep_best(path)
+            else:
+                method = _standard_summary(path)
+        elif spec.raw_rollout_glob:
+            method = _raw_rollout_summary(spec.raw_rollout_glob)
+        else:
+            method = {"missing": True, "source": "", "label": spec.label}
+        method["label"] = spec.label
+        methods[spec.key] = method
+    return methods
+
+
+def _paired_delta(
+    methods: dict[str, dict[str, Any]],
+    left: str,
+    right: str,
+) -> dict[str, Any]:
+    left_values = methods[left].get("seed_success", {})
+    right_values = methods[right].get("seed_success", {})
+    seeds = sorted(set(left_values) & set(right_values))
+    deltas = [float(left_values[seed]) - float(right_values[seed]) for seed in seeds]
+    return {
+        "left": left,
+        "right": right,
+        "seeds": seeds,
+        "mean_delta": _mean(deltas),
+        "std_delta": _sample_std(deltas),
+        "ci95_delta": _ci95(deltas),
+        "seed_deltas": {seed: delta for seed, delta in zip(seeds, deltas)},
+    }
+
+
+def _per_task_delta(
+    methods: dict[str, dict[str, Any]],
+    left: str,
+    right: str,
+) -> dict[str, float]:
+    left_tasks = methods[left].get("per_task_success", {})
+    right_tasks = methods[right].get("per_task_success", {})
+    return {
+        task: float(left_tasks[task]["mean_success"]) - float(right_tasks[task]["mean_success"])
+        for task in sorted(set(left_tasks) & set(right_tasks))
+    }
+
+
+def _pct(value: float) -> str:
+    if math.isnan(value):
+        return "n/a"
+    return f"{value * 100:.2f}%"
+
+
+def _pp(value: float) -> str:
+    if math.isnan(value):
+        return "n/a"
+    return f"{value * 100:+.2f} pp"
+
+
+def _render_markdown(report: dict[str, Any]) -> str:
+    methods = report["methods"]
+    lines = [
+        "# Paper Analysis",
+        "",
+        f"Generated: `{report['generated_utc']}`",
+        "",
+        "## Main Seed Statistics",
+        "",
+        "| key | method | n | success | 95% CI | progress | action MSE | gain vs canonical h16 |",
+        "|---|---|---:|---:|---:|---:|---:|---:|",
+    ]
+    baseline = methods["h16_policy_canonical"]["mean_success"]
+    for key in [spec.key for spec in METHODS]:
+        method = methods[key]
+        if method.get("missing"):
+            lines.append(f"| {key} | {method['label']} | 0 | missing | missing | missing | missing | missing |")
+            continue
+        lines.append(
+            "| {key} | {label} | {n} | {success} +/- {std} | {ci} | {progress} | {mse:.3f} | {gain} |".format(
+                key=key,
+                label=method["label"],
+                n=method["num_completed"],
+                success=_pct(method["mean_success"]),
+                std=f"{method['std_success'] * 100:.2f}",
+                ci=f"+/- {method['ci95_success'] * 100:.2f}",
+                progress=_pct(method["mean_progress"]),
+                mse=method["mean_action_mse_to_best"],
+                gain=_pp(method["mean_success"] - baseline),
+            )
+        )
+    lines.extend(
+        [
+            "",
+            "## Paired Seed Deltas",
+            "",
+            "| comparison | seeds | mean delta | 95% CI | seed deltas |",
+            "|---|---:|---:|---:|---|",
+        ]
+    )
+    for name, delta in report["paired_deltas"].items():
+        seed_deltas = ", ".join(
+            f"{seed}:{value * 100:+.2f}" for seed, value in delta["seed_deltas"].items()
+        )
+        lines.append(
+            f"| {name} | {len(delta['seeds'])} | {_pp(delta['mean_delta'])} | +/- {delta['ci95_delta'] * 100:.2f} | {seed_deltas} |"
+        )
+    lines.extend(
+        [
+            "",
+            "## Per-Task Mean Success",
+            "",
+            "| task | h16 policy | best clean | near-miss lattice | no-expert lattice | full lattice | clean-h16 delta | noexpert-clean gap |",
+            "|---|---:|---:|---:|---:|---:|---:|---:|",
+        ]
+    )
+    tasks = sorted(methods["h16_policy_canonical"].get("per_task_success", {}))
+    for task in tasks:
+        h16 = methods["h16_policy_canonical"]["per_task_success"][task]["mean_success"]
+        clean = methods["best_clean_residual_k2"]["per_task_success"][task]["mean_success"]
+        near = methods["same_state_near_miss"]["per_task_success"][task]["mean_success"]
+        noexpert = methods["same_state_no_expert"]["per_task_success"][task]["mean_success"]
+        full = methods["same_state_full"]["per_task_success"][task]["mean_success"]
+        lines.append(
+            f"| {task} | {_pct(h16)} | {_pct(clean)} | {_pct(near)} | {_pct(noexpert)} | {_pct(full)} | {_pp(clean - h16)} | {_pp(noexpert - clean)} |"
+        )
+    gap = report["mechanism_gap"]
+    lines.extend(
+        [
+            "",
+            "## Mechanism Gap",
+            "",
+            f"- Best clean residual transport improves over canonical h16 by {_pp(gap['best_clean_vs_h16'])}.",
+            f"- Same-state no-expert lattice improves over canonical h16 by {_pp(gap['same_state_no_expert_vs_h16'])}.",
+            f"- Remaining clean-to-same-state proposal gap is {_pp(gap['same_state_no_expert_vs_best_clean'])}.",
+            f"- Full lattice adds expert proposals and reaches {_pct(methods['same_state_full']['mean_success'])}, a {_pp(gap['same_state_full_vs_no_expert'])} gain over no-expert.",
+            "",
+            "## Selection Histograms",
+            "",
+        ]
+    )
+    for key in ["same_state_near_miss", "same_state_no_expert", "same_state_policy_baseline", "same_state_full", "best_clean_residual_k2"]:
+        counts = methods[key].get("selected_candidate_type_counts", {})
+        if counts:
+            total = sum(int(value) for value in counts.values())
+            summary = ", ".join(
+                f"{name}={count} ({count / total * 100:.1f}%)"
+                for name, count in sorted(counts.items(), key=lambda item: (-int(item[1]), item[0]))
+            )
+        else:
+            summary = "not recorded"
+        lines.append(f"- `{key}`: {summary}")
+    scale_counts = methods["best_clean_residual_k2"].get("selected_residual_scale_counts", {})
+    if scale_counts:
+        lines.append(f"- `best_clean_residual_k2` residual scale counts: {scale_counts}")
+    lines.extend(
+        [
+            "",
+            "## Selected-Type Outcomes",
+            "",
+            "These rows are measured from raw rollout rows. In residual retrieval, `policy_residual` is the zero-residual action, i.e. abstaining to the current policy mean.",
+            "",
+            "| method | selected type | count | success | progress |",
+            "|---|---|---:|---:|---:|",
+        ]
+    )
+    for key in ["best_clean_residual_k2", "residual_k4_consensus", "same_state_no_expert", "same_state_policy_baseline"]:
+        for candidate_type, values in methods[key].get("selected_type_outcomes", {}).items():
+            lines.append(
+                f"| {key} | {candidate_type} | {int(values['count'])} | {_pct(values['success_rate'])} | {_pct(values['mean_progress'])} |"
+            )
+    return "\n".join(lines) + "\n"
+
+
+def build_report() -> dict[str, Any]:
+    methods = _load_methods()
+    paired_deltas = {
+        "best_clean - canonical_h16": _paired_delta(methods, "best_clean_residual_k2", "h16_policy_canonical"),
+        "best_clean - direct_same_ckpt": _paired_delta(methods, "best_clean_residual_k2", "near_miss_policy_bc5"),
+        "no_expert_lattice - canonical_h16": _paired_delta(methods, "same_state_no_expert", "h16_policy_canonical"),
+        "full_lattice - no_expert_lattice": _paired_delta(methods, "same_state_full", "same_state_no_expert"),
+        "policy_candidate_lattice - no_expert_lattice": _paired_delta(
+            methods,
+            "same_state_policy_baseline",
+            "same_state_no_expert",
+        ),
+    }
+    mechanism_gap = {
+        "best_clean_vs_h16": methods["best_clean_residual_k2"]["mean_success"]
+        - methods["h16_policy_canonical"]["mean_success"],
+        "best_clean_vs_direct_same_ckpt": methods["best_clean_residual_k2"]["mean_success"]
+        - methods["near_miss_policy_bc5"]["mean_success"],
+        "same_state_no_expert_vs_h16": methods["same_state_no_expert"]["mean_success"]
+        - methods["h16_policy_canonical"]["mean_success"],
+        "same_state_no_expert_vs_best_clean": methods["same_state_no_expert"]["mean_success"]
+        - methods["best_clean_residual_k2"]["mean_success"],
+        "same_state_full_vs_no_expert": methods["same_state_full"]["mean_success"]
+        - methods["same_state_no_expert"]["mean_success"],
+    }
+    return {
+        "generated_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        "methods": methods,
+        "paired_deltas": paired_deltas,
+        "per_task_deltas": {
+            "best_clean_vs_h16": _per_task_delta(methods, "best_clean_residual_k2", "h16_policy_canonical"),
+            "no_expert_vs_best_clean": _per_task_delta(methods, "same_state_no_expert", "best_clean_residual_k2"),
+        },
+        "mechanism_gap": mechanism_gap,
+    }
+
+
+def main() -> int:
+    RESULTS_DIR.mkdir(parents=True, exist_ok=True)
+    report = build_report()
+    OUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
+    OUT_MD.write_text(_render_markdown(report), encoding="utf-8")
+    print(f"Wrote {OUT_JSON}")
+    print(f"Wrote {OUT_MD}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/slurm/build_paper_table_status.sbatch

@@ -16,3 +16,4 @@ cd "$PROJECT_DIR"
 mkdir -p outputs/hpc/logs results
 
 "$PYTHON" scripts/build_paper_table_status.py
+"$PYTHON" scripts/build_paper_analysis.py