ctt reproduce v0 artifacts

workspace/runs/reproduce_v0/leakage_audit_train.json

@@ -0,0 +1,17 @@
+{
+  "indexes": [
+    {
+      "audience": "train_retrieval",
+      "include_outcomes": true,
+      "num_groups": 2873,
+      "num_rows": 45968,
+      "path": "data/cil_charts/train/index.json",
+      "retrieval_index_allowed": true,
+      "split": "train"
+    }
+  ],
+  "num_indexes": 1,
+  "status": "pass",
+  "violations": [],
+  "warnings": []
+}

workspace/runs/reproduce_v0/metrics.json

@@ -0,0 +1,306 @@
+{
+  "acceptance_gates": [
+    {
+      "direction": "gte",
+      "name": "selected_clean_success >= 47.45%",
+      "passed": false,
+      "threshold": 0.4745,
+      "value": 0.3889855072463768
+    },
+    {
+      "direction": "gte",
+      "name": "selected_clean_success >= 50%",
+      "passed": false,
+      "threshold": 0.5,
+      "value": 0.3889855072463768
+    },
+    {
+      "direction": "gte",
+      "name": "proposal_oracle_success >= 50%",
+      "passed": false,
+      "threshold": 0.5,
+      "value": 0.44347826086956516
+    },
+    {
+      "direction": "lt",
+      "name": "support_gap < 7 pts",
+      "passed": false,
+      "threshold": 0.07,
+      "value": 0.12637681159420305
+    },
+    {
+      "direction": "lt",
+      "name": "selector_gap < 3 pts",
+      "passed": false,
+      "threshold": 0.03,
+      "value": 0.05449275362318834
+    },
+    {
+      "direction": "gte",
+      "name": "strict PTR@16/0.20 >= 35%",
+      "passed": false,
+      "threshold": 0.35,
+      "value": 0.23655913978494625
+    },
+    {
+      "direction": "gte",
+      "name": "loose PTR@16/0.40 >= 60%",
+      "passed": false,
+      "threshold": 0.6,
+      "value": 0.5268817204301075
+    },
+    {
+      "direction": "lte",
+      "name": "negative-near@16/0.20 <= 5%",
+      "passed": false,
+      "threshold": 0.05,
+      "value": 0.05333333333333334
+    }
+  ],
+  "best_candidate_oracle_key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
+  "best_clean_key": "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005",
+  "best_support_proxy": {
+    "label": "chart-synthesis",
+    "mean_positive_min_rms_l2_at_16": 0.5247493039060185,
+    "missing": false,
+    "negative_near_at_16_thr_0p2": 0.05333333333333334,
+    "num_groups": 93,
+    "positive_closer_than_negative_rate_at_16": 0.6533333333333333,
+    "ptr_at_16_thr_0p2": 0.23655913978494625,
+    "ptr_at_16_thr_0p4": 0.5268817204301075,
+    "source": "results/generator_v2_positive_tangent_chart_synthesis_d15_w1_tau0p5_eval.json"
+  },
+  "causal_action_decomposition": {
+    "base": 0.29739130434782607,
+    "clean_gain": 0.09159420289855075,
+    "closed_fraction_of_noexpert_gap": 0.3361702127659574,
+    "full_oracle": 0.6933333333333334,
+    "full_oracle_gap": 0.30434782608695654,
+    "proposal_oracle": 0.44347826086956516,
+    "same_state_gain_over_base": 0.27246376811594214,
+    "same_state_oracle": 0.5698550724637682,
+    "selected": 0.3889855072463768,
+    "selector_gap": 0.05449275362318834,
+    "support_gap": 0.12637681159420305,
+    "total_gap_to_same_state": 0.1808695652173914
+  },
+  "causal_action_targets": {
+    "best_paper_target_range": [
+      0.47,
+      0.52
+    ],
+    "selected_success_for_65pct_gap_closure": 0.47449275362318843,
+    "selected_success_for_75pct_gap_closure": 0.5017391304347827
+  },
+  "chart_export": {
+    "audience": "train_retrieval",
+    "content_hash": "4b815cf64a7c295982a70268111abf4637f42ca079b4084024056433e3ea60c0",
+    "include_outcomes": true,
+    "label_counts": {
+      "negative": 35225,
+      "neutral": 9183,
+      "positive": 1560
+    },
+    "missing": false,
+    "num_groups_exported": 2873,
+    "num_rows": 45968,
+    "path": "data/cil_charts/train/index.json",
+    "retrieval_index_allowed": true,
+    "shard_count": 1,
+    "split": "train",
+    "task_counts": {
+      "LiftPegUpright-v1": 8000,
+      "PickCube-v1": 16000,
+      "PullCube-v1": 5968,
+      "PushCube-v1": 8000,
+      "StackCube-v1": 8000
+    }
+  },
+  "command": "python scripts/reproduce_v0_report.py",
+  "data_hash": "1286909a5b6ab7018641a5f02ccdeed5860d9c85efaf8b8f13d37fcbc27cb18b",
+  "generated_utc": "2026-07-02T23:01:22.292526+00:00",
+  "git": {
+    "commit": "14e81292e2004925f81e0bb4810f6f144bfd87b7",
+    "dirty": true,
+    "dirty_line_count": 100
+  },
+  "host": {
+    "hostname": "rorqual4",
+    "machine": "x86_64",
+    "platform": "Linux-5.14.0-687.15.1.el9_8.x86_64-x86_64-AMD_EPYC_9654_96-Core_Processor-with-glibc2.37"
+  },
+  "input_files": [
+    {
+      "bytes": 748066,
+      "path": "results/paper_analysis.json",
+      "sha256": "d1eceb0f62469a28c827d4e15a48b4ba2f10bf4ddc2683ea175c76a645b6bcfc"
+    },
+    {
+      "bytes": 57175,
+      "path": "results/paper_analysis.md",
+      "sha256": "dcc26b0692fafe3c80d204c1210a47bfd7df694ddd9bdd3ea2bde93b61e418e3"
+    },
+    {
+      "bytes": 1300,
+      "path": "results/generator_v2_positive_tangent_targets_summary.json",
+      "sha256": "e865dde7889d442990dc3290f1a4d38d0b502cea7d7c4f729d35a87bd772244d"
+    },
+    {
+      "bytes": 1534508,
+      "path": "results/generator_v2_positive_tangent_memory_eval.json",
+      "sha256": "a18307de406d7cacd272d6b5a266656dd1100cc0ac9f631e29e1e5bc81eec0c2"
+    },
+    {
+      "bytes": 6024,
+      "path": "results/generator_v2_positive_tangent_local_atlas_sweep_summary.json",
+      "sha256": "cd00b4604eea036d77475e830690469eafb65779708af957d2ca72097fd3789f"
+    },
+    {
+      "bytes": 7242,
+      "path": "results/generator_v2_positive_tangent_chart_synthesis_sweep_summary.json",
+      "sha256": "730bdfed34d819e31dbe7c8592fc9764294f1ec19d54cf1bea56134452b6d74d"
+    },
+    {
+      "bytes": 4257,
+      "path": "results/generator_v2_positive_tangent_cvae_sweep_summary.json",
+      "sha256": "bef9ac66b1a6765a5a5cc72b1a404c75d91cf26a52c72902b083d05b40a49758"
+    },
+    {
+      "bytes": 4825,
+      "path": "results/generator_v2_positive_tangent_spline_cvae_sweep_summary.json",
+      "sha256": "af82ba12af293a2775909f2c2f9a8882d0693ac9baab37f89a298af8160b7ca7"
+    },
+    {
+      "bytes": 3855,
+      "path": "results/generator_v2_positive_tangent_spline_flow_sweep_summary.json",
+      "sha256": "6b25d488876dcdf184129446d9d9cf89fd6f7c40fea09062b9b1ce70e86fd04f"
+    },
+    {
+      "bytes": 9994,
+      "path": "results/generator_v2_positive_tangent_guided_spline_flow_sweep_summary.json",
+      "sha256": "0abdbb6e1901c95828123f847e80084305c4bbfbef8762787839fa994d72349e"
+    },
+    {
+      "bytes": 16002,
+      "path": "latex/main.tex",
+      "sha256": "e5d21a4aaa9193e12187154ad776d2335b5e39d81f873185439e166b76195f4c"
+    },
+    {
+      "bytes": 662,
+      "path": "latex/tables/car_decomposition.tex",
+      "sha256": "b91502d96f1ccf0b8e943d2432612820b498962b02872abd8a659aede47bcb73"
+    },
+    {
+      "bytes": 414971,
+      "path": "data/cil_charts/train/index.json",
+      "sha256": "b0d684b6c63b2ace1071f1a737c6ebc743bdab995b4727f8965e1955217eb3b8"
+    },
+    {
+      "bytes": 345,
+      "path": "runs/reproduce_v0/leakage_audit_train.json",
+      "sha256": "7e32288520ea60bccb7ab6975138e8972564b1fffd2a6629a9791a8a871fc4e1"
+    }
+  ],
+  "leakage_audit": {
+    "missing": false,
+    "num_indexes": 1,
+    "path": "runs/reproduce_v0/leakage_audit_train.json",
+    "status": "pass",
+    "violations": [],
+    "warnings": []
+  },
+  "leakage_contract": {
+    "deployment_clean_disallows": [
+      "validation/test same-state rewards",
+      "validation/test same-state proposals",
+      "expert proposals at deployment",
+      "hidden simulator state at selection time"
+    ],
+    "retrieval_index": "train-only",
+    "test_outcomes": "evaluator-only"
+  },
+  "paper_analysis_generated_utc": "2026-07-02T22:43:39+00:00",
+  "python": "3.11.4 (main, Nov  4 2023, 03:43:13) [GCC 12.3.1 20230526]",
+  "report_type": "reproduce_v0",
+  "source": "results/paper_analysis.json",
+  "split_hash": "94d8fabb1c8a0e0f0f2dadd1a4eea4941db0d0dab986abf42dc645159df8a2f6",
+  "support_proxy_rows": [
+    {
+      "label": "memory",
+      "mean_positive_min_rms_l2_at_16": 0.6388166329321816,
+      "missing": false,
+      "negative_near_at_16_thr_0p2": 0.08,
+      "num_groups": 93,
+      "positive_closer_than_negative_rate_at_16": 0.6133333333333333,
+      "ptr_at_16_thr_0p2": 0.11827956989247312,
+      "ptr_at_16_thr_0p4": 0.41935483870967744,
+      "source": "results/generator_v2_positive_tangent_memory_eval.json"
+    },
+    {
+      "label": "local-atlas",
+      "mean_positive_min_rms_l2_at_16": 0.5330067923630918,
+      "missing": false,
+      "negative_near_at_16_thr_0p2": 0.05333333333333334,
+      "num_groups": 93,
+      "positive_closer_than_negative_rate_at_16": 0.64,
+      "ptr_at_16_thr_0p2": 0.23655913978494625,
+      "ptr_at_16_thr_0p4": 0.5268817204301075,
+      "source": "results/generator_v2_positive_tangent_local_atlas_pool16_eval.json"
+    },
+    {
+      "label": "chart-synthesis",
+      "mean_positive_min_rms_l2_at_16": 0.5247493039060185,
+      "missing": false,
+      "negative_near_at_16_thr_0p2": 0.05333333333333334,
+      "num_groups": 93,
+      "positive_closer_than_negative_rate_at_16": 0.6533333333333333,
+      "ptr_at_16_thr_0p2": 0.23655913978494625,
+      "ptr_at_16_thr_0p4": 0.5268817204301075,
+      "source": "results/generator_v2_positive_tangent_chart_synthesis_d15_w1_tau0p5_eval.json"
+    },
+    {
+      "label": "raw-cvae",
+      "mean_positive_min_rms_l2_at_16": 0.641030304113575,
+      "missing": false,
+      "negative_near_at_16_thr_0p2": 0.0,
+      "num_groups": 93,
+      "positive_closer_than_negative_rate_at_16": 0.6533333333333333,
+      "ptr_at_16_thr_0p2": 0.07526881720430108,
+      "ptr_at_16_thr_0p4": 0.34408602150537637,
+      "source": "results/generator_v2_positive_tangent_cvae_temp0p5_eval.json"
+    },
+    {
+      "label": "spline-cvae",
+      "mean_positive_min_rms_l2_at_16": 0.6786195585806957,
+      "missing": false,
+      "negative_near_at_16_thr_0p2": 0.013333333333333334,
+      "num_groups": 93,
+      "positive_closer_than_negative_rate_at_16": 0.6666666666666666,
+      "ptr_at_16_thr_0p2": 0.0967741935483871,
+      "ptr_at_16_thr_0p4": 0.34408602150537637,
+      "source": "results/generator_v2_positive_tangent_spline_cvae_eval.json"
+    },
+    {
+      "label": "spline-flow",
+      "mean_positive_min_rms_l2_at_16": 0.7541321921929743,
+      "missing": false,
+      "negative_near_at_16_thr_0p2": 0.0,
+      "num_groups": 93,
+      "positive_closer_than_negative_rate_at_16": 0.6666666666666666,
+      "ptr_at_16_thr_0p2": 0.010752688172043012,
+      "ptr_at_16_thr_0p4": 0.2903225806451613,
+      "source": "results/generator_v2_positive_tangent_spline_flow_noise0p5_eval.json"
+    },
+    {
+      "label": "guided-spline-flow",
+      "mean_positive_min_rms_l2_at_16": 0.7866852985588353,
+      "missing": false,
+      "negative_near_at_16_thr_0p2": 0.0,
+      "num_groups": 93,
+      "positive_closer_than_negative_rate_at_16": 0.6933333333333334,
+      "ptr_at_16_thr_0p2": 0.010752688172043012,
+      "ptr_at_16_thr_0p4": 0.25806451612903225,
+      "source": "results/generator_v2_positive_tangent_guided_spline_flow_gs0p5_pool2_eval.json"
+    }
+  ]
+}

workspace/runs/reproduce_v0/report.md

@@ -0,0 +1,60 @@
+# Reproduce V0: Counterfactual Action Atlas Diagnostic
+
+Generated UTC: `2026-07-02T23:01:22.292526+00:00`
+Git commit: `14e81292e2004925f81e0bb4810f6f144bfd87b7`; dirty: `True` (100 dirty paths).
+Data hash: `1286909a5b6ab7018641a5f02ccdeed5860d9c85efaf8b8f13d37fcbc27cb18b`
+Split hash: `94d8fabb1c8a0e0f0f2dadd1a4eea4941db0d0dab986abf42dc645159df8a2f6`
+
+## Core Decomposition
+
+| Quantity | Value |
+| --- | ---: |
+| Base h=16 success | 29.74% |
+| Selected clean V0 success | 38.90% |
+| Clean proposal oracle | 44.35% |
+| Hidden same-state no-expert oracle | 56.99% |
+| Selector gap | 5.45% |
+| Support gap | 12.64% |
+| Closed BC-to-no-expert gap | 33.62% |
+
+Interpretation: current V0 proves same-state causal geometry is useful, but the
+larger remaining error is proposal support, not selector calibration.
+
+## Support Proxies
+
+| Proxy | N | PTR@16/0.20 | PTR@16/0.40 | NegNear@16/0.20 | Pos<Neg@16 | Mean +dist | Source |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- |
+| memory | 93 | 11.83% | 41.94% | 8.00% | 61.33% | 0.6388 | `results/generator_v2_positive_tangent_memory_eval.json` |
+| local-atlas | 93 | 23.66% | 52.69% | 5.33% | 64.00% | 0.5330 | `results/generator_v2_positive_tangent_local_atlas_pool16_eval.json` |
+| chart-synthesis | 93 | 23.66% | 52.69% | 5.33% | 65.33% | 0.5247 | `results/generator_v2_positive_tangent_chart_synthesis_d15_w1_tau0p5_eval.json` |
+| raw-cvae | 93 | 7.53% | 34.41% | 0.00% | 65.33% | 0.6410 | `results/generator_v2_positive_tangent_cvae_temp0p5_eval.json` |
+| spline-cvae | 93 | 9.68% | 34.41% | 1.33% | 66.67% | 0.6786 | `results/generator_v2_positive_tangent_spline_cvae_eval.json` |
+| spline-flow | 93 | 1.08% | 29.03% | 0.00% | 66.67% | 0.7541 | `results/generator_v2_positive_tangent_spline_flow_noise0p5_eval.json` |
+| guided-spline-flow | 93 | 1.08% | 25.81% | 0.00% | 69.33% | 0.7867 | `results/generator_v2_positive_tangent_guided_spline_flow_gs0p5_pool2_eval.json` |
+
+## Train Chart DB Export
+
+Index: `data/cil_charts/train/index.json`
+Split/audience: `train` / `train_retrieval`
+Rows: 45968; charts: 2873; shards: 1
+Content hash: `4b815cf64a7c295982a70268111abf4637f42ca079b4084024056433e3ea60c0`
+Leakage audit: `pass`; violations: 0; warnings: 0.
+
+## Acceptance Gates
+
+| Gate | Value | Threshold | Status |
+| --- | ---: | ---: | --- |
+| selected_clean_success >= 47.45% | 38.90% | 47.45% | FAIL |
+| selected_clean_success >= 50% | 38.90% | 50.00% | FAIL |
+| proposal_oracle_success >= 50% | 44.35% | 50.00% | FAIL |
+| support_gap < 7 pts | 12.64% | 7.00% | FAIL |
+| selector_gap < 3 pts | 5.45% | 3.00% | FAIL |
+| strict PTR@16/0.20 >= 35% | 23.66% | 35.00% | FAIL |
+| loose PTR@16/0.40 >= 60% | 52.69% | 60.00% | FAIL |
+| negative-near@16/0.20 <= 5% | 5.33% | 5.00% | FAIL |
+
+## Leakage Contract
+
+- Retrieval and generator training use train-only chart indexes.
+- Deployment-clean rows do not use validation/test same-state outcomes, validation/test proposals, expert proposals, or hidden simulator state.
+- Validation/test outcomes are evaluator-only and must not enter candidate generation or selection.

workspace/runs/reproduce_v0/tables.tex

@@ -0,0 +1,22 @@
+% Auto-generated by scripts/reproduce_v0_report.py
+\begin{tabular}{lrrrrrr}
+\toprule
+Method & Base & Proposal Oracle & Selected & Same-State Oracle & Support Gap & Selector Gap \\
+\midrule
+Current CIL-Atlas V0 & 29.74 & 44.35 & 38.90 & 56.99 & 12.64 & 5.45 \\
+\bottomrule
+\end{tabular}
+
+\begin{tabular}{lrrrrr}
+\toprule
+Support proxy & PTR@16/0.20 & PTR@16/0.40 & NegNear@16/0.20 & Pos<Neg@16 & Mean +dist \\
+\midrule
+memory & 11.83 & 41.94 & 8.00 & 61.33 & 0.6388 \\
+local-atlas & 23.66 & 52.69 & 5.33 & 64.00 & 0.5330 \\
+chart-synthesis & 23.66 & 52.69 & 5.33 & 65.33 & 0.5247 \\
+raw-cvae & 7.53 & 34.41 & 0.00 & 65.33 & 0.6410 \\
+spline-cvae & 9.68 & 34.41 & 1.33 & 66.67 & 0.6786 \\
+spline-flow & 1.08 & 29.03 & 0.00 & 66.67 & 0.7541 \\
+guided-spline-flow & 1.08 & 25.81 & 0.00 & 69.33 & 0.7867 \\
+\bottomrule
+\end{tabular}