ctt artifacts 2026-07-02 workspace/runs/ctt_residual_smoke_proxy

workspace/runs/ctt_residual_smoke_proxy/command.txt

@@ -0,0 +1 @@
+python scripts/eval_ctt_proxy.py --checkpoint runs/ctt_residual_smoke/model.pt --source-index data/cil_charts/train/index.json --target-index data/cil_charts/train/index.json --out-dir runs/ctt_residual_smoke_proxy --k 16 --max-target-charts 16 --neighbors 4 --thresholds 0.20,0.40

workspace/runs/ctt_residual_smoke_proxy/config.yaml

@@ -0,0 +1,8 @@
+checkpoint: runs/ctt_residual_smoke/model.pt
+k: 16
+max_target_charts: 16
+neighbors: 4
+out_dir: runs/ctt_residual_smoke_proxy
+source_index: data/cil_charts/train/index.json
+target_index: data/cil_charts/train/index.json
+thresholds: 0.20,0.40

workspace/runs/ctt_residual_smoke_proxy/data_hash.txt

@@ -0,0 +1 @@
+90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957

workspace/runs/ctt_residual_smoke_proxy/eval.log

@@ -0,0 +1,3 @@
+source_charts=301 target_charts=16 k=16
+source_index=data/cil_charts/train/index.json
+target_index=data/cil_charts/train/index.json

workspace/runs/ctt_residual_smoke_proxy/git_hash.txt

@@ -0,0 +1 @@
+14e81292e2004925f81e0bb4810f6f144bfd87b7

workspace/runs/ctt_residual_smoke_proxy/metrics.json

@@ -0,0 +1,472 @@
+{
+  "data_hash": "90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957",
+  "k": 16,
+  "num_rows": 16,
+  "report_type": "ctt_proxy_eval",
+  "rows": [
+    {
+      "chart_id": "ms-003a2225c0892dcda683",
+      "negative_near_at_16_thr_0p20": 0.0,
+      "negative_near_at_16_thr_0p40": 0.0,
+      "num_proposals": 8,
+      "pos_closer_than_neg_at_16": 1.0,
+      "pptc_at_16_thr_0p20": 1.0,
+      "pptc_at_16_thr_0p40": 1.0,
+      "proxy_support_distance_at_16": 0.06552680286217681,
+      "seed": "unknown",
+      "task_id": "PullCube-v1"
+    },
+    {
+      "chart_id": "ms-0093b06390890d181104",
+      "negative_near_at_16_thr_0p20": 0.0,
+      "negative_near_at_16_thr_0p40": 0.0,
+      "num_proposals": 16,
+      "pos_closer_than_neg_at_16": 1.0,
+      "pptc_at_16_thr_0p20": 1.0,
+      "pptc_at_16_thr_0p40": 1.0,
+      "proxy_support_distance_at_16": 0.13042893616823228,
+      "seed": "unknown",
+      "task_id": "PullCube-v1"
+    },
+    {
+      "chart_id": "ms-016c5de39ed3c111889a",
+      "negative_near_at_16_thr_0p20": 0.25,
+      "negative_near_at_16_thr_0p40": 0.75,
+      "num_proposals": 4,
+      "pos_closer_than_neg_at_16": 1.0,
+      "pptc_at_16_thr_0p20": 1.0,
+      "pptc_at_16_thr_0p40": 1.0,
+      "proxy_support_distance_at_16": 0.11890084844087313,
+      "seed": "unknown",
+      "task_id": "StackCube-v1"
+    },
+    {
+      "chart_id": "ms-01e6eaaaf40875e0ef5a",
+      "negative_near_at_16_thr_0p20": 0.0,
+      "negative_near_at_16_thr_0p40": 0.45454545454545453,
+      "num_proposals": 11,
+      "pos_closer_than_neg_at_16": 0.45454545454545453,
+      "pptc_at_16_thr_0p20": 1.0,
+      "pptc_at_16_thr_0p40": 1.0,
+      "proxy_support_distance_at_16": 0.1019671272273881,
+      "seed": "unknown",
+      "task_id": "StackCube-v1"
+    },
+    {
+      "chart_id": "ms-02311c7783bc67e188b4",
+      "negative_near_at_16_thr_0p20": 0.0,
+      "negative_near_at_16_thr_0p40": 0.0,
+      "num_proposals": 13,
+      "pos_closer_than_neg_at_16": null,
+      "pptc_at_16_thr_0p20": 1.0,
+      "pptc_at_16_thr_0p40": 1.0,
+      "proxy_support_distance_at_16": 0.05320919721120968,
+      "seed": "unknown",
+      "task_id": "PullCube-v1"
+    },
+    {
+      "chart_id": "ms-0285f5815c210ac00bec",
+      "negative_near_at_16_thr_0p20": 0.0,
+      "negative_near_at_16_thr_0p40": 0.0,
+      "num_proposals": 7,
+      "pos_closer_than_neg_at_16": 0.5714285714285714,
+      "pptc_at_16_thr_0p20": 1.0,
+      "pptc_at_16_thr_0p40": 1.0,
+      "proxy_support_distance_at_16": 0.07181674356769384,
+      "seed": "unknown",
+      "task_id": "PullCube-v1"
+    },
+    {
+      "chart_id": "ms-02fd0338e80ef19ceec1",
+      "negative_near_at_16_thr_0p20": 0.0,
+      "negative_near_at_16_thr_0p40": 0.07142857142857142,
+      "num_proposals": 14,
+      "pos_closer_than_neg_at_16": 0.14285714285714285,
+      "pptc_at_16_thr_0p20": 1.0,
+      "pptc_at_16_thr_0p40": 1.0,
+      "proxy_support_distance_at_16": 0.1280471585918045,
+      "seed": "unknown",
+      "task_id": "PullCube-v1"
+    },
+    {
+      "chart_id": "ms-0350672b45f92c437c22",
+      "negative_near_at_16_thr_0p20": 0.0,
+      "negative_near_at_16_thr_0p40": 0.3125,
+      "num_proposals": 16,
+      "pos_closer_than_neg_at_16": 1.0,
+      "pptc_at_16_thr_0p20": 1.0,
+      "pptc_at_16_thr_0p40": 1.0,
+      "proxy_support_distance_at_16": 0.05777542542198933,
+      "seed": "unknown",
+      "task_id": "PushCube-v1"
+    },
+    {
+      "chart_id": "ms-03df3e12e1d61aa9ebbf",
+      "negative_near_at_16_thr_0p20": 0.3333333333333333,
+      "negative_near_at_16_thr_0p40": 0.8333333333333334,
+      "num_proposals": 6,
+      "pos_closer_than_neg_at_16": 0.5,
+      "pptc_at_16_thr_0p20": 1.0,
+      "pptc_at_16_thr_0p40": 1.0,
+      "proxy_support_distance_at_16": 0.1066669801008382,
+      "seed": "unknown",
+      "task_id": "StackCube-v1"
+    },
+    {
+      "chart_id": "ms-04eec0abab5f55620e4e",
+      "negative_near_at_16_thr_0p20": 0.0,
+      "negative_near_at_16_thr_0p40": 0.75,
+      "num_proposals": 8,
+      "pos_closer_than_neg_at_16": 0.875,
+      "pptc_at_16_thr_0p20": 1.0,
+      "pptc_at_16_thr_0p40": 1.0,
+      "proxy_support_distance_at_16": 0.10213291452149058,
+      "seed": "unknown",
+      "task_id": "StackCube-v1"
+    },
+    {
+      "chart_id": "ms-0541a144afe57b5a7ae6",
+      "negative_near_at_16_thr_0p20": 0.5,
+      "negative_near_at_16_thr_0p40": 1.0,
+      "num_proposals": 4,
+      "pos_closer_than_neg_at_16": 1.0,
+      "pptc_at_16_thr_0p20": 1.0,
+      "pptc_at_16_thr_0p40": 1.0,
+      "proxy_support_distance_at_16": 0.05142830766767363,
+      "seed": "unknown",
+      "task_id": "PickCube-v1"
+    },
+    {
+      "chart_id": "ms-0547201761e8e1873482",
+      "negative_near_at_16_thr_0p20": 0.0,
+      "negative_near_at_16_thr_0p40": 1.0,
+      "num_proposals": 5,
+      "pos_closer_than_neg_at_16": 0.8,
+      "pptc_at_16_thr_0p20": 1.0,
+      "pptc_at_16_thr_0p40": 1.0,
+      "proxy_support_distance_at_16": 0.09290951723595935,
+      "seed": "unknown",
+      "task_id": "StackCube-v1"
+    },
+    {
+      "chart_id": "ms-059d7962a5f998edf668",
+      "negative_near_at_16_thr_0p20": 0.16666666666666666,
+      "negative_near_at_16_thr_0p40": 0.8333333333333334,
+      "num_proposals": 12,
+      "pos_closer_than_neg_at_16": 0.8333333333333334,
+      "pptc_at_16_thr_0p20": 1.0,
+      "pptc_at_16_thr_0p40": 1.0,
+      "proxy_support_distance_at_16": 0.061852177177210546,
+      "seed": "unknown",
+      "task_id": "LiftPegUpright-v1"
+    },
+    {
+      "chart_id": "ms-062addb9da32a0c2ea97",
+      "negative_near_at_16_thr_0p20": 0.16666666666666666,
+      "negative_near_at_16_thr_0p40": 0.16666666666666666,
+      "num_proposals": 6,
+      "pos_closer_than_neg_at_16": 0.8333333333333334,
+      "pptc_at_16_thr_0p20": 1.0,
+      "pptc_at_16_thr_0p40": 1.0,
+      "proxy_support_distance_at_16": 0.05492816531591116,
+      "seed": "unknown",
+      "task_id": "LiftPegUpright-v1"
+    },
+    {
+      "chart_id": "ms-0660d5f65df5fc4b6132",
+      "negative_near_at_16_thr_0p20": 0.2,
+      "negative_near_at_16_thr_0p40": 1.0,
+      "num_proposals": 5,
+      "pos_closer_than_neg_at_16": 0.8,
+      "pptc_at_16_thr_0p20": 1.0,
+      "pptc_at_16_thr_0p40": 1.0,
+      "proxy_support_distance_at_16": 0.07564185058326081,
+      "seed": "unknown",
+      "task_id": "LiftPegUpright-v1"
+    },
+    {
+      "chart_id": "ms-067cac2e4dbeddb12239",
+      "negative_near_at_16_thr_0p20": 0.0,
+      "negative_near_at_16_thr_0p40": 0.0,
+      "num_proposals": 5,
+      "pos_closer_than_neg_at_16": 0.4,
+      "pptc_at_16_thr_0p20": 1.0,
+      "pptc_at_16_thr_0p40": 1.0,
+      "proxy_support_distance_at_16": 0.10653714300147019,
+      "seed": "unknown",
+      "task_id": "PullCube-v1"
+    }
+  ],
+  "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372",
+  "summary": {
+    "negative_near_at_16_thr_0p20": {
+      "macro_by_seed": {
+        "ci": {
+          "confidence": 0.95,
+          "high": 0.10104166666666667,
+          "low": 0.10104166666666667,
+          "mean": 0.10104166666666667,
+          "n": 1,
+          "num_samples": 200
+        },
+        "mean": 0.10104166666666667,
+        "per_seed": {
+          "unknown": 0.10104166666666667
+        }
+      },
+      "macro_by_task": {
+        "ci": {
+          "confidence": 0.95,
+          "high": 0.33555555555555555,
+          "low": 0.02333333333333333,
+          "mean": 0.1588888888888889,
+          "n": 5,
+          "num_samples": 200
+        },
+        "mean": 0.1588888888888889,
+        "per_task": {
+          "LiftPegUpright-v1": 0.17777777777777778,
+          "PickCube-v1": 0.5,
+          "PullCube-v1": 0.0,
+          "PushCube-v1": 0.0,
+          "StackCube-v1": 0.11666666666666665
+        }
+      },
+      "metric": "negative_near_at_16_thr_0p20",
+      "micro": {
+        "confidence": 0.95,
+        "high": 0.17916666666666667,
+        "low": 0.04375,
+        "mean": 0.10104166666666667,
+        "n": 16,
+        "num_samples": 200
+      },
+      "n": 16
+    },
+    "negative_near_at_16_thr_0p40": {
+      "macro_by_seed": {
+        "ci": {
+          "confidence": 0.95,
+          "high": 0.44823795995670995,
+          "low": 0.44823795995670995,
+          "mean": 0.44823795995670995,
+          "n": 1,
+          "num_samples": 200
+        },
+        "mean": 0.44823795995670995,
+        "per_seed": {
+          "unknown": 0.44823795995670995
+        }
+      },
+      "macro_by_task": {
+        "ci": {
+          "confidence": 0.95,
+          "high": 0.8140151515151516,
+          "low": 0.2630952380952381,
+          "mean": 0.5497294372294371,
+          "n": 5,
+          "num_samples": 200
+        },
+        "mean": 0.5497294372294371,
+        "per_task": {
+          "LiftPegUpright-v1": 0.6666666666666666,
+          "PickCube-v1": 1.0,
+          "PullCube-v1": 0.011904761904761904,
+          "PushCube-v1": 0.3125,
+          "StackCube-v1": 0.7575757575757576
+        }
+      },
+      "metric": "negative_near_at_16_thr_0p40",
+      "micro": {
+        "confidence": 0.95,
+        "high": 0.6190476190476191,
+        "low": 0.2700892857142857,
+        "mean": 0.44823795995670995,
+        "n": 16,
+        "num_samples": 200
+      },
+      "n": 16
+    },
+    "pos_closer_than_neg_at_16": {
+      "macro_by_seed": {
+        "ci": {
+          "confidence": 0.95,
+          "high": 0.7473665223665226,
+          "low": 0.7473665223665226,
+          "mean": 0.7473665223665226,
+          "n": 1,
+          "num_samples": 200
+        },
+        "mean": 0.7473665223665226,
+        "per_seed": {
+          "unknown": 0.7473665223665226
+        }
+      },
+      "macro_by_task": {
+        "ci": {
+          "confidence": 0.95,
+          "high": 0.9644444444444444,
+          "low": 0.7188961038961039,
+          "mean": 0.8341976911976913,
+          "n": 5,
+          "num_samples": 200
+        },
+        "mean": 0.8341976911976913,
+        "per_task": {
+          "LiftPegUpright-v1": 0.8222222222222223,
+          "PickCube-v1": 1.0,
+          "PullCube-v1": 0.6228571428571428,
+          "PushCube-v1": 1.0,
+          "StackCube-v1": 0.725909090909091
+        }
+      },
+      "metric": "pos_closer_than_neg_at_16",
+      "micro": {
+        "confidence": 0.95,
+        "high": 0.8745238095238096,
+        "low": 0.6316883116883116,
+        "mean": 0.7473665223665226,
+        "n": 15,
+        "num_samples": 200
+      },
+      "n": 15
+    },
+    "pptc_at_16_thr_0p20": {
+      "macro_by_seed": {
+        "ci": {
+          "confidence": 0.95,
+          "high": 1.0,
+          "low": 1.0,
+          "mean": 1.0,
+          "n": 1,
+          "num_samples": 200
+        },
+        "mean": 1.0,
+        "per_seed": {
+          "unknown": 1.0
+        }
+      },
+      "macro_by_task": {
+        "ci": {
+          "confidence": 0.95,
+          "high": 1.0,
+          "low": 1.0,
+          "mean": 1.0,
+          "n": 5,
+          "num_samples": 200
+        },
+        "mean": 1.0,
+        "per_task": {
+          "LiftPegUpright-v1": 1.0,
+          "PickCube-v1": 1.0,
+          "PullCube-v1": 1.0,
+          "PushCube-v1": 1.0,
+          "StackCube-v1": 1.0
+        }
+      },
+      "metric": "pptc_at_16_thr_0p20",
+      "micro": {
+        "confidence": 0.95,
+        "high": 1.0,
+        "low": 1.0,
+        "mean": 1.0,
+        "n": 16,
+        "num_samples": 200
+      },
+      "n": 16
+    },
+    "pptc_at_16_thr_0p40": {
+      "macro_by_seed": {
+        "ci": {
+          "confidence": 0.95,
+          "high": 1.0,
+          "low": 1.0,
+          "mean": 1.0,
+          "n": 1,
+          "num_samples": 200
+        },
+        "mean": 1.0,
+        "per_seed": {
+          "unknown": 1.0
+        }
+      },
+      "macro_by_task": {
+        "ci": {
+          "confidence": 0.95,
+          "high": 1.0,
+          "low": 1.0,
+          "mean": 1.0,
+          "n": 5,
+          "num_samples": 200
+        },
+        "mean": 1.0,
+        "per_task": {
+          "LiftPegUpright-v1": 1.0,
+          "PickCube-v1": 1.0,
+          "PullCube-v1": 1.0,
+          "PushCube-v1": 1.0,
+          "StackCube-v1": 1.0
+        }
+      },
+      "metric": "pptc_at_16_thr_0p40",
+      "micro": {
+        "confidence": 0.95,
+        "high": 1.0,
+        "low": 1.0,
+        "mean": 1.0,
+        "n": 16,
+        "num_samples": 200
+      },
+      "n": 16
+    },
+    "proxy_support_distance_at_16": {
+      "macro_by_seed": {
+        "ci": {
+          "confidence": 0.95,
+          "high": 0.08623558094344885,
+          "low": 0.08623558094344885,
+          "mean": 0.08623558094344885,
+          "n": 1,
+          "num_samples": 200
+        },
+        "mean": 0.08623558094344885,
+        "per_seed": {
+          "unknown": 0.08623558094344885
+        }
+      },
+      "macro_by_task": {
+        "ci": {
+          "confidence": 0.95,
+          "high": 0.094056298755031,
+          "low": 0.05650963944095736,
+          "mean": 0.07409085437083965,
+          "n": 5,
+          "num_samples": 200
+        },
+        "mean": 0.07409085437083965,
+        "per_task": {
+          "LiftPegUpright-v1": 0.06414073102546085,
+          "PickCube-v1": 0.05142830766767363,
+          "PullCube-v1": 0.09259433023376455,
+          "PushCube-v1": 0.05777542542198933,
+          "StackCube-v1": 0.10451547750530987
+        }
+      },
+      "metric": "proxy_support_distance_at_16",
+      "micro": {
+        "confidence": 0.95,
+        "high": 0.09839497744854023,
+        "low": 0.07223141092515181,
+        "mean": 0.08623558094344885,
+        "n": 16,
+        "num_samples": 200
+      },
+      "n": 16
+    }
+  },
+  "target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372",
+  "thresholds": [
+    0.2,
+    0.4
+  ]
+}

workspace/runs/ctt_residual_smoke_proxy/metrics_by_seed.json

@@ -0,0 +1 @@
+{}

workspace/runs/ctt_residual_smoke_proxy/metrics_by_task.json

@@ -0,0 +1,42 @@
+{
+  "LiftPegUpright-v1": {
+    "negative_near_at_16_thr_0p20": 0.17777777777777778,
+    "negative_near_at_16_thr_0p40": 0.6666666666666666,
+    "pos_closer_than_neg_at_16": 0.8222222222222223,
+    "pptc_at_16_thr_0p20": 1.0,
+    "pptc_at_16_thr_0p40": 1.0,
+    "proxy_support_distance_at_16": 0.06414073102546085
+  },
+  "PickCube-v1": {
+    "negative_near_at_16_thr_0p20": 0.5,
+    "negative_near_at_16_thr_0p40": 1.0,
+    "pos_closer_than_neg_at_16": 1.0,
+    "pptc_at_16_thr_0p20": 1.0,
+    "pptc_at_16_thr_0p40": 1.0,
+    "proxy_support_distance_at_16": 0.05142830766767363
+  },
+  "PullCube-v1": {
+    "negative_near_at_16_thr_0p20": 0.0,
+    "negative_near_at_16_thr_0p40": 0.011904761904761904,
+    "pos_closer_than_neg_at_16": 0.6228571428571428,
+    "pptc_at_16_thr_0p20": 1.0,
+    "pptc_at_16_thr_0p40": 1.0,
+    "proxy_support_distance_at_16": 0.09259433023376455
+  },
+  "PushCube-v1": {
+    "negative_near_at_16_thr_0p20": 0.0,
+    "negative_near_at_16_thr_0p40": 0.3125,
+    "pos_closer_than_neg_at_16": 1.0,
+    "pptc_at_16_thr_0p20": 1.0,
+    "pptc_at_16_thr_0p40": 1.0,
+    "proxy_support_distance_at_16": 0.05777542542198933
+  },
+  "StackCube-v1": {
+    "negative_near_at_16_thr_0p20": 0.11666666666666665,
+    "negative_near_at_16_thr_0p40": 0.7575757575757576,
+    "pos_closer_than_neg_at_16": 0.725909090909091,
+    "pptc_at_16_thr_0p20": 1.0,
+    "pptc_at_16_thr_0p40": 1.0,
+    "proxy_support_distance_at_16": 0.10451547750530987
+  }
+}

workspace/runs/ctt_residual_smoke_proxy/report.md

@@ -0,0 +1,14 @@
+# CTT Proxy Evaluation
+
+K: `16`
+
+| Metric | N | Mean | 95% CI |
+| --- | ---: | ---: | ---: |
+| negative_near_at_16_thr_0p20 | 16 | 0.1010 | [0.0437, 0.1792] |
+| negative_near_at_16_thr_0p40 | 16 | 0.4482 | [0.2701, 0.6190] |
+| pos_closer_than_neg_at_16 | 15 | 0.7474 | [0.6317, 0.8745] |
+| pptc_at_16_thr_0p20 | 16 | 1.0000 | [1.0000, 1.0000] |
+| pptc_at_16_thr_0p40 | 16 | 1.0000 | [1.0000, 1.0000] |
+| proxy_support_distance_at_16 | 16 | 0.0862 | [0.0722, 0.0984] |
+
+This is PPTC/proxy support geometry, not OutcomePTR or rollout success.

workspace/runs/ctt_residual_smoke_proxy/split_hash.txt

@@ -0,0 +1 @@
+867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372

workspace/runs/ctt_residual_smoke_proxy/table.tex

@@ -0,0 +1,13 @@
+% Auto-generated by scripts/eval_ctt_proxy.py
+\begin{tabular}{lrrr}
+\toprule
+Metric & N & Mean & CI high \\
+\midrule
+negative\_near\_at\_16\_thr\_0p20 & 16 & 0.1010 & 0.1792 \\
+negative\_near\_at\_16\_thr\_0p40 & 16 & 0.4482 & 0.6190 \\
+pos\_closer\_than\_neg\_at\_16 & 15 & 0.7474 & 0.8745 \\
+pptc\_at\_16\_thr\_0p20 & 16 & 1.0000 & 1.0000 \\
+pptc\_at\_16\_thr\_0p40 & 16 & 1.0000 & 1.0000 \\
+proxy\_support\_distance\_at\_16 & 16 & 0.0862 & 0.0984 \\
+\bottomrule
+\end{tabular}

workspace/runs/ctt_residual_smoke_proxy/train.log

@@ -0,0 +1 @@
+see checkpoint run