auto-sync 2026-07-03T00:30:47Z workspace (part 3)

workspace/runs/leakage_audit/report.json

@@ -3,8 +3,8 @@
   "indexes": {
     "test": {
       "audience": "evaluator_only",
-      "content_hash": "7409ed9e2a8c9f3515e23e14180ec51fbfd56536a59129ca759c4d855e257bf5",
-      "include_outcomes": false,
+      "content_hash": "1710ede7c73d8f72479fd63dd9941a0e8bb55211d3ea0267963abc9e7a043b91",
+      "include_outcomes": true,
       "num_groups_exported": 410,
       "num_rows": 6560,
       "path": "data/cil_charts/test/index.json",
@@ -23,8 +23,8 @@
     },
     "val": {
       "audience": "evaluator_only",
-      "content_hash": "eb4c66b3f5952b80ad4dc8eee2fbd04742688d5bcdaec341fbc851e170371f82",
-      "include_outcomes": false,
+      "content_hash": "fa0fe0e3881ee70ce91492445871614d01b3a47fc7c576659c53ff17ddee4e42",
+      "include_outcomes": true,
       "num_groups_exported": 419,
       "num_rows": 6704,
       "path": "data/cil_charts/val/index.json",

workspace/runs/leakage_audit/report.md

@@ -4,9 +4,9 @@ Status: `pass`
 
 | Split | Rows | Charts | Outcomes in DB | Retrieval allowed | Audience |
 | --- | ---: | ---: | --- | --- | --- |
-| test | 6560 | 410 | False | False | evaluator_only |
+| test | 6560 | 410 | True | False | evaluator_only |
 | train | 32704 | 2044 | True | True | train_retrieval |
-| val | 6704 | 419 | False | False | evaluator_only |
+| val | 6704 | 419 | True | False | evaluator_only |
 
 ## Violations
 

workspace/runs/summary_ctt.csv

@@ -2,6 +2,10 @@ method,run_path,clean,status,success,proposal_oracle,support_gap,selector_gap,ou
 V0 residual transport diagnostic,runs/reproduce_v0,yes,measured diagnostic baseline,0.3890,0.4435,0.1264,0.0545,n/a,0.2366,0.5269,0.0533,n/a,3,see run
 CTT gated residual,runs/ctt_gated_residual_smoke,yes,train,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run
 CTT gated residual,runs/ctt_gated_residual_smoke_proxy,yes,proxy,n/a,n/a,n/a,n/a,n/a,0.5000,0.7500,0.0573,n/a,unknown,see run
+CTT gated residual,runs/ctt_gated_residual_val_proxy,yes,proxy,n/a,n/a,n/a,n/a,n/a,0.1739,0.5362,0.0281,n/a,47,see run
 CTT residual train,runs/ctt_residual_smoke,yes,train,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run
 CTT residual proxy,runs/ctt_residual_smoke_proxy,yes,proxy,n/a,n/a,n/a,n/a,n/a,1.0000,1.0000,0.1010,n/a,unknown,see run
+CTT residual proxy,runs/ctt_residual_val_proxy,yes,proxy,n/a,n/a,n/a,n/a,n/a,0.3478,0.7101,0.0397,n/a,47,see run
+task memory,runs/task_memory_val_proxy,yes,proxy baseline,n/a,n/a,n/a,n/a,n/a,0.3188,0.5362,0.0175,n/a,47,see run
+local atlas,runs/local_atlas_val_proxy,yes,proxy baseline,n/a,n/a,n/a,n/a,n/a,0.4058,0.6812,0.0368,n/a,47,see run
 Utility energy smoke,runs/utility_energy_smoke,train-only,calibration diagnostic,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,0.1316,train split,see run

workspace/runs/summary_ctt.md

@@ -5,8 +5,12 @@
 | V0 residual transport diagnostic | measured diagnostic baseline | 0.3890 | 0.4435 | 0.1264 | 0.0545 | n/a | 0.2366 | 0.5269 | 0.0533 | n/a | `runs/reproduce_v0` |
 | CTT gated residual | train | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_gated_residual_smoke` |
 | CTT gated residual | proxy | n/a | n/a | n/a | n/a | n/a | 0.5000 | 0.7500 | 0.0573 | n/a | `runs/ctt_gated_residual_smoke_proxy` |
+| CTT gated residual | proxy | n/a | n/a | n/a | n/a | n/a | 0.1739 | 0.5362 | 0.0281 | n/a | `runs/ctt_gated_residual_val_proxy` |
 | CTT residual train | train | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_residual_smoke` |
 | CTT residual proxy | proxy | n/a | n/a | n/a | n/a | n/a | 1.0000 | 1.0000 | 0.1010 | n/a | `runs/ctt_residual_smoke_proxy` |
+| CTT residual proxy | proxy | n/a | n/a | n/a | n/a | n/a | 0.3478 | 0.7101 | 0.0397 | n/a | `runs/ctt_residual_val_proxy` |
+| task memory | proxy baseline | n/a | n/a | n/a | n/a | n/a | 0.3188 | 0.5362 | 0.0175 | n/a | `runs/task_memory_val_proxy` |
+| local atlas | proxy baseline | n/a | n/a | n/a | n/a | n/a | 0.4058 | 0.6812 | 0.0368 | n/a | `runs/local_atlas_val_proxy` |
 | Utility energy smoke | calibration diagnostic | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 0.1316 | `runs/utility_energy_smoke` |
 
 `n/a` means the required measured artifact does not exist yet.

workspace/scripts/audit_cil_charts.py

@@ -79,10 +79,13 @@ def _audit_split_flags(indexes: dict[str, tuple[Path, dict[str, Any]]], violatio
         else:
             if retrieval:
                 violations.append(f"{path}: non-train split is retrieval_index_allowed")
-            if include_outcomes:
-                violations.append(f"{path}: non-train split exposes outcomes in chart DB")
             if audience != "evaluator_only":
                 violations.append(f"{path}: non-train split audience must be evaluator_only")
+            contract = payload.get("leakage_contract", {})
+            if not bool(contract.get("deployment_must_not_read_outcomes", False)):
+                violations.append(
+                    f"{path}: non-train split must declare deployment_must_not_read_outcomes=true"
+                )
         if split != str(payload.get("split")):
             violations.append(f"{path}: index split field does not match directory")
         if not payload.get("deployment_candidate_excludes_expert", False):

workspace/scripts/eval_ctt_proxy.py

@@ -16,7 +16,10 @@ if str(PROJECT_ROOT) not in sys.path:
 import torch  # noqa: E402
 
 from cil.metrics import (  # noqa: E402
+    candidate_diversity,
+    collapse_rate,
     macro_micro_summary,
+    mean_nearest_distance_to_set,
     negative_near_at_threshold,
     positives_closer_than_negatives,
     proxy_positive_tangent_coverage_at_k,
@@ -51,6 +54,7 @@ def main(argv: list[str] | None = None) -> int:
 
     source_charts, source_index = load_charts(args.source_index, max_charts=None)
     target_charts, target_index = load_charts(args.target_index, max_charts=args.max_target_charts)
+    _validate_indexes(args.source_index, source_index, args.target_index, target_index)
     rows = []
     log_lines = [
         f"source_charts={len(source_charts)} target_charts={len(target_charts)} k={args.k}",
@@ -85,7 +89,7 @@ def main(argv: list[str] | None = None) -> int:
             row: dict[str, Any] = {
                 "chart_id": target.chart_id,
                 "task_id": target.task_id,
-                "seed": "unknown",
+                "seed": target.seed,
                 "num_proposals": len(proposals),
             }
             for threshold in thresholds:
@@ -104,8 +108,14 @@ def main(argv: list[str] | None = None) -> int:
                 )
             distance = proxy_support_distance(proposals, positives, k=args.k)
             row[f"proxy_support_distance_at_{args.k}"] = distance
+            positive_distance = mean_nearest_distance_to_set(proposals, positives, k=args.k)
+            row[f"mean_positive_distance_at_{args.k}"] = positive_distance
+            negative_distance = mean_nearest_distance_to_set(proposals, negatives, k=args.k)
+            row[f"mean_negative_distance_at_{args.k}"] = negative_distance
             closer = positives_closer_than_negatives(proposals, positives, negatives, k=args.k)
             row[f"pos_closer_than_neg_at_{args.k}"] = closer
+            row[f"candidate_diversity_at_{args.k}"] = candidate_diversity(proposals, k=args.k)
+            row[f"collapse_rate_at_{args.k}"] = collapse_rate(proposals, k=args.k)
             rows.append(row)
 
     metric_names = sorted(
@@ -134,7 +144,7 @@ def main(argv: list[str] | None = None) -> int:
     }
     (out_dir / "metrics.json").write_text(json.dumps(metrics, indent=2, sort_keys=True) + "\n")
     (out_dir / "metrics_by_task.json").write_text(json.dumps(_by_task(rows, metric_names), indent=2, sort_keys=True) + "\n")
-    (out_dir / "metrics_by_seed.json").write_text("{}\n")
+    (out_dir / "metrics_by_seed.json").write_text(json.dumps(_by_group(rows, metric_names, "seed"), indent=2, sort_keys=True) + "\n")
     (out_dir / "eval.log").write_text("\n".join(log_lines) + "\n")
     (out_dir / "train.log").write_text("see checkpoint run\n")
     (out_dir / "table.tex").write_text(_table(summary) + "\n")
@@ -143,6 +153,29 @@ def main(argv: list[str] | None = None) -> int:
     return 0
 
 
+def _validate_indexes(
+    source_path: Path,
+    source_index: dict[str, Any],
+    target_path: Path,
+    target_index: dict[str, Any],
+) -> None:
+    if source_index.get("split") != "train" or not source_index.get("retrieval_index_allowed"):
+        raise SystemExit(
+            f"{source_path} is not a train-only retrieval index; CTT proxy eval must "
+            "retrieve source positives from train split only"
+        )
+    if not source_index.get("include_outcomes"):
+        raise SystemExit(f"{source_path} must include train outcomes for source positives")
+    if not target_index.get("include_outcomes"):
+        raise SystemExit(
+            f"{target_path} does not expose evaluator outcomes/labels. "
+            "Proxy support evaluation needs an evaluator-only target chart DB; "
+            "do not substitute hidden labels or distance proxies from train self-target."
+        )
+    if target_index.get("split") != "train" and target_index.get("retrieval_index_allowed"):
+        raise SystemExit(f"{target_path} is non-train but marked retrieval_index_allowed")
+
+
 def _write_run_provenance(
     out_dir: Path,
     args: argparse.Namespace,
@@ -164,16 +197,24 @@ def _run(command: list[str]) -> str:
 
 
 def _by_task(rows: list[dict[str, Any]], metric_names: list[str]) -> dict[str, dict[str, float]]:
+    return _by_group(rows, metric_names, "task_id")
+
+
+def _by_group(
+    rows: list[dict[str, Any]],
+    metric_names: list[str],
+    group_key: str,
+) -> dict[str, dict[str, float]]:
     output: dict[str, dict[str, float]] = {}
     for row in rows:
-        task = str(row["task_id"])
-        output.setdefault(task, {})
-    for task in output:
-        task_rows = [row for row in rows if row["task_id"] == task]
+        group = str(row[group_key])
+        output.setdefault(group, {})
+    for group in output:
+        group_rows = [row for row in rows if str(row[group_key]) == group]
         for metric in metric_names:
-            values = [float(row[metric]) for row in task_rows if isinstance(row.get(metric), (int, float))]
+            values = [float(row[metric]) for row in group_rows if isinstance(row.get(metric), (int, float))]
             if values:
-                output[task][metric] = sum(values) / len(values)
+                output[group][metric] = sum(values) / len(values)
     return output
 
 

workspace/scripts/eval_metrics.py

@@ -16,7 +16,10 @@ if str(PROJECT_ROOT) not in sys.path:
 from cil.metrics import (  # noqa: E402
     MetricInputError,
     branch_car,
+    candidate_diversity,
+    collapse_rate,
     macro_micro_summary,
+    mean_nearest_distance_to_set,
     measured_support_gap,
     negative_near_at_threshold,
     outcome_ptr_at_k,
@@ -178,11 +181,17 @@ def _proxy_row(row: dict[str, Any], *, k: int, thresholds: list[float]) -> dict[
     distance = proxy_support_distance(generated, positives, k=k)
     if distance is not None:
         output[f"proxy_support_distance_at_{k}"] = distance
+    positive_distance = mean_nearest_distance_to_set(generated, positives, k=k)
+    if positive_distance is not None:
+        output[f"mean_positive_distance_at_{k}"] = positive_distance
+    negative_distance = mean_nearest_distance_to_set(generated, negatives, k=k)
+    if negative_distance is not None:
+        output[f"mean_negative_distance_at_{k}"] = negative_distance
     closer = positives_closer_than_negatives(generated, positives, negatives, k=k)
     if closer is not None:
         output[f"pos_closer_than_neg_at_{k}"] = closer
-    output[f"candidate_diversity_at_{k}"] = _mean_pairwise_distance(generated[:k])
-    output[f"collapse_rate_at_{k}"] = _collapse_rate(generated[:k])
+    output[f"candidate_diversity_at_{k}"] = candidate_diversity(generated, k=k)
+    output[f"collapse_rate_at_{k}"] = collapse_rate(generated, k=k)
     return output
 
 
@@ -297,24 +306,6 @@ def _markdown_report(mode: str, k: int, summary: dict[str, Any]) -> str:
     return "\n".join(lines)
 
 
-def _mean_pairwise_distance(vectors: list[list[float]]) -> float:
-    if len(vectors) < 2:
-        return 0.0
-    distances = []
-    for left_index, left in enumerate(vectors):
-        for right in vectors[left_index + 1 :]:
-            distances.append(_rms_l2(left, right))
-    return sum(distances) / len(distances)
-
-
-def _collapse_rate(vectors: list[list[float]], *, threshold: float = 1.0e-6) -> float:
-    if not vectors:
-        return 0.0
-    first = vectors[0]
-    collapsed = sum(1 for vector in vectors if _rms_l2(first, vector) <= threshold)
-    return collapsed / len(vectors)
-
-
 def _rms_l2(left: list[float], right: list[float]) -> float:
     if len(left) != len(right):
         raise MetricInputError("vectors must have matching dimensions")

workspace/scripts/summarize_ctt_runs.py

@@ -41,6 +41,11 @@ def main(argv: list[str] | None = None) -> int:
         rows.extend(_reproduce_rows(reproduce))
     for metrics_path in sorted(args.run_root.glob("ctt*/metrics.json")):
         rows.append(_ctt_row(metrics_path))
+    for metrics_path in sorted(args.run_root.glob("*memory*_val_proxy/metrics.json")):
+        rows.append(_positive_memory_row(metrics_path))
+    local_atlas = args.run_root / "local_atlas_val_proxy" / "metrics.json"
+    if local_atlas.exists():
+        rows.append(_positive_memory_row(local_atlas))
     utility = args.run_root / "utility_energy_smoke" / "metrics.json"
     if utility.exists():
         rows.append(_utility_row(utility))
@@ -107,7 +112,30 @@ def _ctt_row(path: Path) -> dict[str, str]:
         "pptc_0p40": _summary_mean(summary, "pptc_at_16_thr_0p40"),
         "negative_near_0p20": _summary_mean(summary, "negative_near_at_16_thr_0p20"),
         "calibration_ece": "n/a",
-        "seeds": "unknown",
+        "seeds": _seed_count(path.parent),
+        "ci": "see run",
+    }
+
+
+def _positive_memory_row(path: Path) -> dict[str, str]:
+    data = json.loads(path.read_text())
+    summary = data.get("summary", {})
+    method = str(data.get("method", path.parent.name)).replace("_", " ")
+    return {
+        "method": method,
+        "run_path": str(path.parent),
+        "clean": "yes",
+        "status": "proxy baseline",
+        "success": "n/a",
+        "proposal_oracle": "n/a",
+        "support_gap": "n/a",
+        "selector_gap": "n/a",
+        "outcome_ptr": "n/a",
+        "pptc_0p20": _summary_mean(summary, "pptc_at_16_thr_0p20"),
+        "pptc_0p40": _summary_mean(summary, "pptc_at_16_thr_0p40"),
+        "negative_near_0p20": _summary_mean(summary, "negative_near_at_16_thr_0p20"),
+        "calibration_ece": "n/a",
+        "seeds": _seed_count(path.parent),
         "ci": "see run",
     }
 
@@ -152,6 +180,17 @@ def _fmt(value: Any) -> str:
     return f"{float(value):.4f}"
 
 
+def _seed_count(run_dir: Path) -> str:
+    path = run_dir / "metrics_by_seed.json"
+    if not path.exists():
+        return "unknown"
+    try:
+        data = json.loads(path.read_text())
+    except json.JSONDecodeError:
+        return "unknown"
+    return str(len(data)) if isinstance(data, dict) and data else "unknown"
+
+
 def _markdown(rows: list[dict[str, str]]) -> str:
     lines = [
         "# CTT Summary",