vla / workspace /scripts /build_paper_analysis.py
anhtld's picture
manual-sync chart-synthesis 2026-07-02T22:45:08Z workspace/scripts/build_paper_analysis.py
0c096c5 verified
Raw
History Blame Contribute Delete
129 kB
#!/usr/bin/env python
from __future__ import annotations
import json
import math
import sys
from collections import Counter
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
ROOT_DIR = Path(__file__).resolve().parents[1]
if str(ROOT_DIR) not in sys.path:
sys.path.insert(0, str(ROOT_DIR))
from dovla_cil.eval.metrics import (
candidate_prefix_causal_metrics,
causal_action_decomposition,
finite_mean,
)
RESULTS_DIR = Path("results")
OUT_JSON = RESULTS_DIR / "paper_analysis.json"
OUT_MD = RESULTS_DIR / "paper_analysis.md"
LATEX_TABLES_DIR = Path("latex") / "tables"
OUT_CAR_TABLE = LATEX_TABLES_DIR / "car_decomposition.tex"
GENERATOR_V2_MEMORY_EVAL = RESULTS_DIR / "generator_v2_positive_tangent_memory_eval.json"
GENERATOR_V2_LOCAL_ATLAS_EVAL = RESULTS_DIR / "generator_v2_positive_tangent_local_atlas_pool16_eval.json"
GENERATOR_V2_LOCAL_ATLAS_SWEEP = RESULTS_DIR / "generator_v2_positive_tangent_local_atlas_sweep_summary.json"
GENERATOR_V2_CHART_SYNTHESIS_EVAL = RESULTS_DIR / "generator_v2_positive_tangent_chart_synthesis_eval.json"
GENERATOR_V2_CHART_SYNTHESIS_SWEEP = RESULTS_DIR / "generator_v2_positive_tangent_chart_synthesis_sweep_summary.json"
GENERATOR_V2_CVAE_EVAL = RESULTS_DIR / "generator_v2_positive_tangent_cvae_eval.json"
GENERATOR_V2_CVAE_SWEEP = RESULTS_DIR / "generator_v2_positive_tangent_cvae_sweep_summary.json"
GENERATOR_V2_SPLINE_CVAE_EVAL = RESULTS_DIR / "generator_v2_positive_tangent_spline_cvae_eval.json"
GENERATOR_V2_SPLINE_CVAE_SWEEP = RESULTS_DIR / "generator_v2_positive_tangent_spline_cvae_sweep_summary.json"
GENERATOR_V2_SPLINE_FLOW_EVAL = RESULTS_DIR / "generator_v2_positive_tangent_spline_flow_eval.json"
GENERATOR_V2_SPLINE_FLOW_SWEEP = RESULTS_DIR / "generator_v2_positive_tangent_spline_flow_sweep_summary.json"
GENERATOR_V2_GUIDED_SPLINE_FLOW_EVAL = RESULTS_DIR / "generator_v2_positive_tangent_guided_spline_flow_eval.json"
GENERATOR_V2_GUIDED_SPLINE_FLOW_SWEEP = RESULTS_DIR / "generator_v2_positive_tangent_guided_spline_flow_sweep_summary.json"
CANONICAL_H16_ROLLOUT = Path("/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs")
FALLBACK_BEST_CLEAN_KEY = "residual_k4_consensus_grid035040045_noopbonus003"
NON_DEPLOYMENT_KEYS = {
"same_state_near_miss",
"same_state_no_expert",
"same_state_policy_baseline",
"same_state_full",
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8",
"residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace",
"transport_field_reground_fieldonly_k6matched_b12_clean_k6_oraclek8",
"transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_oraclek8",
"transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
}
@dataclass(frozen=True)
class MethodSpec:
key: str
label: str
summary_path: str | None = None
raw_rollout_glob: str | None = None
summary_mode: str = "standard"
headline_metric: str = "mean_success"
METHODS = [
MethodSpec(
key="h16_policy_canonical",
label="Direct h=16 policy, canonical rollout",
raw_rollout_glob=str(CANONICAL_H16_ROLLOUT / "seed_*/online_rollout.json"),
),
MethodSpec(
key="gaussian_field",
label="Gaussian field search",
summary_path="h16_field_sweep_summary.json",
summary_mode="field_sweep_best",
),
MethodSpec(
key="near_miss_policy_bc5",
label="Near-miss proposal policy, direct",
summary_path="h16_policy_ckpt_near_miss_policy_bc5_summary.json",
),
MethodSpec(
key="best_clean_residual_k2",
label="K2 residual transport, safe + margin 0.20",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"knn2_scale0p40_safe_types_margin0p20_summary.json"
),
),
MethodSpec(
key="residual_taskrelative_k2",
label="K2 task-relative residual transport, safe + margin 0.20",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"taskrelative_knn2_scale0p40_safe_types_margin0p20_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus",
label="K4 mean-by-type tangent consensus",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_summary.json"
),
),
MethodSpec(
key="residual_k4_kernel_consensus",
label="K4 kernel-weighted tangent consensus",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_kernel_mean_by_type_summary.json"
),
),
MethodSpec(
key="residual_k4_kernel_consensus_noopbonus003",
label="K4 kernel-weighted tangent consensus, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_kernel_mean_by_type_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_kernel_consensus_s035_noopbonus003",
label="K4 kernel-weighted tangent consensus, scale 0.35, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s035_safe_margin0p20_kernel_mean_by_type_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_kernel_consensus_s045_noopbonus003",
label="K4 kernel-weighted tangent consensus, scale 0.45, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s045_safe_margin0p20_kernel_mean_by_type_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_fieldsoftmax_grid",
label="K4 field-softmax tangent transport, scales 0.35/0.40/0.45",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_fieldsoftmax_grid_safe_margin0p20_summary.json"
),
),
MethodSpec(
key="residual_k4_fieldsoftmax_grid_noopbonus003",
label="K4 field-softmax tangent transport, scales 0.35/0.40/0.45, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_fieldsoftmax_grid_safe_margin0p20_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_fieldsoftmax_grid_margin010_noopbonus003",
label="K4 field-softmax tangent transport, margin 0.10, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_fieldsoftmax_grid_safe_margin0p10_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_fieldsoftmax_grid_margin005_noopbonus003",
label="K4 field-softmax tangent transport, margin 0.05, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_fieldsoftmax_grid_safe_margin0p05_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_fieldsoftmax_grid_margin000_noopbonus003",
label="K4 field-softmax tangent transport, margin 0.00, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_fieldsoftmax_grid_safe_margin0p00_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k8_fieldsoftmax_grid_noopbonus003",
label="K8 field-softmax tangent transport, scales 0.35/0.40/0.45, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k8_fieldsoftmax_grid_safe_margin0p20_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_noopbonus003",
label="K4 mean-by-type tangent consensus, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_noopbonus003_srcprog025",
label="K4 mean-by-type tangent consensus, no-op bonus 0.03, source progress >= 0.25",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_noopbonus0p03_srcprog0p25_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_margin015_noopbonus003",
label="K4 mean-by-type tangent consensus, margin 0.15, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p15_mean_by_type_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_margin025_noopbonus003",
label="K4 mean-by-type tangent consensus, margin 0.25, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p25_mean_by_type_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_margin015_srcscorebonus002",
label="K4 mean-by-type tangent consensus, margin 0.15, source-score bonus 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p15_mean_by_type_srcscorebonus0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_margin025_srcscorebonus002",
label="K4 mean-by-type tangent consensus, margin 0.25, source-score bonus 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p25_mean_by_type_srcscorebonus0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_noopbonus003",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_srcscorebonus002",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, source-score bonus 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_srcscorebonus0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_srcadvbonus002",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, source-advantage bonus 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_srcadvbonus0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_srcadvbonus005",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, source-advantage bonus 0.05",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_srcadvbonus0p05_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_noopbonus003_srcadvbonus002",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, source-advantage bonus 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_srcadvbonus0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_srcadvgate000",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, source-advantage gate >= 0.0",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_srcadvgate0p0_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_noopbonus003_srcadvgate000",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, source-advantage gate >= 0.0",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_srcadvgate0p0_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_typesuccessbonus002",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, train family-success bonus 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_typesuccessbonus0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_typesuccessbonus003",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, train family-success bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_typesuccessbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_typesuccessbonus005",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, train family-success bonus 0.05",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_typesuccessbonus0p05_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_noopbonus003_typesuccessbonus002",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, train family-success bonus 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_typesuccessbonus0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_consensus005",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, consensus penalty 0.05",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_consensus0p05_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_noopbonus003_consensus002",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, consensus penalty 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_consensus0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_noopbonus003_consensus005",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, consensus penalty 0.05",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_consensus0p05_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_noopbonus003_consensus010",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, consensus penalty 0.10",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_consensus0p10_summary.json"
),
),
MethodSpec(
key="residual_k4_compose_grid035040045",
label="K4 composed type-consensus tangents, scales 0.35/0.40/0.45",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_compose_grid035040045_safe_margin0p20_summary.json"
),
),
MethodSpec(
key="residual_k4_compose_grid035040045_noopbonus003",
label="K4 composed type-consensus tangents, scales 0.35/0.40/0.45, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_compose_grid035040045_safe_margin0p20_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_grid035040045",
label="K4 composed type-consensus tangents, masked, scales 0.35/0.40/0.45",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_grid035040045_safe_margin0p20_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_grid035040045_noopbonus003",
label="K4 composed type-consensus tangents, masked, scales 0.35/0.40/0.45, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_grid035040045_safe_margin0p20_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045",
label="K4 composed type-consensus tangents, masked, drop near-miss+no-op composite",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003",
label="K4 composed type-consensus tangents, masked, drop near-miss+no-op composite",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_margin010_noopbonus003",
label="K4 compatible tangents, margin 0.10, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p10_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8",
label="K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 diagnostic",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_oraclek8_summary.json"
),
headline_metric="mean_candidate_oracle_success_rate",
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace",
label="K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 branch trace",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_oraclek8trace_summary.json"
),
headline_metric="mean_candidate_oracle_success_rate",
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger002",
label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001",
label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_latest_margin000_clean",
label="Transported residual field re-grounding, no bonus/challenger, margin 0.00",
summary_path=(
"h16_transport_field_reground_fieldonly_latest_margin0p00_"
"nobonus_nochallenger_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_latest_margin000_clean_k6",
label="Transported residual field re-grounding, no bonus/challenger, K6 support",
summary_path=(
"h16_transport_field_reground_fieldonly_latest_margin0p00_"
"clean_k6_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6",
label="K6-matched transported residual field re-grounding, no bonus/challenger",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_b12_v2_"
"besttransport_margin0p00_k6_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg",
label="K6-matched transported residual field re-grounding, drop no-op+wrong-gripper composite",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_b12_v2_"
"besttransport_margin0p00_k6_dropnoopwg_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted",
label="K6-matched transported residual field re-grounding, exact drop-mask target map",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_fieldsoftmax",
label="K6-matched transported residual field re-grounding, exact drop-mask field-softmax reducer",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_fieldsoftmax_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_typesuccess001",
label="K6-matched transported residual field re-grounding, exact drop-mask train-type prior 0.01",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_typesuccess001_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_typesuccess002",
label="K6-matched transported residual field re-grounding, exact drop-mask train-type prior 0.02",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_typesuccess002_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_typesuccess003",
label="K6-matched transported residual field re-grounding, exact drop-mask train-type prior 0.03",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_typesuccess003_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_rankcal005",
label="K6-matched transported residual field re-grounding, exact drop-mask train rank calibration 0.05",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_rankcal005_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001",
label="K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.01",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore001_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_typecal005",
label="K6-matched transported residual field re-grounding, source-score 0.01 + transport-outcome type calibration 0.05",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore001_typecal005_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_typecal010",
label="K6-matched transported residual field re-grounding, source-score 0.01 + transport-outcome type calibration 0.10",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore001_typecal010_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_typecal020",
label="K6-matched transported residual field re-grounding, source-score 0.01 + transport-outcome type calibration 0.20",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore001_typecal020_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_zscore",
label="K6-matched transported residual field re-grounding, source-score 0.01, z-score retrieval chart",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore001_metric_zscore_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrel",
label="K6-matched transported residual field re-grounding, source-score 0.01, task-relative retrieval chart",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore001_metric_taskrel_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrelz",
label="K6-matched transported residual field re-grounding, source-score 0.01, task-relative z-score retrieval chart",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore001_metric_taskrelz_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005",
label="K6-matched transported residual field re-grounding, task-conditioned source-score prior",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_advw1p0",
label="Generator V1 utility-weighted tangents, expert anchor, advantage weight 1.0",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_advw1p0_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_advw2p0",
label="Generator V1 utility-weighted tangents, expert anchor, advantage weight 2.0",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_advw2p0_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_policyanchor_advw2p0",
label="Generator V1 positive tangents, policy anchor, advantage weight 2.0",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_policyanchor_advw2p0_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_advw2p0_gate0",
label="Generator V1 utility-weighted tangents, expert anchor, source advantage gate >= 0.0",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_advw2p0_gate0_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_policyanchor_advw2p0_gate0",
label="Generator V1 positive tangents, policy anchor, source advantage gate >= 0.0",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_policyanchor_advw2p0_gate0_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_advw1p0_oraclek8",
label="Generator V1 expert-anchor advantage weight 1.0, candidate-oracle K8",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_advw1p0_oraclek8_summary.json"
),
headline_metric="mean_candidate_oracle_success_rate",
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_advw2p0_oraclek8",
label="Generator V1 expert-anchor advantage weight 2.0, candidate-oracle K8",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_advw2p0_oraclek8_summary.json"
),
headline_metric="mean_candidate_oracle_success_rate",
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_policyanchor_advw2p0_oraclek8",
label="Generator V1 policy-anchor advantage weight 2.0, candidate-oracle K8",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_policyanchor_advw2p0_oraclek8_summary.json"
),
headline_metric="mean_candidate_oracle_success_rate",
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_advw0p5",
label="Generator V1 utility-weighted tangents, expert anchor, advantage weight 0.5",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_advw0p5_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_advw4p0",
label="Generator V1 utility-weighted tangents, expert anchor, advantage weight 4.0",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_advw4p0_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_policyanchor_advw1p0",
label="Generator V1 positive tangents, policy anchor, advantage weight 1.0",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_policyanchor_advw1p0_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_policyanchor_advw4p0",
label="Generator V1 positive tangents, policy anchor, advantage weight 4.0",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_policyanchor_advw4p0_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_scaleoracle_trainall_k4_scale0025",
label="K6-matched transported residual field re-grounding, all-train tangent-length prior 0.025",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_"
"scaleoracle_trainall_k4_scale0025_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_scaleoracle_trainall_k4_scale005",
label="K6-matched transported residual field re-grounding, all-train tangent-length prior 0.05",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_"
"scaleoracle_trainall_k4_scale005_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_scaleoracle_trainall_k4_scale010",
label="K6-matched transported residual field re-grounding, all-train tangent-length prior 0.10",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_"
"scaleoracle_trainall_k4_scale010_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005",
label="K6-matched transported residual field re-grounding, train-oracle calibrated selector",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_"
"oraclecal_train800_k4_r005_t005_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025",
label="K6-matched transported residual field re-grounding, small train-oracle selector calibration",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_"
"oraclecal_train800_k4_r0025_t0025_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005",
label="K6-matched transported residual field re-grounding, train-oracle rank calibration",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_"
"oraclecal_train800_k4_rank005_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005",
label="K6-matched transported residual field re-grounding, train-oracle type calibration",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore_task_pick001_stack005_"
"oraclecal_train800_k4_type005_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8",
label="K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore001_oraclek8_summary.json"
),
headline_metric="mean_candidate_oracle_success_rate",
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0005",
label="K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.005",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore0005_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0015",
label="K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.015",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore0015_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore002",
label="K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.02",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore002_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore005",
label="K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.05",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_srcscore005_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_oraclek8",
label="K6-matched transported residual field re-grounding, exact drop-mask candidate-oracle K8",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_"
"besttransport_margin0p00_k6_oraclek8_summary.json"
),
headline_metric="mean_candidate_oracle_success_rate",
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b24_clean_k6_dropnoopwg_retargeted",
label="K6-matched transported residual field re-grounding, exact drop-mask target map, B24",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b24_v1_"
"besttransport_margin0p00_k6_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b24_clean_k6_dropnoopwg_retargeted_latest",
label="K6-matched transported residual field re-grounding, exact drop-mask target map, B24 latest",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b24_v1_"
"latest_margin0p00_k6_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_oraclek8",
label="K6-matched transported residual field re-grounding, candidate-oracle K8 diagnostic",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_b12_v2_"
"besttransport_margin0p00_k6_oraclek8_summary.json"
),
headline_metric="mean_candidate_oracle_success_rate",
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_best",
label="K6-matched transported residual field re-grounding, validation-rank checkpoint",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_b12_v2_"
"best_margin0p00_k6_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_prependpolicy",
label="K6-matched transported residual field re-grounding, policy candidate prepended",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_b12_v2_"
"besttransport_margin0p00_k6_prependpolicy_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_prependpolicy_margin003",
label="K6-matched transported residual field re-grounding, policy candidate prepended, margin 0.03",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_b12_v2_"
"besttransport_margin0p03_k6_prependpolicy_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopfamily",
label="K6-matched transported residual field re-grounding, drop no-op residual family",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_b12_v2_"
"besttransport_margin0p00_k6_dropnoopfamily_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_droppolicy_dropnoopwg",
label="K6-matched transported residual field re-grounding, drop policy residual and no-op+wrong-gripper",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_b12_v2_"
"besttransport_margin0p00_k6_droppolicy_dropnoopwg_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_e24_besttransport",
label="K6-matched transported residual field re-grounding, 24 epochs, transport-rank checkpoint",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_b12_e24_v1_"
"besttransport_margin0p00_k6_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_e24_latest",
label="K6-matched transported residual field re-grounding, 24 epochs, latest checkpoint",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_b12_e24_v1_"
"latest_margin0p00_k6_summary.json"
),
),
MethodSpec(
key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_e24_best",
label="K6-matched transported residual field re-grounding, 24 epochs, validation-rank checkpoint",
summary_path=(
"h16_transport_field_reground_fieldonly_k6clean_b12_e24_v1_"
"best_margin0p00_k6_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_stacknowg",
label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, no wrong-gripper component on Stack",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_stacknowg_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0005",
label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.005",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p005_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0015",
label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.015",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p015_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_scale035",
label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, scale-gated 0.35",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_scale035_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_scales035040",
label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, scale-gated 0.35/0.40",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_scales035040_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger003",
label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001",
label="K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmwgchallenger0p01_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgmargin003_challenger001",
label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger 0.01, wrong-gripper margin 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmwgmargin0p03_challenger0p01_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgmargin005_challenger001",
label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger 0.01, wrong-gripper margin 0.05",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmwgmargin0p05_challenger0p01_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull",
label="K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmwgchallenger0p01_pickpull_norm_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_challenger001",
label="K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_challenger0p01_norm_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin003_challenger001",
label="K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper Pick/Pull margin 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_wgmargin0p03_challenger0p01_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin005_challenger001",
label="K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper Pick/Pull margin 0.05",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_wgmargin0p05_challenger0p01_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpullstack_challenger001",
label="K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper challenger on Pick/Pull/Stack",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpullstack_challenger0p01_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_noopwgcontact_challenger001",
label="K4 compatible tangents, no-op bonus 0.03, near-miss global + no-op/wrong-gripper contact challenger",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_noopwgcontact_challenger0p01_summary.json"
),
),
MethodSpec(
key="residual_k6_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001",
label="K6 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k6_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_summary.json"
),
),
MethodSpec(
key="residual_k8_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001",
label="K8 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k8_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_typesuccessbonus002_nmchallenger001",
label="K4 compatible tangents, no-op bonus 0.03, train family-success bonus 0.02, near-miss challenger gate 0.01",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_typesuccessbonus0p02_nmchallenger0p01_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_typesuccessbonus005_nmchallenger001",
label="K4 compatible tangents, no-op bonus 0.03, train family-success bonus 0.05, near-miss challenger gate 0.01",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_typesuccessbonus0p05_nmchallenger0p01_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_taskrelz_nmchallenger001",
label="K4 compatible tangents, actor-relative z-score retrieval, no-op bonus 0.03, near-miss challenger gate 0.01",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_taskrelz_nmchallenger0p01_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_srcscore005",
label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, train source-score certificate 0.05",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_srcscore0p05_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_srcscore010",
label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, train source-score certificate 0.10",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_srcscore0p10_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_srcadv005",
label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, train source-advantage certificate 0.05",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_srcadv0p05_summary.json"
),
),
MethodSpec(
key="distill_direct_bestpolicy",
label="Direct policy distilled from clean residual-tangent teacher",
summary_path="h16_distill_direct_bestpolicy_summary.json",
),
MethodSpec(
key="distill_retrieval_residual_teacher_bestfield",
label="Distilled policy + clean residual-tangent teacher at inference",
summary_path="h16_distill_retrieval_residual_teacher_bestfield_summary.json",
),
MethodSpec(
key="typed_proposal_lattice_types6_prepend_margin000",
label="Typed proposal lattice head, six families, policy-prepended margin 0.00",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_typedprop_p2_bestpt_"
"proposal_lattice_types6_prepend_margin0p00_summary.json"
),
),
MethodSpec(
key="typed_proposal_lattice_types6_prepend_margin005",
label="Typed proposal lattice head, six families, policy-prepended margin 0.05",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_typedprop_p2_bestpt_"
"proposal_lattice_types6_prepend_margin0p05_summary.json"
),
),
MethodSpec(
key="typed_proposal_lattice_types4safe_prepend_margin005",
label="Typed proposal lattice head, safe four families, policy-prepended margin 0.05",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_typedprop_p2_bestpt_"
"proposal_lattice_types4safe_prepend_margin0p05_summary.json"
),
),
MethodSpec(
key="typed_proposal_lattice_types2sparse_prepend_margin005",
label="Typed proposal lattice head, sparse no-op/wrong-gripper families, policy-prepended margin 0.05",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_typedprop_p2_bestpt_"
"proposal_lattice_types2sparse_prepend_margin0p05_summary.json"
),
),
MethodSpec(
key="typed_proposal_lattice_nooponly_prepend_margin005",
label="Typed proposal lattice head, no-op family only, policy-prepended margin 0.05",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_typedprop_p2_bestpt_"
"proposal_lattice_nooponly_prepend_margin0p05_summary.json"
),
),
MethodSpec(
key="typed_proposal_lattice_types2sparse_bestpolicy_prepend_margin005",
label="Typed proposal lattice head, sparse families, best-policy checkpoint, policy-prepended margin 0.05",
summary_path="h16_bestpolicy_types2sparse_prepend_margin0p05_summary.json",
),
MethodSpec(
key="typed_proposal_lattice_nooponly_bestpolicy_prepend_margin005",
label="Typed proposal lattice head, no-op family only, best-policy checkpoint, policy-prepended margin 0.05",
summary_path="h16_bestpolicy_nooponly_prepend_margin0p05_summary.json",
),
MethodSpec(
key="typed_proposal_lattice_nooponly_sparsehead_prepend_margin005",
label="Typed proposal lattice head, no-op-only head retrain, policy-prepended margin 0.05",
summary_path="h16_typedprop_nooponly_sparsehead_prepend_margin0p05_summary.json",
),
MethodSpec(
key="typed_proposal_lattice_nooponly_sparsehead_prepend_margin000",
label="Typed proposal lattice head, no-op-only head retrain, policy-prepended margin 0.00",
summary_path="h16_typedprop_nooponly_sparsehead_prepend_margin0p00_summary.json",
),
MethodSpec(
key="typed_proposal_lattice_nooponly_sparsehead_prepend_margin010",
label="Typed proposal lattice head, no-op-only head retrain, policy-prepended margin 0.10",
summary_path="h16_typedprop_nooponly_sparsehead_prepend_margin0p10_summary.json",
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001",
label="K4 composed compatible tangents, no-op bonus 0.03, singleton near-miss bonus 0.01",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmbonus0p01_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus002",
label="K4 composed compatible tangents, no-op bonus 0.03, singleton near-miss bonus 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmbonus0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_srcscorebonus002",
label="K4 composed type-consensus tangents, masked, drop near-miss+no-op composite, source-score bonus 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_srcscorebonus0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_srcscorebonus002",
label="K4 composed type-consensus tangents, masked, drop near-miss+no-op composite, no-op bonus 0.03, source-score bonus 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_srcscorebonus0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_dropnmnoop_l2comp002_grid035040045_noopbonus003",
label="K4 composed type-consensus tangents, masked, drop near-miss+no-op composite, composite L2 penalty 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_dropnmnoop_l2comp002_grid035040045_safe_margin0p20_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_compbonus_grid035040045_noopbonus003",
label="K4 composed type-consensus tangents, masked, component no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_compbonus_grid035040045_safe_margin0p20_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_l2comp002_grid035040045_noopbonus003",
label="K4 composed type-consensus tangents, masked, composite L2 penalty 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_l2comp002_grid035040045_safe_margin0p20_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_composemasked_l2comp005_grid035040045_noopbonus003",
label="K4 composed type-consensus tangents, masked, composite L2 penalty 0.05",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_composemasked_l2comp005_grid035040045_safe_margin0p20_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="repair_nearmiss_k4_grid025035050_margin020",
label="K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.20",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_repair_"
"nearmiss_k4_grid025035050_margin0p20_summary.json"
),
),
MethodSpec(
key="repair_nearmiss_k4_grid035050075_margin020",
label="K4 near-miss-to-expert repair tangent, scales 0.35/0.50/0.75, margin 0.20",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_repair_"
"nearmiss_k4_grid035050075_margin0p20_summary.json"
),
),
MethodSpec(
key="repair_nearmiss_k4_grid025035050_margin010",
label="K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.10",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_repair_"
"nearmiss_k4_grid025035050_margin0p10_summary.json"
),
),
MethodSpec(
key="repair_safe_k4_grid025035050_margin020",
label="K4 safe-family-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.20",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_repair_"
"safe_k4_grid025035050_margin0p20_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_noopbonus003_l2penalty005",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, action L2 penalty 0.05",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_l2penalty0p05_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_noopbonus003_l2penalty010",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, action L2 penalty 0.10",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_l2penalty0p10_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035040045_noopbonus003_l2penalty020",
label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, action L2 penalty 0.20",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_l2penalty0p20_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid040045050_noopbonus003",
label="K4 mean-by-type tangent consensus, scales 0.40/0.45/0.50, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid040045050_safe_margin0p20_mean_by_type_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid040045050_srcscorebonus002",
label="K4 mean-by-type tangent consensus, scales 0.40/0.45/0.50, source-score bonus 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid040045050_safe_margin0p20_mean_by_type_srcscorebonus0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_grid035045055_noopbonus003",
label="K4 mean-by-type tangent consensus, scales 0.35/0.45/0.55, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4_grid035045055_safe_margin0p20_mean_by_type_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_nooponly_noopbonus003",
label="K4 mean-by-type tangent consensus, no-op-only residuals, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_nooponly_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_nooponly_srcscorebonus002",
label="K4 mean-by-type tangent consensus, no-op-only residuals, source-score bonus 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_nooponly_srcscorebonus0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_noopbonus003_srcprog050",
label="K4 mean-by-type tangent consensus, no-op bonus 0.03, source progress >= 0.50",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_noopbonus0p03_srcprog0p50_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_noopbonus003_srcprog075",
label="K4 mean-by-type tangent consensus, no-op bonus 0.03, source progress >= 0.75",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_noopbonus0p03_srcprog0p75_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_srcprogbonus003",
label="K4 mean-by-type tangent consensus, source-progress bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_srcprogbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_srcprogbonus005",
label="K4 mean-by-type tangent consensus, source-progress bonus 0.05",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_srcprogbonus0p05_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_srcscorebonus0015",
label="K4 mean-by-type tangent consensus, source-score bonus 0.015",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_srcscorebonus0p015_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_srcscorebonus002",
label="K4 mean-by-type tangent consensus, source-score bonus 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_srcscorebonus0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_srcscorebonus0025",
label="K4 mean-by-type tangent consensus, source-score bonus 0.025",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_srcscorebonus0p025_summary.json"
),
),
MethodSpec(
key="residual_taskrelative_k4_consensus_noopbonus003",
label="K4 task-relative tangent consensus, no-op bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"taskrelative_k4s040_safe_margin0p20_mean_by_type_noopbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_noopbonus001",
label="K4 mean-by-type tangent consensus, no-op bonus 0.01",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_noopbonus0p01_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_noopbonus002",
label="K4 mean-by-type tangent consensus, no-op bonus 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_noopbonus0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_noopbonus0025",
label="K4 mean-by-type tangent consensus, no-op bonus 0.025",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_noopbonus0p025_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_noopbonus0035",
label="K4 mean-by-type tangent consensus, no-op bonus 0.035",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_noopbonus0p035_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_wgbonus003",
label="K4 mean-by-type tangent consensus, wrong-gripper bonus 0.03",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_wgbonus0p03_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_noop003_wg002",
label="K4 mean-by-type tangent consensus, no-op 0.03 + wrong-gripper 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_noop0p03_wg0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_noop003_wg004",
label="K4 mean-by-type tangent consensus, no-op 0.03 + wrong-gripper 0.04",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_noop0p03_wg0p04_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_noop0025_wg002",
label="K4 mean-by-type tangent consensus, no-op 0.025 + wrong-gripper 0.02",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_noop0p025_wg0p02_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_noopbonus005",
label="K4 mean-by-type tangent consensus, no-op bonus 0.05",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_noopbonus0p05_summary.json"
),
),
MethodSpec(
key="residual_k4_consensus_noopbonus008",
label="K4 mean-by-type tangent consensus, no-op bonus 0.08",
summary_path=(
"h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_"
"k4s040_safe_margin0p20_mean_by_type_noopbonus0p08_summary.json"
),
),
MethodSpec(
key="same_state_near_miss",
label="Same-state lattice, near-miss only",
summary_path="h16_lattice_near_miss_only_v2_summary.json",
),
MethodSpec(
key="same_state_no_expert",
label="Same-state lattice, no expert",
summary_path="h16_lattice_no_expert_summary.json",
),
MethodSpec(
key="same_state_policy_baseline",
label="Same-state no-expert + policy candidate",
summary_path="h16_lattice_no_expert_policy_baseline_margin000_summary.json",
),
MethodSpec(
key="same_state_full",
label="Same-state lattice, full",
summary_path="h16_lattice_summary.json",
),
]
def _load_json(path: Path) -> dict[str, Any]:
with path.open("r", encoding="utf-8") as handle:
return json.load(handle)
def _mean(values: list[float]) -> float:
return sum(values) / len(values) if values else float("nan")
def _sample_std(values: list[float]) -> float:
if len(values) <= 1:
return 0.0
mean = _mean(values)
return math.sqrt(sum((value - mean) ** 2 for value in values) / (len(values) - 1))
def _ci95(values: list[float]) -> float:
if len(values) <= 1:
return 0.0
t_crit = {
1: 12.706,
2: 4.303,
3: 3.182,
4: 2.776,
5: 2.571,
6: 2.447,
7: 2.365,
8: 2.306,
9: 2.262,
10: 2.228,
}.get(len(values) - 1, 1.96)
return t_crit * _sample_std(values) / math.sqrt(len(values))
def _success(row: dict[str, Any]) -> float:
return float(row["policy_rollout_success_rate"])
def _progress(row: dict[str, Any]) -> float:
return float(row.get("policy_rollout_progress", float("nan")))
def _action_mse(row: dict[str, Any]) -> float:
return float(row.get("action_mse_to_best", float("nan")))
def _seed(row: dict[str, Any], fallback: int) -> int:
return int(row.get("seed", fallback))
def _standard_summary(path: Path) -> dict[str, Any]:
data = _load_json(path)
rows = list(data.get("rows", []))
return _normalize_summary(data, rows, source=str(path))
def _field_sweep_best(path: Path) -> dict[str, Any]:
data = _load_json(path)
best_config = data.get("best", {}).get("config")
rows = [row for row in data.get("rows", []) if row.get("config") == best_config]
normalized = _normalize_summary(data.get("best", data), rows, source=str(path))
normalized["best_config"] = best_config
return normalized
def _raw_rollout_summary(pattern: str) -> dict[str, Any]:
rows = []
for index, path in enumerate(sorted(Path().glob(pattern) if not pattern.startswith("/") else Path("/").glob(pattern[1:]))):
row = _load_json(path)
row = dict(row)
row["path"] = str(path)
row["seed"] = _seed(row, index)
rows.append(row)
return _normalize_summary({}, rows, source=pattern)
def _normalize_summary(data: dict[str, Any], rows: list[dict[str, Any]], *, source: str) -> dict[str, Any]:
successes = [_success(row) for row in rows]
progress = [_progress(row) for row in rows]
action_mse = [_action_mse(row) for row in rows]
selected_counts = Counter()
selected_counts.update(data.get("selected_candidate_type_counts", {}))
selected_scale_counts = Counter()
selected_scale_counts.update(data.get("selected_residual_scale_counts", {}))
expected_count = sum(int(row.get("num_groups", 0)) for row in rows)
has_top_level_selected_counts = (
bool(selected_counts) and sum(int(value) for value in selected_counts.values()) == expected_count
)
has_top_level_scale_counts = (
bool(selected_scale_counts)
and sum(int(value) for value in selected_scale_counts.values()) == expected_count
)
if not has_top_level_selected_counts:
selected_counts = Counter()
if not has_top_level_scale_counts:
selected_scale_counts = Counter()
if not has_top_level_selected_counts or not has_top_level_scale_counts:
for row in rows:
path = row.get("path")
if not path:
continue
raw_path = Path(str(path))
if not raw_path.exists():
continue
raw = _load_json(raw_path)
if not has_top_level_selected_counts:
selected_counts.update(raw.get("selected_candidate_type_counts", {}))
if not has_top_level_scale_counts:
selected_scale_counts.update(raw.get("selected_residual_scale_counts", {}))
output = {
"source": source,
"num_completed": len(rows),
"mean_success": _mean(successes),
"std_success": _sample_std(successes),
"ci95_success": _ci95(successes),
"mean_progress": _mean(progress),
"mean_action_mse_to_best": _mean(action_mse),
"seed_success": {_seed(row, index): _success(row) for index, row in enumerate(rows)},
"seed_progress": {_seed(row, index): _progress(row) for index, row in enumerate(rows)},
"seed_action_mse_to_best": {_seed(row, index): _action_mse(row) for index, row in enumerate(rows)},
"per_task_success": _per_task(rows),
"selected_candidate_type_counts": dict(selected_counts),
"selected_residual_scale_counts": dict(selected_scale_counts),
"selected_type_outcomes": _selected_type_outcomes(rows),
}
candidate_oracle_success = [
float(row["candidate_oracle_success_rate"])
for row in rows
if row.get("candidate_oracle_success_rate") is not None
]
if candidate_oracle_success:
raw_prefix_metrics = _candidate_prefix_metrics_from_raw(rows)
selected_branch_success = [
float(row["candidate_oracle_selected_branch_success_rate"])
for row in rows
if row.get("candidate_oracle_selected_branch_success_rate") is not None
]
selected_branch_progress = [
float(row["candidate_oracle_selected_branch_progress"])
for row in rows
if row.get("candidate_oracle_selected_branch_progress") is not None
]
oracle_progress = [
float(row["candidate_oracle_progress"])
for row in rows
if row.get("candidate_oracle_progress") is not None
]
selected_success_mean = _mean(selected_branch_success)
oracle_success_mean = _mean(candidate_oracle_success)
selected_progress_mean = _mean(selected_branch_progress)
oracle_progress_mean = _mean(oracle_progress)
output.update(
{
"candidate_oracle_rollouts": int(data.get("candidate_oracle_rollouts") or 0),
"candidate_oracle_unique_tolerance": data.get(
"candidate_oracle_unique_tolerance"
),
"mean_candidate_oracle_success_rate": oracle_success_mean,
"std_candidate_oracle_success_rate": _sample_std(
candidate_oracle_success
),
"ci95_candidate_oracle_success_rate": _ci95(candidate_oracle_success),
"seed_candidate_oracle_success": {
_seed(row, index): float(row["candidate_oracle_success_rate"])
for index, row in enumerate(rows)
if row.get("candidate_oracle_success_rate") is not None
},
"mean_candidate_oracle_progress": oracle_progress_mean,
"mean_candidate_oracle_selected_branch_success_rate": (
selected_success_mean
),
"mean_candidate_oracle_selected_branch_progress": (
selected_progress_mean
),
"candidate_oracle_selector_gap_success": (
oracle_success_mean - selected_success_mean
),
"candidate_oracle_selector_gap_progress": (
oracle_progress_mean - selected_progress_mean
),
"mean_candidate_oracle_score_gain_over_selected": _mean(
[
float(row["candidate_oracle_score_gain_over_selected"])
for row in rows
if row.get("candidate_oracle_score_gain_over_selected")
is not None
]
),
"mean_candidate_oracle_unique_count": _mean(
[
float(row.get("candidate_oracle_unique_count") or 0.0)
for row in rows
if row.get("candidate_oracle_unique_count") is not None
]
),
"mean_candidate_oracle_improvement_rate": _mean(
[
float(row["candidate_oracle_improvement_rate"])
for row in rows
if row.get("candidate_oracle_improvement_rate") is not None
]
),
"candidate_oracle_type_counts": data.get(
"candidate_oracle_type_counts", {}
),
}
)
if raw_prefix_metrics:
output.update(raw_prefix_metrics)
for key in (
"mean_candidate_oracle_best_branch_rank",
"candidate_oracle_best_branch_rank_counts",
"mean_candidate_oracle_branch_success_rates",
"mean_candidate_oracle_branch_progress",
"mean_candidate_oracle_branch_score_gains_over_selected",
):
if key in data:
output[key] = data[key]
return output
def _candidate_prefix_metrics_from_raw(rows: list[dict[str, Any]]) -> dict[str, Any]:
prefix_metrics: list[dict[str, Any]] = []
for row in rows:
path = row.get("path")
if not path:
continue
raw_path = Path(str(path))
if not raw_path.exists():
continue
raw = _load_json(raw_path)
for item in raw.get("rows", []):
scores = item.get("candidate_oracle_branch_scores") or []
if not scores:
continue
metrics = candidate_prefix_causal_metrics(
branch_scores=[float(value) for value in scores],
selected_score=(
float(item["candidate_oracle_selected_branch_score"])
if item.get("candidate_oracle_selected_branch_score") is not None
else None
),
branch_types=[str(value) for value in item.get("candidate_oracle_types", [])],
valid_mask=[bool(value) for value in item.get("candidate_oracle_valid_mask", [])]
or None,
)
if metrics:
prefix_metrics.append(metrics)
if not prefix_metrics:
return {}
ptr = finite_mean([item.get("ptr_at_k") for item in prefix_metrics])
ncar = finite_mean([item.get("ncar_to_proposal_oracle") for item in prefix_metrics])
return {
"mean_candidate_oracle_car_to_proposal_oracle": _mean(
[float(item["car_to_proposal_oracle"]) for item in prefix_metrics]
),
"mean_candidate_oracle_selector_regret_at_k": _mean(
[float(item["selector_regret_at_k"]) for item in prefix_metrics]
),
"mean_candidate_oracle_ptr_at_k": ptr,
"mean_candidate_oracle_ncar_to_proposal_oracle": ncar,
"candidate_oracle_base_trace_coverage": (
sum(1 for item in prefix_metrics if item.get("base_utility") is not None)
/ len(prefix_metrics)
),
}
def _per_task(rows: list[dict[str, Any]]) -> dict[str, dict[str, float]]:
task_values: dict[str, list[float]] = {}
task_counts: dict[str, list[int]] = {}
for row in rows:
for task, metrics in row.get("per_task", {}).items():
task_values.setdefault(task, []).append(float(metrics["policy_rollout_success_rate"]))
task_counts.setdefault(task, []).append(int(metrics.get("num_groups", 0)))
return {
task: {
"mean_success": _mean(values),
"std_success": _sample_std(values),
"mean_num_groups": _mean([float(value) for value in task_counts.get(task, [])]),
}
for task, values in sorted(task_values.items())
}
def _selected_type_outcomes(rows: list[dict[str, Any]]) -> dict[str, dict[str, float]]:
grouped: dict[str, dict[str, float]] = {}
for row in rows:
path = row.get("path")
if not path:
continue
raw_path = Path(str(path))
if not raw_path.exists():
continue
raw = _load_json(raw_path)
for item in raw.get("rows", []):
candidate_type = str(item.get("nearest_candidate_type") or "unknown")
stats = grouped.setdefault(
candidate_type,
{"count": 0.0, "success_sum": 0.0, "progress_sum": 0.0},
)
stats["count"] += 1.0
stats["success_sum"] += 1.0 if item.get("success") else 0.0
stats["progress_sum"] += float(item.get("progress", 0.0))
return {
candidate_type: {
"count": values["count"],
"success_rate": values["success_sum"] / values["count"] if values["count"] else float("nan"),
"mean_progress": values["progress_sum"] / values["count"] if values["count"] else float("nan"),
}
for candidate_type, values in sorted(
grouped.items(),
key=lambda item: (-item[1]["count"], item[0]),
)
}
def _load_methods() -> dict[str, dict[str, Any]]:
methods: dict[str, dict[str, Any]] = {}
for spec in METHODS:
if spec.summary_path:
path = RESULTS_DIR / spec.summary_path
if not path.exists():
methods[spec.key] = {
"missing": True,
"source": str(path),
"label": spec.label,
"headline_metric": spec.headline_metric,
"headline_metric_label": _headline_metric_label(
spec.headline_metric
),
}
continue
if spec.summary_mode == "field_sweep_best":
method = _field_sweep_best(path)
else:
method = _standard_summary(path)
elif spec.raw_rollout_glob:
method = _raw_rollout_summary(spec.raw_rollout_glob)
else:
method = {"missing": True, "source": "", "label": spec.label}
method["label"] = spec.label
_attach_headline_fields(method, spec)
methods[spec.key] = method
return methods
def _load_generator_v2_support_proxy() -> dict[str, Any]:
if not GENERATOR_V2_MEMORY_EVAL.exists():
return {"missing": True, "source": str(GENERATOR_V2_MEMORY_EVAL)}
data = _load_json(GENERATOR_V2_MEMORY_EVAL)
return {
"missing": False,
"source": str(GENERATOR_V2_MEMORY_EVAL),
"report_type": data.get("report_type"),
"metric_scope": data.get("metric_scope"),
"note": data.get("note"),
"targets": data.get("targets"),
"seed": data.get("seed"),
"val_fraction": data.get("val_fraction"),
"diversity_weight": data.get("diversity_weight"),
"num_examples": data.get("num_examples"),
"num_train_examples": data.get("num_train_examples"),
"num_val_examples": data.get("num_val_examples"),
"num_groups": data.get("num_groups"),
"num_val_groups": data.get("num_val_groups"),
"num_eval_groups": data.get("num_eval_groups"),
"num_eval_groups_with_positive": data.get("num_eval_groups_with_positive"),
"label_counts": data.get("label_counts", {}),
"train_positive_by_task": data.get("train_positive_by_task", {}),
"proposal_count_by_task": data.get("proposal_count_by_task", {}),
"overall": data.get("overall", {}),
"per_task": data.get("per_task", {}),
}
def _load_generator_v2_local_atlas_support_proxy() -> dict[str, Any]:
source = GENERATOR_V2_LOCAL_ATLAS_EVAL
sweep_best: dict[str, Any] | None = None
if GENERATOR_V2_LOCAL_ATLAS_SWEEP.exists():
sweep = _load_json(GENERATOR_V2_LOCAL_ATLAS_SWEEP)
if sweep.get("best", {}).get("path"):
candidate = Path(str(sweep["best"]["path"]))
if candidate.exists():
source = candidate
sweep_best = sweep["best"]
if not source.exists():
return {"missing": True, "source": str(source)}
data = _load_json(source)
return {
"missing": False,
"source": str(source),
"sweep_summary": str(GENERATOR_V2_LOCAL_ATLAS_SWEEP)
if GENERATOR_V2_LOCAL_ATLAS_SWEEP.exists()
else None,
"sweep_best": sweep_best,
"report_type": data.get("report_type"),
"metric_scope": data.get("metric_scope"),
"note": data.get("note"),
"targets": data.get("targets"),
"config": data.get("config", {}),
"code_dim": data.get("code_dim"),
"horizon": data.get("horizon"),
"action_dim": data.get("action_dim"),
"num_examples": data.get("num_examples"),
"num_groups": data.get("num_groups"),
"num_train_examples": data.get("num_train_examples"),
"num_val_examples": data.get("num_val_examples"),
"num_train_positive": data.get("num_train_positive"),
"num_val_groups_with_positive": data.get("num_val_groups_with_positive"),
"label_counts": data.get("label_counts", {}),
"train_positive_by_task": data.get("train_positive_by_task", {}),
"overall": data.get("overall", {}),
"per_task": data.get("per_task", {}),
}
def _load_generator_v2_chart_synthesis_support_proxy() -> dict[str, Any]:
source = GENERATOR_V2_CHART_SYNTHESIS_EVAL
sweep_best: dict[str, Any] | None = None
if GENERATOR_V2_CHART_SYNTHESIS_SWEEP.exists():
sweep = _load_json(GENERATOR_V2_CHART_SYNTHESIS_SWEEP)
if sweep.get("best", {}).get("path"):
candidate = Path(str(sweep["best"]["path"]))
if candidate.exists():
source = candidate
sweep_best = sweep["best"]
if not source.exists():
return {"missing": True, "source": str(source)}
data = _load_json(source)
return {
"missing": False,
"source": str(source),
"sweep_summary": str(GENERATOR_V2_CHART_SYNTHESIS_SWEEP)
if GENERATOR_V2_CHART_SYNTHESIS_SWEEP.exists()
else None,
"sweep_best": sweep_best,
"report_type": data.get("report_type"),
"metric_scope": data.get("metric_scope"),
"note": data.get("note"),
"targets": data.get("targets"),
"config": data.get("config", {}),
"code_dim": data.get("code_dim"),
"horizon": data.get("horizon"),
"action_dim": data.get("action_dim"),
"num_examples": data.get("num_examples"),
"num_groups": data.get("num_groups"),
"num_train_examples": data.get("num_train_examples"),
"num_val_examples": data.get("num_val_examples"),
"num_train_positive": data.get("num_train_positive"),
"num_val_groups_with_positive": data.get("num_val_groups_with_positive"),
"label_counts": data.get("label_counts", {}),
"train_positive_by_task": data.get("train_positive_by_task", {}),
"overall": data.get("overall", {}),
"per_task": data.get("per_task", {}),
}
def _load_generator_v2_cvae_support_proxy() -> dict[str, Any]:
source = GENERATOR_V2_CVAE_EVAL
sweep_best: dict[str, Any] | None = None
if GENERATOR_V2_CVAE_SWEEP.exists():
sweep = _load_json(GENERATOR_V2_CVAE_SWEEP)
if sweep.get("best", {}).get("path"):
candidate = Path(str(sweep["best"]["path"]))
if candidate.exists():
source = candidate
sweep_best = sweep["best"]
if not source.exists():
return {"missing": True, "source": str(source)}
data = _load_json(source)
return {
"missing": False,
"source": str(source),
"sweep_summary": str(GENERATOR_V2_CVAE_SWEEP)
if GENERATOR_V2_CVAE_SWEEP.exists()
else None,
"sweep_best": sweep_best,
"report_type": data.get("report_type"),
"metric_scope": data.get("metric_scope"),
"note": data.get("note"),
"targets": data.get("targets"),
"config": data.get("config", {}),
"num_examples": data.get("num_examples"),
"num_groups": data.get("num_groups"),
"num_train_examples": data.get("num_train_examples"),
"num_val_examples": data.get("num_val_examples"),
"num_train_positive": data.get("num_train_positive"),
"num_val_groups_with_positive": data.get("num_val_groups_with_positive"),
"label_counts": data.get("label_counts", {}),
"train_positive_by_task": data.get("train_positive_by_task", {}),
"overall": data.get("overall", {}),
"per_task": data.get("per_task", {}),
"train_history": data.get("train_history", []),
}
def _load_generator_v2_spline_cvae_support_proxy() -> dict[str, Any]:
source = GENERATOR_V2_SPLINE_CVAE_EVAL
sweep_best: dict[str, Any] | None = None
if GENERATOR_V2_SPLINE_CVAE_SWEEP.exists():
sweep = _load_json(GENERATOR_V2_SPLINE_CVAE_SWEEP)
if sweep.get("best", {}).get("path"):
candidate = Path(str(sweep["best"]["path"]))
if candidate.exists():
source = candidate
sweep_best = sweep["best"]
if not source.exists():
return {"missing": True, "source": str(source)}
data = _load_json(source)
return {
"missing": False,
"source": str(source),
"sweep_summary": str(GENERATOR_V2_SPLINE_CVAE_SWEEP)
if GENERATOR_V2_SPLINE_CVAE_SWEEP.exists()
else None,
"sweep_best": sweep_best,
"report_type": data.get("report_type"),
"metric_scope": data.get("metric_scope"),
"note": data.get("note"),
"targets": data.get("targets"),
"config": data.get("config", {}),
"code_dim": data.get("code_dim"),
"horizon": data.get("horizon"),
"action_dim": data.get("action_dim"),
"num_examples": data.get("num_examples"),
"num_groups": data.get("num_groups"),
"num_train_examples": data.get("num_train_examples"),
"num_val_examples": data.get("num_val_examples"),
"num_train_positive": data.get("num_train_positive"),
"num_val_groups_with_positive": data.get("num_val_groups_with_positive"),
"label_counts": data.get("label_counts", {}),
"train_positive_by_task": data.get("train_positive_by_task", {}),
"overall": data.get("overall", {}),
"per_task": data.get("per_task", {}),
"train_history": data.get("train_history", []),
}
def _load_generator_v2_spline_flow_support_proxy() -> dict[str, Any]:
source = GENERATOR_V2_SPLINE_FLOW_EVAL
sweep_best: dict[str, Any] | None = None
if GENERATOR_V2_SPLINE_FLOW_SWEEP.exists():
sweep = _load_json(GENERATOR_V2_SPLINE_FLOW_SWEEP)
if sweep.get("best", {}).get("path"):
candidate = Path(str(sweep["best"]["path"]))
if candidate.exists():
source = candidate
sweep_best = sweep["best"]
if not source.exists():
return {"missing": True, "source": str(source)}
data = _load_json(source)
return {
"missing": False,
"source": str(source),
"sweep_summary": str(GENERATOR_V2_SPLINE_FLOW_SWEEP)
if GENERATOR_V2_SPLINE_FLOW_SWEEP.exists()
else None,
"sweep_best": sweep_best,
"report_type": data.get("report_type"),
"metric_scope": data.get("metric_scope"),
"note": data.get("note"),
"targets": data.get("targets"),
"config": data.get("config", {}),
"code_dim": data.get("code_dim"),
"horizon": data.get("horizon"),
"action_dim": data.get("action_dim"),
"num_examples": data.get("num_examples"),
"num_groups": data.get("num_groups"),
"num_train_examples": data.get("num_train_examples"),
"num_val_examples": data.get("num_val_examples"),
"num_train_positive": data.get("num_train_positive"),
"num_val_groups_with_positive": data.get("num_val_groups_with_positive"),
"overall": data.get("overall", {}),
"per_task": data.get("per_task", {}),
"train_history": data.get("train_history", []),
}
def _load_generator_v2_guided_spline_flow_support_proxy() -> dict[str, Any]:
source = GENERATOR_V2_GUIDED_SPLINE_FLOW_EVAL
sweep_best: dict[str, Any] | None = None
if GENERATOR_V2_GUIDED_SPLINE_FLOW_SWEEP.exists():
sweep = _load_json(GENERATOR_V2_GUIDED_SPLINE_FLOW_SWEEP)
if sweep.get("best", {}).get("path"):
candidate = Path(str(sweep["best"]["path"]))
if candidate.exists():
source = candidate
sweep_best = sweep["best"]
if not source.exists():
return {"missing": True, "source": str(source)}
data = _load_json(source)
return {
"missing": False,
"source": str(source),
"sweep_summary": str(GENERATOR_V2_GUIDED_SPLINE_FLOW_SWEEP)
if GENERATOR_V2_GUIDED_SPLINE_FLOW_SWEEP.exists()
else None,
"sweep_best": sweep_best,
"report_type": data.get("report_type"),
"metric_scope": data.get("metric_scope"),
"note": data.get("note"),
"targets": data.get("targets"),
"config": data.get("config", {}),
"code_dim": data.get("code_dim"),
"horizon": data.get("horizon"),
"action_dim": data.get("action_dim"),
"num_examples": data.get("num_examples"),
"num_groups": data.get("num_groups"),
"num_train_examples": data.get("num_train_examples"),
"num_val_examples": data.get("num_val_examples"),
"num_train_positive": data.get("num_train_positive"),
"num_train_negative": data.get("num_train_negative"),
"num_val_groups_with_positive": data.get("num_val_groups_with_positive"),
"label_counts": data.get("label_counts", {}),
"train_positive_by_task": data.get("train_positive_by_task", {}),
"train_negative_by_task": data.get("train_negative_by_task", {}),
"overall": data.get("overall", {}),
"per_task": data.get("per_task", {}),
"train_history": data.get("train_history", []),
"utility_train_history": data.get("utility_train_history", []),
}
def _headline_metric_label(metric: str) -> str:
if metric == "mean_candidate_oracle_success_rate":
return "candidate-oracle"
if metric == "mean_success":
return "deployment"
return metric
def _attach_headline_fields(method: dict[str, Any], spec: MethodSpec) -> None:
metric = spec.headline_metric
method["headline_metric"] = metric
method["headline_metric_label"] = _headline_metric_label(metric)
if metric == "mean_candidate_oracle_success_rate":
method["headline_success"] = method.get(
"mean_candidate_oracle_success_rate", method.get("mean_success")
)
method["headline_std_success"] = method.get(
"std_candidate_oracle_success_rate", method.get("std_success", 0.0)
)
method["headline_ci95_success"] = method.get(
"ci95_candidate_oracle_success_rate", method.get("ci95_success", 0.0)
)
return
method["headline_success"] = method.get("mean_success")
method["headline_std_success"] = method.get("std_success", 0.0)
method["headline_ci95_success"] = method.get("ci95_success", 0.0)
def _best_clean_key(methods: dict[str, dict[str, Any]]) -> str:
best_key = FALLBACK_BEST_CLEAN_KEY
best_success = float("-inf")
for spec in METHODS:
if spec.key in NON_DEPLOYMENT_KEYS:
continue
method = methods.get(spec.key, {})
if method.get("missing"):
continue
success = method.get("mean_success")
if isinstance(success, (int, float)) and math.isfinite(float(success)):
success = float(success)
if success > best_success + 1.0e-12:
best_success = success
best_key = spec.key
return best_key
def _paired_delta(
methods: dict[str, dict[str, Any]],
left: str,
right: str,
) -> dict[str, Any]:
left_values = methods[left].get("seed_success", {})
right_values = methods[right].get("seed_success", {})
seeds = sorted(set(left_values) & set(right_values))
deltas = [float(left_values[seed]) - float(right_values[seed]) for seed in seeds]
return {
"left": left,
"right": right,
"seeds": seeds,
"mean_delta": _mean(deltas),
"std_delta": _sample_std(deltas),
"ci95_delta": _ci95(deltas),
"seed_deltas": {seed: delta for seed, delta in zip(seeds, deltas)},
}
def _per_task_delta(
methods: dict[str, dict[str, Any]],
left: str,
right: str,
) -> dict[str, float]:
left_tasks = methods[left].get("per_task_success", {})
right_tasks = methods[right].get("per_task_success", {})
return {
task: float(left_tasks[task]["mean_success"]) - float(right_tasks[task]["mean_success"])
for task in sorted(set(left_tasks) & set(right_tasks))
}
def _pct(value: float) -> str:
if value is None:
return "n/a"
value = float(value)
if math.isnan(value):
return "n/a"
return f"{value * 100:.2f}%"
def _pp(value: float) -> str:
if value is None:
return "n/a"
value = float(value)
if math.isnan(value):
return "n/a"
return f"{value * 100:+.2f} pp"
def _best_candidate_oracle(
methods: dict[str, dict[str, Any]]
) -> tuple[str | None, dict[str, Any] | None]:
best_key: str | None = None
best_method: dict[str, Any] | None = None
best_success = float("-inf")
for key, method in methods.items():
if method.get("missing") or not method.get("num_completed"):
continue
value = method.get("mean_candidate_oracle_success_rate")
if not isinstance(value, (int, float)):
continue
success = float(value)
if math.isfinite(success) and success > best_success:
best_success = success
best_key = key
best_method = method
return best_key, best_method
def _render_markdown(report: dict[str, Any]) -> str:
methods = report["methods"]
best_clean_key = report["best_clean_key"]
lines = [
"# Paper Analysis",
"",
f"Generated: `{report['generated_utc']}`",
"",
"## Main Seed Statistics",
"",
"| key | method | n | headline metric | headline success | selected/deployed success | 95% CI | progress | action MSE | headline gain vs canonical h16 |",
"|---|---|---:|---|---:|---:|---:|---:|---:|---:|",
]
baseline = methods["h16_policy_canonical"]["mean_success"]
for key in [spec.key for spec in METHODS]:
method = methods[key]
if method.get("missing"):
lines.append(
f"| {key} | {method['label']} | 0 | {method.get('headline_metric_label', 'deployment')} | missing | missing | missing | missing | missing | missing |"
)
continue
headline_success = float(
method.get("headline_success", method.get("mean_success", float("nan")))
)
headline_std = float(method.get("headline_std_success", method.get("std_success", 0.0)))
headline_ci = float(method.get("headline_ci95_success", method.get("ci95_success", 0.0)))
lines.append(
"| {key} | {label} | {n} | {metric} | {headline} +/- {std} | {selected} | {ci} | {progress} | {mse:.3f} | {gain} |".format(
key=key,
label=method["label"],
n=method["num_completed"],
metric=method.get("headline_metric_label", "deployment"),
headline=_pct(headline_success),
std=f"{headline_std * 100:.2f}",
selected=_pct(method["mean_success"]),
ci=f"+/- {headline_ci * 100:.2f}",
progress=_pct(method["mean_progress"]),
mse=method["mean_action_mse_to_best"],
gain=_pp(headline_success - baseline),
)
)
lines.extend(
[
"",
"## Paired Seed Deltas",
"",
"| comparison | seeds | mean delta | 95% CI | seed deltas |",
"|---|---:|---:|---:|---|",
]
)
for name, delta in report["paired_deltas"].items():
seed_deltas = ", ".join(
f"{seed}:{value * 100:+.2f}" for seed, value in delta["seed_deltas"].items()
)
lines.append(
f"| {name} | {len(delta['seeds'])} | {_pp(delta['mean_delta'])} | +/- {delta['ci95_delta'] * 100:.2f} | {seed_deltas} |"
)
lines.extend(
[
"",
"## Per-Task Mean Success",
"",
"| task | h16 policy | best clean | near-miss lattice | no-expert lattice | full lattice | clean-h16 delta | noexpert-clean gap |",
"|---|---:|---:|---:|---:|---:|---:|---:|",
]
)
tasks = sorted(methods["h16_policy_canonical"].get("per_task_success", {}))
for task in tasks:
h16 = methods["h16_policy_canonical"]["per_task_success"][task]["mean_success"]
clean = methods[best_clean_key]["per_task_success"][task]["mean_success"]
near = methods["same_state_near_miss"]["per_task_success"][task]["mean_success"]
noexpert = methods["same_state_no_expert"]["per_task_success"][task]["mean_success"]
full = methods["same_state_full"]["per_task_success"][task]["mean_success"]
lines.append(
f"| {task} | {_pct(h16)} | {_pct(clean)} | {_pct(near)} | {_pct(noexpert)} | {_pct(full)} | {_pp(clean - h16)} | {_pp(noexpert - clean)} |"
)
gap = report["mechanism_gap"]
lines.extend(
[
"",
"## Mechanism Gap",
"",
f"- Best clean residual transport improves over canonical h16 by {_pp(gap['best_clean_vs_h16'])}.",
f"- Same-state no-expert lattice improves over canonical h16 by {_pp(gap['same_state_no_expert_vs_h16'])}.",
f"- Remaining clean-to-same-state proposal gap is {_pp(gap['same_state_no_expert_vs_best_clean'])}.",
f"- Full lattice adds expert proposals and reaches {_pct(methods['same_state_full']['mean_success'])}, a {_pp(gap['same_state_full_vs_no_expert'])} gain over no-expert.",
]
)
decomposition = report["causal_action_decomposition"]
targets = report["causal_action_targets"]
lines.extend(
[
"",
"## Causal Action Regret Decomposition",
"",
"| base | selected clean | proposal oracle | same-state no-expert oracle | support gap | selector gap | gap closed |",
"|---:|---:|---:|---:|---:|---:|---:|",
(
f"| {_pct(decomposition['base'])} | {_pct(decomposition['selected'])} | "
f"{_pct(decomposition['proposal_oracle'])} | "
f"{_pct(decomposition['same_state_oracle'])} | "
f"{_pp(decomposition['support_gap'])} | "
f"{_pp(decomposition['selector_gap'])} | "
f"{decomposition['closed_fraction_of_noexpert_gap'] * 100:.1f}% |"
),
"",
(
"- Current clean policy closes "
f"{decomposition['closed_fraction_of_noexpert_gap'] * 100:.1f}% "
"of the h16-to-same-state-no-expert gap."
),
(
"- Closing 65--75% of that gap implies selected success targets of "
f"{_pct(targets['selected_success_for_65pct_gap_closure'])}--"
f"{_pct(targets['selected_success_for_75pct_gap_closure'])}."
),
]
)
oracle_key, oracle = _best_candidate_oracle(methods)
if oracle and oracle.get("num_completed"):
branch_success = oracle.get("mean_candidate_oracle_branch_success_rates") or []
branch_gains = (
oracle.get("mean_candidate_oracle_branch_score_gains_over_selected")
or []
)
lines.extend(
[
"",
"## Candidate-Oracle Diagnostic",
"",
(
f"- Best diagnostic prefix (`{oracle_key}`) reaches "
f"{_pct(oracle.get('mean_candidate_oracle_success_rate', 0.0))} "
f"with mean progress {_pct(oracle.get('mean_candidate_oracle_progress', 0.0))}; "
"this is diagnostic-only because it uses measured rollout outcomes "
"after generating candidates."
),
(
"- Mean oracle-prefix score gain over the selected branch is "
f"{oracle.get('mean_candidate_oracle_score_gain_over_selected', 0.0):+.3f}, "
"which isolates ranking/abstention headroom inside the clean proposal set."
),
(
"- CAR-to-proposal-oracle from raw prefix traces is "
f"{oracle.get('mean_candidate_oracle_car_to_proposal_oracle', 0.0):+.3f}; "
f"PTR@K is {_pct(oracle.get('mean_candidate_oracle_ptr_at_k'))} "
f"over rows with base trace coverage {_pct(oracle.get('candidate_oracle_base_trace_coverage'))}."
),
(
"- Mean unique candidates in the prefix: "
f"{oracle.get('mean_candidate_oracle_unique_count', 0.0):.2f}."
),
(
"- Candidate-oracle best type counts: "
f"{oracle.get('candidate_oracle_type_counts', {})}."
),
]
)
if oracle.get("mean_candidate_oracle_best_branch_rank") is not None:
lines.extend(
[
(
"- Mean best branch rank in the field-ordered prefix: "
f"{oracle['mean_candidate_oracle_best_branch_rank']:.2f}; "
"rank histogram "
f"{oracle.get('candidate_oracle_best_branch_rank_counts', {})}."
),
(
"- Branch success by prefix rank: "
+ ", ".join(_pct(value) for value in branch_success)
+ "."
),
(
"- Branch score gain by prefix rank: "
+ ", ".join(f"{value:+.3f}" for value in branch_gains)
+ "."
),
]
)
support_proxy = report.get("generator_v2_support_proxy", {})
local_atlas_proxy = report.get("generator_v2_local_atlas_support_proxy", {})
chart_synthesis_proxy = report.get("generator_v2_chart_synthesis_support_proxy", {})
cvae_proxy = report.get("generator_v2_cvae_support_proxy", {})
spline_proxy = report.get("generator_v2_spline_cvae_support_proxy", {})
flow_proxy = report.get("generator_v2_spline_flow_support_proxy", {})
guided_flow_proxy = report.get("generator_v2_guided_spline_flow_support_proxy", {})
lines.extend(
[
"",
"## Generator V2 Support Proxy",
"",
]
)
if support_proxy.get("missing", True):
lines.append(
f"- Pending: `{support_proxy.get('source', GENERATOR_V2_MEMORY_EVAL)}` has not been generated yet."
)
else:
overall = support_proxy.get("overall", {})
lines.extend(
[
(
f"- Artifact `{support_proxy['source']}` evaluates train-only positive "
f"tangent memory proposals on {support_proxy.get('num_eval_groups_with_positive', 0)} "
"heldout groups with positive support."
),
(
f"- Train positives by task: {support_proxy.get('train_positive_by_task', {})}; "
f"prototype count by task: {support_proxy.get('proposal_count_by_task', {})}."
),
"",
"| metric | K1 | K2 | K4 | K8 | K16 |",
"|---|---:|---:|---:|---:|---:|",
_support_proxy_row(overall, "PTR proxy @ RMS<=0.10", "ptr_proxy", "0p1"),
_support_proxy_row(overall, "PTR proxy @ RMS<=0.20", "ptr_proxy", "0p2"),
_support_proxy_row(
overall,
"Negative-near @ RMS<=0.20",
"negative_near",
"0p2",
),
_support_proxy_row(
overall,
"Positive closer than negative",
"positive_closer_than_negative_rate",
None,
),
"",
"| task | eval groups | K8 PTR@0.20 | K16 PTR@0.20 | K16 pos<neg |",
"|---|---:|---:|---:|---:|",
]
)
for task_id, values in sorted(support_proxy.get("per_task", {}).items()):
lines.append(
"| {task} | {groups} | {k8} | {k16} | {closer} |".format(
task=task_id,
groups=int(values.get("num_groups", 0)),
k8=_pct(values.get("ptr_proxy_at_8_thr_0p2")),
k16=_pct(values.get("ptr_proxy_at_16_thr_0p2")),
closer=_pct(values.get("positive_closer_than_negative_rate_at_16")),
)
)
lines.extend(["", "### Trainable CVAE Diagnostic", ""])
if cvae_proxy.get("missing", True):
lines.append(
f"- Pending: `{cvae_proxy.get('source', GENERATOR_V2_CVAE_EVAL)}` has not been generated yet."
)
else:
cvae_history = cvae_proxy.get("train_history", [])
last_epoch = cvae_history[-1] if cvae_history else {}
lines.extend(
[
(
f"- Artifact `{cvae_proxy['source']}` samples from a train-only "
f"positive-tangent CVAE trained on {cvae_proxy.get('num_train_positive', 0)} "
"positive targets."
),
(
f"- Final training snapshot: epoch {int(last_epoch.get('epoch', 0))}, "
f"loss {float(last_epoch.get('loss', float('nan'))):.4f}, "
f"reconstruction MSE {float(last_epoch.get('reconstruction_mse', float('nan'))):.4f}, "
f"KL {float(last_epoch.get('kl', float('nan'))):.4f}."
),
"",
"| generator | heldout groups | K16 PTR@0.20 | K16 PTR@0.40 | K16 neg@0.20 | K16 pos<neg |",
"|---|---:|---:|---:|---:|---:|",
_generator_support_compare_row("memory", support_proxy),
_generator_support_compare_row("local-atlas", local_atlas_proxy),
_generator_support_compare_row("chart-synthesis", chart_synthesis_proxy),
_generator_support_compare_row("raw-cvae", cvae_proxy),
_generator_support_compare_row("spline-cvae", spline_proxy),
_generator_support_compare_row("spline-flow", flow_proxy),
_generator_support_compare_row("guided-spline-flow", guided_flow_proxy),
]
)
if not spline_proxy.get("missing", True):
lines.append(
f"- Spline-CVAE source `{spline_proxy['source']}` uses "
f"{spline_proxy.get('code_dim')}D keyframe codes decoded to "
f"{spline_proxy.get('horizon')}x{spline_proxy.get('action_dim')} chunks."
)
lines.extend(
[
"",
"## Selection Histograms",
"",
]
)
for key in ["same_state_near_miss", "same_state_no_expert", "same_state_policy_baseline", "same_state_full", best_clean_key]:
counts = methods[key].get("selected_candidate_type_counts", {})
if counts:
total = sum(int(value) for value in counts.values())
summary = ", ".join(
f"{name}={count} ({count / total * 100:.1f}%)"
for name, count in sorted(counts.items(), key=lambda item: (-int(item[1]), item[0]))
)
else:
summary = "not recorded"
lines.append(f"- `{key}`: {summary}")
scale_counts = methods[best_clean_key].get("selected_residual_scale_counts", {})
if scale_counts:
lines.append(f"- `{best_clean_key}` residual scale counts: {scale_counts}")
lines.extend(
[
"",
"## Selected-Type Outcomes",
"",
"These rows are measured from raw rollout rows. In residual retrieval, `policy_residual` is the zero-residual action, i.e. abstaining to the current policy mean.",
"",
"| method | selected type | count | success | progress |",
"|---|---|---:|---:|---:|",
]
)
for key in [
"best_clean_residual_k2",
"residual_k4_consensus",
"residual_k4_kernel_consensus",
"residual_k4_kernel_consensus_noopbonus003",
"residual_k4_kernel_consensus_s035_noopbonus003",
"residual_k4_kernel_consensus_s045_noopbonus003",
"residual_k4_fieldsoftmax_grid",
"residual_k4_fieldsoftmax_grid_noopbonus003",
"residual_k4_fieldsoftmax_grid_margin010_noopbonus003",
"residual_k4_fieldsoftmax_grid_margin005_noopbonus003",
"residual_k4_fieldsoftmax_grid_margin000_noopbonus003",
"residual_k8_fieldsoftmax_grid_noopbonus003",
"residual_k4_consensus_noopbonus003",
"residual_k4_consensus_noopbonus001",
"residual_k4_consensus_noopbonus002",
"residual_k4_consensus_noopbonus0025",
"residual_k4_consensus_noopbonus0035",
"residual_k4_consensus_wgbonus003",
"residual_k4_consensus_noop003_wg002",
"residual_k4_consensus_noop003_wg004",
"residual_k4_consensus_noop0025_wg002",
"residual_k4_consensus_noopbonus005",
"residual_k4_consensus_noopbonus008",
"same_state_no_expert",
"same_state_policy_baseline",
]:
for candidate_type, values in methods[key].get("selected_type_outcomes", {}).items():
lines.append(
f"| {key} | {candidate_type} | {int(values['count'])} | {_pct(values['success_rate'])} | {_pct(values['mean_progress'])} |"
)
return "\n".join(lines) + "\n"
def build_report() -> dict[str, Any]:
methods = _load_methods()
best_clean_key = _best_clean_key(methods)
oracle_key, oracle_method = _best_candidate_oracle(methods)
proposal_oracle_success = (
float(oracle_method["mean_candidate_oracle_success_rate"])
if oracle_method is not None
and oracle_method.get("mean_candidate_oracle_success_rate") is not None
else float("nan")
)
paired_deltas = {
"best_clean - canonical_h16": _paired_delta(methods, best_clean_key, "h16_policy_canonical"),
"best_clean - direct_same_ckpt": _paired_delta(methods, best_clean_key, "near_miss_policy_bc5"),
"no_expert_lattice - canonical_h16": _paired_delta(methods, "same_state_no_expert", "h16_policy_canonical"),
"full_lattice - no_expert_lattice": _paired_delta(methods, "same_state_full", "same_state_no_expert"),
"policy_candidate_lattice - no_expert_lattice": _paired_delta(
methods,
"same_state_policy_baseline",
"same_state_no_expert",
),
}
mechanism_gap = {
"best_clean_vs_h16": methods[best_clean_key]["mean_success"]
- methods["h16_policy_canonical"]["mean_success"],
"best_clean_vs_direct_same_ckpt": methods[best_clean_key]["mean_success"]
- methods["near_miss_policy_bc5"]["mean_success"],
"same_state_no_expert_vs_h16": methods["same_state_no_expert"]["mean_success"]
- methods["h16_policy_canonical"]["mean_success"],
"same_state_no_expert_vs_best_clean": methods["same_state_no_expert"]["mean_success"]
- methods[best_clean_key]["mean_success"],
"same_state_full_vs_no_expert": methods["same_state_full"]["mean_success"]
- methods["same_state_no_expert"]["mean_success"],
}
decomposition = causal_action_decomposition(
base=methods["h16_policy_canonical"]["mean_success"],
selected=methods[best_clean_key]["mean_success"],
proposal_oracle=proposal_oracle_success,
same_state_oracle=methods["same_state_no_expert"]["mean_success"],
full_oracle=methods["same_state_full"]["mean_success"],
)
return {
"generated_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"methods": methods,
"paired_deltas": paired_deltas,
"per_task_deltas": {
"best_clean_vs_h16": _per_task_delta(methods, best_clean_key, "h16_policy_canonical"),
"no_expert_vs_best_clean": _per_task_delta(methods, "same_state_no_expert", best_clean_key),
},
"mechanism_gap": mechanism_gap,
"causal_action_decomposition": decomposition.to_dict(),
"causal_action_targets": {
"selected_success_for_65pct_gap_closure": decomposition.target_for_gap_closure(0.65),
"selected_success_for_75pct_gap_closure": decomposition.target_for_gap_closure(0.75),
"best_paper_target_range": [0.47, 0.52],
},
"best_candidate_oracle_key": oracle_key,
"best_clean_key": best_clean_key,
"generator_v2_support_proxy": _load_generator_v2_support_proxy(),
"generator_v2_local_atlas_support_proxy": (
_load_generator_v2_local_atlas_support_proxy()
),
"generator_v2_chart_synthesis_support_proxy": (
_load_generator_v2_chart_synthesis_support_proxy()
),
"generator_v2_cvae_support_proxy": _load_generator_v2_cvae_support_proxy(),
"generator_v2_spline_cvae_support_proxy": (
_load_generator_v2_spline_cvae_support_proxy()
),
"generator_v2_spline_flow_support_proxy": (
_load_generator_v2_spline_flow_support_proxy()
),
"generator_v2_guided_spline_flow_support_proxy": (
_load_generator_v2_guided_spline_flow_support_proxy()
),
}
def _generator_support_compare_row(name: str, proxy: dict[str, Any]) -> str:
if proxy.get("missing", True):
return f"| {name} | missing | missing | missing | missing | missing |"
overall = proxy.get("overall", {})
return (
f"| {name} | {int(proxy.get('num_val_groups_with_positive') or proxy.get('num_eval_groups_with_positive') or 0)} | "
f"{_pct(overall.get('ptr_proxy_at_16_thr_0p2'))} | "
f"{_pct(overall.get('ptr_proxy_at_16_thr_0p4'))} | "
f"{_pct(overall.get('negative_near_at_16_thr_0p2'))} | "
f"{_pct(overall.get('positive_closer_than_negative_rate_at_16'))} |"
)
def _support_proxy_row(
values: dict[str, Any],
label: str,
metric_prefix: str,
threshold_key: str | None,
) -> str:
cells = []
for k in (1, 2, 4, 8, 16):
if threshold_key is None:
key = f"{metric_prefix}_at_{k}"
else:
key = f"{metric_prefix}_at_{k}_thr_{threshold_key}"
cells.append(_pct(values.get(key)))
return f"| {label} | " + " | ".join(cells) + " |"
def _latex_pct(value: float | None) -> str:
if value is None:
return "--"
value = float(value)
if math.isnan(value):
return "--"
return f"{value * 100:.2f}"
def _latex_pp(value: float | None) -> str:
if value is None:
return "--"
value = float(value)
if math.isnan(value):
return "--"
return f"{value * 100:.2f}"
def _render_car_decomposition_table(report: dict[str, Any]) -> str:
decomposition = report["causal_action_decomposition"]
targets = report["causal_action_targets"]
closed = decomposition["closed_fraction_of_noexpert_gap"]
closed_text = "--" if math.isnan(float(closed)) else f"{float(closed) * 100:.1f}"
return (
"\\begin{table}[t]\n"
"\\centering\n"
"\\caption{Causal Action Regret decomposition on the current six-task "
"diagnostic. Support is the proposal-generation gap to the hidden "
"same-state no-expert oracle; selector is the clean proposal-oracle "
"headroom left by the deployed selector.}\n"
"\\label{tab:car-decomposition}\n"
"\\small\n"
"\\begin{tabular}{@{}lrrrrrr@{}}\n"
"\\toprule\n"
"Method & Base & Prop. oracle & Selected & State oracle & "
"Support & Selector \\\\\n"
"\\midrule\n"
"Current CIL-Atlas V0 & "
f"{_latex_pct(decomposition['base'])} & "
f"{_latex_pct(decomposition['proposal_oracle'])} & "
f"{_latex_pct(decomposition['selected'])} & "
f"{_latex_pct(decomposition['same_state_oracle'])} & "
f"{_latex_pp(decomposition['support_gap'])} & "
f"{_latex_pp(decomposition['selector_gap'])} \\\\\n"
"\\bottomrule\n"
"\\end{tabular}\n"
"\\vspace{2pt}\n"
"\\footnotesize Clean gain is "
f"{_latex_pp(decomposition['clean_gain'])} points; gap closed is "
f"{closed_text}\\%; 65--75\\% closure targets are "
f"{_latex_pct(targets['selected_success_for_65pct_gap_closure'])}--"
f"{_latex_pct(targets['selected_success_for_75pct_gap_closure'])}.\n"
"\\end{table}\n"
)
def main() -> int:
RESULTS_DIR.mkdir(parents=True, exist_ok=True)
report = build_report()
OUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
OUT_MD.write_text(_render_markdown(report), encoding="utf-8")
LATEX_TABLES_DIR.mkdir(parents=True, exist_ok=True)
OUT_CAR_TABLE.write_text(_render_car_decomposition_table(report), encoding="utf-8")
print(f"Wrote {OUT_JSON}")
print(f"Wrote {OUT_MD}")
print(f"Wrote {OUT_CAR_TABLE}")
return 0
if __name__ == "__main__":
raise SystemExit(main())