| |
| from __future__ import annotations |
|
|
| import json |
| import math |
| import sys |
| from collections import Counter |
| from dataclasses import dataclass |
| from datetime import datetime, timezone |
| from pathlib import Path |
| from typing import Any |
|
|
| ROOT_DIR = Path(__file__).resolve().parents[1] |
| if str(ROOT_DIR) not in sys.path: |
| sys.path.insert(0, str(ROOT_DIR)) |
|
|
| from dovla_cil.eval.metrics import ( |
| candidate_prefix_causal_metrics, |
| causal_action_decomposition, |
| finite_mean, |
| ) |
|
|
|
|
| RESULTS_DIR = Path("results") |
| OUT_JSON = RESULTS_DIR / "paper_analysis.json" |
| OUT_MD = RESULTS_DIR / "paper_analysis.md" |
| LATEX_TABLES_DIR = Path("latex") / "tables" |
| OUT_CAR_TABLE = LATEX_TABLES_DIR / "car_decomposition.tex" |
| GENERATOR_V2_MEMORY_EVAL = RESULTS_DIR / "generator_v2_positive_tangent_memory_eval.json" |
| GENERATOR_V2_LOCAL_ATLAS_EVAL = RESULTS_DIR / "generator_v2_positive_tangent_local_atlas_pool16_eval.json" |
| GENERATOR_V2_LOCAL_ATLAS_SWEEP = RESULTS_DIR / "generator_v2_positive_tangent_local_atlas_sweep_summary.json" |
| GENERATOR_V2_CHART_SYNTHESIS_EVAL = RESULTS_DIR / "generator_v2_positive_tangent_chart_synthesis_eval.json" |
| GENERATOR_V2_CHART_SYNTHESIS_SWEEP = RESULTS_DIR / "generator_v2_positive_tangent_chart_synthesis_sweep_summary.json" |
| GENERATOR_V2_CVAE_EVAL = RESULTS_DIR / "generator_v2_positive_tangent_cvae_eval.json" |
| GENERATOR_V2_CVAE_SWEEP = RESULTS_DIR / "generator_v2_positive_tangent_cvae_sweep_summary.json" |
| GENERATOR_V2_SPLINE_CVAE_EVAL = RESULTS_DIR / "generator_v2_positive_tangent_spline_cvae_eval.json" |
| GENERATOR_V2_SPLINE_CVAE_SWEEP = RESULTS_DIR / "generator_v2_positive_tangent_spline_cvae_sweep_summary.json" |
| GENERATOR_V2_SPLINE_FLOW_EVAL = RESULTS_DIR / "generator_v2_positive_tangent_spline_flow_eval.json" |
| GENERATOR_V2_SPLINE_FLOW_SWEEP = RESULTS_DIR / "generator_v2_positive_tangent_spline_flow_sweep_summary.json" |
| GENERATOR_V2_GUIDED_SPLINE_FLOW_EVAL = RESULTS_DIR / "generator_v2_positive_tangent_guided_spline_flow_eval.json" |
| GENERATOR_V2_GUIDED_SPLINE_FLOW_SWEEP = RESULTS_DIR / "generator_v2_positive_tangent_guided_spline_flow_sweep_summary.json" |
| CANONICAL_H16_ROLLOUT = Path("/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs") |
| FALLBACK_BEST_CLEAN_KEY = "residual_k4_consensus_grid035040045_noopbonus003" |
| NON_DEPLOYMENT_KEYS = { |
| "same_state_near_miss", |
| "same_state_no_expert", |
| "same_state_policy_baseline", |
| "same_state_full", |
| "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8", |
| "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace", |
| "transport_field_reground_fieldonly_k6matched_b12_clean_k6_oraclek8", |
| "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_oraclek8", |
| "transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8", |
| } |
|
|
|
|
| @dataclass(frozen=True) |
| class MethodSpec: |
| key: str |
| label: str |
| summary_path: str | None = None |
| raw_rollout_glob: str | None = None |
| summary_mode: str = "standard" |
| headline_metric: str = "mean_success" |
|
|
|
|
| METHODS = [ |
| MethodSpec( |
| key="h16_policy_canonical", |
| label="Direct h=16 policy, canonical rollout", |
| raw_rollout_glob=str(CANONICAL_H16_ROLLOUT / "seed_*/online_rollout.json"), |
| ), |
| MethodSpec( |
| key="gaussian_field", |
| label="Gaussian field search", |
| summary_path="h16_field_sweep_summary.json", |
| summary_mode="field_sweep_best", |
| ), |
| MethodSpec( |
| key="near_miss_policy_bc5", |
| label="Near-miss proposal policy, direct", |
| summary_path="h16_policy_ckpt_near_miss_policy_bc5_summary.json", |
| ), |
| MethodSpec( |
| key="best_clean_residual_k2", |
| label="K2 residual transport, safe + margin 0.20", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "knn2_scale0p40_safe_types_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_taskrelative_k2", |
| label="K2 task-relative residual transport, safe + margin 0.20", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "taskrelative_knn2_scale0p40_safe_types_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus", |
| label="K4 mean-by-type tangent consensus", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_kernel_consensus", |
| label="K4 kernel-weighted tangent consensus", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_kernel_mean_by_type_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_kernel_consensus_noopbonus003", |
| label="K4 kernel-weighted tangent consensus, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_kernel_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_kernel_consensus_s035_noopbonus003", |
| label="K4 kernel-weighted tangent consensus, scale 0.35, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s035_safe_margin0p20_kernel_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_kernel_consensus_s045_noopbonus003", |
| label="K4 kernel-weighted tangent consensus, scale 0.45, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s045_safe_margin0p20_kernel_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_fieldsoftmax_grid", |
| label="K4 field-softmax tangent transport, scales 0.35/0.40/0.45", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_fieldsoftmax_grid_safe_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_fieldsoftmax_grid_noopbonus003", |
| label="K4 field-softmax tangent transport, scales 0.35/0.40/0.45, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_fieldsoftmax_grid_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_fieldsoftmax_grid_margin010_noopbonus003", |
| label="K4 field-softmax tangent transport, margin 0.10, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_fieldsoftmax_grid_safe_margin0p10_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_fieldsoftmax_grid_margin005_noopbonus003", |
| label="K4 field-softmax tangent transport, margin 0.05, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_fieldsoftmax_grid_safe_margin0p05_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_fieldsoftmax_grid_margin000_noopbonus003", |
| label="K4 field-softmax tangent transport, margin 0.00, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_fieldsoftmax_grid_safe_margin0p00_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k8_fieldsoftmax_grid_noopbonus003", |
| label="K8 field-softmax tangent transport, scales 0.35/0.40/0.45, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k8_fieldsoftmax_grid_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus003", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus003_srcprog025", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.03, source progress >= 0.25", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p03_srcprog0p25_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_margin015_noopbonus003", |
| label="K4 mean-by-type tangent consensus, margin 0.15, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p15_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_margin025_noopbonus003", |
| label="K4 mean-by-type tangent consensus, margin 0.25, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p25_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_margin015_srcscorebonus002", |
| label="K4 mean-by-type tangent consensus, margin 0.15, source-score bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p15_mean_by_type_srcscorebonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_margin025_srcscorebonus002", |
| label="K4 mean-by-type tangent consensus, margin 0.25, source-score bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p25_mean_by_type_srcscorebonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_srcscorebonus002", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, source-score bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_srcscorebonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_srcadvbonus002", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, source-advantage bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_srcadvbonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_srcadvbonus005", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, source-advantage bonus 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_srcadvbonus0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_srcadvbonus002", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, source-advantage bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_srcadvbonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_srcadvgate000", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, source-advantage gate >= 0.0", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_srcadvgate0p0_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_srcadvgate000", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, source-advantage gate >= 0.0", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_srcadvgate0p0_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_typesuccessbonus002", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, train family-success bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_typesuccessbonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_typesuccessbonus003", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, train family-success bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_typesuccessbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_typesuccessbonus005", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, train family-success bonus 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_typesuccessbonus0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_typesuccessbonus002", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, train family-success bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_typesuccessbonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_consensus005", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, consensus penalty 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_consensus0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_consensus002", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, consensus penalty 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_consensus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_consensus005", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, consensus penalty 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_consensus0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_consensus010", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, consensus penalty 0.10", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_consensus0p10_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_compose_grid035040045", |
| label="K4 composed type-consensus tangents, scales 0.35/0.40/0.45", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_compose_grid035040045_safe_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_compose_grid035040045_noopbonus003", |
| label="K4 composed type-consensus tangents, scales 0.35/0.40/0.45, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_compose_grid035040045_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_grid035040045", |
| label="K4 composed type-consensus tangents, masked, scales 0.35/0.40/0.45", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_grid035040045_safe_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_grid035040045_noopbonus003", |
| label="K4 composed type-consensus tangents, masked, scales 0.35/0.40/0.45, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_grid035040045_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045", |
| label="K4 composed type-consensus tangents, masked, drop near-miss+no-op composite", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003", |
| label="K4 composed type-consensus tangents, masked, drop near-miss+no-op composite", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_margin010_noopbonus003", |
| label="K4 compatible tangents, margin 0.10, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p10_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8", |
| label="K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 diagnostic", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_oraclek8_summary.json" |
| ), |
| headline_metric="mean_candidate_oracle_success_rate", |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace", |
| label="K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 branch trace", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_oraclek8trace_summary.json" |
| ), |
| headline_metric="mean_candidate_oracle_success_rate", |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger002", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_latest_margin000_clean", |
| label="Transported residual field re-grounding, no bonus/challenger, margin 0.00", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_latest_margin0p00_" |
| "nobonus_nochallenger_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_latest_margin000_clean_k6", |
| label="Transported residual field re-grounding, no bonus/challenger, K6 support", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_latest_margin0p00_" |
| "clean_k6_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6", |
| label="K6-matched transported residual field re-grounding, no bonus/challenger", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_b12_v2_" |
| "besttransport_margin0p00_k6_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg", |
| label="K6-matched transported residual field re-grounding, drop no-op+wrong-gripper composite", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_b12_v2_" |
| "besttransport_margin0p00_k6_dropnoopwg_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted", |
| label="K6-matched transported residual field re-grounding, exact drop-mask target map", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_fieldsoftmax", |
| label="K6-matched transported residual field re-grounding, exact drop-mask field-softmax reducer", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_fieldsoftmax_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_typesuccess001", |
| label="K6-matched transported residual field re-grounding, exact drop-mask train-type prior 0.01", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_typesuccess001_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_typesuccess002", |
| label="K6-matched transported residual field re-grounding, exact drop-mask train-type prior 0.02", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_typesuccess002_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_typesuccess003", |
| label="K6-matched transported residual field re-grounding, exact drop-mask train-type prior 0.03", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_typesuccess003_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_rankcal005", |
| label="K6-matched transported residual field re-grounding, exact drop-mask train rank calibration 0.05", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_rankcal005_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001", |
| label="K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.01", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore001_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_typecal005", |
| label="K6-matched transported residual field re-grounding, source-score 0.01 + transport-outcome type calibration 0.05", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore001_typecal005_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_typecal010", |
| label="K6-matched transported residual field re-grounding, source-score 0.01 + transport-outcome type calibration 0.10", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore001_typecal010_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_typecal020", |
| label="K6-matched transported residual field re-grounding, source-score 0.01 + transport-outcome type calibration 0.20", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore001_typecal020_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_zscore", |
| label="K6-matched transported residual field re-grounding, source-score 0.01, z-score retrieval chart", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore001_metric_zscore_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrel", |
| label="K6-matched transported residual field re-grounding, source-score 0.01, task-relative retrieval chart", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore001_metric_taskrel_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_metric_taskrelz", |
| label="K6-matched transported residual field re-grounding, source-score 0.01, task-relative z-score retrieval chart", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore001_metric_taskrelz_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005", |
| label="K6-matched transported residual field re-grounding, task-conditioned source-score prior", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_advw1p0", |
| label="Generator V1 utility-weighted tangents, expert anchor, advantage weight 1.0", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_advw1p0_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_advw2p0", |
| label="Generator V1 utility-weighted tangents, expert anchor, advantage weight 2.0", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_advw2p0_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_policyanchor_advw2p0", |
| label="Generator V1 positive tangents, policy anchor, advantage weight 2.0", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_policyanchor_advw2p0_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_advw2p0_gate0", |
| label="Generator V1 utility-weighted tangents, expert anchor, source advantage gate >= 0.0", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_advw2p0_gate0_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_policyanchor_advw2p0_gate0", |
| label="Generator V1 positive tangents, policy anchor, source advantage gate >= 0.0", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_policyanchor_advw2p0_gate0_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_advw1p0_oraclek8", |
| label="Generator V1 expert-anchor advantage weight 1.0, candidate-oracle K8", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_advw1p0_oraclek8_summary.json" |
| ), |
| headline_metric="mean_candidate_oracle_success_rate", |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_advw2p0_oraclek8", |
| label="Generator V1 expert-anchor advantage weight 2.0, candidate-oracle K8", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_advw2p0_oraclek8_summary.json" |
| ), |
| headline_metric="mean_candidate_oracle_success_rate", |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_policyanchor_advw2p0_oraclek8", |
| label="Generator V1 policy-anchor advantage weight 2.0, candidate-oracle K8", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_policyanchor_advw2p0_oraclek8_summary.json" |
| ), |
| headline_metric="mean_candidate_oracle_success_rate", |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_advw0p5", |
| label="Generator V1 utility-weighted tangents, expert anchor, advantage weight 0.5", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_advw0p5_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_advw4p0", |
| label="Generator V1 utility-weighted tangents, expert anchor, advantage weight 4.0", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_advw4p0_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_policyanchor_advw1p0", |
| label="Generator V1 positive tangents, policy anchor, advantage weight 1.0", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_policyanchor_advw1p0_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_policyanchor_advw4p0", |
| label="Generator V1 positive tangents, policy anchor, advantage weight 4.0", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_policyanchor_advw4p0_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_scaleoracle_trainall_k4_scale0025", |
| label="K6-matched transported residual field re-grounding, all-train tangent-length prior 0.025", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_" |
| "scaleoracle_trainall_k4_scale0025_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_scaleoracle_trainall_k4_scale005", |
| label="K6-matched transported residual field re-grounding, all-train tangent-length prior 0.05", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_" |
| "scaleoracle_trainall_k4_scale005_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_scaleoracle_trainall_k4_scale010", |
| label="K6-matched transported residual field re-grounding, all-train tangent-length prior 0.10", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_" |
| "scaleoracle_trainall_k4_scale010_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r005_t005", |
| label="K6-matched transported residual field re-grounding, train-oracle calibrated selector", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_" |
| "oraclecal_train800_k4_r005_t005_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_r0025_t0025", |
| label="K6-matched transported residual field re-grounding, small train-oracle selector calibration", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_" |
| "oraclecal_train800_k4_r0025_t0025_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_rank005", |
| label="K6-matched transported residual field re-grounding, train-oracle rank calibration", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_" |
| "oraclecal_train800_k4_rank005_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore_task_pick001_stack005_oraclecal_train800_k4_type005", |
| label="K6-matched transported residual field re-grounding, train-oracle type calibration", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore_task_pick001_stack005_" |
| "oraclecal_train800_k4_type005_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore001_oraclek8", |
| label="K6-matched transported residual field re-grounding, source-score 0.01 candidate-oracle K8", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore001_oraclek8_summary.json" |
| ), |
| headline_metric="mean_candidate_oracle_success_rate", |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0005", |
| label="K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.005", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore0005_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore0015", |
| label="K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.015", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore0015_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore002", |
| label="K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.02", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore002_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_srcscore005", |
| label="K6-matched transported residual field re-grounding, exact drop-mask train source-score prior 0.05", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_srcscore005_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopwg_retargeted_oraclek8", |
| label="K6-matched transported residual field re-grounding, exact drop-mask candidate-oracle K8", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b12_v1_" |
| "besttransport_margin0p00_k6_oraclek8_summary.json" |
| ), |
| headline_metric="mean_candidate_oracle_success_rate", |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b24_clean_k6_dropnoopwg_retargeted", |
| label="K6-matched transported residual field re-grounding, exact drop-mask target map, B24", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b24_v1_" |
| "besttransport_margin0p00_k6_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b24_clean_k6_dropnoopwg_retargeted_latest", |
| label="K6-matched transported residual field re-grounding, exact drop-mask target map, B24 latest", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_dropnoopwg_b24_v1_" |
| "latest_margin0p00_k6_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_oraclek8", |
| label="K6-matched transported residual field re-grounding, candidate-oracle K8 diagnostic", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_b12_v2_" |
| "besttransport_margin0p00_k6_oraclek8_summary.json" |
| ), |
| headline_metric="mean_candidate_oracle_success_rate", |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_best", |
| label="K6-matched transported residual field re-grounding, validation-rank checkpoint", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_b12_v2_" |
| "best_margin0p00_k6_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_prependpolicy", |
| label="K6-matched transported residual field re-grounding, policy candidate prepended", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_b12_v2_" |
| "besttransport_margin0p00_k6_prependpolicy_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_prependpolicy_margin003", |
| label="K6-matched transported residual field re-grounding, policy candidate prepended, margin 0.03", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_b12_v2_" |
| "besttransport_margin0p03_k6_prependpolicy_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_dropnoopfamily", |
| label="K6-matched transported residual field re-grounding, drop no-op residual family", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_b12_v2_" |
| "besttransport_margin0p00_k6_dropnoopfamily_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_droppolicy_dropnoopwg", |
| label="K6-matched transported residual field re-grounding, drop policy residual and no-op+wrong-gripper", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_b12_v2_" |
| "besttransport_margin0p00_k6_droppolicy_dropnoopwg_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_e24_besttransport", |
| label="K6-matched transported residual field re-grounding, 24 epochs, transport-rank checkpoint", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_b12_e24_v1_" |
| "besttransport_margin0p00_k6_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_e24_latest", |
| label="K6-matched transported residual field re-grounding, 24 epochs, latest checkpoint", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_b12_e24_v1_" |
| "latest_margin0p00_k6_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="transport_field_reground_fieldonly_k6matched_b12_clean_k6_e24_best", |
| label="K6-matched transported residual field re-grounding, 24 epochs, validation-rank checkpoint", |
| summary_path=( |
| "h16_transport_field_reground_fieldonly_k6clean_b12_e24_v1_" |
| "best_margin0p00_k6_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_stacknowg", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, no wrong-gripper component on Stack", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_stacknowg_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0005", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.005", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p005_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0015", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.015", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p015_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_scale035", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, scale-gated 0.35", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_scale035_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_scales035040", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, scale-gated 0.35/0.40", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_scales035040_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger003", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmwgchallenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgmargin003_challenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger 0.01, wrong-gripper margin 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmwgmargin0p03_challenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgmargin005_challenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger 0.01, wrong-gripper margin 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmwgmargin0p05_challenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmwgchallenger0p01_pickpull_norm_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_challenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_challenger0p01_norm_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin003_challenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper Pick/Pull margin 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_wgmargin0p03_challenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin005_challenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper Pick/Pull margin 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_wgmargin0p05_challenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpullstack_challenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper challenger on Pick/Pull/Stack", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpullstack_challenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_noopwgcontact_challenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss global + no-op/wrong-gripper contact challenger", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_noopwgcontact_challenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k6_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001", |
| label="K6 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k6_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k8_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001", |
| label="K8 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k8_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_typesuccessbonus002_nmchallenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, train family-success bonus 0.02, near-miss challenger gate 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_typesuccessbonus0p02_nmchallenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_typesuccessbonus005_nmchallenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, train family-success bonus 0.05, near-miss challenger gate 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_typesuccessbonus0p05_nmchallenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_taskrelz_nmchallenger001", |
| label="K4 compatible tangents, actor-relative z-score retrieval, no-op bonus 0.03, near-miss challenger gate 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_taskrelz_nmchallenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_srcscore005", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, train source-score certificate 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_srcscore0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_srcscore010", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, train source-score certificate 0.10", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_srcscore0p10_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_srcadv005", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, train source-advantage certificate 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_srcadv0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="distill_direct_bestpolicy", |
| label="Direct policy distilled from clean residual-tangent teacher", |
| summary_path="h16_distill_direct_bestpolicy_summary.json", |
| ), |
| MethodSpec( |
| key="distill_retrieval_residual_teacher_bestfield", |
| label="Distilled policy + clean residual-tangent teacher at inference", |
| summary_path="h16_distill_retrieval_residual_teacher_bestfield_summary.json", |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_types6_prepend_margin000", |
| label="Typed proposal lattice head, six families, policy-prepended margin 0.00", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_typedprop_p2_bestpt_" |
| "proposal_lattice_types6_prepend_margin0p00_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_types6_prepend_margin005", |
| label="Typed proposal lattice head, six families, policy-prepended margin 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_typedprop_p2_bestpt_" |
| "proposal_lattice_types6_prepend_margin0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_types4safe_prepend_margin005", |
| label="Typed proposal lattice head, safe four families, policy-prepended margin 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_typedprop_p2_bestpt_" |
| "proposal_lattice_types4safe_prepend_margin0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_types2sparse_prepend_margin005", |
| label="Typed proposal lattice head, sparse no-op/wrong-gripper families, policy-prepended margin 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_typedprop_p2_bestpt_" |
| "proposal_lattice_types2sparse_prepend_margin0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_nooponly_prepend_margin005", |
| label="Typed proposal lattice head, no-op family only, policy-prepended margin 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_typedprop_p2_bestpt_" |
| "proposal_lattice_nooponly_prepend_margin0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_types2sparse_bestpolicy_prepend_margin005", |
| label="Typed proposal lattice head, sparse families, best-policy checkpoint, policy-prepended margin 0.05", |
| summary_path="h16_bestpolicy_types2sparse_prepend_margin0p05_summary.json", |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_nooponly_bestpolicy_prepend_margin005", |
| label="Typed proposal lattice head, no-op family only, best-policy checkpoint, policy-prepended margin 0.05", |
| summary_path="h16_bestpolicy_nooponly_prepend_margin0p05_summary.json", |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_nooponly_sparsehead_prepend_margin005", |
| label="Typed proposal lattice head, no-op-only head retrain, policy-prepended margin 0.05", |
| summary_path="h16_typedprop_nooponly_sparsehead_prepend_margin0p05_summary.json", |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_nooponly_sparsehead_prepend_margin000", |
| label="Typed proposal lattice head, no-op-only head retrain, policy-prepended margin 0.00", |
| summary_path="h16_typedprop_nooponly_sparsehead_prepend_margin0p00_summary.json", |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_nooponly_sparsehead_prepend_margin010", |
| label="Typed proposal lattice head, no-op-only head retrain, policy-prepended margin 0.10", |
| summary_path="h16_typedprop_nooponly_sparsehead_prepend_margin0p10_summary.json", |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001", |
| label="K4 composed compatible tangents, no-op bonus 0.03, singleton near-miss bonus 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmbonus0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus002", |
| label="K4 composed compatible tangents, no-op bonus 0.03, singleton near-miss bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmbonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_srcscorebonus002", |
| label="K4 composed type-consensus tangents, masked, drop near-miss+no-op composite, source-score bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_srcscorebonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_srcscorebonus002", |
| label="K4 composed type-consensus tangents, masked, drop near-miss+no-op composite, no-op bonus 0.03, source-score bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_srcscorebonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_l2comp002_grid035040045_noopbonus003", |
| label="K4 composed type-consensus tangents, masked, drop near-miss+no-op composite, composite L2 penalty 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_l2comp002_grid035040045_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_compbonus_grid035040045_noopbonus003", |
| label="K4 composed type-consensus tangents, masked, component no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_compbonus_grid035040045_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_l2comp002_grid035040045_noopbonus003", |
| label="K4 composed type-consensus tangents, masked, composite L2 penalty 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_l2comp002_grid035040045_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_l2comp005_grid035040045_noopbonus003", |
| label="K4 composed type-consensus tangents, masked, composite L2 penalty 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_l2comp005_grid035040045_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="repair_nearmiss_k4_grid025035050_margin020", |
| label="K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.20", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_repair_" |
| "nearmiss_k4_grid025035050_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="repair_nearmiss_k4_grid035050075_margin020", |
| label="K4 near-miss-to-expert repair tangent, scales 0.35/0.50/0.75, margin 0.20", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_repair_" |
| "nearmiss_k4_grid035050075_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="repair_nearmiss_k4_grid025035050_margin010", |
| label="K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.10", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_repair_" |
| "nearmiss_k4_grid025035050_margin0p10_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="repair_safe_k4_grid025035050_margin020", |
| label="K4 safe-family-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.20", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_repair_" |
| "safe_k4_grid025035050_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_l2penalty005", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, action L2 penalty 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_l2penalty0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_l2penalty010", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, action L2 penalty 0.10", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_l2penalty0p10_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_l2penalty020", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, action L2 penalty 0.20", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_l2penalty0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid040045050_noopbonus003", |
| label="K4 mean-by-type tangent consensus, scales 0.40/0.45/0.50, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid040045050_safe_margin0p20_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid040045050_srcscorebonus002", |
| label="K4 mean-by-type tangent consensus, scales 0.40/0.45/0.50, source-score bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid040045050_safe_margin0p20_mean_by_type_srcscorebonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035045055_noopbonus003", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.45/0.55, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035045055_safe_margin0p20_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_nooponly_noopbonus003", |
| label="K4 mean-by-type tangent consensus, no-op-only residuals, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_nooponly_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_nooponly_srcscorebonus002", |
| label="K4 mean-by-type tangent consensus, no-op-only residuals, source-score bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_nooponly_srcscorebonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus003_srcprog050", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.03, source progress >= 0.50", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p03_srcprog0p50_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus003_srcprog075", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.03, source progress >= 0.75", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p03_srcprog0p75_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_srcprogbonus003", |
| label="K4 mean-by-type tangent consensus, source-progress bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_srcprogbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_srcprogbonus005", |
| label="K4 mean-by-type tangent consensus, source-progress bonus 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_srcprogbonus0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_srcscorebonus0015", |
| label="K4 mean-by-type tangent consensus, source-score bonus 0.015", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_srcscorebonus0p015_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_srcscorebonus002", |
| label="K4 mean-by-type tangent consensus, source-score bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_srcscorebonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_srcscorebonus0025", |
| label="K4 mean-by-type tangent consensus, source-score bonus 0.025", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_srcscorebonus0p025_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_taskrelative_k4_consensus_noopbonus003", |
| label="K4 task-relative tangent consensus, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "taskrelative_k4s040_safe_margin0p20_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus001", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus002", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus0025", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.025", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p025_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus0035", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.035", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p035_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_wgbonus003", |
| label="K4 mean-by-type tangent consensus, wrong-gripper bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_wgbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noop003_wg002", |
| label="K4 mean-by-type tangent consensus, no-op 0.03 + wrong-gripper 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noop0p03_wg0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noop003_wg004", |
| label="K4 mean-by-type tangent consensus, no-op 0.03 + wrong-gripper 0.04", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noop0p03_wg0p04_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noop0025_wg002", |
| label="K4 mean-by-type tangent consensus, no-op 0.025 + wrong-gripper 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noop0p025_wg0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus005", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus008", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.08", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p08_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="same_state_near_miss", |
| label="Same-state lattice, near-miss only", |
| summary_path="h16_lattice_near_miss_only_v2_summary.json", |
| ), |
| MethodSpec( |
| key="same_state_no_expert", |
| label="Same-state lattice, no expert", |
| summary_path="h16_lattice_no_expert_summary.json", |
| ), |
| MethodSpec( |
| key="same_state_policy_baseline", |
| label="Same-state no-expert + policy candidate", |
| summary_path="h16_lattice_no_expert_policy_baseline_margin000_summary.json", |
| ), |
| MethodSpec( |
| key="same_state_full", |
| label="Same-state lattice, full", |
| summary_path="h16_lattice_summary.json", |
| ), |
| ] |
|
|
|
|
| def _load_json(path: Path) -> dict[str, Any]: |
| with path.open("r", encoding="utf-8") as handle: |
| return json.load(handle) |
|
|
|
|
| def _mean(values: list[float]) -> float: |
| return sum(values) / len(values) if values else float("nan") |
|
|
|
|
| def _sample_std(values: list[float]) -> float: |
| if len(values) <= 1: |
| return 0.0 |
| mean = _mean(values) |
| return math.sqrt(sum((value - mean) ** 2 for value in values) / (len(values) - 1)) |
|
|
|
|
| def _ci95(values: list[float]) -> float: |
| if len(values) <= 1: |
| return 0.0 |
| t_crit = { |
| 1: 12.706, |
| 2: 4.303, |
| 3: 3.182, |
| 4: 2.776, |
| 5: 2.571, |
| 6: 2.447, |
| 7: 2.365, |
| 8: 2.306, |
| 9: 2.262, |
| 10: 2.228, |
| }.get(len(values) - 1, 1.96) |
| return t_crit * _sample_std(values) / math.sqrt(len(values)) |
|
|
|
|
| def _success(row: dict[str, Any]) -> float: |
| return float(row["policy_rollout_success_rate"]) |
|
|
|
|
| def _progress(row: dict[str, Any]) -> float: |
| return float(row.get("policy_rollout_progress", float("nan"))) |
|
|
|
|
| def _action_mse(row: dict[str, Any]) -> float: |
| return float(row.get("action_mse_to_best", float("nan"))) |
|
|
|
|
| def _seed(row: dict[str, Any], fallback: int) -> int: |
| return int(row.get("seed", fallback)) |
|
|
|
|
| def _standard_summary(path: Path) -> dict[str, Any]: |
| data = _load_json(path) |
| rows = list(data.get("rows", [])) |
| return _normalize_summary(data, rows, source=str(path)) |
|
|
|
|
| def _field_sweep_best(path: Path) -> dict[str, Any]: |
| data = _load_json(path) |
| best_config = data.get("best", {}).get("config") |
| rows = [row for row in data.get("rows", []) if row.get("config") == best_config] |
| normalized = _normalize_summary(data.get("best", data), rows, source=str(path)) |
| normalized["best_config"] = best_config |
| return normalized |
|
|
|
|
| def _raw_rollout_summary(pattern: str) -> dict[str, Any]: |
| rows = [] |
| for index, path in enumerate(sorted(Path().glob(pattern) if not pattern.startswith("/") else Path("/").glob(pattern[1:]))): |
| row = _load_json(path) |
| row = dict(row) |
| row["path"] = str(path) |
| row["seed"] = _seed(row, index) |
| rows.append(row) |
| return _normalize_summary({}, rows, source=pattern) |
|
|
|
|
| def _normalize_summary(data: dict[str, Any], rows: list[dict[str, Any]], *, source: str) -> dict[str, Any]: |
| successes = [_success(row) for row in rows] |
| progress = [_progress(row) for row in rows] |
| action_mse = [_action_mse(row) for row in rows] |
| selected_counts = Counter() |
| selected_counts.update(data.get("selected_candidate_type_counts", {})) |
| selected_scale_counts = Counter() |
| selected_scale_counts.update(data.get("selected_residual_scale_counts", {})) |
| expected_count = sum(int(row.get("num_groups", 0)) for row in rows) |
| has_top_level_selected_counts = ( |
| bool(selected_counts) and sum(int(value) for value in selected_counts.values()) == expected_count |
| ) |
| has_top_level_scale_counts = ( |
| bool(selected_scale_counts) |
| and sum(int(value) for value in selected_scale_counts.values()) == expected_count |
| ) |
| if not has_top_level_selected_counts: |
| selected_counts = Counter() |
| if not has_top_level_scale_counts: |
| selected_scale_counts = Counter() |
| if not has_top_level_selected_counts or not has_top_level_scale_counts: |
| for row in rows: |
| path = row.get("path") |
| if not path: |
| continue |
| raw_path = Path(str(path)) |
| if not raw_path.exists(): |
| continue |
| raw = _load_json(raw_path) |
| if not has_top_level_selected_counts: |
| selected_counts.update(raw.get("selected_candidate_type_counts", {})) |
| if not has_top_level_scale_counts: |
| selected_scale_counts.update(raw.get("selected_residual_scale_counts", {})) |
| output = { |
| "source": source, |
| "num_completed": len(rows), |
| "mean_success": _mean(successes), |
| "std_success": _sample_std(successes), |
| "ci95_success": _ci95(successes), |
| "mean_progress": _mean(progress), |
| "mean_action_mse_to_best": _mean(action_mse), |
| "seed_success": {_seed(row, index): _success(row) for index, row in enumerate(rows)}, |
| "seed_progress": {_seed(row, index): _progress(row) for index, row in enumerate(rows)}, |
| "seed_action_mse_to_best": {_seed(row, index): _action_mse(row) for index, row in enumerate(rows)}, |
| "per_task_success": _per_task(rows), |
| "selected_candidate_type_counts": dict(selected_counts), |
| "selected_residual_scale_counts": dict(selected_scale_counts), |
| "selected_type_outcomes": _selected_type_outcomes(rows), |
| } |
| candidate_oracle_success = [ |
| float(row["candidate_oracle_success_rate"]) |
| for row in rows |
| if row.get("candidate_oracle_success_rate") is not None |
| ] |
| if candidate_oracle_success: |
| raw_prefix_metrics = _candidate_prefix_metrics_from_raw(rows) |
| selected_branch_success = [ |
| float(row["candidate_oracle_selected_branch_success_rate"]) |
| for row in rows |
| if row.get("candidate_oracle_selected_branch_success_rate") is not None |
| ] |
| selected_branch_progress = [ |
| float(row["candidate_oracle_selected_branch_progress"]) |
| for row in rows |
| if row.get("candidate_oracle_selected_branch_progress") is not None |
| ] |
| oracle_progress = [ |
| float(row["candidate_oracle_progress"]) |
| for row in rows |
| if row.get("candidate_oracle_progress") is not None |
| ] |
| selected_success_mean = _mean(selected_branch_success) |
| oracle_success_mean = _mean(candidate_oracle_success) |
| selected_progress_mean = _mean(selected_branch_progress) |
| oracle_progress_mean = _mean(oracle_progress) |
| output.update( |
| { |
| "candidate_oracle_rollouts": int(data.get("candidate_oracle_rollouts") or 0), |
| "candidate_oracle_unique_tolerance": data.get( |
| "candidate_oracle_unique_tolerance" |
| ), |
| "mean_candidate_oracle_success_rate": oracle_success_mean, |
| "std_candidate_oracle_success_rate": _sample_std( |
| candidate_oracle_success |
| ), |
| "ci95_candidate_oracle_success_rate": _ci95(candidate_oracle_success), |
| "seed_candidate_oracle_success": { |
| _seed(row, index): float(row["candidate_oracle_success_rate"]) |
| for index, row in enumerate(rows) |
| if row.get("candidate_oracle_success_rate") is not None |
| }, |
| "mean_candidate_oracle_progress": oracle_progress_mean, |
| "mean_candidate_oracle_selected_branch_success_rate": ( |
| selected_success_mean |
| ), |
| "mean_candidate_oracle_selected_branch_progress": ( |
| selected_progress_mean |
| ), |
| "candidate_oracle_selector_gap_success": ( |
| oracle_success_mean - selected_success_mean |
| ), |
| "candidate_oracle_selector_gap_progress": ( |
| oracle_progress_mean - selected_progress_mean |
| ), |
| "mean_candidate_oracle_score_gain_over_selected": _mean( |
| [ |
| float(row["candidate_oracle_score_gain_over_selected"]) |
| for row in rows |
| if row.get("candidate_oracle_score_gain_over_selected") |
| is not None |
| ] |
| ), |
| "mean_candidate_oracle_unique_count": _mean( |
| [ |
| float(row.get("candidate_oracle_unique_count") or 0.0) |
| for row in rows |
| if row.get("candidate_oracle_unique_count") is not None |
| ] |
| ), |
| "mean_candidate_oracle_improvement_rate": _mean( |
| [ |
| float(row["candidate_oracle_improvement_rate"]) |
| for row in rows |
| if row.get("candidate_oracle_improvement_rate") is not None |
| ] |
| ), |
| "candidate_oracle_type_counts": data.get( |
| "candidate_oracle_type_counts", {} |
| ), |
| } |
| ) |
| if raw_prefix_metrics: |
| output.update(raw_prefix_metrics) |
| for key in ( |
| "mean_candidate_oracle_best_branch_rank", |
| "candidate_oracle_best_branch_rank_counts", |
| "mean_candidate_oracle_branch_success_rates", |
| "mean_candidate_oracle_branch_progress", |
| "mean_candidate_oracle_branch_score_gains_over_selected", |
| ): |
| if key in data: |
| output[key] = data[key] |
| return output |
|
|
|
|
| def _candidate_prefix_metrics_from_raw(rows: list[dict[str, Any]]) -> dict[str, Any]: |
| prefix_metrics: list[dict[str, Any]] = [] |
| for row in rows: |
| path = row.get("path") |
| if not path: |
| continue |
| raw_path = Path(str(path)) |
| if not raw_path.exists(): |
| continue |
| raw = _load_json(raw_path) |
| for item in raw.get("rows", []): |
| scores = item.get("candidate_oracle_branch_scores") or [] |
| if not scores: |
| continue |
| metrics = candidate_prefix_causal_metrics( |
| branch_scores=[float(value) for value in scores], |
| selected_score=( |
| float(item["candidate_oracle_selected_branch_score"]) |
| if item.get("candidate_oracle_selected_branch_score") is not None |
| else None |
| ), |
| branch_types=[str(value) for value in item.get("candidate_oracle_types", [])], |
| valid_mask=[bool(value) for value in item.get("candidate_oracle_valid_mask", [])] |
| or None, |
| ) |
| if metrics: |
| prefix_metrics.append(metrics) |
| if not prefix_metrics: |
| return {} |
| ptr = finite_mean([item.get("ptr_at_k") for item in prefix_metrics]) |
| ncar = finite_mean([item.get("ncar_to_proposal_oracle") for item in prefix_metrics]) |
| return { |
| "mean_candidate_oracle_car_to_proposal_oracle": _mean( |
| [float(item["car_to_proposal_oracle"]) for item in prefix_metrics] |
| ), |
| "mean_candidate_oracle_selector_regret_at_k": _mean( |
| [float(item["selector_regret_at_k"]) for item in prefix_metrics] |
| ), |
| "mean_candidate_oracle_ptr_at_k": ptr, |
| "mean_candidate_oracle_ncar_to_proposal_oracle": ncar, |
| "candidate_oracle_base_trace_coverage": ( |
| sum(1 for item in prefix_metrics if item.get("base_utility") is not None) |
| / len(prefix_metrics) |
| ), |
| } |
|
|
|
|
| def _per_task(rows: list[dict[str, Any]]) -> dict[str, dict[str, float]]: |
| task_values: dict[str, list[float]] = {} |
| task_counts: dict[str, list[int]] = {} |
| for row in rows: |
| for task, metrics in row.get("per_task", {}).items(): |
| task_values.setdefault(task, []).append(float(metrics["policy_rollout_success_rate"])) |
| task_counts.setdefault(task, []).append(int(metrics.get("num_groups", 0))) |
| return { |
| task: { |
| "mean_success": _mean(values), |
| "std_success": _sample_std(values), |
| "mean_num_groups": _mean([float(value) for value in task_counts.get(task, [])]), |
| } |
| for task, values in sorted(task_values.items()) |
| } |
|
|
|
|
| def _selected_type_outcomes(rows: list[dict[str, Any]]) -> dict[str, dict[str, float]]: |
| grouped: dict[str, dict[str, float]] = {} |
| for row in rows: |
| path = row.get("path") |
| if not path: |
| continue |
| raw_path = Path(str(path)) |
| if not raw_path.exists(): |
| continue |
| raw = _load_json(raw_path) |
| for item in raw.get("rows", []): |
| candidate_type = str(item.get("nearest_candidate_type") or "unknown") |
| stats = grouped.setdefault( |
| candidate_type, |
| {"count": 0.0, "success_sum": 0.0, "progress_sum": 0.0}, |
| ) |
| stats["count"] += 1.0 |
| stats["success_sum"] += 1.0 if item.get("success") else 0.0 |
| stats["progress_sum"] += float(item.get("progress", 0.0)) |
| return { |
| candidate_type: { |
| "count": values["count"], |
| "success_rate": values["success_sum"] / values["count"] if values["count"] else float("nan"), |
| "mean_progress": values["progress_sum"] / values["count"] if values["count"] else float("nan"), |
| } |
| for candidate_type, values in sorted( |
| grouped.items(), |
| key=lambda item: (-item[1]["count"], item[0]), |
| ) |
| } |
|
|
|
|
| def _load_methods() -> dict[str, dict[str, Any]]: |
| methods: dict[str, dict[str, Any]] = {} |
| for spec in METHODS: |
| if spec.summary_path: |
| path = RESULTS_DIR / spec.summary_path |
| if not path.exists(): |
| methods[spec.key] = { |
| "missing": True, |
| "source": str(path), |
| "label": spec.label, |
| "headline_metric": spec.headline_metric, |
| "headline_metric_label": _headline_metric_label( |
| spec.headline_metric |
| ), |
| } |
| continue |
| if spec.summary_mode == "field_sweep_best": |
| method = _field_sweep_best(path) |
| else: |
| method = _standard_summary(path) |
| elif spec.raw_rollout_glob: |
| method = _raw_rollout_summary(spec.raw_rollout_glob) |
| else: |
| method = {"missing": True, "source": "", "label": spec.label} |
| method["label"] = spec.label |
| _attach_headline_fields(method, spec) |
| methods[spec.key] = method |
| return methods |
|
|
|
|
| def _load_generator_v2_support_proxy() -> dict[str, Any]: |
| if not GENERATOR_V2_MEMORY_EVAL.exists(): |
| return {"missing": True, "source": str(GENERATOR_V2_MEMORY_EVAL)} |
| data = _load_json(GENERATOR_V2_MEMORY_EVAL) |
| return { |
| "missing": False, |
| "source": str(GENERATOR_V2_MEMORY_EVAL), |
| "report_type": data.get("report_type"), |
| "metric_scope": data.get("metric_scope"), |
| "note": data.get("note"), |
| "targets": data.get("targets"), |
| "seed": data.get("seed"), |
| "val_fraction": data.get("val_fraction"), |
| "diversity_weight": data.get("diversity_weight"), |
| "num_examples": data.get("num_examples"), |
| "num_train_examples": data.get("num_train_examples"), |
| "num_val_examples": data.get("num_val_examples"), |
| "num_groups": data.get("num_groups"), |
| "num_val_groups": data.get("num_val_groups"), |
| "num_eval_groups": data.get("num_eval_groups"), |
| "num_eval_groups_with_positive": data.get("num_eval_groups_with_positive"), |
| "label_counts": data.get("label_counts", {}), |
| "train_positive_by_task": data.get("train_positive_by_task", {}), |
| "proposal_count_by_task": data.get("proposal_count_by_task", {}), |
| "overall": data.get("overall", {}), |
| "per_task": data.get("per_task", {}), |
| } |
|
|
|
|
| def _load_generator_v2_local_atlas_support_proxy() -> dict[str, Any]: |
| source = GENERATOR_V2_LOCAL_ATLAS_EVAL |
| sweep_best: dict[str, Any] | None = None |
| if GENERATOR_V2_LOCAL_ATLAS_SWEEP.exists(): |
| sweep = _load_json(GENERATOR_V2_LOCAL_ATLAS_SWEEP) |
| if sweep.get("best", {}).get("path"): |
| candidate = Path(str(sweep["best"]["path"])) |
| if candidate.exists(): |
| source = candidate |
| sweep_best = sweep["best"] |
| if not source.exists(): |
| return {"missing": True, "source": str(source)} |
| data = _load_json(source) |
| return { |
| "missing": False, |
| "source": str(source), |
| "sweep_summary": str(GENERATOR_V2_LOCAL_ATLAS_SWEEP) |
| if GENERATOR_V2_LOCAL_ATLAS_SWEEP.exists() |
| else None, |
| "sweep_best": sweep_best, |
| "report_type": data.get("report_type"), |
| "metric_scope": data.get("metric_scope"), |
| "note": data.get("note"), |
| "targets": data.get("targets"), |
| "config": data.get("config", {}), |
| "code_dim": data.get("code_dim"), |
| "horizon": data.get("horizon"), |
| "action_dim": data.get("action_dim"), |
| "num_examples": data.get("num_examples"), |
| "num_groups": data.get("num_groups"), |
| "num_train_examples": data.get("num_train_examples"), |
| "num_val_examples": data.get("num_val_examples"), |
| "num_train_positive": data.get("num_train_positive"), |
| "num_val_groups_with_positive": data.get("num_val_groups_with_positive"), |
| "label_counts": data.get("label_counts", {}), |
| "train_positive_by_task": data.get("train_positive_by_task", {}), |
| "overall": data.get("overall", {}), |
| "per_task": data.get("per_task", {}), |
| } |
|
|
|
|
| def _load_generator_v2_chart_synthesis_support_proxy() -> dict[str, Any]: |
| source = GENERATOR_V2_CHART_SYNTHESIS_EVAL |
| sweep_best: dict[str, Any] | None = None |
| if GENERATOR_V2_CHART_SYNTHESIS_SWEEP.exists(): |
| sweep = _load_json(GENERATOR_V2_CHART_SYNTHESIS_SWEEP) |
| if sweep.get("best", {}).get("path"): |
| candidate = Path(str(sweep["best"]["path"])) |
| if candidate.exists(): |
| source = candidate |
| sweep_best = sweep["best"] |
| if not source.exists(): |
| return {"missing": True, "source": str(source)} |
| data = _load_json(source) |
| return { |
| "missing": False, |
| "source": str(source), |
| "sweep_summary": str(GENERATOR_V2_CHART_SYNTHESIS_SWEEP) |
| if GENERATOR_V2_CHART_SYNTHESIS_SWEEP.exists() |
| else None, |
| "sweep_best": sweep_best, |
| "report_type": data.get("report_type"), |
| "metric_scope": data.get("metric_scope"), |
| "note": data.get("note"), |
| "targets": data.get("targets"), |
| "config": data.get("config", {}), |
| "code_dim": data.get("code_dim"), |
| "horizon": data.get("horizon"), |
| "action_dim": data.get("action_dim"), |
| "num_examples": data.get("num_examples"), |
| "num_groups": data.get("num_groups"), |
| "num_train_examples": data.get("num_train_examples"), |
| "num_val_examples": data.get("num_val_examples"), |
| "num_train_positive": data.get("num_train_positive"), |
| "num_val_groups_with_positive": data.get("num_val_groups_with_positive"), |
| "label_counts": data.get("label_counts", {}), |
| "train_positive_by_task": data.get("train_positive_by_task", {}), |
| "overall": data.get("overall", {}), |
| "per_task": data.get("per_task", {}), |
| } |
|
|
|
|
| def _load_generator_v2_cvae_support_proxy() -> dict[str, Any]: |
| source = GENERATOR_V2_CVAE_EVAL |
| sweep_best: dict[str, Any] | None = None |
| if GENERATOR_V2_CVAE_SWEEP.exists(): |
| sweep = _load_json(GENERATOR_V2_CVAE_SWEEP) |
| if sweep.get("best", {}).get("path"): |
| candidate = Path(str(sweep["best"]["path"])) |
| if candidate.exists(): |
| source = candidate |
| sweep_best = sweep["best"] |
| if not source.exists(): |
| return {"missing": True, "source": str(source)} |
| data = _load_json(source) |
| return { |
| "missing": False, |
| "source": str(source), |
| "sweep_summary": str(GENERATOR_V2_CVAE_SWEEP) |
| if GENERATOR_V2_CVAE_SWEEP.exists() |
| else None, |
| "sweep_best": sweep_best, |
| "report_type": data.get("report_type"), |
| "metric_scope": data.get("metric_scope"), |
| "note": data.get("note"), |
| "targets": data.get("targets"), |
| "config": data.get("config", {}), |
| "num_examples": data.get("num_examples"), |
| "num_groups": data.get("num_groups"), |
| "num_train_examples": data.get("num_train_examples"), |
| "num_val_examples": data.get("num_val_examples"), |
| "num_train_positive": data.get("num_train_positive"), |
| "num_val_groups_with_positive": data.get("num_val_groups_with_positive"), |
| "label_counts": data.get("label_counts", {}), |
| "train_positive_by_task": data.get("train_positive_by_task", {}), |
| "overall": data.get("overall", {}), |
| "per_task": data.get("per_task", {}), |
| "train_history": data.get("train_history", []), |
| } |
|
|
|
|
| def _load_generator_v2_spline_cvae_support_proxy() -> dict[str, Any]: |
| source = GENERATOR_V2_SPLINE_CVAE_EVAL |
| sweep_best: dict[str, Any] | None = None |
| if GENERATOR_V2_SPLINE_CVAE_SWEEP.exists(): |
| sweep = _load_json(GENERATOR_V2_SPLINE_CVAE_SWEEP) |
| if sweep.get("best", {}).get("path"): |
| candidate = Path(str(sweep["best"]["path"])) |
| if candidate.exists(): |
| source = candidate |
| sweep_best = sweep["best"] |
| if not source.exists(): |
| return {"missing": True, "source": str(source)} |
| data = _load_json(source) |
| return { |
| "missing": False, |
| "source": str(source), |
| "sweep_summary": str(GENERATOR_V2_SPLINE_CVAE_SWEEP) |
| if GENERATOR_V2_SPLINE_CVAE_SWEEP.exists() |
| else None, |
| "sweep_best": sweep_best, |
| "report_type": data.get("report_type"), |
| "metric_scope": data.get("metric_scope"), |
| "note": data.get("note"), |
| "targets": data.get("targets"), |
| "config": data.get("config", {}), |
| "code_dim": data.get("code_dim"), |
| "horizon": data.get("horizon"), |
| "action_dim": data.get("action_dim"), |
| "num_examples": data.get("num_examples"), |
| "num_groups": data.get("num_groups"), |
| "num_train_examples": data.get("num_train_examples"), |
| "num_val_examples": data.get("num_val_examples"), |
| "num_train_positive": data.get("num_train_positive"), |
| "num_val_groups_with_positive": data.get("num_val_groups_with_positive"), |
| "label_counts": data.get("label_counts", {}), |
| "train_positive_by_task": data.get("train_positive_by_task", {}), |
| "overall": data.get("overall", {}), |
| "per_task": data.get("per_task", {}), |
| "train_history": data.get("train_history", []), |
| } |
|
|
|
|
| def _load_generator_v2_spline_flow_support_proxy() -> dict[str, Any]: |
| source = GENERATOR_V2_SPLINE_FLOW_EVAL |
| sweep_best: dict[str, Any] | None = None |
| if GENERATOR_V2_SPLINE_FLOW_SWEEP.exists(): |
| sweep = _load_json(GENERATOR_V2_SPLINE_FLOW_SWEEP) |
| if sweep.get("best", {}).get("path"): |
| candidate = Path(str(sweep["best"]["path"])) |
| if candidate.exists(): |
| source = candidate |
| sweep_best = sweep["best"] |
| if not source.exists(): |
| return {"missing": True, "source": str(source)} |
| data = _load_json(source) |
| return { |
| "missing": False, |
| "source": str(source), |
| "sweep_summary": str(GENERATOR_V2_SPLINE_FLOW_SWEEP) |
| if GENERATOR_V2_SPLINE_FLOW_SWEEP.exists() |
| else None, |
| "sweep_best": sweep_best, |
| "report_type": data.get("report_type"), |
| "metric_scope": data.get("metric_scope"), |
| "note": data.get("note"), |
| "targets": data.get("targets"), |
| "config": data.get("config", {}), |
| "code_dim": data.get("code_dim"), |
| "horizon": data.get("horizon"), |
| "action_dim": data.get("action_dim"), |
| "num_examples": data.get("num_examples"), |
| "num_groups": data.get("num_groups"), |
| "num_train_examples": data.get("num_train_examples"), |
| "num_val_examples": data.get("num_val_examples"), |
| "num_train_positive": data.get("num_train_positive"), |
| "num_val_groups_with_positive": data.get("num_val_groups_with_positive"), |
| "overall": data.get("overall", {}), |
| "per_task": data.get("per_task", {}), |
| "train_history": data.get("train_history", []), |
| } |
|
|
|
|
| def _load_generator_v2_guided_spline_flow_support_proxy() -> dict[str, Any]: |
| source = GENERATOR_V2_GUIDED_SPLINE_FLOW_EVAL |
| sweep_best: dict[str, Any] | None = None |
| if GENERATOR_V2_GUIDED_SPLINE_FLOW_SWEEP.exists(): |
| sweep = _load_json(GENERATOR_V2_GUIDED_SPLINE_FLOW_SWEEP) |
| if sweep.get("best", {}).get("path"): |
| candidate = Path(str(sweep["best"]["path"])) |
| if candidate.exists(): |
| source = candidate |
| sweep_best = sweep["best"] |
| if not source.exists(): |
| return {"missing": True, "source": str(source)} |
| data = _load_json(source) |
| return { |
| "missing": False, |
| "source": str(source), |
| "sweep_summary": str(GENERATOR_V2_GUIDED_SPLINE_FLOW_SWEEP) |
| if GENERATOR_V2_GUIDED_SPLINE_FLOW_SWEEP.exists() |
| else None, |
| "sweep_best": sweep_best, |
| "report_type": data.get("report_type"), |
| "metric_scope": data.get("metric_scope"), |
| "note": data.get("note"), |
| "targets": data.get("targets"), |
| "config": data.get("config", {}), |
| "code_dim": data.get("code_dim"), |
| "horizon": data.get("horizon"), |
| "action_dim": data.get("action_dim"), |
| "num_examples": data.get("num_examples"), |
| "num_groups": data.get("num_groups"), |
| "num_train_examples": data.get("num_train_examples"), |
| "num_val_examples": data.get("num_val_examples"), |
| "num_train_positive": data.get("num_train_positive"), |
| "num_train_negative": data.get("num_train_negative"), |
| "num_val_groups_with_positive": data.get("num_val_groups_with_positive"), |
| "label_counts": data.get("label_counts", {}), |
| "train_positive_by_task": data.get("train_positive_by_task", {}), |
| "train_negative_by_task": data.get("train_negative_by_task", {}), |
| "overall": data.get("overall", {}), |
| "per_task": data.get("per_task", {}), |
| "train_history": data.get("train_history", []), |
| "utility_train_history": data.get("utility_train_history", []), |
| } |
|
|
|
|
| def _headline_metric_label(metric: str) -> str: |
| if metric == "mean_candidate_oracle_success_rate": |
| return "candidate-oracle" |
| if metric == "mean_success": |
| return "deployment" |
| return metric |
|
|
|
|
| def _attach_headline_fields(method: dict[str, Any], spec: MethodSpec) -> None: |
| metric = spec.headline_metric |
| method["headline_metric"] = metric |
| method["headline_metric_label"] = _headline_metric_label(metric) |
| if metric == "mean_candidate_oracle_success_rate": |
| method["headline_success"] = method.get( |
| "mean_candidate_oracle_success_rate", method.get("mean_success") |
| ) |
| method["headline_std_success"] = method.get( |
| "std_candidate_oracle_success_rate", method.get("std_success", 0.0) |
| ) |
| method["headline_ci95_success"] = method.get( |
| "ci95_candidate_oracle_success_rate", method.get("ci95_success", 0.0) |
| ) |
| return |
| method["headline_success"] = method.get("mean_success") |
| method["headline_std_success"] = method.get("std_success", 0.0) |
| method["headline_ci95_success"] = method.get("ci95_success", 0.0) |
|
|
|
|
| def _best_clean_key(methods: dict[str, dict[str, Any]]) -> str: |
| best_key = FALLBACK_BEST_CLEAN_KEY |
| best_success = float("-inf") |
| for spec in METHODS: |
| if spec.key in NON_DEPLOYMENT_KEYS: |
| continue |
| method = methods.get(spec.key, {}) |
| if method.get("missing"): |
| continue |
| success = method.get("mean_success") |
| if isinstance(success, (int, float)) and math.isfinite(float(success)): |
| success = float(success) |
| if success > best_success + 1.0e-12: |
| best_success = success |
| best_key = spec.key |
| return best_key |
|
|
|
|
| def _paired_delta( |
| methods: dict[str, dict[str, Any]], |
| left: str, |
| right: str, |
| ) -> dict[str, Any]: |
| left_values = methods[left].get("seed_success", {}) |
| right_values = methods[right].get("seed_success", {}) |
| seeds = sorted(set(left_values) & set(right_values)) |
| deltas = [float(left_values[seed]) - float(right_values[seed]) for seed in seeds] |
| return { |
| "left": left, |
| "right": right, |
| "seeds": seeds, |
| "mean_delta": _mean(deltas), |
| "std_delta": _sample_std(deltas), |
| "ci95_delta": _ci95(deltas), |
| "seed_deltas": {seed: delta for seed, delta in zip(seeds, deltas)}, |
| } |
|
|
|
|
| def _per_task_delta( |
| methods: dict[str, dict[str, Any]], |
| left: str, |
| right: str, |
| ) -> dict[str, float]: |
| left_tasks = methods[left].get("per_task_success", {}) |
| right_tasks = methods[right].get("per_task_success", {}) |
| return { |
| task: float(left_tasks[task]["mean_success"]) - float(right_tasks[task]["mean_success"]) |
| for task in sorted(set(left_tasks) & set(right_tasks)) |
| } |
|
|
|
|
| def _pct(value: float) -> str: |
| if value is None: |
| return "n/a" |
| value = float(value) |
| if math.isnan(value): |
| return "n/a" |
| return f"{value * 100:.2f}%" |
|
|
|
|
| def _pp(value: float) -> str: |
| if value is None: |
| return "n/a" |
| value = float(value) |
| if math.isnan(value): |
| return "n/a" |
| return f"{value * 100:+.2f} pp" |
|
|
|
|
| def _best_candidate_oracle( |
| methods: dict[str, dict[str, Any]] |
| ) -> tuple[str | None, dict[str, Any] | None]: |
| best_key: str | None = None |
| best_method: dict[str, Any] | None = None |
| best_success = float("-inf") |
| for key, method in methods.items(): |
| if method.get("missing") or not method.get("num_completed"): |
| continue |
| value = method.get("mean_candidate_oracle_success_rate") |
| if not isinstance(value, (int, float)): |
| continue |
| success = float(value) |
| if math.isfinite(success) and success > best_success: |
| best_success = success |
| best_key = key |
| best_method = method |
| return best_key, best_method |
|
|
|
|
| def _render_markdown(report: dict[str, Any]) -> str: |
| methods = report["methods"] |
| best_clean_key = report["best_clean_key"] |
| lines = [ |
| "# Paper Analysis", |
| "", |
| f"Generated: `{report['generated_utc']}`", |
| "", |
| "## Main Seed Statistics", |
| "", |
| "| key | method | n | headline metric | headline success | selected/deployed success | 95% CI | progress | action MSE | headline gain vs canonical h16 |", |
| "|---|---|---:|---|---:|---:|---:|---:|---:|---:|", |
| ] |
| baseline = methods["h16_policy_canonical"]["mean_success"] |
| for key in [spec.key for spec in METHODS]: |
| method = methods[key] |
| if method.get("missing"): |
| lines.append( |
| f"| {key} | {method['label']} | 0 | {method.get('headline_metric_label', 'deployment')} | missing | missing | missing | missing | missing | missing |" |
| ) |
| continue |
| headline_success = float( |
| method.get("headline_success", method.get("mean_success", float("nan"))) |
| ) |
| headline_std = float(method.get("headline_std_success", method.get("std_success", 0.0))) |
| headline_ci = float(method.get("headline_ci95_success", method.get("ci95_success", 0.0))) |
| lines.append( |
| "| {key} | {label} | {n} | {metric} | {headline} +/- {std} | {selected} | {ci} | {progress} | {mse:.3f} | {gain} |".format( |
| key=key, |
| label=method["label"], |
| n=method["num_completed"], |
| metric=method.get("headline_metric_label", "deployment"), |
| headline=_pct(headline_success), |
| std=f"{headline_std * 100:.2f}", |
| selected=_pct(method["mean_success"]), |
| ci=f"+/- {headline_ci * 100:.2f}", |
| progress=_pct(method["mean_progress"]), |
| mse=method["mean_action_mse_to_best"], |
| gain=_pp(headline_success - baseline), |
| ) |
| ) |
| lines.extend( |
| [ |
| "", |
| "## Paired Seed Deltas", |
| "", |
| "| comparison | seeds | mean delta | 95% CI | seed deltas |", |
| "|---|---:|---:|---:|---|", |
| ] |
| ) |
| for name, delta in report["paired_deltas"].items(): |
| seed_deltas = ", ".join( |
| f"{seed}:{value * 100:+.2f}" for seed, value in delta["seed_deltas"].items() |
| ) |
| lines.append( |
| f"| {name} | {len(delta['seeds'])} | {_pp(delta['mean_delta'])} | +/- {delta['ci95_delta'] * 100:.2f} | {seed_deltas} |" |
| ) |
| lines.extend( |
| [ |
| "", |
| "## Per-Task Mean Success", |
| "", |
| "| task | h16 policy | best clean | near-miss lattice | no-expert lattice | full lattice | clean-h16 delta | noexpert-clean gap |", |
| "|---|---:|---:|---:|---:|---:|---:|---:|", |
| ] |
| ) |
| tasks = sorted(methods["h16_policy_canonical"].get("per_task_success", {})) |
| for task in tasks: |
| h16 = methods["h16_policy_canonical"]["per_task_success"][task]["mean_success"] |
| clean = methods[best_clean_key]["per_task_success"][task]["mean_success"] |
| near = methods["same_state_near_miss"]["per_task_success"][task]["mean_success"] |
| noexpert = methods["same_state_no_expert"]["per_task_success"][task]["mean_success"] |
| full = methods["same_state_full"]["per_task_success"][task]["mean_success"] |
| lines.append( |
| f"| {task} | {_pct(h16)} | {_pct(clean)} | {_pct(near)} | {_pct(noexpert)} | {_pct(full)} | {_pp(clean - h16)} | {_pp(noexpert - clean)} |" |
| ) |
| gap = report["mechanism_gap"] |
| lines.extend( |
| [ |
| "", |
| "## Mechanism Gap", |
| "", |
| f"- Best clean residual transport improves over canonical h16 by {_pp(gap['best_clean_vs_h16'])}.", |
| f"- Same-state no-expert lattice improves over canonical h16 by {_pp(gap['same_state_no_expert_vs_h16'])}.", |
| f"- Remaining clean-to-same-state proposal gap is {_pp(gap['same_state_no_expert_vs_best_clean'])}.", |
| f"- Full lattice adds expert proposals and reaches {_pct(methods['same_state_full']['mean_success'])}, a {_pp(gap['same_state_full_vs_no_expert'])} gain over no-expert.", |
| ] |
| ) |
| decomposition = report["causal_action_decomposition"] |
| targets = report["causal_action_targets"] |
| lines.extend( |
| [ |
| "", |
| "## Causal Action Regret Decomposition", |
| "", |
| "| base | selected clean | proposal oracle | same-state no-expert oracle | support gap | selector gap | gap closed |", |
| "|---:|---:|---:|---:|---:|---:|---:|", |
| ( |
| f"| {_pct(decomposition['base'])} | {_pct(decomposition['selected'])} | " |
| f"{_pct(decomposition['proposal_oracle'])} | " |
| f"{_pct(decomposition['same_state_oracle'])} | " |
| f"{_pp(decomposition['support_gap'])} | " |
| f"{_pp(decomposition['selector_gap'])} | " |
| f"{decomposition['closed_fraction_of_noexpert_gap'] * 100:.1f}% |" |
| ), |
| "", |
| ( |
| "- Current clean policy closes " |
| f"{decomposition['closed_fraction_of_noexpert_gap'] * 100:.1f}% " |
| "of the h16-to-same-state-no-expert gap." |
| ), |
| ( |
| "- Closing 65--75% of that gap implies selected success targets of " |
| f"{_pct(targets['selected_success_for_65pct_gap_closure'])}--" |
| f"{_pct(targets['selected_success_for_75pct_gap_closure'])}." |
| ), |
| ] |
| ) |
| oracle_key, oracle = _best_candidate_oracle(methods) |
| if oracle and oracle.get("num_completed"): |
| branch_success = oracle.get("mean_candidate_oracle_branch_success_rates") or [] |
| branch_gains = ( |
| oracle.get("mean_candidate_oracle_branch_score_gains_over_selected") |
| or [] |
| ) |
| lines.extend( |
| [ |
| "", |
| "## Candidate-Oracle Diagnostic", |
| "", |
| ( |
| f"- Best diagnostic prefix (`{oracle_key}`) reaches " |
| f"{_pct(oracle.get('mean_candidate_oracle_success_rate', 0.0))} " |
| f"with mean progress {_pct(oracle.get('mean_candidate_oracle_progress', 0.0))}; " |
| "this is diagnostic-only because it uses measured rollout outcomes " |
| "after generating candidates." |
| ), |
| ( |
| "- Mean oracle-prefix score gain over the selected branch is " |
| f"{oracle.get('mean_candidate_oracle_score_gain_over_selected', 0.0):+.3f}, " |
| "which isolates ranking/abstention headroom inside the clean proposal set." |
| ), |
| ( |
| "- CAR-to-proposal-oracle from raw prefix traces is " |
| f"{oracle.get('mean_candidate_oracle_car_to_proposal_oracle', 0.0):+.3f}; " |
| f"PTR@K is {_pct(oracle.get('mean_candidate_oracle_ptr_at_k'))} " |
| f"over rows with base trace coverage {_pct(oracle.get('candidate_oracle_base_trace_coverage'))}." |
| ), |
| ( |
| "- Mean unique candidates in the prefix: " |
| f"{oracle.get('mean_candidate_oracle_unique_count', 0.0):.2f}." |
| ), |
| ( |
| "- Candidate-oracle best type counts: " |
| f"{oracle.get('candidate_oracle_type_counts', {})}." |
| ), |
| ] |
| ) |
| if oracle.get("mean_candidate_oracle_best_branch_rank") is not None: |
| lines.extend( |
| [ |
| ( |
| "- Mean best branch rank in the field-ordered prefix: " |
| f"{oracle['mean_candidate_oracle_best_branch_rank']:.2f}; " |
| "rank histogram " |
| f"{oracle.get('candidate_oracle_best_branch_rank_counts', {})}." |
| ), |
| ( |
| "- Branch success by prefix rank: " |
| + ", ".join(_pct(value) for value in branch_success) |
| + "." |
| ), |
| ( |
| "- Branch score gain by prefix rank: " |
| + ", ".join(f"{value:+.3f}" for value in branch_gains) |
| + "." |
| ), |
| ] |
| ) |
| support_proxy = report.get("generator_v2_support_proxy", {}) |
| local_atlas_proxy = report.get("generator_v2_local_atlas_support_proxy", {}) |
| chart_synthesis_proxy = report.get("generator_v2_chart_synthesis_support_proxy", {}) |
| cvae_proxy = report.get("generator_v2_cvae_support_proxy", {}) |
| spline_proxy = report.get("generator_v2_spline_cvae_support_proxy", {}) |
| flow_proxy = report.get("generator_v2_spline_flow_support_proxy", {}) |
| guided_flow_proxy = report.get("generator_v2_guided_spline_flow_support_proxy", {}) |
| lines.extend( |
| [ |
| "", |
| "## Generator V2 Support Proxy", |
| "", |
| ] |
| ) |
| if support_proxy.get("missing", True): |
| lines.append( |
| f"- Pending: `{support_proxy.get('source', GENERATOR_V2_MEMORY_EVAL)}` has not been generated yet." |
| ) |
| else: |
| overall = support_proxy.get("overall", {}) |
| lines.extend( |
| [ |
| ( |
| f"- Artifact `{support_proxy['source']}` evaluates train-only positive " |
| f"tangent memory proposals on {support_proxy.get('num_eval_groups_with_positive', 0)} " |
| "heldout groups with positive support." |
| ), |
| ( |
| f"- Train positives by task: {support_proxy.get('train_positive_by_task', {})}; " |
| f"prototype count by task: {support_proxy.get('proposal_count_by_task', {})}." |
| ), |
| "", |
| "| metric | K1 | K2 | K4 | K8 | K16 |", |
| "|---|---:|---:|---:|---:|---:|", |
| _support_proxy_row(overall, "PTR proxy @ RMS<=0.10", "ptr_proxy", "0p1"), |
| _support_proxy_row(overall, "PTR proxy @ RMS<=0.20", "ptr_proxy", "0p2"), |
| _support_proxy_row( |
| overall, |
| "Negative-near @ RMS<=0.20", |
| "negative_near", |
| "0p2", |
| ), |
| _support_proxy_row( |
| overall, |
| "Positive closer than negative", |
| "positive_closer_than_negative_rate", |
| None, |
| ), |
| "", |
| "| task | eval groups | K8 PTR@0.20 | K16 PTR@0.20 | K16 pos<neg |", |
| "|---|---:|---:|---:|---:|", |
| ] |
| ) |
| for task_id, values in sorted(support_proxy.get("per_task", {}).items()): |
| lines.append( |
| "| {task} | {groups} | {k8} | {k16} | {closer} |".format( |
| task=task_id, |
| groups=int(values.get("num_groups", 0)), |
| k8=_pct(values.get("ptr_proxy_at_8_thr_0p2")), |
| k16=_pct(values.get("ptr_proxy_at_16_thr_0p2")), |
| closer=_pct(values.get("positive_closer_than_negative_rate_at_16")), |
| ) |
| ) |
| lines.extend(["", "### Trainable CVAE Diagnostic", ""]) |
| if cvae_proxy.get("missing", True): |
| lines.append( |
| f"- Pending: `{cvae_proxy.get('source', GENERATOR_V2_CVAE_EVAL)}` has not been generated yet." |
| ) |
| else: |
| cvae_history = cvae_proxy.get("train_history", []) |
| last_epoch = cvae_history[-1] if cvae_history else {} |
| lines.extend( |
| [ |
| ( |
| f"- Artifact `{cvae_proxy['source']}` samples from a train-only " |
| f"positive-tangent CVAE trained on {cvae_proxy.get('num_train_positive', 0)} " |
| "positive targets." |
| ), |
| ( |
| f"- Final training snapshot: epoch {int(last_epoch.get('epoch', 0))}, " |
| f"loss {float(last_epoch.get('loss', float('nan'))):.4f}, " |
| f"reconstruction MSE {float(last_epoch.get('reconstruction_mse', float('nan'))):.4f}, " |
| f"KL {float(last_epoch.get('kl', float('nan'))):.4f}." |
| ), |
| "", |
| "| generator | heldout groups | K16 PTR@0.20 | K16 PTR@0.40 | K16 neg@0.20 | K16 pos<neg |", |
| "|---|---:|---:|---:|---:|---:|", |
| _generator_support_compare_row("memory", support_proxy), |
| _generator_support_compare_row("local-atlas", local_atlas_proxy), |
| _generator_support_compare_row("chart-synthesis", chart_synthesis_proxy), |
| _generator_support_compare_row("raw-cvae", cvae_proxy), |
| _generator_support_compare_row("spline-cvae", spline_proxy), |
| _generator_support_compare_row("spline-flow", flow_proxy), |
| _generator_support_compare_row("guided-spline-flow", guided_flow_proxy), |
| ] |
| ) |
| if not spline_proxy.get("missing", True): |
| lines.append( |
| f"- Spline-CVAE source `{spline_proxy['source']}` uses " |
| f"{spline_proxy.get('code_dim')}D keyframe codes decoded to " |
| f"{spline_proxy.get('horizon')}x{spline_proxy.get('action_dim')} chunks." |
| ) |
| lines.extend( |
| [ |
| "", |
| "## Selection Histograms", |
| "", |
| ] |
| ) |
| for key in ["same_state_near_miss", "same_state_no_expert", "same_state_policy_baseline", "same_state_full", best_clean_key]: |
| counts = methods[key].get("selected_candidate_type_counts", {}) |
| if counts: |
| total = sum(int(value) for value in counts.values()) |
| summary = ", ".join( |
| f"{name}={count} ({count / total * 100:.1f}%)" |
| for name, count in sorted(counts.items(), key=lambda item: (-int(item[1]), item[0])) |
| ) |
| else: |
| summary = "not recorded" |
| lines.append(f"- `{key}`: {summary}") |
| scale_counts = methods[best_clean_key].get("selected_residual_scale_counts", {}) |
| if scale_counts: |
| lines.append(f"- `{best_clean_key}` residual scale counts: {scale_counts}") |
| lines.extend( |
| [ |
| "", |
| "## Selected-Type Outcomes", |
| "", |
| "These rows are measured from raw rollout rows. In residual retrieval, `policy_residual` is the zero-residual action, i.e. abstaining to the current policy mean.", |
| "", |
| "| method | selected type | count | success | progress |", |
| "|---|---|---:|---:|---:|", |
| ] |
| ) |
| for key in [ |
| "best_clean_residual_k2", |
| "residual_k4_consensus", |
| "residual_k4_kernel_consensus", |
| "residual_k4_kernel_consensus_noopbonus003", |
| "residual_k4_kernel_consensus_s035_noopbonus003", |
| "residual_k4_kernel_consensus_s045_noopbonus003", |
| "residual_k4_fieldsoftmax_grid", |
| "residual_k4_fieldsoftmax_grid_noopbonus003", |
| "residual_k4_fieldsoftmax_grid_margin010_noopbonus003", |
| "residual_k4_fieldsoftmax_grid_margin005_noopbonus003", |
| "residual_k4_fieldsoftmax_grid_margin000_noopbonus003", |
| "residual_k8_fieldsoftmax_grid_noopbonus003", |
| "residual_k4_consensus_noopbonus003", |
| "residual_k4_consensus_noopbonus001", |
| "residual_k4_consensus_noopbonus002", |
| "residual_k4_consensus_noopbonus0025", |
| "residual_k4_consensus_noopbonus0035", |
| "residual_k4_consensus_wgbonus003", |
| "residual_k4_consensus_noop003_wg002", |
| "residual_k4_consensus_noop003_wg004", |
| "residual_k4_consensus_noop0025_wg002", |
| "residual_k4_consensus_noopbonus005", |
| "residual_k4_consensus_noopbonus008", |
| "same_state_no_expert", |
| "same_state_policy_baseline", |
| ]: |
| for candidate_type, values in methods[key].get("selected_type_outcomes", {}).items(): |
| lines.append( |
| f"| {key} | {candidate_type} | {int(values['count'])} | {_pct(values['success_rate'])} | {_pct(values['mean_progress'])} |" |
| ) |
| return "\n".join(lines) + "\n" |
|
|
|
|
| def build_report() -> dict[str, Any]: |
| methods = _load_methods() |
| best_clean_key = _best_clean_key(methods) |
| oracle_key, oracle_method = _best_candidate_oracle(methods) |
| proposal_oracle_success = ( |
| float(oracle_method["mean_candidate_oracle_success_rate"]) |
| if oracle_method is not None |
| and oracle_method.get("mean_candidate_oracle_success_rate") is not None |
| else float("nan") |
| ) |
| paired_deltas = { |
| "best_clean - canonical_h16": _paired_delta(methods, best_clean_key, "h16_policy_canonical"), |
| "best_clean - direct_same_ckpt": _paired_delta(methods, best_clean_key, "near_miss_policy_bc5"), |
| "no_expert_lattice - canonical_h16": _paired_delta(methods, "same_state_no_expert", "h16_policy_canonical"), |
| "full_lattice - no_expert_lattice": _paired_delta(methods, "same_state_full", "same_state_no_expert"), |
| "policy_candidate_lattice - no_expert_lattice": _paired_delta( |
| methods, |
| "same_state_policy_baseline", |
| "same_state_no_expert", |
| ), |
| } |
| mechanism_gap = { |
| "best_clean_vs_h16": methods[best_clean_key]["mean_success"] |
| - methods["h16_policy_canonical"]["mean_success"], |
| "best_clean_vs_direct_same_ckpt": methods[best_clean_key]["mean_success"] |
| - methods["near_miss_policy_bc5"]["mean_success"], |
| "same_state_no_expert_vs_h16": methods["same_state_no_expert"]["mean_success"] |
| - methods["h16_policy_canonical"]["mean_success"], |
| "same_state_no_expert_vs_best_clean": methods["same_state_no_expert"]["mean_success"] |
| - methods[best_clean_key]["mean_success"], |
| "same_state_full_vs_no_expert": methods["same_state_full"]["mean_success"] |
| - methods["same_state_no_expert"]["mean_success"], |
| } |
| decomposition = causal_action_decomposition( |
| base=methods["h16_policy_canonical"]["mean_success"], |
| selected=methods[best_clean_key]["mean_success"], |
| proposal_oracle=proposal_oracle_success, |
| same_state_oracle=methods["same_state_no_expert"]["mean_success"], |
| full_oracle=methods["same_state_full"]["mean_success"], |
| ) |
| return { |
| "generated_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"), |
| "methods": methods, |
| "paired_deltas": paired_deltas, |
| "per_task_deltas": { |
| "best_clean_vs_h16": _per_task_delta(methods, best_clean_key, "h16_policy_canonical"), |
| "no_expert_vs_best_clean": _per_task_delta(methods, "same_state_no_expert", best_clean_key), |
| }, |
| "mechanism_gap": mechanism_gap, |
| "causal_action_decomposition": decomposition.to_dict(), |
| "causal_action_targets": { |
| "selected_success_for_65pct_gap_closure": decomposition.target_for_gap_closure(0.65), |
| "selected_success_for_75pct_gap_closure": decomposition.target_for_gap_closure(0.75), |
| "best_paper_target_range": [0.47, 0.52], |
| }, |
| "best_candidate_oracle_key": oracle_key, |
| "best_clean_key": best_clean_key, |
| "generator_v2_support_proxy": _load_generator_v2_support_proxy(), |
| "generator_v2_local_atlas_support_proxy": ( |
| _load_generator_v2_local_atlas_support_proxy() |
| ), |
| "generator_v2_chart_synthesis_support_proxy": ( |
| _load_generator_v2_chart_synthesis_support_proxy() |
| ), |
| "generator_v2_cvae_support_proxy": _load_generator_v2_cvae_support_proxy(), |
| "generator_v2_spline_cvae_support_proxy": ( |
| _load_generator_v2_spline_cvae_support_proxy() |
| ), |
| "generator_v2_spline_flow_support_proxy": ( |
| _load_generator_v2_spline_flow_support_proxy() |
| ), |
| "generator_v2_guided_spline_flow_support_proxy": ( |
| _load_generator_v2_guided_spline_flow_support_proxy() |
| ), |
| } |
|
|
|
|
| def _generator_support_compare_row(name: str, proxy: dict[str, Any]) -> str: |
| if proxy.get("missing", True): |
| return f"| {name} | missing | missing | missing | missing | missing |" |
| overall = proxy.get("overall", {}) |
| return ( |
| f"| {name} | {int(proxy.get('num_val_groups_with_positive') or proxy.get('num_eval_groups_with_positive') or 0)} | " |
| f"{_pct(overall.get('ptr_proxy_at_16_thr_0p2'))} | " |
| f"{_pct(overall.get('ptr_proxy_at_16_thr_0p4'))} | " |
| f"{_pct(overall.get('negative_near_at_16_thr_0p2'))} | " |
| f"{_pct(overall.get('positive_closer_than_negative_rate_at_16'))} |" |
| ) |
|
|
|
|
| def _support_proxy_row( |
| values: dict[str, Any], |
| label: str, |
| metric_prefix: str, |
| threshold_key: str | None, |
| ) -> str: |
| cells = [] |
| for k in (1, 2, 4, 8, 16): |
| if threshold_key is None: |
| key = f"{metric_prefix}_at_{k}" |
| else: |
| key = f"{metric_prefix}_at_{k}_thr_{threshold_key}" |
| cells.append(_pct(values.get(key))) |
| return f"| {label} | " + " | ".join(cells) + " |" |
|
|
|
|
| def _latex_pct(value: float | None) -> str: |
| if value is None: |
| return "--" |
| value = float(value) |
| if math.isnan(value): |
| return "--" |
| return f"{value * 100:.2f}" |
|
|
|
|
| def _latex_pp(value: float | None) -> str: |
| if value is None: |
| return "--" |
| value = float(value) |
| if math.isnan(value): |
| return "--" |
| return f"{value * 100:.2f}" |
|
|
|
|
| def _render_car_decomposition_table(report: dict[str, Any]) -> str: |
| decomposition = report["causal_action_decomposition"] |
| targets = report["causal_action_targets"] |
| closed = decomposition["closed_fraction_of_noexpert_gap"] |
| closed_text = "--" if math.isnan(float(closed)) else f"{float(closed) * 100:.1f}" |
| return ( |
| "\\begin{table}[t]\n" |
| "\\centering\n" |
| "\\caption{Causal Action Regret decomposition on the current six-task " |
| "diagnostic. Support is the proposal-generation gap to the hidden " |
| "same-state no-expert oracle; selector is the clean proposal-oracle " |
| "headroom left by the deployed selector.}\n" |
| "\\label{tab:car-decomposition}\n" |
| "\\small\n" |
| "\\begin{tabular}{@{}lrrrrrr@{}}\n" |
| "\\toprule\n" |
| "Method & Base & Prop. oracle & Selected & State oracle & " |
| "Support & Selector \\\\\n" |
| "\\midrule\n" |
| "Current CIL-Atlas V0 & " |
| f"{_latex_pct(decomposition['base'])} & " |
| f"{_latex_pct(decomposition['proposal_oracle'])} & " |
| f"{_latex_pct(decomposition['selected'])} & " |
| f"{_latex_pct(decomposition['same_state_oracle'])} & " |
| f"{_latex_pp(decomposition['support_gap'])} & " |
| f"{_latex_pp(decomposition['selector_gap'])} \\\\\n" |
| "\\bottomrule\n" |
| "\\end{tabular}\n" |
| "\\vspace{2pt}\n" |
| "\\footnotesize Clean gain is " |
| f"{_latex_pp(decomposition['clean_gain'])} points; gap closed is " |
| f"{closed_text}\\%; 65--75\\% closure targets are " |
| f"{_latex_pct(targets['selected_success_for_65pct_gap_closure'])}--" |
| f"{_latex_pct(targets['selected_success_for_75pct_gap_closure'])}.\n" |
| "\\end{table}\n" |
| ) |
|
|
|
|
| def main() -> int: |
| RESULTS_DIR.mkdir(parents=True, exist_ok=True) |
| report = build_report() |
| OUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") |
| OUT_MD.write_text(_render_markdown(report), encoding="utf-8") |
| LATEX_TABLES_DIR.mkdir(parents=True, exist_ok=True) |
| OUT_CAR_TABLE.write_text(_render_car_decomposition_table(report), encoding="utf-8") |
| print(f"Wrote {OUT_JSON}") |
| print(f"Wrote {OUT_MD}") |
| print(f"Wrote {OUT_CAR_TABLE}") |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|