| |
| from __future__ import annotations |
|
|
| import json |
| import math |
| from collections import Counter |
| from dataclasses import dataclass |
| from datetime import datetime, timezone |
| from pathlib import Path |
| from typing import Any |
|
|
|
|
| RESULTS_DIR = Path("results") |
| OUT_JSON = RESULTS_DIR / "paper_analysis.json" |
| OUT_MD = RESULTS_DIR / "paper_analysis.md" |
| CANONICAL_H16_ROLLOUT = Path("/scratch/knguy52/dovla/experiments/dovla_h16_rollout_runs") |
| FALLBACK_BEST_CLEAN_KEY = "residual_k4_consensus_grid035040045_noopbonus003" |
| NON_DEPLOYMENT_KEYS = { |
| "same_state_near_miss", |
| "same_state_no_expert", |
| "same_state_policy_baseline", |
| "same_state_full", |
| } |
|
|
|
|
| @dataclass(frozen=True) |
| class MethodSpec: |
| key: str |
| label: str |
| summary_path: str | None = None |
| raw_rollout_glob: str | None = None |
| summary_mode: str = "standard" |
|
|
|
|
| METHODS = [ |
| MethodSpec( |
| key="h16_policy_canonical", |
| label="Direct h=16 policy, canonical rollout", |
| raw_rollout_glob=str(CANONICAL_H16_ROLLOUT / "seed_*/online_rollout.json"), |
| ), |
| MethodSpec( |
| key="gaussian_field", |
| label="Gaussian field search", |
| summary_path="h16_field_sweep_summary.json", |
| summary_mode="field_sweep_best", |
| ), |
| MethodSpec( |
| key="near_miss_policy_bc5", |
| label="Near-miss proposal policy, direct", |
| summary_path="h16_policy_ckpt_near_miss_policy_bc5_summary.json", |
| ), |
| MethodSpec( |
| key="best_clean_residual_k2", |
| label="K2 residual transport, safe + margin 0.20", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "knn2_scale0p40_safe_types_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_taskrelative_k2", |
| label="K2 task-relative residual transport, safe + margin 0.20", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "taskrelative_knn2_scale0p40_safe_types_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus", |
| label="K4 mean-by-type tangent consensus", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_kernel_consensus", |
| label="K4 kernel-weighted tangent consensus", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_kernel_mean_by_type_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_kernel_consensus_noopbonus003", |
| label="K4 kernel-weighted tangent consensus, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_kernel_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_kernel_consensus_s035_noopbonus003", |
| label="K4 kernel-weighted tangent consensus, scale 0.35, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s035_safe_margin0p20_kernel_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_kernel_consensus_s045_noopbonus003", |
| label="K4 kernel-weighted tangent consensus, scale 0.45, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s045_safe_margin0p20_kernel_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_fieldsoftmax_grid", |
| label="K4 field-softmax tangent transport, scales 0.35/0.40/0.45", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_fieldsoftmax_grid_safe_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_fieldsoftmax_grid_noopbonus003", |
| label="K4 field-softmax tangent transport, scales 0.35/0.40/0.45, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_fieldsoftmax_grid_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_fieldsoftmax_grid_margin010_noopbonus003", |
| label="K4 field-softmax tangent transport, margin 0.10, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_fieldsoftmax_grid_safe_margin0p10_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_fieldsoftmax_grid_margin005_noopbonus003", |
| label="K4 field-softmax tangent transport, margin 0.05, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_fieldsoftmax_grid_safe_margin0p05_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_fieldsoftmax_grid_margin000_noopbonus003", |
| label="K4 field-softmax tangent transport, margin 0.00, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_fieldsoftmax_grid_safe_margin0p00_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k8_fieldsoftmax_grid_noopbonus003", |
| label="K8 field-softmax tangent transport, scales 0.35/0.40/0.45, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k8_fieldsoftmax_grid_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus003", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus003_srcprog025", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.03, source progress >= 0.25", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p03_srcprog0p25_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_margin015_noopbonus003", |
| label="K4 mean-by-type tangent consensus, margin 0.15, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p15_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_margin025_noopbonus003", |
| label="K4 mean-by-type tangent consensus, margin 0.25, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p25_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_margin015_srcscorebonus002", |
| label="K4 mean-by-type tangent consensus, margin 0.15, source-score bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p15_mean_by_type_srcscorebonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_margin025_srcscorebonus002", |
| label="K4 mean-by-type tangent consensus, margin 0.25, source-score bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p25_mean_by_type_srcscorebonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_srcscorebonus002", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, source-score bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_srcscorebonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_srcadvbonus002", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, source-advantage bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_srcadvbonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_srcadvbonus005", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, source-advantage bonus 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_srcadvbonus0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_srcadvbonus002", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, source-advantage bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_srcadvbonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_srcadvgate000", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, source-advantage gate >= 0.0", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_srcadvgate0p0_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_srcadvgate000", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, source-advantage gate >= 0.0", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_srcadvgate0p0_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_typesuccessbonus002", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, train family-success bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_typesuccessbonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_typesuccessbonus003", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, train family-success bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_typesuccessbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_typesuccessbonus005", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, train family-success bonus 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_typesuccessbonus0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_typesuccessbonus002", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, train family-success bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_typesuccessbonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_consensus005", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, consensus penalty 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_consensus0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_consensus002", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, consensus penalty 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_consensus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_consensus005", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, consensus penalty 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_consensus0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_consensus010", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, consensus penalty 0.10", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_consensus0p10_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_compose_grid035040045", |
| label="K4 composed type-consensus tangents, scales 0.35/0.40/0.45", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_compose_grid035040045_safe_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_compose_grid035040045_noopbonus003", |
| label="K4 composed type-consensus tangents, scales 0.35/0.40/0.45, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_compose_grid035040045_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_grid035040045", |
| label="K4 composed type-consensus tangents, masked, scales 0.35/0.40/0.45", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_grid035040045_safe_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_grid035040045_noopbonus003", |
| label="K4 composed type-consensus tangents, masked, scales 0.35/0.40/0.45, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_grid035040045_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045", |
| label="K4 composed type-consensus tangents, masked, drop near-miss+no-op composite", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003", |
| label="K4 composed type-consensus tangents, masked, drop near-miss+no-op composite", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_margin010_noopbonus003", |
| label="K4 compatible tangents, margin 0.10, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p10_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8", |
| label="K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 diagnostic", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_oraclek8_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace", |
| label="K4 compatible tangents, no-op bonus 0.03, unique candidate-oracle prefix K=8 branch trace", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_oraclek8trace_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger002", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_stacknowg", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, no wrong-gripper component on Stack", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_stacknowg_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0005", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.005", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p005_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger0015", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.015", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p015_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_scale035", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, scale-gated 0.35", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_scale035_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001_scales035040", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01, scale-gated 0.35/0.40", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_scales035040_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger003", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmwgchallenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgmargin003_challenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger 0.01, wrong-gripper margin 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmwgmargin0p03_challenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgmargin005_challenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss challenger 0.01, wrong-gripper margin 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmwgmargin0p05_challenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmwgchallenger001_pickpull", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss/wrong-gripper challenger gate 0.01 on Pick/Pull only", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmwgchallenger0p01_pickpull_norm_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_challenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper challenger on Pick/Pull, gate 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_challenger0p01_norm_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin003_challenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper Pick/Pull margin 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_wgmargin0p03_challenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpull_wgmargin005_challenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper Pick/Pull margin 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpull_wgmargin0p05_challenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_wgpickpullstack_challenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss global + wrong-gripper challenger on Pick/Pull/Stack", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_wgpickpullstack_challenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmglobal_noopwgcontact_challenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, near-miss global + no-op/wrong-gripper contact challenger", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmglobal_noopwgcontact_challenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k6_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001", |
| label="K6 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k6_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k8_composemasked_dropnmnoop_grid035040045_noopbonus003_nmchallenger001", |
| label="K8 compatible tangents, no-op bonus 0.03, near-miss challenger gate 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k8_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmchallenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_typesuccessbonus002_nmchallenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, train family-success bonus 0.02, near-miss challenger gate 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_typesuccessbonus0p02_nmchallenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_typesuccessbonus005_nmchallenger001", |
| label="K4 compatible tangents, no-op bonus 0.03, train family-success bonus 0.05, near-miss challenger gate 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_typesuccessbonus0p05_nmchallenger0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_types6_prepend_margin000", |
| label="Typed proposal lattice head, six families, policy-prepended margin 0.00", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_typedprop_p2_bestpt_" |
| "proposal_lattice_types6_prepend_margin0p00_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_types6_prepend_margin005", |
| label="Typed proposal lattice head, six families, policy-prepended margin 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_typedprop_p2_bestpt_" |
| "proposal_lattice_types6_prepend_margin0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_types4safe_prepend_margin005", |
| label="Typed proposal lattice head, safe four families, policy-prepended margin 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_typedprop_p2_bestpt_" |
| "proposal_lattice_types4safe_prepend_margin0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_types2sparse_prepend_margin005", |
| label="Typed proposal lattice head, sparse no-op/wrong-gripper families, policy-prepended margin 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_typedprop_p2_bestpt_" |
| "proposal_lattice_types2sparse_prepend_margin0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_nooponly_prepend_margin005", |
| label="Typed proposal lattice head, no-op family only, policy-prepended margin 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_typedprop_p2_bestpt_" |
| "proposal_lattice_nooponly_prepend_margin0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_types2sparse_bestpolicy_prepend_margin005", |
| label="Typed proposal lattice head, sparse families, best-policy checkpoint, policy-prepended margin 0.05", |
| summary_path="h16_bestpolicy_types2sparse_prepend_margin0p05_summary.json", |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_nooponly_bestpolicy_prepend_margin005", |
| label="Typed proposal lattice head, no-op family only, best-policy checkpoint, policy-prepended margin 0.05", |
| summary_path="h16_bestpolicy_nooponly_prepend_margin0p05_summary.json", |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_nooponly_sparsehead_prepend_margin005", |
| label="Typed proposal lattice head, no-op-only head retrain, policy-prepended margin 0.05", |
| summary_path="h16_typedprop_nooponly_sparsehead_prepend_margin0p05_summary.json", |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_nooponly_sparsehead_prepend_margin000", |
| label="Typed proposal lattice head, no-op-only head retrain, policy-prepended margin 0.00", |
| summary_path="h16_typedprop_nooponly_sparsehead_prepend_margin0p00_summary.json", |
| ), |
| MethodSpec( |
| key="typed_proposal_lattice_nooponly_sparsehead_prepend_margin010", |
| label="Typed proposal lattice head, no-op-only head retrain, policy-prepended margin 0.10", |
| summary_path="h16_typedprop_nooponly_sparsehead_prepend_margin0p10_summary.json", |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus001", |
| label="K4 composed compatible tangents, no-op bonus 0.03, singleton near-miss bonus 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmbonus0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_nmbonus002", |
| label="K4 composed compatible tangents, no-op bonus 0.03, singleton near-miss bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_nmbonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_srcscorebonus002", |
| label="K4 composed type-consensus tangents, masked, drop near-miss+no-op composite, source-score bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_srcscorebonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_srcscorebonus002", |
| label="K4 composed type-consensus tangents, masked, drop near-miss+no-op composite, no-op bonus 0.03, source-score bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_grid035040045_safe_margin0p20_noopbonus0p03_srcscorebonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_dropnmnoop_l2comp002_grid035040045_noopbonus003", |
| label="K4 composed type-consensus tangents, masked, drop near-miss+no-op composite, composite L2 penalty 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_dropnmnoop_l2comp002_grid035040045_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_compbonus_grid035040045_noopbonus003", |
| label="K4 composed type-consensus tangents, masked, component no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_compbonus_grid035040045_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_l2comp002_grid035040045_noopbonus003", |
| label="K4 composed type-consensus tangents, masked, composite L2 penalty 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_l2comp002_grid035040045_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_composemasked_l2comp005_grid035040045_noopbonus003", |
| label="K4 composed type-consensus tangents, masked, composite L2 penalty 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_composemasked_l2comp005_grid035040045_safe_margin0p20_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="repair_nearmiss_k4_grid025035050_margin020", |
| label="K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.20", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_repair_" |
| "nearmiss_k4_grid025035050_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="repair_nearmiss_k4_grid035050075_margin020", |
| label="K4 near-miss-to-expert repair tangent, scales 0.35/0.50/0.75, margin 0.20", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_repair_" |
| "nearmiss_k4_grid035050075_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="repair_nearmiss_k4_grid025035050_margin010", |
| label="K4 near-miss-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.10", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_repair_" |
| "nearmiss_k4_grid025035050_margin0p10_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="repair_safe_k4_grid025035050_margin020", |
| label="K4 safe-family-to-expert repair tangent, scales 0.25/0.35/0.50, margin 0.20", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_repair_" |
| "safe_k4_grid025035050_margin0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_l2penalty005", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, action L2 penalty 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_l2penalty0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_l2penalty010", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, action L2 penalty 0.10", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_l2penalty0p10_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035040045_noopbonus003_l2penalty020", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.40/0.45, no-op bonus 0.03, action L2 penalty 0.20", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035040045_safe_margin0p20_mean_by_type_noopbonus0p03_l2penalty0p20_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid040045050_noopbonus003", |
| label="K4 mean-by-type tangent consensus, scales 0.40/0.45/0.50, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid040045050_safe_margin0p20_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid040045050_srcscorebonus002", |
| label="K4 mean-by-type tangent consensus, scales 0.40/0.45/0.50, source-score bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid040045050_safe_margin0p20_mean_by_type_srcscorebonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_grid035045055_noopbonus003", |
| label="K4 mean-by-type tangent consensus, scales 0.35/0.45/0.55, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4_grid035045055_safe_margin0p20_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_nooponly_noopbonus003", |
| label="K4 mean-by-type tangent consensus, no-op-only residuals, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_nooponly_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_nooponly_srcscorebonus002", |
| label="K4 mean-by-type tangent consensus, no-op-only residuals, source-score bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_nooponly_srcscorebonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus003_srcprog050", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.03, source progress >= 0.50", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p03_srcprog0p50_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus003_srcprog075", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.03, source progress >= 0.75", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p03_srcprog0p75_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_srcprogbonus003", |
| label="K4 mean-by-type tangent consensus, source-progress bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_srcprogbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_srcprogbonus005", |
| label="K4 mean-by-type tangent consensus, source-progress bonus 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_srcprogbonus0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_srcscorebonus0015", |
| label="K4 mean-by-type tangent consensus, source-score bonus 0.015", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_srcscorebonus0p015_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_srcscorebonus002", |
| label="K4 mean-by-type tangent consensus, source-score bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_srcscorebonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_srcscorebonus0025", |
| label="K4 mean-by-type tangent consensus, source-score bonus 0.025", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_srcscorebonus0p025_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_taskrelative_k4_consensus_noopbonus003", |
| label="K4 task-relative tangent consensus, no-op bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "taskrelative_k4s040_safe_margin0p20_mean_by_type_noopbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus001", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.01", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p01_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus002", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus0025", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.025", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p025_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus0035", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.035", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p035_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_wgbonus003", |
| label="K4 mean-by-type tangent consensus, wrong-gripper bonus 0.03", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_wgbonus0p03_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noop003_wg002", |
| label="K4 mean-by-type tangent consensus, no-op 0.03 + wrong-gripper 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noop0p03_wg0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noop003_wg004", |
| label="K4 mean-by-type tangent consensus, no-op 0.03 + wrong-gripper 0.04", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noop0p03_wg0p04_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noop0025_wg002", |
| label="K4 mean-by-type tangent consensus, no-op 0.025 + wrong-gripper 0.02", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noop0p025_wg0p02_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus005", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.05", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p05_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="residual_k4_consensus_noopbonus008", |
| label="K4 mean-by-type tangent consensus, no-op bonus 0.08", |
| summary_path=( |
| "h16_policy_ckpt_near_miss_policy_bc5_bestpt_retrieval_residual_" |
| "k4s040_safe_margin0p20_mean_by_type_noopbonus0p08_summary.json" |
| ), |
| ), |
| MethodSpec( |
| key="same_state_near_miss", |
| label="Same-state lattice, near-miss only", |
| summary_path="h16_lattice_near_miss_only_v2_summary.json", |
| ), |
| MethodSpec( |
| key="same_state_no_expert", |
| label="Same-state lattice, no expert", |
| summary_path="h16_lattice_no_expert_summary.json", |
| ), |
| MethodSpec( |
| key="same_state_policy_baseline", |
| label="Same-state no-expert + policy candidate", |
| summary_path="h16_lattice_no_expert_policy_baseline_margin000_summary.json", |
| ), |
| MethodSpec( |
| key="same_state_full", |
| label="Same-state lattice, full", |
| summary_path="h16_lattice_summary.json", |
| ), |
| ] |
|
|
|
|
| def _load_json(path: Path) -> dict[str, Any]: |
| with path.open("r", encoding="utf-8") as handle: |
| return json.load(handle) |
|
|
|
|
| def _mean(values: list[float]) -> float: |
| return sum(values) / len(values) if values else float("nan") |
|
|
|
|
| def _sample_std(values: list[float]) -> float: |
| if len(values) <= 1: |
| return 0.0 |
| mean = _mean(values) |
| return math.sqrt(sum((value - mean) ** 2 for value in values) / (len(values) - 1)) |
|
|
|
|
| def _ci95(values: list[float]) -> float: |
| if len(values) <= 1: |
| return 0.0 |
| t_crit = { |
| 1: 12.706, |
| 2: 4.303, |
| 3: 3.182, |
| 4: 2.776, |
| 5: 2.571, |
| 6: 2.447, |
| 7: 2.365, |
| 8: 2.306, |
| 9: 2.262, |
| 10: 2.228, |
| }.get(len(values) - 1, 1.96) |
| return t_crit * _sample_std(values) / math.sqrt(len(values)) |
|
|
|
|
| def _success(row: dict[str, Any]) -> float: |
| return float(row["policy_rollout_success_rate"]) |
|
|
|
|
| def _progress(row: dict[str, Any]) -> float: |
| return float(row.get("policy_rollout_progress", float("nan"))) |
|
|
|
|
| def _action_mse(row: dict[str, Any]) -> float: |
| return float(row.get("action_mse_to_best", float("nan"))) |
|
|
|
|
| def _seed(row: dict[str, Any], fallback: int) -> int: |
| return int(row.get("seed", fallback)) |
|
|
|
|
| def _standard_summary(path: Path) -> dict[str, Any]: |
| data = _load_json(path) |
| rows = list(data.get("rows", [])) |
| return _normalize_summary(data, rows, source=str(path)) |
|
|
|
|
| def _field_sweep_best(path: Path) -> dict[str, Any]: |
| data = _load_json(path) |
| best_config = data.get("best", {}).get("config") |
| rows = [row for row in data.get("rows", []) if row.get("config") == best_config] |
| normalized = _normalize_summary(data.get("best", data), rows, source=str(path)) |
| normalized["best_config"] = best_config |
| return normalized |
|
|
|
|
| def _raw_rollout_summary(pattern: str) -> dict[str, Any]: |
| rows = [] |
| for index, path in enumerate(sorted(Path().glob(pattern) if not pattern.startswith("/") else Path("/").glob(pattern[1:]))): |
| row = _load_json(path) |
| row = dict(row) |
| row["path"] = str(path) |
| row["seed"] = _seed(row, index) |
| rows.append(row) |
| return _normalize_summary({}, rows, source=pattern) |
|
|
|
|
| def _normalize_summary(data: dict[str, Any], rows: list[dict[str, Any]], *, source: str) -> dict[str, Any]: |
| successes = [_success(row) for row in rows] |
| progress = [_progress(row) for row in rows] |
| action_mse = [_action_mse(row) for row in rows] |
| selected_counts = Counter() |
| selected_counts.update(data.get("selected_candidate_type_counts", {})) |
| selected_scale_counts = Counter() |
| selected_scale_counts.update(data.get("selected_residual_scale_counts", {})) |
| expected_count = sum(int(row.get("num_groups", 0)) for row in rows) |
| has_top_level_selected_counts = ( |
| bool(selected_counts) and sum(int(value) for value in selected_counts.values()) == expected_count |
| ) |
| has_top_level_scale_counts = ( |
| bool(selected_scale_counts) |
| and sum(int(value) for value in selected_scale_counts.values()) == expected_count |
| ) |
| if not has_top_level_selected_counts: |
| selected_counts = Counter() |
| if not has_top_level_scale_counts: |
| selected_scale_counts = Counter() |
| if not has_top_level_selected_counts or not has_top_level_scale_counts: |
| for row in rows: |
| path = row.get("path") |
| if not path: |
| continue |
| raw_path = Path(str(path)) |
| if not raw_path.exists(): |
| continue |
| raw = _load_json(raw_path) |
| if not has_top_level_selected_counts: |
| selected_counts.update(raw.get("selected_candidate_type_counts", {})) |
| if not has_top_level_scale_counts: |
| selected_scale_counts.update(raw.get("selected_residual_scale_counts", {})) |
| output = { |
| "source": source, |
| "num_completed": len(rows), |
| "mean_success": _mean(successes), |
| "std_success": _sample_std(successes), |
| "ci95_success": _ci95(successes), |
| "mean_progress": _mean(progress), |
| "mean_action_mse_to_best": _mean(action_mse), |
| "seed_success": {_seed(row, index): _success(row) for index, row in enumerate(rows)}, |
| "seed_progress": {_seed(row, index): _progress(row) for index, row in enumerate(rows)}, |
| "seed_action_mse_to_best": {_seed(row, index): _action_mse(row) for index, row in enumerate(rows)}, |
| "per_task_success": _per_task(rows), |
| "selected_candidate_type_counts": dict(selected_counts), |
| "selected_residual_scale_counts": dict(selected_scale_counts), |
| "selected_type_outcomes": _selected_type_outcomes(rows), |
| } |
| candidate_oracle_success = [ |
| float(row["candidate_oracle_success_rate"]) |
| for row in rows |
| if row.get("candidate_oracle_success_rate") is not None |
| ] |
| if candidate_oracle_success: |
| output.update( |
| { |
| "candidate_oracle_rollouts": int(data.get("candidate_oracle_rollouts") or 0), |
| "candidate_oracle_unique_tolerance": data.get( |
| "candidate_oracle_unique_tolerance" |
| ), |
| "mean_candidate_oracle_success_rate": _mean(candidate_oracle_success), |
| "mean_candidate_oracle_progress": _mean( |
| [ |
| float(row["candidate_oracle_progress"]) |
| for row in rows |
| if row.get("candidate_oracle_progress") is not None |
| ] |
| ), |
| "mean_candidate_oracle_score_gain_over_selected": _mean( |
| [ |
| float(row["candidate_oracle_score_gain_over_selected"]) |
| for row in rows |
| if row.get("candidate_oracle_score_gain_over_selected") |
| is not None |
| ] |
| ), |
| "mean_candidate_oracle_unique_count": _mean( |
| [ |
| float(row.get("candidate_oracle_unique_count") or 0.0) |
| for row in rows |
| if row.get("candidate_oracle_unique_count") is not None |
| ] |
| ), |
| "mean_candidate_oracle_improvement_rate": _mean( |
| [ |
| float(row["candidate_oracle_improvement_rate"]) |
| for row in rows |
| if row.get("candidate_oracle_improvement_rate") is not None |
| ] |
| ), |
| "candidate_oracle_type_counts": data.get( |
| "candidate_oracle_type_counts", {} |
| ), |
| } |
| ) |
| for key in ( |
| "mean_candidate_oracle_best_branch_rank", |
| "candidate_oracle_best_branch_rank_counts", |
| "mean_candidate_oracle_branch_success_rates", |
| "mean_candidate_oracle_branch_progress", |
| "mean_candidate_oracle_branch_score_gains_over_selected", |
| ): |
| if key in data: |
| output[key] = data[key] |
| return output |
|
|
|
|
| def _per_task(rows: list[dict[str, Any]]) -> dict[str, dict[str, float]]: |
| task_values: dict[str, list[float]] = {} |
| task_counts: dict[str, list[int]] = {} |
| for row in rows: |
| for task, metrics in row.get("per_task", {}).items(): |
| task_values.setdefault(task, []).append(float(metrics["policy_rollout_success_rate"])) |
| task_counts.setdefault(task, []).append(int(metrics.get("num_groups", 0))) |
| return { |
| task: { |
| "mean_success": _mean(values), |
| "std_success": _sample_std(values), |
| "mean_num_groups": _mean([float(value) for value in task_counts.get(task, [])]), |
| } |
| for task, values in sorted(task_values.items()) |
| } |
|
|
|
|
| def _selected_type_outcomes(rows: list[dict[str, Any]]) -> dict[str, dict[str, float]]: |
| grouped: dict[str, dict[str, float]] = {} |
| for row in rows: |
| path = row.get("path") |
| if not path: |
| continue |
| raw_path = Path(str(path)) |
| if not raw_path.exists(): |
| continue |
| raw = _load_json(raw_path) |
| for item in raw.get("rows", []): |
| candidate_type = str(item.get("nearest_candidate_type") or "unknown") |
| stats = grouped.setdefault( |
| candidate_type, |
| {"count": 0.0, "success_sum": 0.0, "progress_sum": 0.0}, |
| ) |
| stats["count"] += 1.0 |
| stats["success_sum"] += 1.0 if item.get("success") else 0.0 |
| stats["progress_sum"] += float(item.get("progress", 0.0)) |
| return { |
| candidate_type: { |
| "count": values["count"], |
| "success_rate": values["success_sum"] / values["count"] if values["count"] else float("nan"), |
| "mean_progress": values["progress_sum"] / values["count"] if values["count"] else float("nan"), |
| } |
| for candidate_type, values in sorted( |
| grouped.items(), |
| key=lambda item: (-item[1]["count"], item[0]), |
| ) |
| } |
|
|
|
|
| def _load_methods() -> dict[str, dict[str, Any]]: |
| methods: dict[str, dict[str, Any]] = {} |
| for spec in METHODS: |
| if spec.summary_path: |
| path = RESULTS_DIR / spec.summary_path |
| if not path.exists(): |
| methods[spec.key] = {"missing": True, "source": str(path), "label": spec.label} |
| continue |
| if spec.summary_mode == "field_sweep_best": |
| method = _field_sweep_best(path) |
| else: |
| method = _standard_summary(path) |
| elif spec.raw_rollout_glob: |
| method = _raw_rollout_summary(spec.raw_rollout_glob) |
| else: |
| method = {"missing": True, "source": "", "label": spec.label} |
| method["label"] = spec.label |
| methods[spec.key] = method |
| return methods |
|
|
|
|
| def _best_clean_key(methods: dict[str, dict[str, Any]]) -> str: |
| best_key = FALLBACK_BEST_CLEAN_KEY |
| best_success = float("-inf") |
| for spec in METHODS: |
| if spec.key in NON_DEPLOYMENT_KEYS: |
| continue |
| method = methods.get(spec.key, {}) |
| if method.get("missing"): |
| continue |
| success = method.get("mean_success") |
| if isinstance(success, (int, float)) and math.isfinite(float(success)): |
| success = float(success) |
| if success > best_success + 1.0e-12: |
| best_success = success |
| best_key = spec.key |
| return best_key |
|
|
|
|
| def _paired_delta( |
| methods: dict[str, dict[str, Any]], |
| left: str, |
| right: str, |
| ) -> dict[str, Any]: |
| left_values = methods[left].get("seed_success", {}) |
| right_values = methods[right].get("seed_success", {}) |
| seeds = sorted(set(left_values) & set(right_values)) |
| deltas = [float(left_values[seed]) - float(right_values[seed]) for seed in seeds] |
| return { |
| "left": left, |
| "right": right, |
| "seeds": seeds, |
| "mean_delta": _mean(deltas), |
| "std_delta": _sample_std(deltas), |
| "ci95_delta": _ci95(deltas), |
| "seed_deltas": {seed: delta for seed, delta in zip(seeds, deltas)}, |
| } |
|
|
|
|
| def _per_task_delta( |
| methods: dict[str, dict[str, Any]], |
| left: str, |
| right: str, |
| ) -> dict[str, float]: |
| left_tasks = methods[left].get("per_task_success", {}) |
| right_tasks = methods[right].get("per_task_success", {}) |
| return { |
| task: float(left_tasks[task]["mean_success"]) - float(right_tasks[task]["mean_success"]) |
| for task in sorted(set(left_tasks) & set(right_tasks)) |
| } |
|
|
|
|
| def _pct(value: float) -> str: |
| if math.isnan(value): |
| return "n/a" |
| return f"{value * 100:.2f}%" |
|
|
|
|
| def _pp(value: float) -> str: |
| if math.isnan(value): |
| return "n/a" |
| return f"{value * 100:+.2f} pp" |
|
|
|
|
| def _render_markdown(report: dict[str, Any]) -> str: |
| methods = report["methods"] |
| best_clean_key = report["best_clean_key"] |
| lines = [ |
| "# Paper Analysis", |
| "", |
| f"Generated: `{report['generated_utc']}`", |
| "", |
| "## Main Seed Statistics", |
| "", |
| "| key | method | n | success | 95% CI | progress | action MSE | gain vs canonical h16 |", |
| "|---|---|---:|---:|---:|---:|---:|---:|", |
| ] |
| baseline = methods["h16_policy_canonical"]["mean_success"] |
| for key in [spec.key for spec in METHODS]: |
| method = methods[key] |
| if method.get("missing"): |
| lines.append(f"| {key} | {method['label']} | 0 | missing | missing | missing | missing | missing |") |
| continue |
| lines.append( |
| "| {key} | {label} | {n} | {success} +/- {std} | {ci} | {progress} | {mse:.3f} | {gain} |".format( |
| key=key, |
| label=method["label"], |
| n=method["num_completed"], |
| success=_pct(method["mean_success"]), |
| std=f"{method['std_success'] * 100:.2f}", |
| ci=f"+/- {method['ci95_success'] * 100:.2f}", |
| progress=_pct(method["mean_progress"]), |
| mse=method["mean_action_mse_to_best"], |
| gain=_pp(method["mean_success"] - baseline), |
| ) |
| ) |
| lines.extend( |
| [ |
| "", |
| "## Paired Seed Deltas", |
| "", |
| "| comparison | seeds | mean delta | 95% CI | seed deltas |", |
| "|---|---:|---:|---:|---|", |
| ] |
| ) |
| for name, delta in report["paired_deltas"].items(): |
| seed_deltas = ", ".join( |
| f"{seed}:{value * 100:+.2f}" for seed, value in delta["seed_deltas"].items() |
| ) |
| lines.append( |
| f"| {name} | {len(delta['seeds'])} | {_pp(delta['mean_delta'])} | +/- {delta['ci95_delta'] * 100:.2f} | {seed_deltas} |" |
| ) |
| lines.extend( |
| [ |
| "", |
| "## Per-Task Mean Success", |
| "", |
| "| task | h16 policy | best clean | near-miss lattice | no-expert lattice | full lattice | clean-h16 delta | noexpert-clean gap |", |
| "|---|---:|---:|---:|---:|---:|---:|---:|", |
| ] |
| ) |
| tasks = sorted(methods["h16_policy_canonical"].get("per_task_success", {})) |
| for task in tasks: |
| h16 = methods["h16_policy_canonical"]["per_task_success"][task]["mean_success"] |
| clean = methods[best_clean_key]["per_task_success"][task]["mean_success"] |
| near = methods["same_state_near_miss"]["per_task_success"][task]["mean_success"] |
| noexpert = methods["same_state_no_expert"]["per_task_success"][task]["mean_success"] |
| full = methods["same_state_full"]["per_task_success"][task]["mean_success"] |
| lines.append( |
| f"| {task} | {_pct(h16)} | {_pct(clean)} | {_pct(near)} | {_pct(noexpert)} | {_pct(full)} | {_pp(clean - h16)} | {_pp(noexpert - clean)} |" |
| ) |
| gap = report["mechanism_gap"] |
| lines.extend( |
| [ |
| "", |
| "## Mechanism Gap", |
| "", |
| f"- Best clean residual transport improves over canonical h16 by {_pp(gap['best_clean_vs_h16'])}.", |
| f"- Same-state no-expert lattice improves over canonical h16 by {_pp(gap['same_state_no_expert_vs_h16'])}.", |
| f"- Remaining clean-to-same-state proposal gap is {_pp(gap['same_state_no_expert_vs_best_clean'])}.", |
| f"- Full lattice adds expert proposals and reaches {_pct(methods['same_state_full']['mean_success'])}, a {_pp(gap['same_state_full_vs_no_expert'])} gain over no-expert.", |
| ] |
| ) |
| trace_oracle_key = ( |
| "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8trace" |
| ) |
| oracle_key = ( |
| "residual_k4_composemasked_dropnmnoop_grid035040045_noopbonus003_oraclek8" |
| ) |
| oracle = methods.get(trace_oracle_key) |
| if not oracle or not oracle.get("num_completed"): |
| oracle = methods.get(oracle_key) |
| if oracle and oracle.get("num_completed"): |
| branch_success = oracle.get("mean_candidate_oracle_branch_success_rates") or [] |
| branch_gains = ( |
| oracle.get("mean_candidate_oracle_branch_score_gains_over_selected") |
| or [] |
| ) |
| lines.extend( |
| [ |
| "", |
| "## Candidate-Oracle Diagnostic", |
| "", |
| ( |
| "- Oracle over the deployed candidate prefix reaches " |
| f"{_pct(oracle.get('mean_candidate_oracle_success_rate', 0.0))} " |
| f"with mean progress {_pct(oracle.get('mean_candidate_oracle_progress', 0.0))}; " |
| "this is diagnostic-only because it uses measured rollout outcomes " |
| "after generating candidates." |
| ), |
| ( |
| "- Mean oracle-prefix score gain over the selected branch is " |
| f"{oracle.get('mean_candidate_oracle_score_gain_over_selected', 0.0):+.3f}, " |
| "which isolates ranking/abstention headroom inside the clean proposal set." |
| ), |
| ( |
| "- Mean unique candidates in the prefix: " |
| f"{oracle.get('mean_candidate_oracle_unique_count', 0.0):.2f}." |
| ), |
| ( |
| "- Candidate-oracle best type counts: " |
| f"{oracle.get('candidate_oracle_type_counts', {})}." |
| ), |
| ] |
| ) |
| if oracle.get("mean_candidate_oracle_best_branch_rank") is not None: |
| lines.extend( |
| [ |
| ( |
| "- Mean best branch rank in the field-ordered prefix: " |
| f"{oracle['mean_candidate_oracle_best_branch_rank']:.2f}; " |
| "rank histogram " |
| f"{oracle.get('candidate_oracle_best_branch_rank_counts', {})}." |
| ), |
| ( |
| "- Branch success by prefix rank: " |
| + ", ".join(_pct(value) for value in branch_success) |
| + "." |
| ), |
| ( |
| "- Branch score gain by prefix rank: " |
| + ", ".join(f"{value:+.3f}" for value in branch_gains) |
| + "." |
| ), |
| ] |
| ) |
| lines.extend( |
| [ |
| "", |
| "## Selection Histograms", |
| "", |
| ] |
| ) |
| for key in ["same_state_near_miss", "same_state_no_expert", "same_state_policy_baseline", "same_state_full", best_clean_key]: |
| counts = methods[key].get("selected_candidate_type_counts", {}) |
| if counts: |
| total = sum(int(value) for value in counts.values()) |
| summary = ", ".join( |
| f"{name}={count} ({count / total * 100:.1f}%)" |
| for name, count in sorted(counts.items(), key=lambda item: (-int(item[1]), item[0])) |
| ) |
| else: |
| summary = "not recorded" |
| lines.append(f"- `{key}`: {summary}") |
| scale_counts = methods[best_clean_key].get("selected_residual_scale_counts", {}) |
| if scale_counts: |
| lines.append(f"- `{best_clean_key}` residual scale counts: {scale_counts}") |
| lines.extend( |
| [ |
| "", |
| "## Selected-Type Outcomes", |
| "", |
| "These rows are measured from raw rollout rows. In residual retrieval, `policy_residual` is the zero-residual action, i.e. abstaining to the current policy mean.", |
| "", |
| "| method | selected type | count | success | progress |", |
| "|---|---|---:|---:|---:|", |
| ] |
| ) |
| for key in [ |
| "best_clean_residual_k2", |
| "residual_k4_consensus", |
| "residual_k4_kernel_consensus", |
| "residual_k4_kernel_consensus_noopbonus003", |
| "residual_k4_kernel_consensus_s035_noopbonus003", |
| "residual_k4_kernel_consensus_s045_noopbonus003", |
| "residual_k4_fieldsoftmax_grid", |
| "residual_k4_fieldsoftmax_grid_noopbonus003", |
| "residual_k4_fieldsoftmax_grid_margin010_noopbonus003", |
| "residual_k4_fieldsoftmax_grid_margin005_noopbonus003", |
| "residual_k4_fieldsoftmax_grid_margin000_noopbonus003", |
| "residual_k8_fieldsoftmax_grid_noopbonus003", |
| "residual_k4_consensus_noopbonus003", |
| "residual_k4_consensus_noopbonus001", |
| "residual_k4_consensus_noopbonus002", |
| "residual_k4_consensus_noopbonus0025", |
| "residual_k4_consensus_noopbonus0035", |
| "residual_k4_consensus_wgbonus003", |
| "residual_k4_consensus_noop003_wg002", |
| "residual_k4_consensus_noop003_wg004", |
| "residual_k4_consensus_noop0025_wg002", |
| "residual_k4_consensus_noopbonus005", |
| "residual_k4_consensus_noopbonus008", |
| "same_state_no_expert", |
| "same_state_policy_baseline", |
| ]: |
| for candidate_type, values in methods[key].get("selected_type_outcomes", {}).items(): |
| lines.append( |
| f"| {key} | {candidate_type} | {int(values['count'])} | {_pct(values['success_rate'])} | {_pct(values['mean_progress'])} |" |
| ) |
| return "\n".join(lines) + "\n" |
|
|
|
|
| def build_report() -> dict[str, Any]: |
| methods = _load_methods() |
| best_clean_key = _best_clean_key(methods) |
| paired_deltas = { |
| "best_clean - canonical_h16": _paired_delta(methods, best_clean_key, "h16_policy_canonical"), |
| "best_clean - direct_same_ckpt": _paired_delta(methods, best_clean_key, "near_miss_policy_bc5"), |
| "no_expert_lattice - canonical_h16": _paired_delta(methods, "same_state_no_expert", "h16_policy_canonical"), |
| "full_lattice - no_expert_lattice": _paired_delta(methods, "same_state_full", "same_state_no_expert"), |
| "policy_candidate_lattice - no_expert_lattice": _paired_delta( |
| methods, |
| "same_state_policy_baseline", |
| "same_state_no_expert", |
| ), |
| } |
| mechanism_gap = { |
| "best_clean_vs_h16": methods[best_clean_key]["mean_success"] |
| - methods["h16_policy_canonical"]["mean_success"], |
| "best_clean_vs_direct_same_ckpt": methods[best_clean_key]["mean_success"] |
| - methods["near_miss_policy_bc5"]["mean_success"], |
| "same_state_no_expert_vs_h16": methods["same_state_no_expert"]["mean_success"] |
| - methods["h16_policy_canonical"]["mean_success"], |
| "same_state_no_expert_vs_best_clean": methods["same_state_no_expert"]["mean_success"] |
| - methods[best_clean_key]["mean_success"], |
| "same_state_full_vs_no_expert": methods["same_state_full"]["mean_success"] |
| - methods["same_state_no_expert"]["mean_success"], |
| } |
| return { |
| "generated_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"), |
| "methods": methods, |
| "paired_deltas": paired_deltas, |
| "per_task_deltas": { |
| "best_clean_vs_h16": _per_task_delta(methods, best_clean_key, "h16_policy_canonical"), |
| "no_expert_vs_best_clean": _per_task_delta(methods, "same_state_no_expert", best_clean_key), |
| }, |
| "mechanism_gap": mechanism_gap, |
| "best_clean_key": best_clean_key, |
| } |
|
|
|
|
| def main() -> int: |
| RESULTS_DIR.mkdir(parents=True, exist_ok=True) |
| report = build_report() |
| OUT_JSON.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") |
| OUT_MD.write_text(_render_markdown(report), encoding="utf-8") |
| print(f"Wrote {OUT_JSON}") |
| print(f"Wrote {OUT_MD}") |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
