Blame for scripts/optimization_fidelity_dense.py

#!/usr/bin/env python3
"""Compute dense-panel gradient targets and sparse proposal fidelity."""

from __future__ import annotations

import argparse
import csv
import hashlib
import json
import math
import os
import random
import time
from datetime import datetime, timezone
from pathlib import Path

import numpy as np
import torch
from omegaconf import OmegaConf

from scripts.validate_stratified_gradient_second_moment import (
    _flow_loss,
    _prepare_conditioning,
)
from starVLA.dataloader.lerobot_datasets import get_vla_dataset
from starVLA.model.framework.base_framework import build_framework
from starVLA.model.framework.share_tools import apply_config_compat
from starVLA.scoring.flow_residual import (
    randomized_antithetic_stratified_flow_samples,
)
from starVLA.scoring.partial_gradient import finite_gradient_l2_norm
from starVLA.scoring.residual_cache import (
    interpolate_grid_scores,
    percentile_intensity_masses,
    quantile_anchor_positions,
)
from starVLA.scoring.utility_audit import gradient_second_moment


SCHEMA_VERSION = 1


def sha256_file(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as source:
        for chunk in iter(lambda: source.read(8 * 1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def canonical_sha256(payload: dict) -> str:
    encoded = json.dumps(
        payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True
    ).encode("utf-8")
    return hashlib.sha256(encoded).hexdigest()


def atomic_json(path: Path, payload: dict) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    temporary = path.with_suffix(path.suffix + ".tmp")
    temporary.write_text(json.dumps(payload, indent=2), encoding="utf-8")
    os.replace(temporary, path)


def seed_all(seed: int) -> None:
    bounded = int(seed) % (2**32)
    random.seed(bounded)
    np.random.seed(bounded)
    torch.manual_seed(bounded)
    torch.cuda.manual_seed_all(bounded)


def load_dense_panel(dense_dir: Path) -> tuple[list[dict], dict]:
    metadata_path = dense_dir / "dense_residual_metadata.json"
    scores_path = dense_dir / "dense_residual_scores.npz"
    metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
    dense = np.load(scores_path, allow_pickle=False)
    scores = np.asarray(dense["scores"], dtype=np.float64)
    offsets = np.asarray(dense["offsets"], dtype=np.int64)
    records: list[dict] = []
    for index in range(len(offsets) - 1):
        dataset_index = int(dense["dataset_index"][index])
        episode_index = int(dense["episode_index"][index])
        trajectory_index = int(dense["trajectory_index"][index])
        trajectory_id = str(dense["trajectory_id"][index])
        shard_path = dense_dir / "dense_shards" / (
            f"dataset_{dataset_index:02d}_episode_{episode_index:06d}.json"
        )
        payload = json.loads(shard_path.read_text(encoding="utf-8"))
        start, stop = int(offsets[index]), int(offsets[index + 1])
        expected = {
            "dataset_index": dataset_index,
            "episode_index": episode_index,
            "trajectory_index": trajectory_index,
            "trajectory_id": trajectory_id,
            "anchor_count": stop - start,
        }
        observed = {
            "dataset_index": int(payload["dataset_index"]),
            "episode_index": int(payload["episode_index"]),
            "trajectory_index": int(payload["trajectory_index"]),
            "trajectory_id": str(payload["trajectory_id"]),
            "anchor_count": int(payload["anchor_count"]),
        }
        if observed != expected:
            raise RuntimeError(f"dense trajectory identity mismatch: {shard_path}")
        if not (
            len(payload["anchor_positions"])
            == len(payload["base_indices"])
            == len(payload["transform_seeds"])
            == stop - start
        ):
            raise RuntimeError(f"dense anchor identity mismatch: {shard_path}")
        records.append(
            {
                **observed,
                "trajectory_id": payload["trajectory_id"],
                "dataset_name": str(payload["dataset_name"]),
                "manifest_sha256": str(payload["manifest_sha256"]),
                "global_start": start,
                "global_stop": stop,
                "anchor_positions": [int(x) for x in payload["anchor_positions"]],
                "base_indices": [int(x) for x in payload["base_indices"]],
                "transform_seeds": [int(x) for x in payload["transform_seeds"]],
                "dense_scores": scores[start:stop].tolist(),
            }
        )
    records.sort(key=lambda item: (item["dataset_index"], item["trajectory_index"]))
    global_offset = 0
    for record in records:
        record["global_start"] = global_offset
        global_offset += record["anchor_count"]
        record["global_stop"] = global_offset
    if len(records) != int(metadata["trajectory_count"]):
        raise RuntimeError("dense trajectory count differs from metadata")
    if global_offset != int(metadata["valid_anchor_count"]):
        raise RuntimeError("dense anchor count differs from metadata")
    manifests = {record["manifest_sha256"] for record in records}
    if len(manifests) != 1:
        raise RuntimeError("dense shards contain multiple manifest identities")
    identity = {
        "dense_metadata_sha256": sha256_file(metadata_path),
        "dense_scores_sha256": sha256_file(scores_path),
        "dense_manifest_sha256": next(iter(manifests)),
        "trajectory_count": len(records),
        "anchor_count": global_offset,
        "trajectory_keys_sha256": canonical_sha256(
            {
                "keys": [
                    [
                        record["dataset_index"],
                        record["dataset_name"],
                        record["trajectory_index"],
                        record["trajectory_id"],
                        record["episode_index"],
                    ]
                    for record in records
                ]
            }
        ),
        "anchor_keys_sha256": canonical_sha256(
            {
                "keys": [
                    [
                        record["dataset_index"],
                        record["trajectory_id"],
                        position,
                        base_index,
                        transform_seed,
                        record["global_start"] + local_index,
                    ]
                    for record in records
                    for local_index, (position, base_index, transform_seed) in enumerate(
                        zip(
                            record["anchor_positions"],
                            record["base_indices"],
                            record["transform_seeds"],
                        )
                    )
                ]
            }
        ),
    }
    return records, identity


def load_runtime(config_path: Path, checkpoint_path: Path, output: Path):
    cfg = apply_config_compat(OmegaConf.load(config_path))
    cfg.output_dir = str(output / "runtime")
    cfg.datasets.vla_data.temporal_proposal_path = None
    cfg.datasets.vla_data.valid_anchor_manifest_path = None
    cfg.datasets.vla_data.valid_anchor_manifest_migration_path = None
    dataset = get_vla_dataset(
        data_cfg=cfg.datasets.vla_data,
        balance_dataset_weights=cfg.datasets.vla_data.get(
            "balance_dataset_weights", False
        ),
        balance_trajectory_weights=cfg.datasets.vla_data.get(
            "balance_trajectory_weights", False
        ),
        seed=int(cfg.get("seed", 42)),
    )
    model = build_framework(cfg)
    checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=True)
    model.load_state_dict(checkpoint, strict=True)
    model.to(device="cuda", dtype=torch.float32)
    return cfg, dataset, model


def measure_trajectory(
    model,
    source,
    record: dict,
    *,
    estimator_seeds: list[int],
    samples_per_estimator: int,
) -> tuple[np.ndarray, np.ndarray, float]:
    anchor_count = record["anchor_count"]
    estimator_values = np.empty(
        (anchor_count, len(estimator_seeds)), dtype=np.float64
    )
    draw_norms = np.empty(
        (anchor_count, len(estimator_seeds), samples_per_estimator),
        dtype=np.float64,
    )
    started = time.perf_counter()
    for local_index, (base_index, transform_seed) in enumerate(
        zip(record["base_indices"], record["transform_seeds"])
    ):
        seed_all(transform_seed)
        raw = source.get_step_data(record["trajectory_id"], base_index)
        transformed = source.transforms(raw)
        example = source._pack_sample(transformed)
        global_index = record["global_start"] + local_index
        for estimator_index, estimator_seed in enumerate(estimator_seeds):
            model.requires_grad_(False)
            model.train()
            model.qwen_vl_interface.eval()
            seed_all(estimator_seed * 1_000_003 + global_index)
            conditioning = _prepare_conditioning(model, example)
            actions = conditioning[1]
            action_cfg = model.action_model.config
            noises, times = randomized_antithetic_stratified_flow_samples(
                actions,
                sample_count=samples_per_estimator,
                seed=estimator_seed * 1_000_033 + global_index,
                beta_alpha=float(action_cfg.noise_beta_alpha),
                beta_beta=float(action_cfg.noise_beta_beta),
                noise_s=float(action_cfg.noise_s),
            )
            model.action_model.requires_grad_(True)
            parameters = list(model.action_model.parameters())
            for draw_index in range(samples_per_estimator):
                seed_all(
                    estimator_seed * 1_000_037
                    + global_index * 101
                    + draw_index // 2
                )
                model.zero_grad(set_to_none=True)
                loss = _flow_loss(
                    model, conditioning, noises[draw_index], times[draw_index]
                )
                if loss.ndim != 0 or not torch.isfinite(loss):
                    raise FloatingPointError("flow loss must be a finite scalar")
                loss.backward()
                draw_norms[local_index, estimator_index, draw_index] = (
                    finite_gradient_l2_norm(parameters)
                )
            estimator_values[local_index, estimator_index] = math.sqrt(
                float(np.mean(np.square(draw_norms[local_index, estimator_index])))
            )
    model.zero_grad(set_to_none=True)
    torch.cuda.synchronize()
    return estimator_values, draw_norms, time.perf_counter() - started


def worker(args: argparse.Namespace) -> None:
    if not 0 <= args.shard_index < args.num_shards:
        raise ValueError("invalid shard index")
    records, dense_identity = load_dense_panel(args.dense_dir)
    cfg_sha = sha256_file(args.config)
    checkpoint_sha = sha256_file(args.checkpoint)
    worker_identity = {
        "schema_version": SCHEMA_VERSION,
        "config_sha256": cfg_sha,
        "checkpoint_sha256": checkpoint_sha,
        "dense_identity": dense_identity,
        "target_estimator_seeds": args.target_seeds,
        "samples_per_estimator": args.samples_per_estimator,
        "gradient_definition": "sqrt(mean over estimator seeds and draws of squared full action-model gradient L2 norm)",
        "parameter_mode": "full_action_model",
        "frozen_backbone_eval": True,
        "action_model_mode": "train",
        "model_dtype": "float32",
        "global_anchor_index_seed_semantics": True,
        "num_shards": args.num_shards,
    }
    identity_sha = canonical_sha256(worker_identity)
    _, dataset, model = load_runtime(args.config, args.checkpoint, args.output)
    if dataset.valid_anchor_manifest is None:
        raise RuntimeError("current dataset has no valid-anchor manifest")
    if dataset.valid_anchor_manifest.sha256 != dense_identity["dense_manifest_sha256"]:
        raise RuntimeError("current dataset manifest differs from dense panel")
    assigned = records[args.shard_index :: args.num_shards]
    cache_dir = args.output / "dense_gradient_targets"
    started = time.perf_counter()
    new_anchors = 0
    completed_anchors = 0
    for item_index, record in enumerate(assigned):
        key = (
            f"dataset_{record['dataset_index']:02d}_"
            f"episode_{record['episode_index']:06d}.json"
        )
        path = cache_dir / key
        if path.exists():
            cached = json.loads(path.read_text(encoding="utf-8"))
            if (
                cached.get("identity_sha256") != identity_sha
                or cached.get("trajectory_identity")
                != {field: record[field] for field in (
                    "dataset_index", "dataset_name", "episode_index",
                    "trajectory_index", "trajectory_id", "anchor_count",
                    "global_start", "global_stop", "anchor_positions",
                    "base_indices", "transform_seeds",
                )}
            ):
                raise RuntimeError(f"incompatible existing cache: {path}")
            completed_anchors += record["anchor_count"]
            continue
        source = dataset.datasets[record["dataset_index"]]
        estimators, draws, elapsed = measure_trajectory(
            model,
            source,
            record,
            estimator_seeds=args.target_seeds,
            samples_per_estimator=args.samples_per_estimator,
        )
        target = np.sqrt(np.mean(np.square(estimators), axis=1))
        if not np.isfinite(target).all() or np.any(target <= 0):
            raise FloatingPointError("gradient targets must be positive and finite")
        trajectory_identity = {field: record[field] for field in (
            "dataset_index", "dataset_name", "episode_index",
            "trajectory_index", "trajectory_id", "anchor_count",
            "global_start", "global_stop", "anchor_positions",
            "base_indices", "transform_seeds",
        )}
        atomic_json(
            path,
            {
                "schema_version": SCHEMA_VERSION,
                "status": "complete",
                "identity_sha256": identity_sha,
                "worker_identity": worker_identity,
                "trajectory_identity": trajectory_identity,
                "target_estimators": estimators.tolist(),
                "target_draw_norms": draws.tolist(),
                "target_gradient_rms": target.tolist(),
                "scoring_seconds": elapsed,
                "completed_at": datetime.now(timezone.utc).isoformat(),
            },
        )
        new_anchors += record["anchor_count"]
        completed_anchors += record["anchor_count"]
        print(
            f"shard={args.shard_index}/{args.num_shards} "
            f"trajectory={item_index + 1}/{len(assigned)} "
            f"anchors={completed_anchors} new={new_anchors} seconds={elapsed:.2f}",
            flush=True,
        )
    atomic_json(
        args.output / "logs" / f"worker_{args.shard_index:02d}_of_{args.num_shards:02d}.json",
        {
            **worker_identity,
            "identity_sha256": identity_sha,
            "status": "complete",
            "shard_index": args.shard_index,
            "assigned_trajectories": len(assigned),
            "completed_anchors": completed_anchors,
            "new_anchors": new_anchors,
            "wall_clock_seconds": time.perf_counter() - started,
            "completed_at": datetime.now(timezone.utc).isoformat(),
        },
    )


def q_from_scores(scores: np.ndarray, alpha: float, epsilon: float) -> np.ndarray:
    masses = percentile_intensity_masses(scores, epsilon=epsilon)
    baseline = np.full(len(scores), 1.0 / len(scores), dtype=np.float64)
    information = masses / masses.sum()
    return (1.0 - alpha) * baseline + alpha * information


def describe(values: list[float]) -> dict:
    array = np.asarray(values, dtype=np.float64)
    return {
        "mean": float(array.mean()),
        "median": float(np.median(array)),
        "std": float(array.std(ddof=1)),
        "p05": float(np.quantile(array, 0.05)),
        "p25": float(np.quantile(array, 0.25)),
        "p75": float(np.quantile(array, 0.75)),
        "p95": float(np.quantile(array, 0.95)),
        "min": float(array.min()),
        "max": float(array.max()),
    }


def write_csv(path: Path, rows: list[dict]) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    with path.open("w", newline="", encoding="utf-8") as handle:
        writer = csv.DictWriter(handle, fieldnames=list(rows[0]))
        writer.writeheader()
        writer.writerows(rows)


def finalize(args: argparse.Namespace) -> None:
    records, dense_identity = load_dense_panel(args.dense_dir)
    cache_dir = args.output / "dense_gradient_targets"
    cache_files = sorted(cache_dir.glob("dataset_*_episode_*.json"))
    if len(cache_files) != len(records):
        raise RuntimeError(
            f"gradient cache file count {len(cache_files)} != {len(records)}"
        )
    per_trajectory: list[dict] = []
    all_targets = np.empty(dense_identity["anchor_count"], dtype=np.float64)
    cache_identity = None
    q_checks = []
    manual_checks = []
    for record_index, record in enumerate(records):
        path = cache_dir / (
            f"dataset_{record['dataset_index']:02d}_"
            f"episode_{record['episode_index']:06d}.json"
        )
        if not path.exists():
            raise FileNotFoundError(path)
        payload = json.loads(path.read_text(encoding="utf-8"))
        if payload.get("status") != "complete":
            raise RuntimeError(f"incomplete cache: {path}")
        if cache_identity is None:
            cache_identity = payload["identity_sha256"]
        if payload["identity_sha256"] != cache_identity:
            raise RuntimeError("gradient caches contain multiple identities")
        observed = payload["trajectory_identity"]
        expected = {field: record[field] for field in (
            "dataset_index", "dataset_name", "episode_index",
            "trajectory_index", "trajectory_id", "anchor_count",
            "global_start", "global_stop", "anchor_positions",
            "base_indices", "transform_seeds",
        )}
        if observed != expected:
            raise RuntimeError(f"gradient anchor alignment mismatch: {path}")
        h = np.asarray(payload["target_gradient_rms"], dtype=np.float64)
        if len(h) != record["anchor_count"]:
            raise RuntimeError(f"gradient anchor count mismatch: {path}")
        if not np.isfinite(h).all() or np.any(h <= 0):
            raise FloatingPointError(f"invalid gradient target: {path}")
        all_targets[record["global_start"] : record["global_stop"]] = h
        dense_scores = np.asarray(record["dense_scores"], dtype=np.float64)
        proposals: dict[str, np.ndarray] = {
            "p": np.full(len(h), 1.0 / len(h), dtype=np.float64),
            "dense": q_from_scores(dense_scores, args.alpha, args.epsilon),
        }
        for k in (4, 8, 16):
            positions = quantile_anchor_positions(len(h), k)
            estimate = interpolate_grid_scores(
                len(h), positions, dense_scores[positions]
            )
            proposals[str(k)] = q_from_scores(estimate, args.alpha, args.epsilon)
        for label, proposal in proposals.items():
            q_checks.append(
                abs(float(proposal.sum()) - 1.0) <= 1e-10
                and np.isfinite(proposal).all()
                and np.all(proposal > 0)
            )
        moments = {
            label: gradient_second_moment(proposal, h)
            for label, proposal in proposals.items()
        }
        if record_index in {0, len(records) // 2, len(records) - 1}:
            direct = float(
                np.sum(np.square(proposals["p"] * h) / proposals["dense"])
            )
            manual_checks.append(abs(direct - moments["dense"]) <= 1e-12)
        row = {
            "dataset_index": record["dataset_index"],
            "dataset_name": record["dataset_name"],
            "trajectory_index": record["trajectory_index"],
            "trajectory_id": record["trajectory_id"],
            "anchor_count": record["anchor_count"],
            "M_p": moments["p"],
            "M_q4": moments["4"],
            "M_q8": moments["8"],
            "M_q16": moments["16"],
            "M_qDense": moments["dense"],
        }
        for label in ("4", "8", "16"):
            row[f"R_{label}"] = moments[label] / moments["dense"]
            row[f"q{label}_over_p"] = moments[label] / moments["p"]
        row["qDense_over_p"] = moments["dense"] / moments["p"]
        per_trajectory.append(row)
    if not np.isfinite(all_targets).all() or np.any(all_targets <= 0):
        raise FloatingPointError("assembled gradient targets are incomplete")
    aggregate = {
        field: float(np.mean([row[field] for row in per_trajectory]))
        for field in ("M_p", "M_q4", "M_q8", "M_q16", "M_qDense")
    }
    ratios = {
        "R_4": aggregate["M_q4"] / aggregate["M_qDense"],
        "R_8": aggregate["M_q8"] / aggregate["M_qDense"],
        "R_16": aggregate["M_q16"] / aggregate["M_qDense"],
        "q4_over_p": aggregate["M_q4"] / aggregate["M_p"],
        "q8_over_p": aggregate["M_q8"] / aggregate["M_p"],
        "q16_over_p": aggregate["M_q16"] / aggregate["M_p"],
        "qDense_over_p": aggregate["M_qDense"] / aggregate["M_p"],
    }
    trajectory_statistics = {}
    for label in ("4", "8", "16"):
        r_values = [row[f"R_{label}"] for row in per_trajectory]
        p_values = [row[f"q{label}_over_p"] for row in per_trajectory]
        trajectory_statistics[label] = {
            "relative_to_dense": describe(r_values),
            "relative_to_uniform": describe(p_values),
            "better_than_dense": sum(value < 1.0 - args.comparison_tolerance for value in r_values),
            "equal_to_dense": sum(abs(value - 1.0) <= args.comparison_tolerance for value in r_values),
            "worse_than_dense": sum(value > 1.0 + args.comparison_tolerance for value in r_values),
            "better_than_uniform": sum(value < 1.0 - args.comparison_tolerance for value in p_values),
            "equal_to_uniform": sum(abs(value - 1.0) <= args.comparison_tolerance for value in p_values),
            "worse_than_uniform": sum(value > 1.0 + args.comparison_tolerance for value in p_values),
        }
    dense_self_ratio = aggregate["M_qDense"] / aggregate["M_qDense"]
    checks = {
        "trajectory_count_512": len(per_trajectory) == 512,
        "anchor_count_19375": len(all_targets) == 19375,
        "cache_file_count_matches_trajectories": len(cache_files) == len(records),
        "trajectory_and_anchor_identity_aligned": True,
        "all_anchor_targets_positive_finite": bool(
            np.isfinite(all_targets).all() and np.all(all_targets > 0)
        ),
        "proposal_normalization_full_support_finite": all(q_checks),
        "dense_self_ratio_one": abs(dense_self_ratio - 1.0) <= 1e-15,
        "manual_second_moment_checks": all(manual_checks),
        "all_moments_finite": all(np.isfinite(list(aggregate.values()))),
    }
    write_csv(args.output / "optimization_fidelity_per_trajectory.csv", per_trajectory)
    readback = list(
        csv.DictReader(
            (args.output / "optimization_fidelity_per_trajectory.csv").open(
                encoding="utf-8"
            )
        )
    )
    reconstructed = {
        field: float(np.mean([float(row[field]) for row in readback]))
        for field in aggregate
    }
    checks["aggregate_reconstructs_from_per_trajectory_csv"] = all(
        math.isclose(
            reconstructed[field], aggregate[field], rel_tol=1e-12, abs_tol=1e-12
        )
        for field in aggregate
    )
    summary_rows = []
    for label, field in (("Uniform", "M_p"), ("K4", "M_q4"), ("K8", "M_q8"), ("K16", "M_q16"), ("Dense", "M_qDense")):
        value = aggregate[field]
        summary_rows.append(
            {
                "proposal": label,
                "aggregate_second_moment": value,
                "M_over_Mp": value / aggregate["M_p"],
                "improvement_vs_uniform_percent": 100.0 * (1.0 - value / aggregate["M_p"]),
                "M_over_Mdense": value / aggregate["M_qDense"],
                "gap_vs_dense_percent": 100.0 * (value / aggregate["M_qDense"] - 1.0),
            }
        )
    write_csv(args.output / "optimization_fidelity_summary.csv", summary_rows)
    np.savez_compressed(
        args.output / "dense_gradient_targets.npz",
        target_gradient_rms=all_targets,
    )
    metadata = {
        "schema_version": SCHEMA_VERSION,
        "status": "complete" if all(checks.values()) else "failed_checks",
        "started_at": args.started_at,
        "completed_at": datetime.now(timezone.utc).isoformat(),
        "git_commit": None,
        "git_status": "unavailable_server_copy_has_no_dot_git",
        "code_path": str(Path(__file__).resolve()),
        "config": str(args.config.resolve()),
        "config_sha256": sha256_file(args.config),
        "checkpoint": str(args.checkpoint.resolve()),
        "checkpoint_sha256": sha256_file(args.checkpoint),
        "dense_dir": str(args.dense_dir.resolve()),
        "dense_identity": dense_identity,
        "gradient_cache_identity_sha256": cache_identity,
        "trajectory_count": len(per_trajectory),
        "anchor_count": len(all_targets),
        "proposal": {
            "alpha": args.alpha,
            "epsilon": args.epsilon,
            "construction": "within-trajectory percentile-rank safe mixture after linear interpolation at deterministic quantile anchors",
            "K": [4, 8, 16],
        },
        "target_estimator": {
            "definition": "sqrt(mean over estimator seeds and draws of squared full action-model gradient L2 norm)",
            "seeds": args.target_seeds,
            "samples_per_estimator": args.samples_per_estimator,
            "randomized_beta_stratification": True,
            "antithetic_gaussian_noise": True,
            "common_dropout_within_antithetic_pair": True,
            "frozen_backbone_eval": True,
            "action_model_mode": "train",
            "model_dtype": "float32",
        },
        "aggregation": "equal empirical trajectory weight; mean per-trajectory second moment, matching hierarchical_alignment_audit",
        "aggregate_second_moments": aggregate,
        "aggregate_ratios": ratios,
        "trajectory_statistics": trajectory_statistics,
        "sanity_checks": checks,
    }
    atomic_json(args.output / "optimization_fidelity_metadata.json", metadata)
    if not all(checks.values()):
        raise RuntimeError(f"optimization fidelity checks failed: {checks}")
    readme = f"""# Optimization Fidelity on the Dense Temporal Panel

- Status: complete
- Panel: {len(per_trajectory)} trajectories, {len(all_targets)} valid anchors
- Checkpoint: `{args.checkpoint.resolve()}`
- Gradient target: full action-model gradient RMS, four estimator seeds and eight draws per seed
- Proposal: within-trajectory percentile safe mixture, alpha={args.alpha}, epsilon={args.epsilon}
- Aggregation: equal empirical trajectory weight, matching `hierarchical_alignment_audit`

This is a 512-trajectory Dense-reference offline audit. It is not a full-manifest
training proposal, a continuation run, or a downstream success-rate evaluation.

Formal values are stored in `optimization_fidelity_summary.csv` and
`optimization_fidelity_metadata.json`. All required sanity checks passed.
"""
    (args.output / "README.md").write_text(readme, encoding="utf-8")
    print(json.dumps(metadata, indent=2), flush=True)


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--dense-dir", type=Path, required=True)
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument("--config", type=Path, required=True)
    parser.add_argument("--checkpoint", type=Path, required=True)
    parser.add_argument(
        "--target-seeds", type=int, nargs="+", default=[27300, 27301, 27302, 27303]
    )
    parser.add_argument("--samples-per-estimator", type=int, default=8)
    subparsers = parser.add_subparsers(dest="command", required=True)
    worker_parser = subparsers.add_parser("worker")
    worker_parser.add_argument("--shard-index", type=int, required=True)
    worker_parser.add_argument("--num-shards", type=int, required=True)
    worker_parser.set_defaults(func=worker)
    final_parser = subparsers.add_parser("finalize")
    final_parser.add_argument("--alpha", type=float, default=0.2)
    final_parser.add_argument("--epsilon", type=float, default=0.1)
    final_parser.add_argument("--comparison-tolerance", type=float, default=1e-12)
    final_parser.add_argument("--started-at", type=str, required=True)
    final_parser.set_defaults(func=finalize)
    args = parser.parse_args()
    if len(args.target_seeds) != 4 or len(set(args.target_seeds)) != 4:
        raise ValueError("exactly four unique target estimator seeds are required")
    if args.samples_per_estimator != 8:
        raise ValueError("the Gate42F contract requires eight samples per estimator")
    args.func(args)


if __name__ == "__main__":
    main()