v3/test_release.py

#!/usr/bin/env python3
"""Deterministic release gates for the Speedup Patch v3 bundle."""

import json
from pathlib import Path


ROOT = Path(__file__).resolve().parent
R = ROOT / "results"
campaign = json.loads((R / "libero_adm_summary.json").read_text())
paper = json.loads((R / "paper_artifact_audit.json").read_text())
derived = json.loads((R / "derived_release_summary.json").read_text())
iql = json.loads((R / "offline_iql_mechanism.json").read_text())

assert campaign["dataset"]["revision"] == "6ce6aaaaabdbe590b1eef5cd29c0d33f14a08551"
assert campaign["dataset"]["sha256"] == "6d6360559a4b71df3bcabd9b55a75451f04375449ca5267616cbc1c21528eac5"
assert campaign["dataset"]["episodes"] == 27
assert campaign["dataset"]["test_any_step_samples"] == 4840
assert campaign["official_adm"]["commit"] == "68c28b9bbf6b95a801ebb30ea47294ad2d8d9cb3"

adm = campaign["mean_results"]["official_adm_ar_model"]
mlp = campaign["mean_results"]["parameter_matched_mlp"]
assert abs(adm["trainable_parameters"] / mlp["trainable_parameters"] - 1) < .02
assert adm["state_mse"] < mlp["state_mse"]
assert adm["eef_rmse"] < mlp["eef_rmse"]
assert adm["displacement_spearman"] > mlp["displacement_spearman"]
assert derived["official_adm_real_libero"]["adm_wins_eef_rmse_long_horizons"] == 4

assert round(paper["claim_2_bigym"]["recomputed_sup_success"], 2) == .67
assert round(paper["claim_3_libero"]["pi05_sup_success"], 3) == .973
assert round(paper["claim_3_libero"]["pi05_sup_speedup"], 2) == 1.35
assert abs(paper["claim_3_libero"]["vla_sup_success"] - .9365) < 1e-12
assert round(paper["claim_3_libero"]["vla_sup_speedup"], 2) == 1.34
assert round(paper["claim_4_real"]["sup_success"], 3) == .611
assert round(paper["claim_4_real"]["demo_success"], 3) == .600
assert round(paper["claim_4_real"]["sup_speedup_from_displayed_steps"], 2) == 2.17
assert paper["claim_5_efficiency"]["parameter_ratio"] == 781.25
assert paper["claim_6_figure7"]["adm_stronger_negative_suites"] == 4
assert paper["claim_6_figure7"]["source_sha256"] == "5295075ab0c1d73fa00566a5c79e5703e84ad5e3cb59df47aa5bf6627436a8a8"
assert (R / "speedup_patch_v3.svg").stat().st_size > 10_000
assert iql["configuration"]["expectile"] == .95
assert iql["configuration"]["gamma"] == .1
assert iql["configuration"]["penalty"] == -2
assert iql["test"]["contexts"] == 462
assert iql["test"]["predicted_violation_rate"] == 0
assert iql["test"]["safe_oracle_action_agreement"] == 1

print("all Speedup Patch v3 release gates passed")