v3/audit_paper_artifacts.py

#!/usr/bin/env python3
"""Recompute Speedup Patch registered values from the exact arXiv source."""

from __future__ import annotations

import hashlib
import json
import re
from pathlib import Path

import numpy as np


ROOT = Path(__file__).resolve().parent
TEX = (ROOT / "paper_source" / "05_experiment.tex").read_text()
APP = (ROOT / "paper_source" / "appendix.tex").read_text()


def sha256(path):
    return hashlib.sha256(Path(path).read_bytes()).hexdigest()


required_source_fragments = [
    r"\cg{\textbf{0.67}, \textbf{2.01}$\times$}",
    r"\cg{\textbf{0.973}, \textbf{1.35}$\times$}",
    r"\cg{0.937, \textbf{1.34}$\times$}",
    r"SuP (Ours)         & 5.12M  & 2h  &  1ms (2\%)",
    r"0.611}, \textbf{2.17}",
]
for fragment in required_source_fragments:
    if fragment not in TEX:
        raise AssertionError(f"registered source fragment missing: {fragment}")
if "https://huggingface.co/datasets/openvla/modified_libero_rlds" not in APP:
    raise AssertionError("paper-linked LIBERO dataset URL missing")

# Table 2 values as encoded in source. Speedup is the ratio of sums of
# per-suite successful-trajectory step means; success is the suite mean.
libero = {
    "pi05_base": {
        "success": [.988, .924, .980, .982],
        "steps": [105.3, 267.9, 113.1, 138.1],
    },
    "pi05_sup": {
        "success": [.972, .940, .986, .994],
        "steps": [70.4, 215.2, 93.4, 83.0],
    },
    "vla_base": {
        "success": [.922, .936, .970, .942],
        "steps": [99.6, 255.1, 107.0, 136.2],
    },
    "vla_sup": {
        "success": [.912, .934, .956, .944],
        "steps": [77.3, 204.6, 74.2, 91.4],
    },
}

# Full Appendix Bigym table, ACT rows, all 20 tasks.
bigym_act = {
    "base_success": [
        .45, .15, .28, 1, .58, .78, .35, 1, .20, 1,
        .25, 1, 1, 1, 1, .51, 1, 1, .08, .53,
    ],
    "sup_success": [
        .64, .20, .38, 1, .49, .88, .27, 1, .22, 1,
        .25, 1, 1, 1, 1, .53, 1, 1, .13, .46,
    ],
    "base_steps": [
        340.5, 288.3, 320.3, 275, 194.8, 334.1, 243.6, 449.8, 406.7, 200.3,
        372, 100, 325.5, 175, 148.8, 454.4, 100, 389, 596.9, 312.7,
    ],
    "sup_steps": [
        155.9, 176.6, 156, 131, 167.2, 174.3, 138.7, 212.1, 194.5, 86,
        317.9, 40, 152.2, 84, 65, 231.2, 54, 157.4, 171.2, 181,
    ],
}

real = {
    "base_counts": [11, 15, 27],
    "base_steps": [537.8, 519.5, 221.5],
    "demo_counts": [12, 14, 28],
    "demo_steps": [267.4, 223.2, 124.4],
    "sup_counts": [13, 16, 26],
    "sup_steps": [258.5, 192.5, 138.5],
    "trials_per_task": 30,
}

figure7 = {
    "source_file": "paper_source/figure7_rnn_mlp_contrast.png",
    "source_sha256": sha256(ROOT / "paper_source" / "figure7_rnn_mlp_contrast.png"),
    "values_read_from_figure_annotations": {
        "object": {"adm": -.20, "mlp": -.03},
        "spatial": {"adm": -.34, "mlp": -.22},
        "long": {"adm": -.30, "mlp": -.25},
        "goal": {"adm": -.05, "mlp": -.03},
    },
}
adm = [x["adm"] for x in figure7["values_read_from_figure_annotations"].values()]
mlp = [x["mlp"] for x in figure7["values_read_from_figure_annotations"].values()]
figure7["mean_correlation"] = {"adm": float(np.mean(adm)), "mlp": float(np.mean(mlp))}
figure7["adm_stronger_negative_suites"] = int(
    sum(abs(a) > abs(m) for a, m in zip(adm, mlp))
)

summary = {
    "source": {
        "arxiv": "2603.20658",
        "source_tar_sha256": sha256(
            ROOT / "paper_source" / "arxiv_2603.20658_source.tar.gz"
        ),
        "suP_code_release_found": False,
    },
    "claim_2_bigym": {
        "tasks": 20,
        "recomputed_base_success": float(np.mean(bigym_act["base_success"])),
        "recomputed_sup_success": float(np.mean(bigym_act["sup_success"])),
        "reported_rounded_success": .67,
        "reported_speedup": 2.01,
        "ratio_of_summed_displayed_steps": float(
            np.sum(bigym_act["base_steps"]) / np.sum(bigym_act["sup_steps"])
        ),
        "mean_of_per_task_displayed_step_ratios": float(
            np.mean(
                np.asarray(bigym_act["base_steps"])
                / np.asarray(bigym_act["sup_steps"])
            )
        ),
        "qualification": (
            "Success recomputes to 0.6725 -> 0.67. Displayed rounded task "
            "lengths do not uniquely reproduce 2.01 (1.979 ratio-of-sums; "
            "2.052 mean ratios), so the exact speedup needs unavailable raw rollouts."
        ),
    },
    "claim_3_libero": {
        "pi05_sup_success": float(np.mean(libero["pi05_sup"]["success"])),
        "pi05_sup_speedup": float(
            np.sum(libero["pi05_base"]["steps"])
            / np.sum(libero["pi05_sup"]["steps"])
        ),
        "vla_sup_success": float(np.mean(libero["vla_sup"]["success"])),
        "vla_sup_speedup": float(
            np.sum(libero["vla_base"]["steps"])
            / np.sum(libero["vla_sup"]["steps"])
        ),
    },
    "claim_4_real": {
        "sup_success": float(np.sum(real["sup_counts"]) / 90),
        "demo_success": float(np.sum(real["demo_counts"]) / 90),
        "sup_speedup_from_displayed_steps": float(
            np.sum(real["base_steps"]) / np.sum(real["sup_steps"])
        ),
        "demo_speedup_from_displayed_steps": float(
            np.sum(real["base_steps"]) / np.sum(real["demo_steps"])
        ),
        "reported_speedups": {"sup": 2.17, "demo": 2.07},
        "qualification": (
            "SuP displayed lengths give 2.169 -> 2.17. DemoSpeedup displayed "
            "lengths give 2.079 -> 2.08, while the paper reports 2.07; raw "
            "unrounded rollout lengths are not released."
        ),
    },
    "claim_5_efficiency": {
        "sup_trainable_parameters": 5_120_000,
        "demospeedup_trainable_parameters": 4_000_000_000,
        "parameter_ratio": 4_000_000_000 / 5_120_000,
        "sup_training_hours": 2,
        "demospeedup_training_hours": 20,
        "sup_inference_overhead_ms": 1,
        "correction": (
            "The 5.12M value is total SuP training parameters in Table 3, "
            "not scheduler-only parameters and not Table 4."
        ),
    },
    "claim_6_figure7": figure7,
    "linked_assets": {
        "libero_dataset": "https://huggingface.co/datasets/openvla/modified_libero_rlds",
        "pi05_code": "https://github.com/Physical-Intelligence/openpi/tree/main/examples/libero",
        "vla_adapter": "https://github.com/OpenHelix-Team/VLA-Adapter",
        "demospeedup": "https://github.com/lingxiao-guo/DemoSpeedup",
        "adm": "https://github.com/HxLyn3/ADMPO",
    },
}

out = ROOT / "results" / "paper_artifact_audit.json"
out.parent.mkdir(exist_ok=True)
out.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n")
print(json.dumps(summary, indent=2, sort_keys=True))