"""CLI: runs all six claim verifiers and writes EVAL.md + raw artifacts. Usage: python -m rhi_repro.cli (the fixed run command is `bash run.sh`) """ from __future__ import annotations import json import os import platform import socket import subprocess import sys import time import numpy as np from . import verifiers as V ART = os.path.join(".openresearch", "artifacts") def _git_sha() -> str: try: return subprocess.check_output( ["git", "rev-parse", "HEAD"], stderr=subprocess.DEVNULL).decode().strip() except Exception: return "unknown" def main(): t0 = time.time() os.makedirs(ART, exist_ok=True) print("=" * 72, flush=True) print("Reproduction: A Finite-Sample Analysis of Distributionally Robust", flush=True) print("Average-Reward RL (arXiv 2505.12462)", flush=True) print("=" * 72, flush=True) print(f"host={socket.gethostname()} py={platform.python_version()} " f"np={np.__version__} sha={_git_sha()[:10]}", flush=True) claims = [ ("c1_thm53_sample_complexity", V.verify_c1_thm53, dict(art_dir=ART)), ("c2_thm42_reduction", V.verify_c2_thm42, dict(art_dir=ART)), ("c3_lemma51", V.verify_c3_lemma51, dict(n_mdp=300, art_dir=ART)), ("c4_thm52_halpern", V.verify_c4_thm52, dict(art_dir=ART)), ("c5_rsample", V.verify_c5_rsample, dict(art_dir=ART)), ("c6_first_finite", V.verify_c6_first_finite, dict(art_dir=ART)), ] summary = {} for name, fn, kw in claims: print(f"\n--- {name} ---", flush=True) ts = time.time() try: res = fn(**kw) res["_runtime_s"] = round(time.time() - ts, 2) res["_claim"] = name summary[name] = res v = res.get("verdict", "?") print(f" verdict: {v} ({res['_runtime_s']}s)", flush=True) for k in res: if k in ("verdict",) or k.startswith("_"): continue val = res[k] if isinstance(val, (int, float, bool, str)): print(f" {k} = {val}", flush=True) except Exception as e: import traceback traceback.print_exc() summary[name] = dict(verdict="BLOCKED", error=str(e), _runtime_s=round(time.time() - ts, 2)) total = round(time.time() - t0, 1) _write_eval(summary, total) _write_manifest(summary, total) # exit nonzero if any non-VERIFIED (so the run "fails" the gate) bad = [n for n, r in summary.items() if r.get("verdict") not in ("VERIFIED", "FALSIFIED")] print(f"\nTotal runtime: {total}s", flush=True) if bad: print(f"BLOCKED claims: {bad}", flush=True) print("Wrote EVAL.md and artifacts under .openresearch/artifacts/", flush=True) def _write_eval(summary, total): path = os.path.join("EVAL.md") lines = ["# EVAL.md — reproduction of arXiv 2505.12462", "", f"- Total runtime: {total}s", f"- Git SHA: {_git_sha()[:10]}", f"- Date: {time.strftime('%Y-%m-%dT%H:%M:%S%z')}", "", "| Claim | Verdict | Key result | Runtime |", "|---|---|---|---|"] for name, res in summary.items(): v = res.get("verdict", "?") key = _key_result(name, res) lines.append(f"| {name} | **{v}** | {key} | {res.get('_runtime_s','?')}s |") lines += ["", "## Raw artifacts", "", "All raw CSV/JSON outputs are under `.openresearch/artifacts/`."] with open(path, "w") as f: f.write("\n".join(lines)) def _key_result(name, res): try: if name.startswith("c1"): return (f"slope={res.get('log_log_slope'):.2f} (expect ~2), " f"all eps-optimal={res.get('all_runs_eps_optimal')}") if name.startswith("c2"): return (f"transfer_ok={res.get('transfer_ok')}, requires_H={res.get('requires_H')}, " f"red_slope={res.get('reduction_scaling_slope'):.2f} > " f"rhi_slope={res.get('rhi_scaling_slope'):.2f}") if name.startswith("c3"): return (f"{res.get('n_checked')} instances, " f"violations={res.get('n_rhs_violations')}, " f"NC effective={res.get('negative_control_effective')}") if name.startswith("c4"): conv = [r["halpern_converges"] for r in res.get("instances", [])] return (f"halpern converges {sum(conv)}/{len(conv)}; " f"non-expansion ratio={res.get('max_contraction_ratio'):.2f}") if name.startswith("c5"): return (f"Prop C.2 holds={res.get('prop_C2_all_within_eps')}, " f"uses_sampled_T={res.get('uses_sampled_T')}, " f"recursive