"""Verify the curated BarunAction-35M public release without network access.""" from __future__ import annotations import argparse import hashlib import json from collections import Counter from pathlib import Path from typing import Any EXPECTED_FILES = { "barunaction-aggregate.json": ( 5841, "5d7244d2fa449a7ce4b2aa3a20fc095fb118a38920b1dd9181f56669a7ddfea1", ), "barunaction-predictions.jsonl": ( 282046, "e5aea59e5090e3aa7c9dbd6b1a810c753410a159ce18dc8bb889fde3a58b8665", ), "int8-paired-outcomes.jsonl": ( 169082, "e63e48dc8149b615ab03a14fb9fff9b1435ab59c6623e72c16b44d34d7c651ad", ), "manifest.json": ( 1908, "d7d4223a58ac0277653c44209d9cb35edc0d98754b0c543019c0be67545bd318", ), "qwen-aggregate.json": ( 5822, "2184b93d383f1136c050d95e71fb129bbe083b3fea4de86da6bdf5ec7f3e1756", ), "qwen-paired-outcomes.jsonl": ( 95113, "ad5d44e741f9b98a0a0ffaadfa147c142cdaf8fea8801a11b62b7d323e93730a", ), "qwen-predictions.jsonl": ( 282394, "836788091d08b931bf37565d197c4053e6e6f4664d379c38e7668ae9b8739b08", ), "qwen-provenance.json": ( 3434, "1a1c19cdfacaa25e296ab1c3f92a42ebc8c08802c0cec8047ab581d50669769e", ), } EXPECTED_MODEL_FILES = { "barun_config.json": "9b3a1d71baa95a198744d250f9629231738d942570b8685c44307fd83dd33565", "checkpoint_manifest.json": ( "c743ab7c4d33ae75c6b0aa4547458a961b92766da8fcf85fd148fda2ebb5530a" ), "model.safetensors": "fdb95ccf58a095e0d321be998924318b35ee59a334f6dd97d8726d2cf80021d3", "tokenizer.json": "70ded9605fccd09c2340ca7e225361eab0ae8b4dbbb0d6e26343ab5183979db6", } EXPECTED_QWEN_OUTCOMES = { "both_correct": 583, "both_wrong": 74, "candidate_only_win": 19, "baseline_only_win": 80, } EXPECTED_MANIFEST_FILES = frozenset(EXPECTED_FILES) - {"manifest.json"} ALLOWED_PREDICTION_KEYS = { "generated_tokens", "generation_failure", "id", "prediction_raw", "prompt_tokens", "truncated", } class ReleaseVerificationError(ValueError): """Raised when a public release commitment does not reproduce.""" def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as source: for block in iter(lambda: source.read(1024 * 1024), b""): digest.update(block) return digest.hexdigest() def _strict_json(path: Path) -> Any: def reject_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: value: dict[str, Any] = {} for key, item in pairs: if key in value: raise ReleaseVerificationError(f"duplicate JSON key {key!r} in {path}") value[key] = item return value try: return json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=reject_pairs) except (OSError, UnicodeError, json.JSONDecodeError) as error: raise ReleaseVerificationError(f"invalid JSON: {path}") from error def _jsonl(path: Path) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): try: row = json.loads(line) except json.JSONDecodeError as error: raise ReleaseVerificationError(f"invalid JSONL at {path}:{line_number}") from error if not isinstance(row, dict): raise ReleaseVerificationError(f"non-object JSONL row at {path}:{line_number}") rows.append(row) return rows def verify_release(root: Path, *, checkpoint: Path | None = None) -> dict[str, Any]: evidence = root / "benchmarks" / "evidence" checked: dict[str, dict[str, int | str]] = {} for name, (expected_bytes, expected_sha256) in EXPECTED_FILES.items(): path = evidence / name if not path.is_file(): raise ReleaseVerificationError(f"missing curated evidence file: {path}") actual_bytes = path.stat().st_size actual_sha256 = _sha256(path) if actual_bytes != expected_bytes or actual_sha256 != expected_sha256: raise ReleaseVerificationError(f"curated evidence hash mismatch: {path}") checked[name] = {"bytes": actual_bytes, "sha256": actual_sha256} predictions: dict[str, list[dict[str, Any]]] = {} for name in ("barunaction-predictions.jsonl", "qwen-predictions.jsonl"): rows = _jsonl(evidence / name) if len(rows) != 756: raise ReleaseVerificationError(f"{name} must contain exactly 756 rows") if any(set(row) != ALLOWED_PREDICTION_KEYS for row in rows): raise ReleaseVerificationError(f"{name} contains a non-public prediction field") ids = [row["id"] for row in rows] if len(ids) != len(set(ids)): raise ReleaseVerificationError(f"{name} contains duplicate sample IDs") predictions[name] = rows barun_ids = [row["id"] for row in predictions["barunaction-predictions.jsonl"]] qwen_ids = [row["id"] for row in predictions["qwen-predictions.jsonl"]] if barun_ids != qwen_ids: raise ReleaseVerificationError("BarunAction and Qwen prediction IDs are not aligned") paired = _jsonl(evidence / "qwen-paired-outcomes.jsonl") if len(paired) != 756 or [row.get("id") for row in paired] != barun_ids: raise ReleaseVerificationError("paired comparison IDs do not align with predictions") outcome_counts = Counter(str(row.get("outcome")) for row in paired) if dict(outcome_counts) != EXPECTED_QWEN_OUTCOMES: raise ReleaseVerificationError("paired comparison outcomes do not reproduce") if any( set(row) != {"baseline_ast_exact", "candidate_ast_exact", "id", "outcome"} for row in paired ): raise ReleaseVerificationError("paired outcomes contain non-public fields") int8_rows = _jsonl(evidence / "int8-paired-outcomes.jsonl") if len(int8_rows) != 756: raise ReleaseVerificationError("int8 paired evidence must contain exactly 756 rows") aggregates = { "barunaction": _strict_json(evidence / "barunaction-aggregate.json"), "qwen": _strict_json(evidence / "qwen-aggregate.json"), } expected_exact = {"barunaction": (602, 756), "qwen": (663, 756)} for name, (numerator, denominator) in expected_exact.items(): metric = aggregates[name].get("ast_exact_match") if not isinstance(metric, dict): raise ReleaseVerificationError(f"{name} aggregate lacks ast_exact_match") if metric.get("numerator") != numerator or metric.get("denominator") != denominator: raise ReleaseVerificationError(f"{name} aggregate exact count changed") checkpoint_result: dict[str, str] | None = None if checkpoint is not None: checkpoint_result = {} for name, expected_sha256 in EXPECTED_MODEL_FILES.items(): path = checkpoint / name if not path.is_file(): raise ReleaseVerificationError(f"missing model file: {path}") actual_sha256 = _sha256(path) if actual_sha256 != expected_sha256: raise ReleaseVerificationError(f"model file hash mismatch: {path}") checkpoint_result[name] = actual_sha256 manifest = _strict_json(evidence / "manifest.json") if manifest.get("schema_version") != "barunaction-curated-benchmark-evidence-v1": raise ReleaseVerificationError("unexpected curated evidence manifest version") benchmark = manifest.get("benchmark") if not isinstance(benchmark, dict): raise ReleaseVerificationError("curated evidence manifest lacks benchmark boundary") expected_benchmark = { "dataset": "google/mobile-actions", "dataset_revision": "e920309bc2acbc2e99a5e3201cf37df2b9fd9151", "dev_manifest_sha256": ("988bdce5874d1f1a775feeb5ba2b58cd2bdc128f57e73cb9a63d535fae7c1d55"), "dev_rows": 756, "evaluator_version": "action-ir-v1.0.0", "official_evaluation_rows_accessed": 0, "scorer_version": "barun-mobile-actions-score-v1", } if benchmark != expected_benchmark: raise ReleaseVerificationError("curated evidence benchmark boundary changed") redistribution = manifest.get("redistribution") expected_redistribution = { "gold_labels_included": False, "official_evaluation_included": False, "prompts_included": False, "raw_or_processed_dataset_rows_included": False, } if redistribution != expected_redistribution: raise ReleaseVerificationError("curated evidence redistribution boundary changed") manifest_files = manifest.get("files") if not isinstance(manifest_files, dict) or set(manifest_files) != EXPECTED_MANIFEST_FILES: raise ReleaseVerificationError("curated evidence manifest inventory changed") for name in sorted(EXPECTED_MANIFEST_FILES): declaration = manifest_files.get(name) if not isinstance(declaration, dict): raise ReleaseVerificationError(f"curated evidence declaration missing: {name}") expected_bytes, expected_sha256 = EXPECTED_FILES[name] if ( declaration.get("bytes") != expected_bytes or declaration.get("sha256") != expected_sha256 ): raise ReleaseVerificationError(f"curated evidence declaration changed: {name}") if name.endswith(("predictions.jsonl", "paired-outcomes.jsonl")): if ( declaration.get("rows") != 756 or declaration.get("contains_gold_labels") is not False ): raise ReleaseVerificationError(f"curated evidence row boundary changed: {name}") elif set(declaration) != {"bytes", "sha256"}: raise ReleaseVerificationError( f"unexpected curated evidence declaration fields: {name}" ) return { "barunaction_ast_exact": {"denominator": 756, "numerator": 602}, "checkpoint": checkpoint_result, "evidence_files": checked, "official_evaluation_rows_accessed": benchmark["official_evaluation_rows_accessed"], "ok": True, "qwen_ast_exact": {"denominator": 756, "numerator": 663}, "qwen_paired_outcomes": dict(sorted(outcome_counts.items())), "schema_version": "barunaction-public-release-verification-v1", } def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) parser.add_argument("--checkpoint", type=Path) args = parser.parse_args() result = verify_release(args.root.resolve(), checkpoint=args.checkpoint) print(json.dumps(result, indent=2, sort_keys=True)) return 0 if __name__ == "__main__": raise SystemExit(main())