| |
| """Run one fused verdict evaluator and seal its complete result units.""" |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import sys |
| from pathlib import Path |
|
|
| sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) |
|
|
| from repro_control.archives import safe_extract_tar_gz |
| from repro_control.artifacts import finalize_attempt, verify_read_back |
| from repro_control.hashing import atomic_write_json, sha256_file |
| from repro_control.heartbeat import heartbeat, marker |
| from repro_control.runtime import ( |
| assert_evaluation_isolated, |
| configure_scientific_runtime, |
| load_job_manifest, |
| require_control_freeze, |
| verify_science_spec, |
| ) |
|
|
|
|
| def main() -> int: |
| parser = argparse.ArgumentParser() |
| parser.add_argument("--science-spec", type=Path, required=True) |
| parser.add_argument("--job-manifest", type=Path, required=True) |
| parser.add_argument("--control-dir", type=Path, required=True) |
| parser.add_argument("--freeze-sha256", required=True) |
| parser.add_argument("--unit-manifest", type=Path) |
| parser.add_argument("--output-dir", type=Path) |
| parser.add_argument("--eval-archive", type=Path) |
| parser.add_argument("--eval-destination", type=Path, default=Path("/data/eval")) |
| parser.add_argument("--imports-archive", type=Path) |
| parser.add_argument("--imports-destination", type=Path, default=Path("/imports")) |
| parser.add_argument("--dry-run", action="store_true") |
| args = parser.parse_args() |
| spec = verify_science_spec(args.science_spec) |
| manifest = load_job_manifest(args.job_manifest, freeze_sha256=args.freeze_sha256) |
| assert_evaluation_isolated(manifest) |
| if manifest["logical_id"] not in spec["evaluators"]: |
| raise SystemExit("unregistered evaluator logical identity") |
| require_control_freeze( |
| args.control_dir, |
| args.freeze_sha256, |
| manifest["science_spec_sha256"], |
| ) |
| configure_scientific_runtime() |
| if args.dry_run: |
| print( |
| json.dumps( |
| { |
| "contract_verified": True, |
| "logical_id": manifest["logical_id"], |
| "outcomes": {}, |
| } |
| ) |
| ) |
| return 0 |
| if args.unit_manifest is None or args.output_dir is None: |
| raise SystemExit("--unit-manifest and --output-dir are required for execution") |
| if sha256_file(args.unit_manifest) != manifest["hashes"]["config"]: |
| raise SystemExit("evaluator unit manifest hash mismatch") |
| args.output_dir.mkdir(parents=True, exist_ok=False) |
| if args.eval_archive: |
| safe_extract_tar_gz(args.eval_archive, args.eval_destination) |
| if args.imports_archive: |
| safe_extract_tar_gz(args.imports_archive, args.imports_destination) |
| marker("GPU_READY", manifest["logical_id"]) |
| from repro_control.evaluation import evaluate_units |
|
|
| with heartbeat(f"SCIENTIFIC_EVAL:{manifest['logical_id']}"): |
| evaluate_units(args.unit_manifest, args.output_dir / "results.json") |
| atomic_write_json( |
| args.output_dir / "evaluation-receipt.json", |
| { |
| "format": 1, |
| "logical_id": manifest["logical_id"], |
| "attempt_id": manifest["attempt_id"], |
| "unit_manifest_sha256": sha256_file(args.unit_manifest), |
| "results_sha256": sha256_file(args.output_dir / "results.json"), |
| "outcomes": {}, |
| }, |
| ) |
| finalize_attempt( |
| args.output_dir, |
| logical_id=manifest["logical_id"], |
| attempt_id=manifest["attempt_id"], |
| expected_outputs=manifest["expected_outputs"], |
| ) |
| verify_read_back( |
| args.output_dir, |
| logical_id=manifest["logical_id"], |
| attempt_id=manifest["attempt_id"], |
| ) |
| marker("DONE", f"{manifest['logical_id']} {manifest['attempt_id']}") |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
