#!/usr/bin/env python3 """ Run RBM inference locally: load a checkpoint from HuggingFace and compute per-frame progress and success for a video (or .npy/.npz frames) and task instruction. Writes rewards .npy, success-probs .npy, and a progress/success plot. Requires the robometer package. Example: python scripts/example_inference_local.py \\ --model-path aliangdw/qwen4b_pref_prog_succ_8_frames_all_part2 \\ --video /path/to/video.mp4 \\ --task "Pick up the red block and place it in the bin" """ from __future__ import annotations import argparse import json from pathlib import Path import matplotlib.pyplot as plt import numpy as np from benchmark_progress_mark_local import ( DEFAULT_INFERENCE_MODE, DEFAULT_MAX_FRAMES, DEFAULT_MIN_FRAMES, DEFAULT_PREFIX_BATCH_SIZE, DEFAULT_PREFIX_SAMPLE_FRAMES, RobometerLocalRunner, build_frame_retry_schedule, is_cuda_oom_error, load_all_video_frames, sample_video_frames_with_indices, ) from robometer.evals.eval_viz_utils import create_combined_progress_success_plot def main() -> None: parser = argparse.ArgumentParser( description="Run RBM inference locally: load model from HuggingFace and compute per-frame progress and success.", epilog="Outputs: .npy (rewards), _success_probs.npy, _progress_success.png", formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--model-path", default="../../models/Robometer-4B", help="HuggingFace model id or local checkpoint path") parser.add_argument("--video", default="example_videos/soar_put_green_stick_in_brown_bowl.mp4", help="Video path/URL or .npy/.npz with frames (T,H,W,C)") parser.add_argument("--task", default="Put green stick in brown bowl", help="Task instruction for the trajectory") parser.add_argument("--fps", type=float, default=1.0, help="FPS when sampling from video (default: 1.0)") parser.add_argument("--max-frames", type=int, default=DEFAULT_MAX_FRAMES, help="Max frames to extract from video (default: 128)") parser.add_argument("--min-frames", type=int, default=DEFAULT_MIN_FRAMES, help="Minimum retry frame budget after OOM (default: 32)") parser.add_argument( "--inference-mode", choices=["frame_steps", "whole"], default=DEFAULT_INFERENCE_MODE, help="frame_steps matches benchmark behavior; whole does a single forward pass on the full sampled trajectory", ) parser.add_argument("--prefix-sample-frames", type=int, default=DEFAULT_PREFIX_SAMPLE_FRAMES, help="Frames per prefix in frame_steps mode (default: 4)") parser.add_argument("--prefix-batch-size", type=int, default=DEFAULT_PREFIX_BATCH_SIZE, help="Batch size for prefix inference in frame_steps mode (default: 1)") parser.add_argument( "--adaptive-max-frames", dest="adaptive_max_frames", action="store_true", help="On CUDA OOM in whole mode, retry with a smaller frame budget", ) parser.add_argument( "--no-adaptive-max-frames", dest="adaptive_max_frames", action="store_false", help="Disable frame-budget retry on CUDA OOM", ) parser.add_argument( "--success-threshold", type=float, default=0.5, help="Threshold for binary success in plot (default: 0.5)", ) parser.add_argument("--out", default=None, help="Output path for rewards .npy (default: _rewards.npy)") parser.set_defaults(adaptive_max_frames=True) args = parser.parse_args() video_path = Path(args.video) # Create output directory: scripts/outputs/{video_name}/ if args.out is not None: out_path = Path(args.out) else: output_dir = Path(__file__).parent / "outputs" / video_path.stem output_dir.mkdir(parents=True, exist_ok=True) out_path = output_dir / f"{video_path.stem}_rewards.npy" runner = RobometerLocalRunner(model_path=args.model_path) all_frames, native_fps = load_all_video_frames(video_path) retry_schedule = ( build_frame_retry_schedule(args.max_frames, args.min_frames, bool(args.adaptive_max_frames)) if args.inference_mode == "whole" else [int(args.max_frames)] ) frames = None rewards = None success_probs = None used_max_frames = retry_schedule[0] for attempt_idx, frame_budget in enumerate(retry_schedule, start=1): frames, _ = sample_video_frames_with_indices( all_frames, native_fps=native_fps, fps=float(args.fps), max_frames=int(frame_budget), required_frames=[], ) print( f"Loaded {len(all_frames)} total frames; sampled {len(frames)} frames at fps={float(args.fps):g} " f"(max_frames={int(frame_budget)}, try {attempt_idx}/{len(retry_schedule)})" ) try: rewards, success_probs = runner.compute_rewards_per_frame( video_frames=frames, task=args.task, inference_mode=args.inference_mode, prefix_sample_frames=int(args.prefix_sample_frames), prefix_batch_size=int(args.prefix_batch_size), ) used_max_frames = int(frame_budget) break except RuntimeError as exc: if args.inference_mode != "whole" or not is_cuda_oom_error(exc) or attempt_idx == len(retry_schedule): raise next_budget = retry_schedule[attempt_idx] print( f"[OOM] whole inference hit CUDA OOM at max_frames={int(frame_budget)}; " f"retrying with max_frames={int(next_budget)}" ) runner.reload_model() if rewards is None or success_probs is None or frames is None: raise RuntimeError("Robometer inference did not produce outputs.") # Save results (directory already created) np.save(str(out_path), rewards) success_path = out_path.with_name(out_path.stem + "_success_probs.npy") np.save(str(success_path), success_probs) show_success = success_probs.size > 0 and success_probs.size == rewards.size success_binary = (success_probs > float(args.success_threshold)).astype(np.int32) if show_success else None fig = create_combined_progress_success_plot( progress_pred=rewards, num_frames=int(frames.shape[0]), success_binary=success_binary, success_probs=success_probs if show_success else None, success_labels=None, title=f"Progress/Success — {video_path.name}", ) plot_path = out_path.with_name(out_path.stem + "_progress_success.png") fig.savefig(str(plot_path), dpi=200) plt.close(fig) summary = { "video": str(video_path), "num_frames": int(frames.shape[0]), "inference_mode": args.inference_mode, "max_frames_used": int(used_max_frames), "model_path": args.model_path, "out_rewards": str(out_path), "out_success_probs": str(success_path), "out_plot": str(plot_path), "reward_min": float(np.min(rewards)) if rewards.size else None, "reward_max": float(np.max(rewards)) if rewards.size else None, "reward_mean": float(np.mean(rewards)) if rewards.size else None, } print(json.dumps(summary, indent=2)) if __name__ == "__main__": main()