scripts/benchmark_verb_object_mp.py

#!/usr/bin/env python3
"""
Grid-test motion planning for VerbObjectColor-v1 (requires working mplib; use CPU if GPU segfaults).

  python scripts/benchmark_verb_object_mp.py -n 5 --obj-xy 0 0.02 -b cpu --render-backend cpu
  python scripts/benchmark_verb_object_mp.py -v -n 8   # per-seed OK/FAIL lines

Some machines still hit mplib segfaults; run this locally to get real failure tables.
Prints any (verb, shape) with at least one failed trial. Exit 1 if any failure.
"""
from __future__ import annotations

import argparse
import os
import sys

script_dir = os.path.dirname(os.path.abspath(__file__))
root = os.path.dirname(script_dir)
if root not in sys.path:
    sys.path.insert(0, root)

import gymnasium as gym

import mani_skill.envs  # noqa: F401
from mani_skill.examples.motionplanning.panda.run import MP_SOLUTIONS


def main():
    p = argparse.ArgumentParser()
    p.add_argument(
        "--verbs",
        nargs="*",
        default=[
            "lift",
            "grasp",
            "push",
            "pull",
            "rotate",
            "slide",
            "nudge",
            "touch",
        ],
    )
    p.add_argument(
        "--shapes",
        nargs="*",
        default=[
            "cube",
            "sphere",
            "cup",
            "ring",
            "car",
            "pyramid",
            "star",
        ],
    )
    p.add_argument("-n", type=int, default=5, help="seeds 0..n-1 per pair")
    p.add_argument("--obj-xy", type=float, nargs=2, default=[0.0, 0.02])
    p.add_argument("-b", "--sim-backend", default="cpu", choices=["cpu", "gpu"])
    p.add_argument("--render-backend", default="cpu", choices=["cpu", "gpu"])
    p.add_argument(
        "-v",
        "--verbose",
        action="store_true",
        help="Print OK/FAIL per trial (verb, shape, seed, reason).",
    )
    args = p.parse_args()

    solve = MP_SOLUTIONS["VerbObjectColor-v1"]
    bad: list[tuple[str, str, int, int, int]] = []

    for verb in args.verbs:
        for shape in args.shapes:
            fails = exc = 0
            for seed in range(args.n):
                env = gym.make(
                    "VerbObjectColor-v1",
                    obs_mode="none",
                    control_mode="pd_joint_pos",
                    render_mode=None,
                    sim_backend=args.sim_backend,
                    render_backend=args.render_backend,
                    verb=verb,
                    object_shape=shape,
                    object_color_id=0,
                    distractor_max=0,
                    object_size_jiggle=0.0,
                    task_difficulty=1.0,
                )
                try:
                    res = solve(
                        env,
                        seed=seed,
                        debug=False,
                        vis=False,
                        options={"obj_xy": list(args.obj_xy)},
                    )
                except Exception as e:
                    exc += 1
                    fails += 1
                    if args.verbose:
                        print(f"FAIL {verb:7} {shape:8} seed={seed} exception={e!r}")
                else:
                    if res == -1 or res is None:
                        fails += 1
                        if args.verbose:
                            print(
                                f"FAIL {verb:7} {shape:8} seed={seed} reason=plan_or_exec_-1"
                            )
                    elif not bool(res[-1]["success"].item()):
                        fails += 1
                        if args.verbose:
                            print(
                                f"FAIL {verb:7} {shape:8} seed={seed} reason=eval_success_false"
                            )
                    elif args.verbose:
                        print(f"OK   {verb:7} {shape:8} seed={seed}")
                finally:
                    env.close()
            if fails:
                bad.append((verb, shape, fails, args.n, exc))

    print(
        f"VerbObjectColor-v1 MP grid: obj_xy={list(args.obj_xy)}, {args.n} seeds/pair, backends={args.sim_backend}/{args.render_backend}"
    )
    if not bad:
        print("All pairs succeeded.")
        return 0
    print("Failures (verb, shape, fail_count, trials, exceptions):")
    for row in bad:
        print(" ", row)
    return 1


if __name__ == "__main__":
    raise SystemExit(main())