twanghcmut/backup-foundation-physics / scripts /render_real2sim_demo.py
twanghcmut's picture
download
raw
25.2 kB
#!/usr/bin/env python
"""S9 real2sim visual deliverable: real video + observed-vs-simulated silhouettes, honestly.
**What this renders, per object.** Two silhouettes per frame, reprojected onto the
real camera image via :func:`fpgm.viz.overlays.project_silhouette_hull` (the same
convex-hull re-projection ``fpgm.datagen.object_poses.compute_silhouette_iou`` and
``write_object_poses_debug_video`` already use -- see this script's own
docstrings below for why nothing here writes a second projector):
* **observed** -- S6's measured pose track (``poses.npz``'s ``T_world_obj``),
amber.
* **simulated** -- the MuJoCo posterior's highest-weight particle's rollout
(:func:`fpgm.physics.simulate.MujocoSimulator.simulate_batched`), magenta.
Plus a small per-frame HUD: frame index, whether S6 actually scored this frame
as an observation, the current observed-vs-simulated translation error in mm,
and (for an object whose simulation is anchored above empty space -- see
below) an explicit, computed-not-hardcoded note of why.
**This is a visualization script, not a new identification run.** It reuses
every piece of ``fpgm.physics`` exactly as ``scripts/run_physics_identification.py``
does (:func:`~fpgm.physics.scene.build_observed_track`,
:func:`~fpgm.physics.scene.build_sim_spec`,
:class:`~fpgm.physics.simulate.MujocoSimulator`,
:func:`~fpgm.physics.likelihood.log_likelihood`,
:func:`~fpgm.physics.inference.accumulate`) and touches no file under
``src/fpgm/physics/``. The one deliberate simplification: the material prior
always comes from :meth:`~fpgm.physics.priors.VlmPriorProposer.fallback_verdict`
(the documented "unknown material" default) rather than a live VLM call --
this script's job is showing what the *simulator* currently produces from a
measured pose track, not re-running the VLM subprocess, and the VLM verdict
only widens/narrows the *prior* particles are drawn from, not the rollout
dynamics themselves. (Confirmed against the demo episode's own prior physics
runs, ``outputs/physics_multibody/.../brick/physics.json`` /
``.../drawer/physics.json``: both were already run this same way, i.e. this is
not a special-cased shortcut invented for this script.)
**If the simulated silhouette falls away from the observed one, most frames,
that is not a bug in this script.** ``fpgm.physics.scene.build_sim_spec`` always
anchors a body at the episode's FIRST PNP frame -- if that frame is not actually
resting near the static-scene heightfield (e.g. the object has not been placed
yet in this phase of the episode), a free body free-falls before any contact is
possible, exactly as it should. See ``fpgm.physics.scene``'s own module
docstring (the heightfield section, and "Scenes are multi-body...") and
:func:`~fpgm.physics.scene.build_sim_spec`'s own
``anchor_height_above_heightfield_m`` diagnostic, which this script reads and
prints into both the HUD and ``summary.json`` rather than re-deriving the
number itself. (Historical note: an earlier version of this module used a
single FITTED PLANE as the scene's only static surface, which was frequently
the wrong surface entirely -- see ``fpgm.physics.scene``'s module docstring for
the measured bug that motivated replacing it with an observed-depth
heightfield.)
Usage::
PYTHONPATH=src python scripts/render_real2sim_demo.py
# non-default episode/objects:
PYTHONPATH=src python scripts/render_real2sim_demo.py \\
--uuid <uuid> --camera <serial> --labels drawer brick --particles 64
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Any
REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO_ROOT / "src"))
import numpy as np # noqa: E402
from scipy.spatial.transform import Rotation # noqa: E402
from fpgm.config_datagen import DatagenProfile # noqa: E402
from fpgm.data.droid_raw import read_mp4_properties # noqa: E402
from fpgm.datagen.robot_buffers import ( # noqa: E402
load_extrinsics_candidates,
read_native_camera_intrinsics,
)
from fpgm.geometry.camera import Camera # noqa: E402
from fpgm.geometry.transforms import transform_points # noqa: E402
from fpgm.physics import inference, likelihood, materials, priors, scene, simulate # noqa: E402
from fpgm.physics.types import ( # noqa: E402
PRISMATIC_PARAMS,
RIGID_PARAMS,
EpisodeLogLik,
ParamSpace,
PhysicsError,
SimResult,
)
from fpgm.utils.io import atomic_write, ensure_dir # noqa: E402
from fpgm.utils.logging import get_logger, setup_logging # noqa: E402
from fpgm.utils.timing import StepTimer # noqa: E402
from fpgm.viz.overlays import ( # noqa: E402
VideoWriter,
draw_hud,
draw_silhouette,
project_silhouette_hull,
read_frames_bgr,
)
logger = get_logger("render_real2sim_demo")
_DEFAULT_UUID = "AUTOLab+0d4edc83+2023-10-21-19h-07m-04s"
_DEFAULT_CAMERA = "22008760"
_DEFAULT_LABELS = ("drawer", "brick")
_DEFAULT_N_PARTICLES = 64
_DEFAULT_SEED = 0
_DEFAULT_OUT_DIR = REPO_ROOT / "outputs" / "real2sim_demo"
# BGR (OpenCV convention), chosen for contrast against both the amber DROID
# tabletop and each other -- not ``fpgm.viz.overlays.color_for``'s indexed
# palette, since these two colours mean "observed" / "simulated" for every
# object, not "which object" (color_for(i) already means the latter, one per
# label, in S6's own object_poses_debug.mp4 -- reusing it here would collide
# meanings).
_OBSERVED_COLOR_BGR = (0, 210, 255) # amber
_SIMULATED_COLOR_BGR = (230, 60, 220) # magenta
#: Heightfield-surface clearance above which fpgm.physics.scene.build_sim_spec itself
#: already flags "anchor frame is above the observed heightfield surface" as a
#: limitation (see that function's own docstring) -- reused verbatim here rather than a
#: second threshold, so this script's "known cause" note fires exactly when
#: build_sim_spec's own diagnostic does.
_ANCHOR_ABOVE_SUPPORT_GATE_M = 0.05
# --------------------------------------------------------------------------- #
# Geometry loading -- observed silhouette needs the SAME canonical points
# fpgm.physics.scene loaded to build the convex-hull collision mesh, at their
# raw (un-rescaled) MESH-vs-OBSERVED_SURFACE convention.
# --------------------------------------------------------------------------- #
def _load_canonical_points(geometry_path: Path, label: str) -> np.ndarray:
"""``(N, 3)`` points in the object's own canonical/local frame.
Mirrors ``fpgm.physics.scene``'s own private ``_canonical_geometry_points``
dispatch (MESH -> ``.glb`` vertices via trimesh, OBSERVED_SURFACE -> a
``.npz``'s ``points_canonical`` key) rather than importing that
underscore-prefixed name across a module boundary this task is not
allowed to modify. Five lines, keyed only on file suffix, matching
``fpgm.datagen.geometry_ops.write_object_mesh_glb`` /
``write_object_point_cloud_npz`` -- the two writers this reads back.
"""
if geometry_path.suffix == ".glb":
import trimesh
mesh = trimesh.load(str(geometry_path), force="mesh", process=False)
return np.asarray(mesh.vertices, dtype=np.float64)
if geometry_path.suffix == ".npz":
with np.load(geometry_path) as npz:
return np.asarray(npz["points_canonical"], dtype=np.float64)
raise PhysicsError(f"{label}: unrecognised geometry file {geometry_path}")
def _load_camera_for_episode(
profile: DatagenProfile,
uuid: str,
camera_serial: str,
video_width: int,
video_height: int,
) -> Camera:
"""The same :class:`Camera` S6 built for this episode/camera.
Native scene-flow intrinsics rescaled to the mp4's own resolution, paired
with whichever extrinsic candidate S2's own alignment gate chose (recorded
in ``robot_buffers/meta.json``'s ``chosen_extrinsics``). Mirrors
``EpisodePipeline._run_s6``'s "camera + extrinsics: mirror S2's own
construction" block (``fpgm/datagen/pipeline.py``) -- duplicated rather
than imported because that block lives inline in a private pipeline
method, not a reusable function; two call sites in that same file
(``_run_s6``, the S8 export path) already duplicate it themselves, so this
is a third occurrence of an established pattern, not a new one.
"""
master_dir = profile.paths.master_dir(uuid, camera_serial)
trajectory_path = profile.paths.episode_dir(uuid) / "trajectory.h5"
camera_intrinsics_native = read_native_camera_intrinsics(
profile.paths.flows_h5(uuid), camera_serial
)
extrinsics_candidates = load_extrinsics_candidates(
profile.paths.cameras_json(uuid), trajectory_path, camera_serial
)
s2_meta = json.loads((master_dir / "robot_buffers" / "meta.json").read_text())
chosen_name = s2_meta["payload"]["chosen_extrinsics"]
chosen = next(c for c in extrinsics_candidates if c.name == chosen_name)
return Camera(camera_intrinsics_native.scaled(video_width, video_height), chosen.world_to_cam)
# --------------------------------------------------------------------------- #
# MuJoCo (w, x, y, z) quaternion -> rotation matrix, for rendering the
# simulated rollout's silhouette (fpgm.physics.likelihood only ever needed the
# inverse direction, matrix -> quat, for its residuals).
# --------------------------------------------------------------------------- #
def _quat_wxyz_to_matrix(quat_wxyz: np.ndarray) -> np.ndarray:
"""``(..., 4)`` MuJoCo-order unit quaternions -> ``(..., 3, 3)`` rotation matrices.
``fpgm.physics.likelihood`` implements the opposite direction
(``_matrix_to_quat_wxyz``) as local numpy specifically to avoid
``scipy.spatial.transform``'s ``(x, y, z, w)`` reordering *and* a measured
ill-conditioning in its own geodesic-angle computation near 0/pi radians
(see that module's docstring). Neither concern applies to a plain
quaternion-to-matrix conversion (no angle extraction, no subtraction of
near-equal floats), so this direction uses ``scipy`` directly rather than
hand-rolling a second Shepperd's-method implementation for one call site.
"""
quat_wxyz = np.asarray(quat_wxyz, dtype=np.float64)
quat_xyzw = quat_wxyz[..., [1, 2, 3, 0]]
flat = Rotation.from_quat(quat_xyzw.reshape(-1, 4)).as_matrix()
return flat.reshape(*quat_wxyz.shape[:-1], 3, 3)
def _divergence_frame(poses_row: np.ndarray) -> int | None:
"""First frame this rollout has no pose (NaN), or ``None`` if it never diverged.
``fpgm.physics.worker_mujoco`` fills ``poses`` with NaN up front and only
overwrites a frame once ``mj_step`` for it succeeds (see that module's own
per-particle loop, which ``break``s out -- without writing that frame's
pose -- the instant it detects divergence). The first NaN row IS the
divergence point by that contract; this reads it directly rather than
re-deriving a divergence test of its own.
"""
finite_per_frame = np.all(np.isfinite(poses_row), axis=-1)
if finite_per_frame.all():
return None
return int(np.argmax(~finite_per_frame))
def _select_best_particle(
track: Any,
sim_result: SimResult,
prior: Any,
particles: np.ndarray,
*,
timer: StepTimer,
) -> tuple[int, float, EpisodeLogLik, str | None]:
"""Highest-weight particle's index, plus its ESS and a degeneracy note if any.
"Posterior-mean rollout" per the task brief means: score every particle's
rollout against the observed track exactly as
``fpgm.physics.inference.accumulate`` would for a real identification run
(one episode, softmax over its log-likelihoods), then take the argmax --
well-defined even when the posterior is degenerate (ESS -> 1, all mass on
one particle), which is the measured, expected case here (both objects'
prior ``physics.json`` runs report ``ess=1.0``).
Falls back to a "least-diverged" ranking -- more frames survived before
hitting NaN, ties broken by lower raw translation+rotation residual via
:func:`~fpgm.physics.likelihood.se3_residuals` (which, unlike
:func:`~fpgm.physics.likelihood.log_likelihood`, does not force ``-inf``
on a diverged particle -- it still has real pre-divergence residuals) --
only in the genuinely degenerate case ``accumulate`` itself refuses to
report a posterior over: every particle's total log-weight is ``-inf``
(see that function's own docstring). This is the brief's own explicitly
anticipated worst case for the brick ("if every brick particle diverges
... still render whatever the least-diverged particle did").
Returns:
``(best_idx, ess, episode_loglik, degeneracy_note)``. ``degeneracy_note``
is ``None`` on the normal path, else the ``PhysicsError`` message
``accumulate`` raised.
"""
loglik = likelihood.log_likelihood(track, sim_result.poses, sim_result.ok, timer=timer)
episode_ll = EpisodeLogLik(
uuid=track.uuid,
camera_serial=track.camera_serial,
label=track.label,
loglik=loglik,
n_obs_frames=track.n_valid,
n_diverged=int(np.sum(~np.asarray(sim_result.ok, dtype=bool))),
)
try:
posterior = inference.accumulate(prior, particles, [episode_ll], timer=timer)
return int(np.argmax(posterior.weights)), float(posterior.ess), episode_ll, None
except PhysicsError as exc:
trans_res, rot_res = likelihood.se3_residuals(track, sim_result.poses, timer=timer)
survived = np.array(
[
sim_result.poses.shape[1] if (df := _divergence_frame(p)) is None else df
for p in sim_result.poses
]
)
residual_sum = np.nansum(trans_res, axis=1) + np.nansum(rot_res, axis=1)
# np.lexsort's LAST key is primary: rank by most-frames-survived first,
# lowest residual second.
order = np.lexsort((residual_sum, -survived))
return int(order[0]), 0.0, episode_ll, str(exc)
# --------------------------------------------------------------------------- #
# Per-object render
# --------------------------------------------------------------------------- #
def render_object(
uuid: str,
camera_serial: str,
label: str,
*,
profile: DatagenProfile,
out_dir: Path,
n_particles: int,
seed: int,
timer: StepTimer,
) -> dict[str, Any]:
"""Score + simulate ``label``, write ``real2sim_<label>.mp4``, return its summary dict."""
logger.info("=== %s: build observed track + sim spec ===", label)
with timer.step("load_track", uuid=uuid, obj_label=label):
track = scene.build_observed_track(uuid, camera_serial, label, profile=profile, timer=timer)
scratch_dir = ensure_dir(out_dir / "scratch" / label)
with timer.step("build_spec", uuid=uuid, obj_label=label):
spec, diagnostics = scene.build_sim_spec(
uuid, camera_serial, label, profile=profile, scratch_dir=scratch_dir, timer=timer,
)
for note in diagnostics.get("limitations", []):
print(f"[{label}] {note}")
body = spec.bodies[0]
space = ParamSpace(RIGID_PARAMS)
if body.kind == "prismatic":
space = space.extended(PRISMATIC_PARAMS)
# See module docstring: fallback (uninformative) material prior, not a
# live VLM call -- this script visualizes the simulator, not the VLM.
verdict = priors.VlmPriorProposer.fallback_verdict(label)
prior = materials.material_prior(verdict, space)
rng = np.random.default_rng(seed)
particles = prior.sample(rng, n_particles)
simulator = simulate.MujocoSimulator(scratch_dir=scratch_dir)
with timer.step("simulate", uuid=uuid, obj_label=label, n=n_particles):
sim_result = simulator.simulate_batched(spec, particles, space, timer=timer)
n_diverged = int(np.sum(~np.asarray(sim_result.ok, dtype=bool)))
with timer.step("score", uuid=uuid, obj_label=label, n=n_particles):
best_idx, ess, episode_ll, degeneracy_note = _select_best_particle(
track, sim_result, prior, particles, timer=timer
)
if degeneracy_note is not None:
print(
f"[{label}] every particle's rollout was inconsistent with the observation "
f"(accumulate() refused a posterior: {degeneracy_note}); rendering the "
"least-diverged particle instead, ESS is reported as 0."
)
best_poses = sim_result.poses[best_idx] # (T, 7)
best_ok = bool(sim_result.ok[best_idx])
div_frame = _divergence_frame(best_poses)
print(
f"{label}: n_particles={n_particles} n_diverged={n_diverged} ess={ess:.3f} "
f"best_idx={best_idx} best_ok={best_ok} divergence_frame={div_frame} "
f"spread_nats={episode_ll.spread_nats:.3f}"
)
# -- translation error, mm, every frame; NaN after the rollout diverges
# propagates through np.linalg.norm honestly (no fabricated "error") -- #
obs_pos_m = track.T_world_obj[:, :3, 3]
sim_pos_m = best_poses[:, :3]
trans_err_mm = np.linalg.norm(obs_pos_m - sim_pos_m, axis=1) * 1000.0
scored = track.valid & np.isfinite(trans_err_mm)
if scored.any():
err_summary = {
"mean_mm": float(np.mean(trans_err_mm[scored])),
"median_mm": float(np.median(trans_err_mm[scored])),
"max_mm": float(np.max(trans_err_mm[scored])),
"n_frames_scored": int(scored.sum()),
}
else:
err_summary = {"mean_mm": None, "median_mm": None, "max_mm": None, "n_frames_scored": 0}
# -- geometry for the two silhouettes (see module-level helpers' docstrings) -- #
meta_path = profile.paths.master_dir(uuid, camera_serial) / "object_poses" / "meta.json"
meta = json.loads(meta_path.read_text())
geometry_path = Path(meta["payload"]["geometry_paths"][label])
canonical_points = _load_canonical_points(geometry_path, label)
scaled_points = canonical_points * diagnostics["scale"]
episode_dir = profile.paths.episode_dir(uuid)
mp4_path = episode_dir / "recordings" / "MP4" / f"{camera_serial}.mp4"
mp4_fps, n_video_frames, (video_w, video_h) = read_mp4_properties(mp4_path)
camera = _load_camera_for_episode(profile, uuid, camera_serial, video_w, video_h)
finite_frame = np.all(np.isfinite(best_poses), axis=-1) # (T,)
sim_rot = np.full((spec.n_frames, 3, 3), np.nan, dtype=np.float64)
if finite_frame.any():
sim_rot[finite_frame] = _quat_wxyz_to_matrix(best_poses[finite_frame, 3:7])
anchor_clear_m = diagnostics.get("anchor_height_above_heightfield_m")
known_cause = None
known_cause_hud = None
if anchor_clear_m is not None and anchor_clear_m > _ANCHOR_ABOVE_SUPPORT_GATE_M:
known_cause = (
f"anchored {anchor_clear_m * 1000:.0f}mm above the observed heightfield surface "
"directly under it at sim start -- free-falls before any contact is possible"
)
# Short enough to fit one HUD line at this video's width -- the full
# sentence above is the one that goes into summary.json; draw_hud has
# no text-wrapping, and a 110-char line at 1280px runs off-frame.
known_cause_hud = (
f"KNOWN CAUSE: anchored {anchor_clear_m * 1000:.0f}mm above heightfield surface "
"-> free-falls (see summary.json)"
)
out_path = out_dir / f"real2sim_{label}.mp4"
with timer.step("render_video", uuid=uuid, obj_label=label, n=n_video_frames):
with VideoWriter(out_path, fps=mp4_fps) as writer:
for t, frame_bgr in enumerate(read_frames_bgr(mp4_path)):
if t >= n_video_frames:
break
out = frame_bgr
verts_obs_world = transform_points(track.T_world_obj[t], canonical_points)
out = draw_silhouette(
out, project_silhouette_hull(verts_obs_world, camera), _OBSERVED_COLOR_BGR
)
if finite_frame[t]:
verts_sim_world = scaled_points @ sim_rot[t].T + best_poses[t, :3]
out = draw_silhouette(
out, project_silhouette_hull(verts_sim_world, camera), _SIMULATED_COLOR_BGR
)
if np.isfinite(trans_err_mm[t]):
err_str = f"{trans_err_mm[t]:.1f} mm"
else:
err_str = "n/a (sim diverged)"
obs_status = "scored (S6 valid)" if track.valid[t] else "NOT scored (gap/low-vis)"
hud = [
f"{label} frame {t}/{n_video_frames - 1}",
f"observed (amber): {obs_status}",
f"simulated (magenta): {'ok' if finite_frame[t] else 'DIVERGED'}"
f" [best of {n_particles}, {n_diverged} diverged]",
f"translation error: {err_str} ESS={ess:.2f}",
]
if known_cause_hud is not None:
hud.append(known_cause_hud)
out = draw_hud(out, hud)
writer.write(out)
logger.info("wrote %s", out_path)
return {
"label": label,
"video_path": str(out_path),
"n_particles": n_particles,
"n_diverged": n_diverged,
"ess": ess,
"degenerate_posterior_note": degeneracy_note,
"best_particle_idx": best_idx,
"best_particle_ok": best_ok,
"divergence_frame": div_frame,
"translation_error_mm": err_summary,
"body_kind": body.kind,
"n_kinematic_bodies": diagnostics["n_kinematic_bodies"],
"anchor_height_above_heightfield_m": anchor_clear_m,
"known_cause": known_cause,
"spread_nats": episode_ll.spread_nats,
"n_obs_frames": episode_ll.n_obs_frames,
}
def render_combined_video(
out_dir: Path, labels: tuple[str, ...], *, timer: StepTimer,
) -> Path | None:
"""Horizontal concat of the per-object videos, frame-by-frame -- cheap: no re-simulation.
``None`` if fewer than two labels were rendered (nothing to place side by
side) or any per-object video is missing (a prior object's render failed)
rather than raising -- the per-object deliverables are the ones that
matter; the combined view is a bonus the task brief calls out as
"if cheap".
"""
if len(labels) < 2:
return None
paths = [out_dir / f"real2sim_{label}.mp4" for label in labels]
if not all(p.exists() for p in paths):
logger.warning("render_combined_video: not all per-object videos exist, skipping")
return None
fps, _, _ = read_mp4_properties(paths[0])
combined_path = out_dir / "real2sim_combined.mp4"
with timer.step("render_combined_video", n=len(labels)):
with VideoWriter(combined_path, fps=fps) as writer:
for frames in zip(*(read_frames_bgr(p) for p in paths), strict=True):
writer.write(np.concatenate(frames, axis=1))
logger.info("wrote %s", combined_path)
return combined_path
# --------------------------------------------------------------------------- #
# CLI
# --------------------------------------------------------------------------- #
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
p.add_argument("--uuid", default=_DEFAULT_UUID)
p.add_argument("--camera", default=_DEFAULT_CAMERA)
p.add_argument("--labels", nargs="+", default=list(_DEFAULT_LABELS))
p.add_argument("--particles", type=int, default=_DEFAULT_N_PARTICLES)
p.add_argument("--seed", type=int, default=_DEFAULT_SEED)
p.add_argument("--out", type=Path, default=_DEFAULT_OUT_DIR)
p.add_argument("--config", type=Path, default=REPO_ROOT / "configs" / "datagen_droid.yaml")
p.add_argument("--log-level", default="INFO")
return p.parse_args()
def main() -> int:
args = parse_args()
setup_logging(args.log_level)
out_dir = ensure_dir(args.out)
profile = DatagenProfile.from_yaml(args.config)
timer = StepTimer("render_real2sim_demo")
objects: dict[str, Any] = {}
for label in args.labels:
objects[label] = render_object(
args.uuid, args.camera, label,
profile=profile, out_dir=out_dir,
n_particles=args.particles, seed=args.seed, timer=timer,
)
combined_path = render_combined_video(out_dir, tuple(args.labels), timer=timer)
summary = {
"uuid": args.uuid,
"camera_serial": args.camera,
"n_particles": args.particles,
"seed": args.seed,
"objects": objects,
"combined_video_path": str(combined_path) if combined_path is not None else None,
"timings": timer.summary(),
}
summary_path = out_dir / "summary.json"
atomic_write(summary_path, json.dumps(summary, indent=2).encode("utf-8"))
print(f"\nwrote {summary_path}")
for label, obj in objects.items():
print(
f" {label}: n_diverged={obj['n_diverged']}/{obj['n_particles']} "
f"ess={obj['ess']:.3f} translation_error_mm={obj['translation_error_mm']}"
)
print()
print(timer.report())
return 0
if __name__ == "__main__":
raise SystemExit(main())

Xet Storage Details

Size:
25.2 kB
·
Xet hash:
c396d868d5a050cab9cf6ca361d44ec4af06bde48a643439e2adfbdf2d5ad1e5

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.