"""Camera-space placement, 3DGS PLY export and orbit rendering for Lift4D. The object->camera convention is taken verbatim from Lift4D's ``sam3d/run_inference.py:render_gs_frame``: p_p3d = compose_transform(scale, quaternion_to_matrix(q_l2c), trans) .transform_points(p_local) # PyTorch3D row-vector p_cam = p_p3d @ diag(-1, -1, 1) PyTorch3D's ``Transform3d.rotate(R)`` applies ``p_row @ R``, i.e. R^T in the usual column-vector sense -- which is why run_inference composes the Gaussian orientations with ``quaternion_invert(q_l2c)``. Written as one rigid map: R_o2c = diag(-1,-1,1) @ R(q_l2c)^T t_o2c = diag(-1,-1,1) @ t p_cam = R_o2c @ (scale * p_local) + t_o2c Camera space is the usual OpenCV/3DGS frame (+x right, +y down, +z forward). """ from __future__ import annotations import math from typing import List import numpy as np import torch SH_C0 = 0.28209479177387814 def _flip(device): """diag(-1, -1, 1): PyTorch3D -> camera, as in run_inference.py.""" return torch.tensor([[-1.0, 0.0, 0.0], [0.0, -1.0, 0.0], [0.0, 0.0, 1.0]], dtype=torch.float32, device=device) def _mat_to_quat(R: torch.Tensor) -> torch.Tensor: from pytorch3d.transforms import matrix_to_quaternion return matrix_to_quaternion(R) def to_camera_space(gs, scale_t, trans_t, rot_t, device="cuda"): """Return (means, quats, scales, colors, opacities) in camera space.""" from pytorch3d.transforms import quaternion_to_matrix, quaternion_multiply xyz = gs.get_xyz.to(device).float() quats = gs.get_rotation.to(device).float() scales = gs.get_scaling.to(device).float() opac = gs.get_opacity.to(device).float().reshape(-1) dc = gs._features_dc.to(device).float() if dc.dim() == 3: dc = dc.squeeze(1) colors = torch.clamp(dc * SH_C0 + 0.5, 0.0, 1.0) s = scale_t.to(device).float().reshape(-1) if s.numel() == 1: s = s.repeat(3) t = trans_t.to(device).float().reshape(3) q = rot_t.to(device).float().reshape(4) M = _flip(device) R_o2c = M @ quaternion_to_matrix(q).transpose(0, 1) t_o2c = M @ t q_o2c = _mat_to_quat(R_o2c) means = (R_o2c @ (xyz * s).transpose(0, 1)).transpose(0, 1) + t_o2c quats = quaternion_multiply(q_o2c.unsqueeze(0).expand(quats.shape[0], -1), quats) scales = scales * s return means, quats, scales, colors, opac # -------------------------------------------------------------------------- # 3DGS PLY export for gr.Model3D # -------------------------------------------------------------------------- # camera space (y down, z forward) -> viewer space (y up, z toward viewer) _VIEW_FLIP = np.array([[1.0, 0.0, 0.0], [0.0, -1.0, 0.0], [0.0, 0.0, -1.0]], dtype=np.float32) _VIEW_QUAT = np.array([0.0, 1.0, 0.0, 0.0], dtype=np.float32) # wxyz, Rx(180 deg) def _quat_mul_np(a, b): aw, ax, ay, az = a bw, bx, by, bz = b[:, 0], b[:, 1], b[:, 2], b[:, 3] return np.stack([ aw * bw - ax * bx - ay * by - az * bz, aw * bx + ax * bw + ay * bz - az * by, aw * by - ax * bz + ay * bw + az * bx, aw * bz + ax * by - ay * bx + az * bw, ], axis=-1) def write_viewer_ply(path, means, quats, scales, colors, opacities, center=None): """Write a standard 3DGS .ply that gr.Model3D / gsplat.js can display. Gaussians come in camera space; they are rotated into the viewer's y-up frame and recentred so the object appears upright and framed. """ m = means.detach().cpu().numpy().astype(np.float32) q = quats.detach().cpu().numpy().astype(np.float32) s = scales.detach().cpu().numpy().astype(np.float32) c = colors.detach().cpu().numpy().astype(np.float32) o = opacities.detach().cpu().numpy().astype(np.float32).reshape(-1) if center is None: center = np.median(m, axis=0) m = (m - center) @ _VIEW_FLIP.T q = _quat_mul_np(_VIEW_QUAT, q) eps = 1e-6 f_dc = (c - 0.5) / SH_C0 opacity = np.log(np.clip(o, eps, 1 - eps) / (1 - np.clip(o, eps, 1 - eps))) scale_log = np.log(np.clip(s, eps, None)) normals = np.zeros_like(m) data = np.concatenate([m, normals, f_dc, opacity[:, None], scale_log, q], axis=1).astype(np.float32) header = ( "ply\nformat binary_little_endian 1.0\n" f"element vertex {data.shape[0]}\n" "property float x\nproperty float y\nproperty float z\n" "property float nx\nproperty float ny\nproperty float nz\n" "property float f_dc_0\nproperty float f_dc_1\nproperty float f_dc_2\n" "property float opacity\n" "property float scale_0\nproperty float scale_1\nproperty float scale_2\n" "property float rot_0\nproperty float rot_1\nproperty float rot_2\nproperty float rot_3\n" "end_header\n" ) with open(path, "wb") as fh: fh.write(header.encode("ascii")) fh.write(data.tobytes()) return str(path) # -------------------------------------------------------------------------- # orbit rendering with the Inria 3DGS rasterizer # -------------------------------------------------------------------------- def _proj_matrix(znear, zfar, fovx, fovy, device): tx, ty = math.tan(fovx / 2), math.tan(fovy / 2) P = torch.zeros(4, 4, dtype=torch.float32, device=device) P[0, 0] = 1.0 / tx P[1, 1] = 1.0 / ty P[2, 2] = zfar / (zfar - znear) P[2, 3] = -(zfar * znear) / (zfar - znear) P[3, 2] = 1.0 return P def _look_at(cam_pos, target, device): """World->camera rotation/translation in the OpenCV (y-down) convention.""" f = target - cam_pos f = f / torch.norm(f).clamp_min(1e-8) up_hint = torch.tensor([0.0, -1.0, 0.0], device=device) if torch.abs(torch.dot(f, up_hint)) > 0.999: up_hint = torch.tensor([0.0, 0.0, 1.0], device=device) r = torch.cross(f, up_hint, dim=0) r = r / torch.norm(r).clamp_min(1e-8) d = torch.cross(f, r, dim=0) R_wc = torch.stack([r, d, f], dim=0) t_wc = -R_wc @ cam_pos return R_wc, t_wc def render_orbit(frames, size=512, orbit_steps=48, fov_deg=42.0, elevation_deg=12.0, bg=(1.0, 1.0, 1.0), device="cuda"): """Render a 360-degree orbit that also plays the reconstructed sequence. ``frames`` is a list of (means, quats, scales, colors, opacities) tuples in camera space, one per reconstructed video frame. Step ``t`` of the orbit shows the frame at ``t / orbit_steps`` of the sequence, so the video shows both the object's deformation and every side of the geometry. Each frame is recentred on its own median so the turntable stays object-centric: the subject's bulk translation through the scene (a running horse crosses far more than its own body length) would otherwise fling it out of frame. The orbit is phased so azimuth 0 starts broadside to the subject's longest horizontal axis (its length): for an elongated subject like the horse, an end-on start reads as an odd, foreshortened perspective, whereas presenting the long side first gives the natural side-profile turntable. """ from diff_gaussian_rasterization import (GaussianRasterizationSettings, GaussianRasterizer) from pytorch3d.transforms import matrix_to_quaternion, quaternion_multiply centers = [f[0].median(dim=0).values for f in frames] rad = max( float(torch.quantile((f[0] - c).norm(dim=1), 0.97)) for f, c in zip(frames, centers) ) rad = max(rad, 1e-3) center = torch.zeros(3, dtype=torch.float32, device=device) # Phase the orbit so it starts broadside to the subject's longest horizontal # axis. Camera space is y-down, so the horizontal (ground) plane is x-z; the # dominant x-z eigenvector of the recentred means is the subject's length. # The horizontal view direction at azimuth a is [-sin a, cos a]; choosing # az_phase = atan2(lz, lx) makes that direction perpendicular to the length # axis at a = 0 (a side-on view) instead of looking straight down its length. xz = torch.cat([(f[0] - c)[:, [0, 2]] for f, c in zip(frames, centers)], dim=0) xz = (xz - xz.mean(dim=0)).to(torch.float32) cov = (xz.transpose(0, 1) @ xz) / max(xz.shape[0] - 1, 1) evals, evecs = torch.linalg.eigh(cov.cpu()) length_axis = evecs[:, int(torch.argmax(evals))] az_phase = math.atan2(float(length_axis[1]), float(length_axis[0])) fov = math.radians(fov_deg) dist = rad / math.tan(fov / 2.0) * 1.12 el = math.radians(elevation_deg) bg_t = torch.tensor(bg, dtype=torch.float32, device=device) P = _proj_matrix(0.01, 100.0, fov, fov, device) tan_half = math.tan(fov / 2.0) out = [] n = len(frames) with torch.no_grad(): for t in range(orbit_steps): az = az_phase + 2.0 * math.pi * t / orbit_steps direction = torch.tensor([ math.sin(az) * math.cos(el), -math.sin(el), -math.cos(az) * math.cos(el), ], dtype=torch.float32, device=device) cam_pos = center + dist * direction R_wc, t_wc = _look_at(cam_pos, center, device) # Gaussians are moved into the orbit camera's frame here, so the # rasterizer gets an identity view matrix and a pure camera->clip # projection. fi = min(int(t * n / orbit_steps), n - 1) means, quats, scales, colors, opac = frames[fi] means = means - centers[fi] q_wc = matrix_to_quaternion(R_wc) quats_v = quaternion_multiply(q_wc.unsqueeze(0).expand(quats.shape[0], -1), quats) means_v = (R_wc @ means.transpose(0, 1)).transpose(0, 1) + t_wc settings = GaussianRasterizationSettings( image_height=size, image_width=size, tanfovx=tan_half, tanfovy=tan_half, bg=bg_t, scale_modifier=1.0, viewmatrix=torch.eye(4, dtype=torch.float32, device=device), projmatrix=P.transpose(0, 1).contiguous(), sh_degree=0, campos=torch.zeros(3, dtype=torch.float32, device=device), prefiltered=False, debug=False, ) rasterizer = GaussianRasterizer(raster_settings=settings) screen = torch.zeros_like(means_v) image, _ = rasterizer( means3D=means_v.contiguous(), means2D=screen, shs=None, colors_precomp=colors.contiguous(), opacities=opac.reshape(-1, 1).contiguous(), scales=scales.contiguous(), rotations=quats_v.contiguous(), cov3D_precomp=None, ) frame = (image.clamp(0, 1).permute(1, 2, 0) * 255).to(torch.uint8).cpu().numpy() out.append(frame) return out def save_video(frames: List[np.ndarray], path, fps=12): import imageio.v2 as imageio imageio.mimwrite(str(path), frames, fps=fps, quality=8, macro_block_size=1, codec="libx264", output_params=["-pix_fmt", "yuv420p"]) return str(path)