import hashlib import os import shutil import time import zipfile from contextlib import nullcontext from pathlib import Path import cv2 import imageio import numpy as np import torch import torch.nn.functional as F from einops import rearrange from PIL import Image from .camera import get_plucker_embedding VISTA4D_REUSE_GENERATED_MAP_IN_MEMORY = True VISTA4D_DA3_PROCESS_RES = 896 VISTA4D_DA3_CHUNK_SIZE = -1 VISTA4D_DA3_CHUNK_OVERLAP = 8 _VISTA4D_MAP_CACHE = {} _VISTA4D_MAP_CACHE_MAX_ITEMS = 2 def _center_slice(length, frame_num): if length < frame_num: raise ValueError(f"Vista4D input needs at least {frame_num} frames, got {length}.") start = (length - frame_num) // 2 return slice(start, start + frame_num) def _tensor_to_video_np(frames): video = frames.detach().cpu().float().permute(1, 2, 3, 0).numpy() return ((video + 1.0) * 127.5).clip(0, 255).astype(np.uint8) def _video_np_to_tensor(video, device, dtype): video = torch.from_numpy(video).permute(3, 0, 1, 2).float().div_(127.5).sub_(1.0) return video.to(device=device, dtype=dtype) def _crop_resize_video(video, height, width, resample=Image.Resampling.LANCZOS): if video.shape[1:3] == (height, width): return video.copy() frames = [] for frame in video: image = Image.fromarray(frame) in_w, in_h = image.size if in_h / in_w > height / width: crop_h = int(in_w * height / width) top = (in_h - crop_h) // 2 image = image.crop((0, top, in_w, top + crop_h)) else: crop_w = int(in_h * width / height) left = (in_w - crop_w) // 2 image = image.crop((left, 0, left + crop_w, in_h)) frames.append(np.asarray(image.resize((width, height), resample))) return np.stack(frames, axis=0) def _resize_intrinsics(intrinsics, height, width, height_input, width_input): intrinsics = intrinsics.copy() if height_input / width_input > height / width: crop_h = int(width_input * height / width) intrinsics[..., 3] -= (height_input - crop_h) / 2 height_input = crop_h else: crop_w = int(height_input * width / height) intrinsics[..., 2] -= (width_input - crop_w) / 2 width_input = crop_w intrinsics[..., 0] *= width / width_input intrinsics[..., 1] *= height / height_input intrinsics[..., 2] *= width / width_input intrinsics[..., 3] *= height / height_input return intrinsics def _infer_intrinsics_size(intrinsics, fallback_height, fallback_width): cx = float(np.nanmedian(intrinsics[..., 2])) cy = float(np.nanmedian(intrinsics[..., 3])) if np.isfinite(cx) and np.isfinite(cy) and cx > 0 and cy > 0: return max(1, int(round(cy * 2))), max(1, int(round(cx * 2))) return fallback_height, fallback_width def _read_video(path): cap = cv2.VideoCapture(os.fspath(path)) if not cap.isOpened(): raise ValueError(f"Could not open Vista4D video: {path}") frames = [] while True: ok, frame = cap.read() if not ok: break frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) cap.release() if not frames: raise ValueError(f"Vista4D video has no frames: {path}") return np.stack(frames, axis=0) def _save_video(path, video, fps): os.makedirs(os.path.dirname(os.fspath(path)), exist_ok=True) writer = imageio.get_writer(os.fspath(path), fps=fps, quality=9, macro_block_size=1) try: for frame in video: writer.append_data(frame) finally: writer.close() def _load_masks(folder, frame_num, height, width, default): folder = Path(folder) if not folder.is_dir(): fill = np.ones if default else np.zeros return fill((frame_num, height, width), dtype=np.bool_) files = sorted(folder.glob("*.png")) if len(files) == 0: raise ValueError(f"Vista4D mask folder is empty: {folder}") masks = [] for file in files: frame = cv2.imread(os.fspath(file), cv2.IMREAD_GRAYSCALE) if frame is None: raise ValueError(f"Could not load Vista4D mask: {file}") masks.append(frame > 127) masks = np.stack(masks, axis=0) masks = masks[_center_slice(masks.shape[0], frame_num)] if masks.shape[1:3] != (height, width): masks = F.interpolate(torch.from_numpy(masks[:, None].astype(np.float32)), size=(height, width), mode="nearest")[:, 0].numpy() > 0.5 return masks def _save_masks(folder, masks): folder = Path(folder) folder.mkdir(parents=True, exist_ok=True) for idx, mask in enumerate(masks): cv2.imwrite(os.fspath(folder / f"{idx:05d}.png"), mask.astype(np.uint8) * 255) def _pixel_grid(height, width): ys, xs = np.meshgrid(np.arange(height, dtype=np.float32), np.arange(width, dtype=np.float32), indexing="ij") return xs.reshape(-1), ys.reshape(-1) def _frame_points(video, depths, sky_mask, dynamic_mask, cam_c2w, intrinsics, frame_idx, xs, ys): depth = depths[frame_idx].reshape(-1).astype(np.float32) valid = np.isfinite(depth) & (depth > 1e-5) if sky_mask is not None: valid &= ~sky_mask[frame_idx].reshape(-1).astype(np.bool_) if not np.any(valid): return np.empty((0, 3), dtype=np.float32), np.empty((0, 3), dtype=np.uint8), np.empty((0,), dtype=np.bool_) fx, fy, cx, cy = intrinsics[frame_idx] z = depth[valid] cam_points = np.stack(((xs[valid] - cx) * z / fx, (ys[valid] - cy) * z / fy, z), axis=1) world_points = cam_points @ cam_c2w[frame_idx, :3, :3].T + cam_c2w[frame_idx, :3, 3] colors = video[frame_idx].reshape(-1, 3)[valid] point_dynamic = dynamic_mask[frame_idx].reshape(-1)[valid] if dynamic_mask is not None else np.zeros(colors.shape[0], dtype=np.bool_) return world_points.astype(np.float32), colors, point_dynamic.astype(np.bool_) def _project_points(points, colors, cam_w2c, intrinsics, height, width): if points.shape[0] == 0: return None cam_points = points @ cam_w2c[:3, :3].T + cam_w2c[:3, 3] z = cam_points[:, 2] valid = z > 1e-5 if not np.any(valid): return None fx, fy, cx, cy = intrinsics valid_idx = np.nonzero(valid)[0] x = np.rint(cam_points[valid, 0] * fx / z[valid] + cx).astype(np.int32) y = np.rint(cam_points[valid, 1] * fy / z[valid] + cy).astype(np.int32) inside = (x >= 0) & (x < width) & (y >= 0) & (y < height) if not np.any(inside): return None return y[inside] * width + x[inside], z[valid][inside], colors[valid][inside], valid_idx[inside] def _render_point_chunks(point_chunks, cam_w2c, intrinsics, height, width): empty_frame = np.zeros((height, width, 3), dtype=np.uint8) empty_mask = np.zeros((height, width), dtype=np.bool_) zbuf = np.full(height * width, np.inf, dtype=np.float32) projected = [] for points, colors, is_dynamic in point_chunks: item = _project_points(points, colors, cam_w2c, intrinsics, height, width) if item is None: continue pix, z, colors, _ = item np.minimum.at(zbuf, pix, z) projected.append((pix, z, colors, is_dynamic)) if len(projected) == 0: return empty_frame, empty_mask, empty_mask.copy() output = np.zeros((height * width, 3), dtype=np.uint8) dynamic = np.zeros(height * width, dtype=np.bool_) for pix, z, colors, is_dynamic in projected: keep = z <= zbuf[pix] + 1e-4 output[pix[keep]] = colors[keep] if is_dynamic: dynamic[pix[keep]] = True mask = np.isfinite(zbuf).reshape(height, width) return output.reshape(height, width, 3), mask, dynamic.reshape(height, width) & mask def _to_torch_point_cache(point_cache, device): return [(torch.from_numpy(points).to(device=device, dtype=torch.float32), torch.from_numpy(colors).to(device=device)) for points, colors in point_cache] def _project_points_torch(points, colors, cam_w2c, intrinsics, height, width): if points.shape[0] == 0: return None cam_points = points @ cam_w2c[:3, :3].T + cam_w2c[:3, 3] z_all = cam_points[:, 2] valid = z_all > 1e-5 if not bool(valid.any()): return None fx, fy, cx, cy = intrinsics.unbind(0) valid_idx = torch.nonzero(valid, as_tuple=False).squeeze(1) z = z_all[valid_idx] x = torch.round(cam_points[valid_idx, 0] * fx / z + cx).to(torch.int64) y = torch.round(cam_points[valid_idx, 1] * fy / z + cy).to(torch.int64) inside = (x >= 0) & (x < width) & (y >= 0) & (y < height) if not bool(inside.any()): return None return y[inside] * width + x[inside], z[inside], colors[valid_idx[inside]] def _render_point_chunks_torch(point_chunks, cam_w2c, intrinsics, height, width): device = cam_w2c.device zbuf = torch.full((height * width,), float("inf"), dtype=torch.float32, device=device) projected = [] order_start = 0 for points, colors, is_dynamic in point_chunks: item = _project_points_torch(points, colors, cam_w2c, intrinsics, height, width) if item is None: order_start += points.shape[0] continue pix, z, colors = item order = torch.arange(order_start, order_start + pix.shape[0], dtype=torch.int32, device=device) zbuf.scatter_reduce_(0, pix, z, reduce="amin", include_self=True) projected.append((pix, z, colors, order, is_dynamic)) order_start += points.shape[0] frame = torch.zeros((height * width, 3), dtype=torch.uint8, device=device) dynamic = torch.zeros((height * width,), dtype=torch.bool, device=device) if len(projected) > 0: winner_order = torch.full((height * width,), -1, dtype=torch.int32, device=device) for pix, z, _, order, is_dynamic in projected: keep = z <= zbuf[pix] + 1e-4 if keep.any(): winner_order.scatter_reduce_(0, pix[keep], order[keep], reduce="amax", include_self=True) if is_dynamic: dynamic[pix[keep]] = True for pix, _, colors, order, _ in projected: selected = order == winner_order[pix] if selected.any(): frame[pix[selected]] = colors[selected] mask = torch.isfinite(zbuf) return frame.reshape(height, width, 3).cpu().numpy(), mask.reshape(height, width).cpu().numpy(), (dynamic & mask).reshape(height, width).cpu().numpy() def _render_point_cloud_video_torch(static_cache, dynamic_cache, target_cam_w2c, target_intrinsics, height, width, window_radius): device = torch.device("cuda") static_cache = _to_torch_point_cache(static_cache, device) dynamic_cache = _to_torch_point_cache(dynamic_cache, device) target_cam_w2c = torch.from_numpy(target_cam_w2c).to(device=device, dtype=torch.float32) target_intrinsics = torch.from_numpy(target_intrinsics).to(device=device, dtype=torch.float32) rendered = [] alpha_masks = [] dynamic_masks = [] context = torch.autocast(device_type="cuda", enabled=False) if device.type == "cuda" else nullcontext() with torch.no_grad(), context: for target_idx in range(len(dynamic_cache)): if window_radius < 0: point_chunks = [(points, colors, False) for points, colors in static_cache] else: start = max(0, target_idx - window_radius) end = min(len(dynamic_cache), target_idx + window_radius + 1) point_chunks = [(static_cache[idx][0], static_cache[idx][1], False) for idx in range(start, end) if static_cache[idx][0].shape[0] > 0] dynamic_points, dynamic_colors = dynamic_cache[target_idx] if dynamic_points.shape[0] > 0: point_chunks.append((dynamic_points, dynamic_colors, True)) frame, mask, rendered_dynamic = _render_point_chunks_torch(point_chunks, target_cam_w2c[target_idx], target_intrinsics[target_idx], height, width) rendered.append(frame) alpha_masks.append(mask) dynamic_masks.append(rendered_dynamic) return np.stack(rendered, axis=0), np.stack(alpha_masks, axis=0), np.stack(dynamic_masks, axis=0) def _render_point_cloud_video(video, depths, sky_mask, dynamic_mask, cam_c2w, intrinsics, target_cam_c2w=None, target_intrinsics=None, window_radius=-1): frame_num, height, width = video.shape[:3] if target_cam_c2w is None: target_cam_c2w = cam_c2w if target_intrinsics is None: target_intrinsics = intrinsics xs, ys = _pixel_grid(height, width) static_cache = [] dynamic_cache = [] for idx in range(frame_num): points, colors, point_dynamic = _frame_points(video, depths, sky_mask, dynamic_mask, cam_c2w, intrinsics, idx, xs, ys) static = ~point_dynamic static_cache.append((points[static], colors[static])) dynamic_cache.append((points[point_dynamic], colors[point_dynamic])) target_cam_w2c = np.linalg.inv(target_cam_c2w).astype(np.float32) rendered = [] alpha_masks = [] dynamic_masks = [] window_radius = int(window_radius) if window_radius < 0: static_points = [points for points, _ in static_cache if points.shape[0] > 0] static_colors = [colors for points, colors in static_cache if points.shape[0] > 0] static_cache = [(np.concatenate(static_points, axis=0), np.concatenate(static_colors, axis=0))] if static_points else [] if torch.cuda.is_available(): return _render_point_cloud_video_torch(static_cache, dynamic_cache, target_cam_w2c, target_intrinsics, height, width, window_radius) for target_idx in range(frame_num): if window_radius < 0: point_chunks = [(points, colors, False) for points, colors in static_cache] else: start = max(0, target_idx - window_radius) end = min(frame_num, target_idx + window_radius + 1) point_chunks = [(static_cache[idx][0], static_cache[idx][1], False) for idx in range(start, end) if static_cache[idx][0].shape[0] > 0] dynamic_points, dynamic_colors = dynamic_cache[target_idx] if dynamic_points.shape[0] > 0: point_chunks.append((dynamic_points, dynamic_colors, True)) frame, mask, rendered_dynamic = _render_point_chunks(point_chunks, target_cam_w2c[target_idx], target_intrinsics[target_idx], height, width) rendered.append(frame) alpha_masks.append(mask) dynamic_masks.append(rendered_dynamic) return np.stack(rendered, axis=0), np.stack(alpha_masks, axis=0), np.stack(dynamic_masks, axis=0) def _load_cameras(path, frame_num): data = np.load(path) cam_c2w = data["cam_c2w"][_center_slice(data["cam_c2w"].shape[0], frame_num)] intrinsics = data["intrinsics"][_center_slice(data["intrinsics"].shape[0], frame_num)] return cam_c2w.astype(np.float32), intrinsics.astype(np.float32) def _save_cameras(path, cam_c2w, intrinsics): os.makedirs(os.path.dirname(os.fspath(path)), exist_ok=True) np.savez(path, cam_c2w=cam_c2w.astype(np.float32), intrinsics=intrinsics.astype(np.float32)) def _is_camera_npz(path): if Path(path).suffix.lower() != ".npz": return False try: with np.load(path) as data: return "cam_c2w" in data.files and "intrinsics" in data.files except Exception: return False def _extract_zip(path): digest = hashlib.sha1(os.path.abspath(path).encode("utf-8")).hexdigest()[:12] target = Path("ckpts") / "temp" / "vista4d_maps" / digest if target.is_dir(): return target target.mkdir(parents=True, exist_ok=True) with zipfile.ZipFile(path) as archive: archive.extractall(target) roots = [item for item in target.iterdir() if item.is_dir()] if len(roots) == 1 and (roots[0] / "video_src.mp4").is_file(): return roots[0] return target def _resolve_custom_input(input_custom): if input_custom is None or str(input_custom).strip() == "": return None, None path = Path(input_custom) if path.is_file() and _is_camera_npz(path): return None, path if path.is_file() and path.suffix.lower() == ".zip": return _extract_zip(path), None if path.is_file(): path = path.parent if not path.is_dir(): raise ValueError(f"Vista4D custom_guide must be a preprocessed map folder/zip or target camera .npz, got: {input_custom}") return path, None def _default_output_dir(): from wgp import save_path return save_path def _hash_array(hasher, array): array = np.ascontiguousarray(array) hasher.update(str(array.shape).encode("utf-8")) hasher.update(str(array.dtype).encode("utf-8")) hasher.update(array.view(np.uint8)) def _hash_file(path): hasher = hashlib.sha1() with open(path, "rb") as reader: for chunk in iter(lambda: reader.read(1024 * 1024), b""): hasher.update(chunk) return hasher.hexdigest() def _make_generated_map_cache_key(video, frame_num, height, width, fps, process_res, chunk_size, chunk_overlap, custom_settings, target_camera_path, model_mode): hasher = hashlib.sha1() hasher.update(b"vista4d-memory-map-v2") hasher.update(f"{frame_num}|{height}|{width}|{fps}|{process_res}|{chunk_size}|{chunk_overlap}".encode("utf-8")) if target_camera_path is None: hasher.update(str(model_mode or "").encode("utf-8")) if isinstance(custom_settings, dict): hasher.update(str(custom_settings.get("vista4d_seg_keywords", "_all_")).encode("utf-8")) hasher.update(str(custom_settings.get("vista4d_scene_scale", 1.0)).encode("utf-8")) if target_camera_path is None: hasher.update(str(custom_settings.get("vista4d_camera_strength", 100.0)).encode("utf-8")) if target_camera_path is not None: hasher.update(os.path.abspath(os.fspath(target_camera_path)).encode("utf-8")) hasher.update(_hash_file(target_camera_path).encode("utf-8")) _hash_array(hasher, video) return hasher.hexdigest() def _cache_generated_map(cache_key, data): _VISTA4D_MAP_CACHE[cache_key] = data while len(_VISTA4D_MAP_CACHE) > _VISTA4D_MAP_CACHE_MAX_ITEMS: _VISTA4D_MAP_CACHE.pop(next(iter(_VISTA4D_MAP_CACHE))) def _parse_keywords(custom_settings): raw = "_all_" if isinstance(custom_settings, dict): raw = str(custom_settings.get("vista4d_seg_keywords", raw)) return [item.strip() for item in raw.replace(";", ",").split(",") if item.strip()] def _camera_strength(custom_settings): if not isinstance(custom_settings, dict): return 1.0 return float(custom_settings.get("vista4d_camera_strength", 100.0) or 0.0) / 100.0 def _get_da3_settings(): return VISTA4D_DA3_PROCESS_RES, VISTA4D_DA3_CHUNK_SIZE, VISTA4D_DA3_CHUNK_OVERLAP def _save_generated_map_to_disk(data, source_cam_c2w, source_intrinsics, depths, sky_mask, fps): map_dir = Path(_default_output_dir()) / f"vista4d_map_{time.strftime('%Y%m%d_%H%M%S')}" map_dir.mkdir(parents=True, exist_ok=True) _save_video(map_dir / "video_src.mp4", data["source_video"], fps=fps) _save_video(map_dir / "video_pc.mp4", data["point_cloud_video"], fps=fps) _save_masks(map_dir / "alpha_mask_src", data["source_alpha_mask"]) _save_masks(map_dir / "dynamic_mask_src", data["source_motion_mask"]) _save_masks(map_dir / "static_mask_src", data["source_alpha_mask"] & ~data["source_motion_mask"]) _save_masks(map_dir / "alpha_mask_pc", data["point_cloud_alpha_mask"]) _save_masks(map_dir / "dynamic_mask_pc", data["point_cloud_motion_mask"]) _save_masks(map_dir / "static_mask_pc", data["point_cloud_alpha_mask"] & ~data["point_cloud_motion_mask"]) _save_masks(map_dir / "sky_mask_src", sky_mask.astype(np.bool_)) _save_cameras(map_dir / "cameras_src.npz", source_cam_c2w, source_intrinsics) _save_cameras(map_dir / "cameras_tgt.npz", data["cam_c2w"], data["intrinsics"]) np.save(map_dir / "depths_da3.npy", depths.astype(np.float16)) shutil.make_archive(os.fspath(map_dir), "zip", root_dir=map_dir) print(f"Vista4D map saved to: {map_dir}") return map_dir def _rotation_x(angle): c, s = np.cos(angle), np.sin(angle) return np.array([[1.0, 0.0, 0.0], [0.0, c, -s], [0.0, s, c]], dtype=np.float32) def _rotation_y(angle): c, s = np.cos(angle), np.sin(angle) return np.array([[c, 0.0, s], [0.0, 1.0, 0.0], [-s, 0.0, c]], dtype=np.float32) def _estimate_focus_depth(depths, sky_mask): valid = np.isfinite(depths) & (depths > 1e-5) if sky_mask is not None: valid &= ~sky_mask.astype(np.bool_) if not np.any(valid): return 1.0 samples = depths[valid] if samples.size > 200000: samples = samples[np.linspace(0, samples.size - 1, 200000, dtype=np.int64)] return max(float(np.median(samples)), 1e-3) def _apply_local_camera_delta(cam_c2w, translations, rotations): target = cam_c2w.copy() for idx in range(cam_c2w.shape[0]): delta = np.eye(4, dtype=np.float32) delta[:3, :3] = rotations[idx] delta[:3, 3] = translations[idx] target[idx] = cam_c2w[idx] @ delta return target def _generate_target_camera_trajectory(cam_c2w, intrinsics, depths, sky_mask, model_mode, strength): mode = str(model_mode or "dolly_zoom") if mode not in { "dolly_zoom", "left_front_zoom", "right_front_zoom", "close_crane_above", "close_crane_below", "arc_right_45", "arc_left_45", "push_in", "pull_back", "truck_right", "truck_left", "pedestal_up", "pedestal_down", "pan_right_45", "pan_left_45", "tilt_up_45", "tilt_down_45", "zoom_in", "zoom_out", "bird_view", "crane_above_right", "crane_above_left", "crane_below_right", "crane_below_left", }: mode = "dolly_zoom" focus = _estimate_focus_depth(depths, sky_mask) frame_num = cam_c2w.shape[0] progress = np.linspace(0.0, 1.0, frame_num, dtype=np.float32) * np.float32(strength) translations = np.zeros((frame_num, 3), dtype=np.float32) rotations = np.repeat(np.eye(3, dtype=np.float32)[None], frame_num, axis=0) target_intrinsics = intrinsics.copy() focal_scale = np.ones(frame_num, dtype=np.float32) if mode == "dolly_zoom": translations[:, 2] = 0.35 * focus * progress focal_scale = np.clip((focus - translations[:, 2]) / focus, 0.25, 4.0).astype(np.float32) elif mode in ("left_front_zoom", "right_front_zoom"): translations[:, 0] = (-0.22 if mode == "left_front_zoom" else 0.22) * focus * progress translations[:, 2] = 0.20 * focus * progress focal_scale = np.clip(1.0 + 0.25 * progress, 0.25, 4.0).astype(np.float32) elif mode in ("close_crane_above", "close_crane_below"): translations[:, 1] = (-0.22 if mode == "close_crane_above" else 0.22) * focus * progress translations[:, 2] = 0.18 * focus * progress for idx, amount in enumerate(progress): rotations[idx] = _rotation_x(np.deg2rad(-15.0 if mode == "close_crane_above" else 15.0) * amount) elif mode in ("arc_right_45", "arc_left_45"): sign = -1.0 if mode == "arc_right_45" else 1.0 angles = np.deg2rad(45.0) * progress * sign translations[:, 0] = -focus * np.sin(angles) translations[:, 2] = focus * (1.0 - np.cos(angles)) for idx, angle in enumerate(angles): rotations[idx] = _rotation_y(angle) elif mode == "push_in": translations[:, 2] = 0.25 * focus * progress elif mode == "pull_back": translations[:, 2] = -0.25 * focus * progress elif mode == "truck_right": translations[:, 0] = 0.25 * focus * progress elif mode == "truck_left": translations[:, 0] = -0.25 * focus * progress elif mode == "pedestal_up": translations[:, 1] = -0.25 * focus * progress elif mode == "pedestal_down": translations[:, 1] = 0.25 * focus * progress elif mode == "pan_right_45": for idx, amount in enumerate(progress): rotations[idx] = _rotation_y(np.deg2rad(45.0) * amount) elif mode == "pan_left_45": for idx, amount in enumerate(progress): rotations[idx] = _rotation_y(np.deg2rad(-45.0) * amount) elif mode == "tilt_up_45": for idx, amount in enumerate(progress): rotations[idx] = _rotation_x(np.deg2rad(45.0) * amount) elif mode == "tilt_down_45": for idx, amount in enumerate(progress): rotations[idx] = _rotation_x(np.deg2rad(-45.0) * amount) elif mode == "zoom_in": focal_scale = np.clip(1.0 + 0.60 * progress, 0.25, 4.0).astype(np.float32) elif mode == "zoom_out": focal_scale = np.clip(1.0 - 0.35 * progress, 0.25, 4.0).astype(np.float32) elif mode == "bird_view": translations[:, 1] = -0.70 * focus * progress translations[:, 2] = -0.12 * focus * progress for idx, amount in enumerate(progress): rotations[idx] = _rotation_x(np.deg2rad(-60.0) * amount) elif mode in ("crane_above_right", "crane_above_left", "crane_below_right", "crane_below_left"): above = "above" in mode right = "right" in mode translations[:, 0] = (0.18 if right else -0.18) * focus * progress translations[:, 1] = (-0.24 if above else 0.24) * focus * progress translations[:, 2] = 0.16 * focus * progress yaw = np.deg2rad(-12.0 if right else 12.0) pitch = np.deg2rad(-18.0 if above else 18.0) for idx, amount in enumerate(progress): rotations[idx] = _rotation_y(yaw * amount) @ _rotation_x(pitch * amount) target_intrinsics[:, 0:2] *= focal_scale[:, None] return _apply_local_camera_delta(cam_c2w, translations, rotations), target_intrinsics def _create_map_from_control_video(input_frames, frame_num, height, width, fps, custom_settings, target_camera_path=None, model_mode=None): video = _tensor_to_video_np(input_frames) video = video[_center_slice(video.shape[0], frame_num)] height_input, width_input = video.shape[1:3] # video = _crop_resize_video(video, height, width) scene_scale = 1.0 if isinstance(custom_settings, dict): scene_scale = float(custom_settings.get("vista4d_scene_scale", 1.0) or 1.0) process_res, chunk_size, chunk_overlap = _get_da3_settings() if process_res <= 0: process_res = width from preprocessing.depth_anything_v3.depth import resolve_da3_chunk_size, run_da3_reconstruction chunk_size = resolve_da3_chunk_size(chunk_size) cache_key = None if VISTA4D_REUSE_GENERATED_MAP_IN_MEMORY: cache_key = _make_generated_map_cache_key(video, frame_num, height, width, fps, process_res, chunk_size, chunk_overlap, custom_settings, target_camera_path, model_mode) cached = _VISTA4D_MAP_CACHE.get(cache_key) if cached is not None: print("Vista4D map reused from memory cache.") return cached depths, sky_mask, cam_c2w, intrinsics = run_da3_reconstruction(video, process_res=process_res, chunk_size=chunk_size, chunk_overlap=chunk_overlap) if scene_scale != 1.0: depths *= scene_scale cam_c2w[:, :3, 3] *= scene_scale source_cam_c2w = cam_c2w source_intrinsics = intrinsics target_cam_c2w = cam_c2w target_intrinsics = intrinsics if target_camera_path is not None: target_cam_c2w, target_intrinsics = _load_cameras(target_camera_path, frame_num) camera_height, camera_width = _infer_intrinsics_size(target_intrinsics, height_input, width_input) if (camera_height, camera_width) != (height, width): target_intrinsics = _resize_intrinsics(target_intrinsics, height, width, camera_height, camera_width) if scene_scale != 1.0: target_cam_c2w[:, :3, 3] *= scene_scale else: target_cam_c2w, target_intrinsics = _generate_target_camera_trajectory(cam_c2w, intrinsics, depths, sky_mask, model_mode, _camera_strength(custom_settings)) keywords = _parse_keywords(custom_settings) if len(keywords) == 0: dynamic_mask = np.zeros((frame_num, height, width), dtype=np.bool_) elif "_all_" in keywords: dynamic_mask = np.ones((frame_num, height, width), dtype=np.bool_) else: from preprocessing.sam3.preprocessor import run_sam3_video dynamic_mask = run_sam3_video(video, keywords) source_alpha_mask = np.ones((frame_num, height, width), dtype=np.bool_) point_cloud_video, point_cloud_alpha_mask, point_cloud_dynamic_mask = _render_point_cloud_video(video, depths, sky_mask, dynamic_mask, cam_c2w, intrinsics, target_cam_c2w, target_intrinsics) data = { "source_video": video, "point_cloud_video": point_cloud_video, "source_alpha_mask": source_alpha_mask, "source_motion_mask": dynamic_mask, "point_cloud_alpha_mask": point_cloud_alpha_mask, "point_cloud_motion_mask": point_cloud_dynamic_mask, "cam_c2w": target_cam_c2w, "intrinsics": target_intrinsics, } if VISTA4D_REUSE_GENERATED_MAP_IN_MEMORY: _cache_generated_map(cache_key, data) print("Vista4D map kept in memory cache.") return data return _save_generated_map_to_disk(data, source_cam_c2w, source_intrinsics, depths, sky_mask, fps) def _load_map(map_dir, frame_num, height, width): video_src = _read_video(map_dir / "video_src.mp4") video_pc = _read_video(map_dir / "video_pc.mp4") video_src = video_src[_center_slice(video_src.shape[0], frame_num)] video_pc = video_pc[_center_slice(video_pc.shape[0], frame_num)] src_h, src_w = video_src.shape[1:3] video_src = _crop_resize_video(video_src, height, width) video_pc = _crop_resize_video(video_pc, height, width) cam_c2w, intrinsics = _load_cameras(map_dir / "cameras_tgt.npz", frame_num) if (src_h, src_w) != (height, width): intrinsics = _resize_intrinsics(intrinsics, height, width, src_h, src_w) return { "source_video": video_src, "point_cloud_video": video_pc, "source_alpha_mask": _load_masks(map_dir / "alpha_mask_src", frame_num, height, width, True), "source_motion_mask": _load_masks(map_dir / "dynamic_mask_src", frame_num, height, width, False), "point_cloud_alpha_mask": _load_masks(map_dir / "alpha_mask_pc", frame_num, height, width, True), "point_cloud_motion_mask": _load_masks(map_dir / "dynamic_mask_pc", frame_num, height, width, False), "cam_c2w": cam_c2w, "intrinsics": intrinsics, } def _pack_masks(alpha_mask, motion_mask, device, dtype): masks = np.stack((alpha_mask, motion_mask), axis=0)[None].astype(np.float32) masks = torch.from_numpy(masks).to(device=device, dtype=dtype) b, c, f, h, w = masks.shape masks = torch.cat((torch.repeat_interleave(masks[:, :, 0:1], repeats=4, dim=2), masks[:, :, 1:]), dim=2) masks = rearrange(masks, "b c (f sf) (h sh) (w sw) -> b (c sf sh sw) f h w", sf=4, sh=8, sw=8) return masks def prepare_vista4d_condition(pipeline, input_frames, input_custom, frame_num, height, width, tile_size, fps=16, custom_settings=None, model_mode=None): map_dir, target_camera_path = _resolve_custom_input(input_custom) data = None if map_dir is None: if input_frames is None: raise ValueError("Vista4D needs a control video or a preprocessed map in custom_guide.") map_ref = _create_map_from_control_video(input_frames, frame_num, height, width, fps, custom_settings, target_camera_path, model_mode=model_mode) if isinstance(map_ref, dict): data = map_ref else: map_dir = map_ref if data is None: data = _load_map(map_dir, frame_num, height, width) device = pipeline.device dtype = pipeline.dtype vae_dtype = pipeline.VAE_dtype source_video = _video_np_to_tensor(data["source_video"], device, vae_dtype) point_video = _video_np_to_tensor(data["point_cloud_video"], device, vae_dtype) source_latents = pipeline.vae.encode([source_video], tile_size=tile_size)[0].unsqueeze(0).to(device=device, dtype=dtype) point_latents = pipeline.vae.encode([point_video], tile_size=tile_size)[0].unsqueeze(0).to(device=device, dtype=dtype) source_masks = _pack_masks(data["source_alpha_mask"], data["source_motion_mask"], device, dtype) point_masks = _pack_masks(data["point_cloud_alpha_mask"], data["point_cloud_motion_mask"], device, dtype) lat_h = height // pipeline.vae_stride[1] lat_w = width // pipeline.vae_stride[2] cam_c2w = torch.from_numpy(data["cam_c2w"][None]).to(device=device, dtype=dtype) intrinsics = torch.from_numpy(data["intrinsics"][None]).to(device=device, dtype=dtype) cam_emb = get_plucker_embedding(intrinsics, cam_c2w, height, width, height_dit=lat_h // 2, width_dit=lat_w // 2) cam_emb = cam_emb[:, ::pipeline.vae_stride[0]].to(dtype=dtype) return { "vista": { "source_latents": source_latents, "point_latents": point_latents, "source_mask_latents": source_masks, "point_mask_latents": point_masks, "cam_emb": cam_emb, } }