#!/usr/bin/env python3 # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Visualization utilities for Panoptic Recon 3D model outputs. This module provides functions for: - 2D segmentation visualization - Depth map visualization - 3D mesh extraction and PLY export """ from pathlib import Path from typing import Optional, Tuple, Union import numpy as np # Optional imports for visualization try: import matplotlib.pyplot as plt import matplotlib.patches as mpatches HAS_MATPLOTLIB = True except ImportError: HAS_MATPLOTLIB = False try: from PIL import Image HAS_PIL = True except ImportError: HAS_PIL = False try: from skimage import measure HAS_SKIMAGE = True except ImportError: HAS_SKIMAGE = False try: from scipy.spatial import KDTree HAS_SCIPY = True except ImportError: HAS_SCIPY = False def create_color_palette() -> np.ndarray: """Create Front3D color palette for semantic classes. Returns: Color palette as numpy array (N, 3) with uint8 RGB values. """ return np.array([ (0, 0, 0), # 0: background (174, 199, 232), # 1: wall (152, 223, 138), # 2: floor (31, 119, 180), # 3: cabinet (255, 187, 120), # 4: bed (188, 189, 34), # 5: chair (140, 86, 75), # 6: sofa (255, 152, 150), # 7: table (214, 39, 40), # 8: door (197, 176, 213), # 9: window (148, 103, 189), # 10: bookshelf (196, 156, 148), # 11: picture (23, 190, 207), # 12: counter (178, 76, 76), # 13 (247, 182, 210), # 14: desk (66, 188, 102), # 15 (219, 219, 141), # 16: curtain (140, 57, 197), # 17 (202, 185, 52), # 18 (51, 176, 203), # 19 (200, 54, 131), # 20 (92, 193, 61), # 21 (78, 71, 183), # 22 (172, 114, 82), # 23 (255, 127, 14), # 24: refrigerator (91, 163, 138), # 25 (153, 98, 156), # 26 (140, 153, 101), # 27 (158, 218, 229), # 28: shower curtain (100, 125, 154), # 29 (178, 127, 135), # 30 (120, 185, 128), # 31 (146, 111, 194), # 32 (44, 160, 44), # 33: toilet (112, 128, 144), # 34: sink (96, 207, 209), # 35 (227, 119, 194), # 36: bathtub (213, 92, 176), # 37 (94, 106, 211), # 38 (82, 84, 163), # 39: otherfurn (100, 85, 144), # 40 (172, 172, 172), # 41 ], dtype=np.uint8) def colorize_segmentation( segmentation: np.ndarray, palette: Optional[np.ndarray] = None, ) -> np.ndarray: """Colorize segmentation map. Args: segmentation: Segmentation map (H, W) with class indices. palette: Color palette (N, 3). Uses default if None. Returns: Colorized image (H, W, 3) as uint8. """ if palette is None: palette = create_color_palette() # Clip indices to valid range seg_clipped = np.clip(segmentation, 0, len(palette) - 1) return palette[seg_clipped] def visualize_2d_segmentation( image: np.ndarray, panoptic_2d: np.ndarray, output_path: Optional[Union[str, Path]] = None, alpha: float = 0.6, figsize: Tuple[int, int] = (18, 6), dpi: int = 150, ) -> Optional[np.ndarray]: """Visualize 2D panoptic segmentation overlaid on image. Args: image: Original RGB image (H, W, C). panoptic_2d: Panoptic segmentation map (H, W). output_path: Path to save visualization. If None, returns array. alpha: Blend alpha for overlay. figsize: Figure size. dpi: DPI for saved figure. Returns: Overlay image as numpy array if output_path is None. """ if not HAS_MATPLOTLIB: raise ImportError("matplotlib required for visualization") if not HAS_PIL: raise ImportError("PIL required for visualization") # Get color palette palette = create_color_palette() colored_seg = colorize_segmentation(panoptic_2d, palette) # Resize image to match segmentation if needed if image.shape[:2] != panoptic_2d.shape: image_pil = Image.fromarray(image) image_pil = image_pil.resize((panoptic_2d.shape[1], panoptic_2d.shape[0]), Image.LANCZOS) image = np.array(image_pil) # Create overlay overlay = (image.astype(np.float32) * (1 - alpha) + colored_seg.astype(np.float32) * alpha) overlay = overlay.clip(0, 255).astype(np.uint8) if output_path is None: return overlay # Create side-by-side visualization fig, axes = plt.subplots(1, 3, figsize=figsize) axes[0].imshow(image) axes[0].set_title('Original Image', fontsize=14, fontweight='bold') axes[0].axis('off') axes[1].imshow(colored_seg) axes[1].set_title('Panoptic Segmentation', fontsize=14, fontweight='bold') axes[1].axis('off') axes[2].imshow(overlay) axes[2].set_title('Overlay', fontsize=14, fontweight='bold') axes[2].axis('off') plt.tight_layout() plt.savefig(output_path, dpi=dpi, bbox_inches='tight') plt.close() print(f"✓ Saved 2D segmentation visualization to: {output_path}") return None def visualize_depth_map( depth_2d: np.ndarray, output_path: Optional[Union[str, Path]] = None, vmin: float = 0.0, vmax: float = 6.0, cmap: str = 'viridis', figsize: Tuple[int, int] = (10, 8), dpi: int = 150, ) -> Optional[np.ndarray]: """Visualize depth map. Args: depth_2d: Depth map (H, W). output_path: Path to save visualization. If None, returns array. vmin: Minimum depth for colormap. vmax: Maximum depth for colormap. cmap: Matplotlib colormap name. figsize: Figure size. dpi: DPI for saved figure. Returns: Colorized depth as numpy array if output_path is None. """ if not HAS_MATPLOTLIB: raise ImportError("matplotlib required for visualization") # Normalize depth depth_norm = (depth_2d - vmin) / (vmax - vmin) depth_norm = np.clip(depth_norm, 0, 1) # Get colormap cm = plt.get_cmap(cmap) depth_colored = (cm(depth_norm)[:, :, :3] * 255).astype(np.uint8) if output_path is None: return depth_colored fig, ax = plt.subplots(1, 1, figsize=figsize) im = ax.imshow(depth_2d, cmap=cmap, vmin=vmin, vmax=vmax) ax.set_title('Depth Map', fontsize=14, fontweight='bold') ax.axis('off') cbar = plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04) cbar.set_label('Depth (m)', rotation=270, labelpad=20, fontsize=12) plt.tight_layout() plt.savefig(output_path, dpi=dpi, bbox_inches='tight') plt.close() print(f"✓ Saved depth map visualization to: {output_path}") return None def get_mesh( distance_field: np.ndarray, iso_value: float = 1.0, spacing: Tuple[float, float, float] = (1.0, 1.0, 1.0), ) -> Tuple[np.ndarray, np.ndarray]: """Extract mesh from distance field using marching cubes. Args: distance_field: 3D distance field (D, H, W). iso_value: Iso-surface value. spacing: Voxel spacing. Returns: vertices: Mesh vertices (N, 3). faces: Mesh faces (M, 3). """ if not HAS_SKIMAGE: raise ImportError("scikit-image required for mesh extraction") vertices, faces, _, _ = measure.marching_cubes( distance_field, level=iso_value, spacing=spacing ) return vertices, faces def write_ply( vertices: np.ndarray, output_file: Union[str, Path], colors: Optional[np.ndarray] = None, faces: Optional[np.ndarray] = None, ) -> None: """Write PLY file. Args: vertices: Vertex positions (N, 3). output_file: Output PLY file path. colors: Optional vertex colors (N, 3) as uint8. faces: Optional face indices (M, 3). """ with open(output_file, "w") as f: f.write("ply\n") f.write("format ascii 1.0\n") f.write(f"element vertex {len(vertices)}\n") f.write("property float x\n") f.write("property float y\n") f.write("property float z\n") if colors is not None: f.write("property uchar red\n") f.write("property uchar green\n") f.write("property uchar blue\n") if faces is not None and len(faces) > 0: f.write(f"element face {len(faces)}\n") f.write("property list uchar uint vertex_indices\n") f.write("end_header\n") # Write vertices if colors is not None: for v, c in zip(vertices, colors): f.write(f"{v[0]} {v[1]} {v[2]} {int(c[0])} {int(c[1])} {int(c[2])}\n") else: for v in vertices: f.write(f"{v[0]} {v[1]} {v[2]}\n") # Write faces if faces is not None: for face in faces: f.write(f"3 {face[0]} {face[1]} {face[2]}\n") def save_3d_mesh( geometry_3d: np.ndarray, semantic_3d: np.ndarray, output_path: Union[str, Path], iso_value: float = 1.0, voxel_size: float = 0.03, ) -> bool: """Extract and save 3D mesh with semantic colors. Args: geometry_3d: 3D geometry/TSDF (D, H, W). semantic_3d: 3D semantic segmentation (D, H, W). output_path: Output PLY file path. iso_value: Iso-surface value for mesh extraction. voxel_size: Voxel size in meters. Returns: True if successful, False otherwise. """ if not HAS_SKIMAGE: print("Warning: scikit-image not installed. Cannot save PLY mesh.") return False if not HAS_SCIPY: print("Warning: scipy not installed. Cannot color mesh by semantics.") try: # Extract mesh vertices, faces = get_mesh( geometry_3d, iso_value=iso_value, spacing=(voxel_size, voxel_size, voxel_size) ) colors = None if HAS_SCIPY and np.any(semantic_3d): # Get non-zero labeled voxels nonzero_coords = np.stack(semantic_3d.nonzero(), axis=-1) if len(nonzero_coords) > 0: # Build KD tree for nearest neighbor lookup labels_kd = KDTree(nonzero_coords) palette = create_color_palette() # Create color volume semantic_clipped = np.clip(semantic_3d, 0, len(palette) - 1).astype(np.uint32) color_volume = palette[semantic_clipped] # Find nearest label for each vertex # Scale vertices to voxel indices vertex_indices = (vertices / voxel_size).astype(int) neighbor_inds = labels_kd.query(vertex_indices)[1] neighbors = labels_kd.data[neighbor_inds].astype(int) # Clip to valid indices neighbors = np.clip(neighbors, 0, np.array(color_volume.shape[:3]) - 1) colors = color_volume[neighbors[:, 0], neighbors[:, 1], neighbors[:, 2]] # Write PLY write_ply(vertices, output_path, colors, faces) print(f"✓ Saved 3D mesh to: {output_path}") print(f" Vertices: {len(vertices)}, Faces: {len(faces)}") return True except Exception as e: print(f"Warning: Failed to save 3D mesh: {e}") return False def save_outputs( outputs, output_dir: Union[str, Path], original_image: Optional[np.ndarray] = None, save_mesh: bool = True, save_depth: bool = True, save_segmentation: bool = True, save_numpy: bool = True, ) -> dict: """Save all model outputs to directory. Args: outputs: PanopticRecon3DOutput from model. output_dir: Output directory. original_image: Optional original input image for visualization. save_mesh: Whether to save 3D mesh PLY files. save_depth: Whether to save depth visualization. save_segmentation: Whether to save segmentation visualization. save_numpy: Whether to save raw numpy arrays. Returns: Dictionary of saved file paths. """ output_dir = Path(output_dir) output_dir.mkdir(parents=True, exist_ok=True) saved_files = {} # Convert outputs to numpy outputs_np = outputs.to_numpy() # Save numpy arrays if save_numpy: for name, arr in outputs_np.items(): npy_path = output_dir / f"{name}.npy" np.save(npy_path, arr) saved_files[f"{name}_npy"] = str(npy_path) # Save 2D segmentation visualization if save_segmentation and original_image is not None: seg_path = output_dir / "panoptic_2d_visualization.png" visualize_2d_segmentation( original_image, outputs_np["panoptic_seg_2d"], seg_path ) saved_files["segmentation_vis"] = str(seg_path) # Save depth visualization if save_depth: depth_path = output_dir / "depth_visualization.png" visualize_depth_map( outputs_np["depth_2d"], depth_path ) saved_files["depth_vis"] = str(depth_path) # Save 3D meshes if save_mesh: # Semantic mesh semantic_mesh_path = output_dir / "mesh_semantic.ply" if save_3d_mesh( outputs_np["geometry_3d"], outputs_np["semantic_seg_3d"], semantic_mesh_path ): saved_files["semantic_mesh"] = str(semantic_mesh_path) # Panoptic mesh panoptic_mesh_path = output_dir / "mesh_panoptic.ply" if save_3d_mesh( outputs_np["geometry_3d"], outputs_np["panoptic_seg_3d"], panoptic_mesh_path ): saved_files["panoptic_mesh"] = str(panoptic_mesh_path) return saved_files