trellis/utils/postprocessing_utils.py

from typing import *
import numpy as np
import torch
import utils3d
import nvdiffrast.torch as dr
from tqdm import tqdm
import trimesh
import trimesh.visual
import xatlas
import pyvista as pv
from pymeshfix import _meshfix
import igraph
import cv2
from PIL import Image
from .random_utils import sphere_hammersley_sequence
from .render_utils import render_multiview
from ..renderers import GaussianRenderer
from ..representations import Strivec, Gaussian, MeshExtractResult

from api_spz.core.exceptions import CancelledException


def postprocess_mesh(
    vertices: Union[np.ndarray, torch.Tensor],
    faces: Union[np.ndarray, torch.Tensor],
    simplify: bool = True,
    simplify_ratio: float = 0.9,
    fill_holes: bool = True,
    fill_holes_max_hole_size: float = 0.04,
    fill_holes_max_hole_nbe: int = 32,
    fill_holes_resolution: int = 1024,
    fill_holes_num_views: int = 1000,
    debug: bool = False,
    verbose: bool = False,
) -> Tuple[torch.Tensor, torch.Tensor]:
    """
    Postprocess a mesh by simplifying, removing invisible faces, and removing isolated pieces.
    Maintains data on GPU where possible and uses fp16 precision.

    Args:
        vertices: Vertices of the mesh. Shape (V, 3). Can be numpy array or torch tensor.
        faces: Faces of the mesh. Shape (F, 3). Can be numpy array or torch tensor.
        simplify: Whether to simplify the mesh, using quadric edge collapse.
        simplify_ratio: Ratio of faces to keep after simplification.
        fill_holes: Whether to fill holes in the mesh.
        fill_holes_max_hole_size: Maximum area of a hole to fill.
        fill_holes_max_hole_nbe: Maximum number of boundary edges of a hole to fill.
        fill_holes_resolution: Resolution of the rasterization.
        fill_holes_num_views: Number of views to rasterize the mesh.
        verbose: Whether to print progress.

    Returns:
        Tuple[torch.Tensor, torch.Tensor]: Processed vertices and faces as torch tensors on GPU
    """
    if verbose:
        tqdm.write(f'Before postprocess: {vertices.shape[0]} vertices, {faces.shape[0]} faces')

    # Convert inputs to torch tensors if needed and ensure float32
    if isinstance(vertices, np.ndarray):
        vertices = torch.from_numpy(vertices).float()
    if isinstance(faces, np.ndarray):
        faces = torch.from_numpy(faces)
    
    # Ensure tensors are on GPU and in float32
    vertices = vertices.cuda().float()
    faces = faces.cuda()
    
    # Simplify
    if simplify and simplify_ratio > 0:
        vertices_cpu = vertices.cpu().numpy()  # Already float32
        faces_cpu = faces.cpu().numpy()
        mesh = pv.PolyData(vertices_cpu, np.concatenate([np.full((faces_cpu.shape[0], 1), 3), faces_cpu], axis=1))
        mesh = mesh.decimate(simplify_ratio, progress_bar=verbose)
        vertices = torch.tensor(mesh.points, device='cuda')  # Will stay float32
        faces = torch.tensor(mesh.faces.reshape(-1, 4)[:, 1:], device='cuda', dtype=torch.int32)
        if verbose:
            tqdm.write(f'After decimate: {vertices.shape[0]} vertices, {faces.shape[0]} faces')

    # Remove invisible faces - already operates on GPU
    if fill_holes:
        vertices, faces = _fill_holes(
            vertices, faces,
            max_hole_size=fill_holes_max_hole_size,
            max_hole_nbe=fill_holes_max_hole_nbe,
            resolution=fill_holes_resolution,
            num_views=fill_holes_num_views,
            debug=debug,
            verbose=verbose,
        )
        if verbose:
            tqdm.write(f'After remove invisible faces: {vertices.shape[0]} vertices, {faces.shape[0]} faces')

    return vertices, faces


def parametrize_mesh(vertices: Union[np.ndarray, torch.Tensor], faces: Union[np.ndarray, torch.Tensor]):
    """
    Parametrize a mesh to a texture space, using xatlas.

    Args:
        vertices: Vertices of the mesh. Shape (V, 3). Can be numpy array or torch tensor.
        faces: Faces of the mesh. Shape (F, 3). Can be numpy array or torch tensor.

    Returns:
        Tuple[np.ndarray, np.ndarray, np.ndarray]: Remapped vertices, faces, and UV coordinates
    """
    # Convert to numpy if needed
    if isinstance(vertices, torch.Tensor):
        vertices = vertices.detach().cpu().numpy()
    if isinstance(faces, torch.Tensor):
        faces = faces.detach().cpu().numpy()
    
    # Ensure correct dtypes for xatlas
    vertices = vertices.astype(np.float32)
    faces = faces.astype(np.uint32)

    # Run parametrization
    vmapping, indices, uvs = xatlas.parametrize(vertices, faces)

    # Apply remapping
    vertices = vertices[vmapping]
    faces = indices

    return vertices, faces, uvs

def bake_texture(
    vertices: np.array,
    faces: np.array,
    uvs: np.array,
    observations: List[np.array],
    masks: List[np.array],
    extrinsics: List[np.array],
    intrinsics: List[np.array],
    texture_size: int = 2048,
    near: float = 0.1,
    far: float = 10.0,
    mode: Literal['fast', 'opt'] = 'opt',
    lambda_tv: float = 1e-2,
    verbose: bool = False,
    cancel_event=None,
):
    """
    Bake texture to a mesh from multiple observations.

    Args:
        vertices (np.array): Vertices of the mesh. Shape (V, 3).
        faces (np.array): Faces of the mesh. Shape (F, 3).
        uvs (np.array): UV coordinates of the mesh. Shape (V, 2).
        observations (List[np.array]): List of observations. Each observation is a 2D image. Shape (H, W, 3).
        masks (List[np.array]): List of masks. Each mask is a 2D image. Shape (H, W).
        extrinsics (List[np.array]): List of extrinsics. Shape (4, 4).
        intrinsics (List[np.array]): List of intrinsics. Shape (3, 3).
        texture_size (int): Size of the texture.
        near (float): Near plane of the camera.
        far (float): Far plane of the camera.
        mode (Literal['fast', 'opt']): Mode of texture baking.
        lambda_tv (float): Weight of total variation loss in optimization.
        verbose (bool): Whether to print progress.
    """
    vertices = torch.tensor(vertices).cuda()
    faces = torch.tensor(faces.astype(np.int32)).cuda()
    uvs = torch.tensor(uvs).cuda()
    observations = [torch.tensor(obs / 255.0, dtype=torch.float16).cuda() for obs in observations] # Keep observations as float16 if desired for memory
    masks = [torch.tensor(m>0).bool().cuda() for m in masks]
    views = [utils3d.torch.extrinsics_to_view(torch.tensor(extr).cuda()) for extr in extrinsics]
    projections = [utils3d.torch.intrinsics_to_perspective(torch.tensor(intr).cuda(), near, far) for intr in intrinsics]

    if mode == 'fast':
        texture = torch.zeros((texture_size * texture_size, 3), dtype=torch.float32).cuda()
        texture_weights = torch.zeros((texture_size * texture_size), dtype=torch.float32).cuda()
        rastctx = utils3d.torch.RastContext(backend='cuda')
        for observation, view, projection, mask_in in tqdm(zip(observations, views, projections, masks), total=len(observations), disable=not verbose, desc='Texture baking (fast)'):
            if cancel_event and cancel_event.is_set(): 
                raise CancelledException(f"Cancelled the texture baking (fast).")
            with torch.no_grad():
                rast = utils3d.torch.rasterize_triangle_faces(
                    rastctx, vertices[None], faces, observation.shape[1], observation.shape[0], uv=uvs[None], view=view, projection=projection
                )
                uv_map = rast['uv'][0].detach().flip(0)
                mask = rast['mask'][0].detach().bool() & mask_in
            
            # nearest neighbor interpolation
            uv_map = (uv_map * texture_size).floor().long()
            obs = observation[mask]
            uv_map = uv_map[mask]
            idx = uv_map[:, 0] + (texture_size - uv_map[:, 1] - 1) * texture_size
            texture = texture.scatter_add(0, idx.view(-1, 1).expand(-1, 3), obs.float())
            texture_weights = texture_weights.scatter_add(0, idx, torch.ones((obs.shape[0]), dtype=torch.float32, device=texture.device))

        mask = texture_weights > 0
        texture[mask] /= texture_weights[mask][:, None]
        texture = np.clip(texture.reshape(texture_size, texture_size, 3).cpu().numpy() * 255, 0, 255).astype(np.uint8)

        # inpaint
        mask_np = (texture_weights == 0).cpu().numpy().astype(np.uint8).reshape(texture_size, texture_size)
        texture = cv2.inpaint(texture, mask_np, 3, cv2.INPAINT_TELEA)

    elif mode == 'opt':
        rastctx = utils3d.torch.RastContext(backend='cuda')
        observations = [observations.flip(0) for observations in observations]
        masks = [m.flip(0) for m in masks]
        _uv = []
        _uv_dr = []
        for observation, view, projection in tqdm(zip(observations, views, projections), total=len(views), disable=not verbose, desc='Texture baking (opt): UV'):
            if cancel_event and cancel_event.is_set(): 
                raise CancelledException(f"Cancelled the texture baking (opt).")
            with torch.no_grad():
                rast = utils3d.torch.rasterize_triangle_faces(
                    rastctx, vertices[None], faces, observation.shape[1], observation.shape[0], uv=uvs[None], view=view, projection=projection
                )
                _uv.append(rast['uv'].detach())
                _uv_dr.append(rast['uv_dr'].detach())

        texture = torch.nn.Parameter(torch.zeros((1, texture_size, texture_size, 3), dtype=torch.float32).cuda())
        optimizer = torch.optim.Adam([texture], betas=(0.5, 0.9), lr=1e-2)

        def exp_anealing(optimizer, step, total_steps, start_lr, end_lr):
            return start_lr * (end_lr / start_lr) ** (step / total_steps)

        def cosine_anealing(optimizer, step, total_steps, start_lr, end_lr):
            return end_lr + 0.5 * (start_lr - end_lr) * (1 + np.cos(np.pi * step / total_steps))
        
        def tv_loss(texture):
            return torch.nn.functional.l1_loss(texture[:, :-1, :, :], texture[:, 1:, :, :]) + \
                   torch.nn.functional.l1_loss(texture[:, :, :-1, :], texture[:, :, 1:, :])
    
        total_steps = 1000
        with tqdm(total=total_steps, disable=not verbose, desc='Texture baking (opt): optimizing') as pbar:
            for step in range(total_steps):
                optimizer.zero_grad()
                selected = np.random.randint(0, len(views))
                uv, uv_dr, observation, mask = _uv[selected], _uv_dr[selected], observations[selected], masks[selected]
                render = dr.texture(texture, uv, uv_dr)[0]
                loss = torch.nn.functional.l1_loss(render[mask], observation[mask])
                if lambda_tv > 0:
                    loss += lambda_tv * tv_loss(texture)
                loss.backward()
                optimizer.step()
                # annealing
                optimizer.param_groups[0]['lr'] = cosine_anealing(optimizer, step, total_steps, 1e-2, 1e-5)
                pbar.set_postfix({'loss': loss.item()})
                pbar.update()
                if cancel_event and cancel_event.is_set(): 
                    raise CancelledException(f"Cancelled texture optimization at step {step}/{total_steps}.")
        texture = np.clip(texture[0].flip(0).detach().cpu().numpy() * 255, 0, 255).astype(np.uint8)
        mask = 1 - utils3d.torch.rasterize_triangle_faces(
            rastctx, (uvs * 2 - 1)[None], faces, texture_size, texture_size
        )['mask'][0].detach().cpu().numpy().astype(np.uint8)
        texture = cv2.inpaint(texture, mask, 3, cv2.INPAINT_TELEA)
    else:
        raise ValueError(f'Unknown mode: {mode}')

    return texture


def to_glb(
    app_rep: Union[Strivec, Gaussian],
    mesh: MeshExtractResult,
    simplify: float = 0.95,
    fill_holes: bool = True,
    fill_holes_max_size: float = 0.04,
    texture_size: int = 1024,
    debug: bool = False,
    verbose: bool = False,
    cancel_event = None,
) -> trimesh.Trimesh:
    """
    Convert a generated asset to a glb file.

    Args:
        app_rep (Union[Strivec, Gaussian]): Appearance representation.
        mesh (MeshExtractResult): Extracted mesh.
        simplify (float): Ratio of faces to remove in simplification.
        fill_holes (bool): Whether to fill holes in the mesh.
        fill_holes_max_size (float): Maximum area of a hole to fill.
        texture_size (int): Size of the texture.
        debug (bool): Whether to print debug information.
        verbose (bool): Whether to print progress.
    """
    vertices = mesh.vertices
    faces = mesh.faces
    
    
    vertices, faces = postprocess_mesh(
        vertices, faces,
        simplify=simplify > 0,
        simplify_ratio=simplify,
        fill_holes=fill_holes,
        fill_holes_max_hole_size=fill_holes_max_size,
        fill_holes_max_hole_nbe=int(250 * np.sqrt(1-simplify)),
        fill_holes_resolution=1024,
        fill_holes_num_views=1000,
        debug=debug,
        verbose=verbose,
    )

    # parametrize mesh (converts to CPU numpy internally)
    vertices, faces, uvs = parametrize_mesh(vertices, faces)

    # bake texture
    observations, extrinsics, intrinsics = render_multiview(app_rep, resolution=1024, nviews=30) # nviews was 100
    masks = [np.any(observation > 0, axis=-1) for observation in observations]
    extrinsics = [extrinsics[i].cpu().numpy() for i in range(len(extrinsics))]
    intrinsics = [intrinsics[i].cpu().numpy() for i in range(len(intrinsics))]
    texture = bake_texture(
        vertices, faces, uvs,
        observations, masks, extrinsics, intrinsics,
        texture_size=texture_size, mode='opt',
        lambda_tv=0.01,
        verbose=verbose,
        cancel_event=cancel_event,
    )
    texture = Image.fromarray(texture)

    # rotate mesh (from z-up to y-up)
    vertices = vertices @ np.array([[1, 0, 0], [0, 0, -1], [0, 1, 0]])
    material = trimesh.visual.material.PBRMaterial(
        roughnessFactor=1.0,
        baseColorTexture=texture,
        baseColorFactor=np.array([255, 255, 255, 255], dtype=np.uint8)
    )
    mesh = trimesh.Trimesh(vertices, faces, visual=trimesh.visual.TextureVisuals(uv=uvs, material=material))
    return mesh


def simplify_gs(
    gs: Gaussian,
    simplify: float = 0.95,
    verbose: bool = True,
):
    """
    Simplify 3D Gaussians with aggressive pruning
    NOTE: this function is not used in the current implementation for the unsatisfactory performance.
    
    Args:
        gs (Gaussian): 3D Gaussian.
        simplify (float): Ratio of Gaussians to remove in simplification.
    """
    # if simplify <= 0:
    #     return gs
    
    try:
        # Move everything to CUDA and ensure float32 precision
        device = torch.device('cuda')
        
        # Convert all gaussian parameters to float32
        gs._features_dc = gs._features_dc.float()
        if gs._features_rest is not None:
            gs._features_rest = gs._features_rest.float()
        gs._opacity = gs._opacity.float()
        gs._rotation = gs._rotation.float()
        gs._scaling = gs._scaling.float()
        gs._xyz = gs._xyz.float()
        
        # Get initial opacity and ensure proper dimensions
        initial_opacity = gs.get_opacity.squeeze()
        if initial_opacity.dim() == 0:
            initial_opacity = initial_opacity.unsqueeze(0)
        
        # More aggressive initial pruning
        with torch.no_grad():
            opacity_threshold = 0.1  # Increased from 0.05
            initial_mask = initial_opacity > opacity_threshold
            
            # Handle case where no points meet threshold
            if not initial_mask.any():
                num_keep = max(int(0.1 * initial_opacity.shape[0]), 1)
                _, top_indices = initial_opacity.topk(num_keep)
                initial_mask = torch.zeros_like(initial_mask, dtype=torch.bool)
                initial_mask[top_indices] = True
            
            # Apply mask and ensure at least one point remains
            if initial_mask.sum() == 0:
                max_idx = torch.argmax(initial_opacity)
                initial_mask[max_idx] = True
            
            gs._xyz = gs._xyz[initial_mask]
            gs._rotation = gs._rotation[initial_mask]
            gs._scaling = gs._scaling[initial_mask]
            gs._opacity = gs._opacity[initial_mask]
            gs._features_dc = gs._features_dc[initial_mask]
            gs._features_rest = gs._features_rest[initial_mask] if gs._features_rest is not None else None
            
            if verbose:
                print(f"Initial pruning: kept {initial_mask.sum().item()} points out of {len(initial_mask)}")
        
        # Early return if too few points
        if gs._xyz.shape[0] < 2:
            if verbose:
                print("Too few points remain after initial pruning, returning original gaussian")
            return gs
        
        # Render multiview observations with reduced views
        observations, extrinsics, intrinsics = render_multiview(gs, resolution=512, nviews=30)
        observations = [torch.tensor(obs / 255.0, dtype=torch.float32, device=device).permute(2, 0, 1) for obs in observations]
        extrinsics = [e.float() for e in extrinsics]
        intrinsics = [i.float() for i in intrinsics]
        
        # Initialize renderer with smaller resolution
        renderer = GaussianRenderer({
            "resolution": 512,
            "near": 0.8,
            "far": 1.6,
            "ssaa": 1,
            "bg_color": (0,0,0),
        })

        # Clone Gaussian parameters
        new_gs = Gaussian(**gs.init_params)
        new_gs._features_dc = gs._features_dc.clone().to(device, dtype=torch.float32)
        new_gs._features_rest = gs._features_rest.clone().to(device, dtype=torch.float32) if gs._features_rest is not None else None
        new_gs._opacity = torch.nn.Parameter(gs._opacity.clone().to(device, dtype=torch.float32))
        new_gs._rotation = torch.nn.Parameter(gs._rotation.clone().to(device, dtype=torch.float32))
        new_gs._scaling = torch.nn.Parameter(gs._scaling.clone().to(device, dtype=torch.float32))
        new_gs._xyz = torch.nn.Parameter(gs._xyz.clone().to(device, dtype=torch.float32))
        
        # Get initial point count and set target
        current_points = new_gs._xyz.shape[0]
        target_ratio = max(0.1, 1 - simplify * 1.2)  # Ensure we keep at least 10% of points
        num_target = max(int(target_ratio * current_points), 2)  # Ensure at least 2 points remain
        
        if verbose:
            print(f"Starting optimization with {current_points} points, target: {num_target}")
        
        # Optimization parameters
        start_lr = [5e-4, 5e-3, 0.025, 0.1]
        end_lr = [5e-6, 5e-5, 0.00025, 0.001]
        optimizer = torch.optim.Adam([
            {"params": new_gs._xyz, "lr": start_lr[0]},
            {"params": new_gs._rotation, "lr": start_lr[1]},
            {"params": new_gs._scaling, "lr": start_lr[2]},
            {"params": new_gs._opacity, "lr": start_lr[3]},
        ], lr=start_lr[0])
        
        def cosine_anealing(optimizer, step, total_steps, start_lr, end_lr):
            return end_lr + 0.5 * (start_lr - end_lr) * (1 + np.cos(np.pi * step / total_steps))
        
        _zeta = new_gs.get_opacity.clone().detach().squeeze().float()
        if _zeta.dim() == 0:
            _zeta = _zeta.unsqueeze(0)
        _lambda = torch.zeros_like(_zeta, dtype=torch.float32)
        _delta = 1e-6
        _interval = 5
        
        total_steps = 500
        with tqdm(total=total_steps, disable=not verbose, desc='Simplifying Gaussian') as pbar:
            for i in range(total_steps):
                try:
                    # More frequent pruning
                    if i % 50 == 0 and new_gs._xyz.shape[0] > 2:  # Only prune if we have enough points
                        with torch.cuda.amp.autocast(enabled=False):
                            opacity = new_gs.get_opacity.squeeze()
                            if opacity.dim() == 0:
                                opacity = opacity.unsqueeze(0)
                            
                            mask = opacity > opacity_threshold
                            if not mask.any():  # If all would be pruned, keep top points
                                num_keep = max(int(0.1 * len(mask)), 2)
                                _, top_indices = opacity.topk(min(num_keep, len(mask)))
                                mask = torch.zeros_like(mask, dtype=torch.bool)
                                mask[top_indices] = True
                            
                            # Ensure we keep at least 2 points
                            if mask.sum() < 2:
                                _, top_indices = opacity.topk(2)
                                mask = torch.zeros_like(mask, dtype=torch.bool)
                                mask[top_indices] = True
                            
                            # Apply mask
                            new_gs._xyz = torch.nn.Parameter(new_gs._xyz[mask].float())
                            new_gs._rotation = torch.nn.Parameter(new_gs._rotation[mask].float())
                            new_gs._scaling = torch.nn.Parameter(new_gs._scaling[mask].float())
                            new_gs._opacity = torch.nn.Parameter(new_gs._opacity[mask].float())
                            new_gs._features_dc = new_gs._features_dc[mask].float()
                            new_gs._features_rest = new_gs._features_rest[mask].float() if new_gs._features_rest is not None else None
                            
                            # Update optimization variables
                            _zeta = _zeta[mask].float()
                            _lambda = _lambda[mask].float()
                            
                            # Update optimizer state
                            optimizer = torch.optim.Adam([
                                {"params": new_gs._xyz, "lr": start_lr[0]},
                                {"params": new_gs._rotation, "lr": start_lr[1]},
                                {"params": new_gs._scaling, "lr": start_lr[2]},
                                {"params": new_gs._opacity, "lr": start_lr[3]},
                            ], lr=start_lr[0])

                    with torch.cuda.amp.autocast(enabled=False):
                        opacity = new_gs.get_opacity.squeeze().float()
                        if opacity.dim() == 0:
                            opacity = opacity.unsqueeze(0)
                        current_points = opacity.shape[0]
                        
                        # Sparsification
                        if i % _interval == 0 and current_points > 2:
                            _zeta = _lambda + opacity.detach()
                            if current_points > num_target:
                                k = min(num_target, current_points - 2)  # Keep at least 2 points
                                if k > 0:
                                    index = _zeta.topk(k)[1]
                                    _m = torch.ones_like(_zeta, dtype=torch.bool)
                                    _m[index] = 0
                                    _zeta[_m] = 0
                            _lambda = _lambda + opacity.detach() - _zeta
                        
                        # Sample random view
                        view_idx = np.random.randint(len(observations))
                        observation = observations[view_idx].float()
                        extrinsic = extrinsics[view_idx].float()
                        intrinsic = intrinsics[view_idx].float()
                        
                        # Render and compute loss
                        color = renderer.render(new_gs, extrinsic, intrinsic)['color'].float()
                        rgb_loss = torch.nn.functional.l1_loss(color, observation)
                        loss = rgb_loss + _delta * torch.sum(torch.pow(_lambda + opacity - _zeta, 2))
                        
                        # Optimization step
                        optimizer.zero_grad()
                        loss.backward()
                        optimizer.step()
                        
                        # Update learning rates
                        for j in range(len(optimizer.param_groups)):
                            optimizer.param_groups[j]['lr'] = cosine_anealing(optimizer, i, total_steps, start_lr[j], end_lr[j])
                        
                        # Update progress bar
                        if not torch.isnan(rgb_loss).any():
                            pbar.set_postfix({
                                'loss': rgb_loss.item(),
                                'num': current_points,
                                'lambda': _lambda.mean().item()
                            })
                        
                    pbar.update()
                    
                except RuntimeError as e:
                    if "out of memory" in str(e):
                        torch.cuda.empty_cache()
                        continue
                    else:
                        raise e

        # Final pruning with safety check
        with torch.no_grad():
            opacity = new_gs.get_opacity.squeeze()
            if opacity.dim() == 0:
                opacity = opacity.unsqueeze(0)
            
            final_mask = opacity > opacity_threshold
            if not final_mask.any() or final_mask.sum() < 2:
                num_keep = max(int(0.1 * len(opacity)), 2)
                _, top_indices = opacity.topk(min(num_keep, len(opacity)))
                final_mask = torch.zeros_like(final_mask, dtype=torch.bool)
                final_mask[top_indices] = True
            
            new_gs._xyz = new_gs._xyz.data[final_mask].float()
            new_gs._rotation = new_gs._rotation.data[final_mask].float()
            new_gs._scaling = new_gs._scaling.data[final_mask].float()
            new_gs._opacity = new_gs._opacity.data[final_mask].float()
            new_gs._features_dc = new_gs._features_dc[final_mask].float()
            new_gs._features_rest = new_gs._features_rest[final_mask].float() if new_gs._features_rest is not None else None
            
            if verbose:
                print(f"Final number of points: {final_mask.sum().item()}")
        
        return new_gs
        
    except Exception as e:
        print(f"Error in simplify_gs: {str(e)}")
        print(f"Error details: {str(e.__class__.__name__)}")
        import traceback
        traceback.print_exc()
        return gs  # Return original gaussian if simplification fails


@torch.no_grad()
def _fill_holes(
    verts,
    faces,
    max_hole_size=0.04,
    max_hole_nbe=32,
    resolution=128,
    num_views=500,
    debug=False,
    verbose=False
):
    """
    Rasterize a mesh from multiple views and remove invisible faces.
    Also includes postprocessing to:
        1. Remove connected components that are have low visibility.
        2. Mincut to remove faces at the inner side of the mesh connected to the outer side with a small hole.

    Args:
        verts (torch.Tensor): Vertices of the mesh. Shape (V, 3).
        faces (torch.Tensor): Faces of the mesh. Shape (F, 3).
        max_hole_size (float): Maximum area of a hole to fill.
        resolution (int): Resolution of the rasterization.
        num_views (int): Number of views to rasterize the mesh.
        verbose (bool): Whether to print progress.
    """
    # Construct cameras
    yaws = []
    pitchs = []
    for i in range(num_views):
        y, p = sphere_hammersley_sequence(i, num_views)
        yaws.append(y)
        pitchs.append(p)
    yaws = torch.tensor(yaws).cuda()
    pitchs = torch.tensor(pitchs).cuda()
    radius = 2.0
    fov = torch.deg2rad(torch.tensor(40)).cuda()
    projection = utils3d.torch.perspective_from_fov_xy(fov, fov, 1, 3)
    views = []
    for (yaw, pitch) in zip(yaws, pitchs):
        orig = torch.tensor([
            torch.sin(yaw) * torch.cos(pitch),
            torch.cos(yaw) * torch.cos(pitch),
            torch.sin(pitch),
        ]).cuda().float() * radius
        view = utils3d.torch.view_look_at(orig, torch.tensor([0, 0, 0]).float().cuda(), torch.tensor([0, 0, 1]).float().cuda())
        views.append(view)
    views = torch.stack(views, dim=0)

    # Rasterize
    visblity = torch.zeros(faces.shape[0], dtype=torch.int32, device=verts.device)
    rastctx = utils3d.torch.RastContext(backend='cuda')
    for i in tqdm(range(views.shape[0]), total=views.shape[0], disable=not verbose, desc='Rasterizing'):
        view = views[i]
        buffers = utils3d.torch.rasterize_triangle_faces(
            rastctx, verts[None], faces, resolution, resolution, view=view, projection=projection
        )
        face_id = buffers['face_id'][0][buffers['mask'][0] > 0.95] - 1
        face_id = torch.unique(face_id).long()
        visblity[face_id] += 1
    visblity = visblity.float() / num_views
    
    # Mincut
    ## construct outer faces
    edges, face2edge, edge_degrees = utils3d.torch.compute_edges(faces)
    boundary_edge_indices = torch.nonzero(edge_degrees == 1).reshape(-1)
    connected_components = utils3d.torch.compute_connected_components(faces, edges, face2edge)
    outer_face_indices = torch.zeros(faces.shape[0], dtype=torch.bool, device=faces.device)
    for i in range(len(connected_components)):
        outer_face_indices[connected_components[i]] = visblity[connected_components[i]] > min(max(visblity[connected_components[i]].quantile(0.75).item(), 0.25), 0.5)
    outer_face_indices = outer_face_indices.nonzero().reshape(-1)
    
    ## construct inner faces
    inner_face_indices = torch.nonzero(visblity == 0).reshape(-1)
    if verbose:
        tqdm.write(f'Found {inner_face_indices.shape[0]} invisible faces')
    if inner_face_indices.shape[0] == 0:
        return verts, faces
    
    ## Construct dual graph (faces as nodes, edges as edges)
    dual_edges, dual_edge2edge = utils3d.torch.compute_dual_graph(face2edge)
    dual_edge2edge = edges[dual_edge2edge]
    dual_edges_weights = torch.norm(verts[dual_edge2edge[:, 0]] - verts[dual_edge2edge[:, 1]], dim=1)
    if verbose:
        tqdm.write(f'Dual graph: {dual_edges.shape[0]} edges')

    ## solve mincut problem
    ### construct main graph
    g = igraph.Graph()
    g.add_vertices(faces.shape[0])
    g.add_edges(dual_edges.cpu().numpy())
    g.es['weight'] = dual_edges_weights.cpu().numpy()
    
    ### source and target
    g.add_vertex('s')
    g.add_vertex('t')
    
    ### connect invisible faces to source
    g.add_edges([(f, 's') for f in inner_face_indices], attributes={'weight': torch.ones(inner_face_indices.shape[0], dtype=torch.float32).cpu().numpy()})
    
    ### connect outer faces to target
    g.add_edges([(f, 't') for f in outer_face_indices], attributes={'weight': torch.ones(outer_face_indices.shape[0], dtype=torch.float32).cpu().numpy()})
                
    ### solve mincut
    cut = g.mincut('s', 't', (np.array(g.es['weight']) * 1000).tolist())
    remove_face_indices = torch.tensor([v for v in cut.partition[0] if v < faces.shape[0]], dtype=torch.long, device=faces.device)
    if verbose:
        tqdm.write(f'Mincut solved, start checking the cut')
    
    ### check if the cut is valid with each connected component
    to_remove_cc = utils3d.torch.compute_connected_components(faces[remove_face_indices])
    if debug:
        tqdm.write(f'Number of connected components of the cut: {len(to_remove_cc)}')
    valid_remove_cc = []
    cutting_edges = []
    for cc in to_remove_cc:
        #### check if the connected component has low visibility
        visblity_median = visblity[remove_face_indices[cc]].median()
        if debug:
            tqdm.write(f'visblity_median: {visblity_median}')
        if visblity_median > 0.25:
            continue
        
        #### check if the cuting loop is small enough
        cc_edge_indices, cc_edges_degree = torch.unique(face2edge[remove_face_indices[cc]], return_counts=True)
        cc_boundary_edge_indices = cc_edge_indices[cc_edges_degree == 1]
        cc_new_boundary_edge_indices = cc_boundary_edge_indices[~torch.isin(cc_boundary_edge_indices, boundary_edge_indices)]
        if len(cc_new_boundary_edge_indices) > 0:
            cc_new_boundary_edge_cc = utils3d.torch.compute_edge_connected_components(edges[cc_new_boundary_edge_indices])
            cc_new_boundary_edges_cc_center = [verts[edges[cc_new_boundary_edge_indices[edge_cc]]].mean(dim=1).mean(dim=0) for edge_cc in cc_new_boundary_edge_cc]
            cc_new_boundary_edges_cc_area = []
            for i, edge_cc in enumerate(cc_new_boundary_edge_cc):
                _e1 = verts[edges[cc_new_boundary_edge_indices[edge_cc]][:, 0]] - cc_new_boundary_edges_cc_center[i]
                _e2 = verts[edges[cc_new_boundary_edge_indices[edge_cc]][:, 1]] - cc_new_boundary_edges_cc_center[i]
                cc_new_boundary_edges_cc_area.append(torch.norm(torch.cross(_e1, _e2, dim=-1), dim=1).sum() * 0.5)
            if debug:
                cutting_edges.append(cc_new_boundary_edge_indices)
                tqdm.write(f'Area of the cutting loop: {cc_new_boundary_edges_cc_area}')
            if any([l > max_hole_size for l in cc_new_boundary_edges_cc_area]):
                continue
            
        valid_remove_cc.append(cc)
        
    if debug:
        face_v = verts[faces].mean(dim=1).cpu().numpy()
        vis_dual_edges = dual_edges.cpu().numpy()
        vis_colors = np.zeros((faces.shape[0], 3), dtype=np.uint8)
        vis_colors[inner_face_indices.cpu().numpy()] = [0, 0, 255]
        vis_colors[outer_face_indices.cpu().numpy()] = [0, 255, 0]
        vis_colors[remove_face_indices.cpu().numpy()] = [255, 0, 255]
        if len(valid_remove_cc) > 0:
            vis_colors[remove_face_indices[torch.cat(valid_remove_cc)].cpu().numpy()] = [255, 0, 0]
        utils3d.io.write_ply('dbg_dual.ply', face_v, edges=vis_dual_edges, vertex_colors=vis_colors)
        
        vis_verts = verts.cpu().numpy()
        vis_edges = edges[torch.cat(cutting_edges)].cpu().numpy()
        utils3d.io.write_ply('dbg_cut.ply', vis_verts, edges=vis_edges)
        
    
    if len(valid_remove_cc) > 0:
        remove_face_indices = remove_face_indices[torch.cat(valid_remove_cc)]
        mask = torch.ones(faces.shape[0], dtype=torch.bool, device=faces.device)
        mask[remove_face_indices] = 0
        faces = faces[mask]
        faces, verts = utils3d.torch.remove_unreferenced_vertices(faces, verts)
        if verbose:
            tqdm.write(f'Removed {(~mask).sum()} faces by mincut')
    else:
        if verbose:
            tqdm.write(f'Removed 0 faces by mincut')
            
    mesh = _meshfix.PyTMesh()
    mesh.load_array(verts.cpu().numpy(), faces.cpu().numpy())
    mesh.fill_small_boundaries(nbe=max_hole_nbe, refine=True)
    verts, faces = mesh.return_arrays()
    verts, faces = torch.tensor(verts, device='cuda', dtype=torch.float32), torch.tensor(faces, device='cuda', dtype=torch.int32)

    return verts, faces