run.py

import os
import argparse
import glm
import numpy as np
import torch
import rembg
from PIL import Image
from torchvision.transforms import v2
import torchvision
from pytorch_lightning import seed_everything
from omegaconf import OmegaConf
from einops import rearrange, repeat
from tqdm import tqdm
from huggingface_hub import hf_hub_download
from diffusers import DiffusionPipeline, EulerAncestralDiscreteScheduler

from src.data.objaverse import load_mipmap
from src.utils import render_utils
from src.utils.train_util import instantiate_from_config
from src.utils.camera_util import (
    FOV_to_intrinsics, 
    center_looking_at_camera_pose,
    get_zero123plus_input_cameras,
    get_circular_camera_poses,
)
from src.utils.mesh_util import save_obj, save_obj_with_mtl
from src.utils.infer_util import remove_background, resize_foreground, save_video

def str_to_tuple(arg_str):
    try:
        return eval(arg_str)
    except:
        raise argparse.ArgumentTypeError("Tuple argument must be in the format (x, y)")
    

def get_render_cameras(batch_size=1, M=120, radius=4.0, elevation=20.0, is_flexicubes=False, fov=50):
    """
    Get the rendering camera parameters.
    """
    train_res = [512, 512]
    cam_near_far = [0.1, 1000.0]
    fovy = np.deg2rad(fov)
    proj_mtx = render_utils.perspective(fovy, train_res[1] / train_res[0], cam_near_far[0], cam_near_far[1])
    all_mv = []
    all_mvp = []
    all_campos = []
    if isinstance(elevation, tuple):
        elevation_0 = np.deg2rad(elevation[0])
        elevation_1 = np.deg2rad(elevation[1])
        for i in range(M//2):
            azimuth = 2 * np.pi * i / (M // 2)
            z = radius * np.cos(azimuth) * np.sin(elevation_0)
            x = radius * np.sin(azimuth) * np.sin(elevation_0)
            y = radius * np.cos(elevation_0)

            eye = glm.vec3(x, y, z)
            at = glm.vec3(0.0, 0.0, 0.0)
            up = glm.vec3(0.0, 1.0, 0.0)
            view_matrix = glm.lookAt(eye, at, up)
            mv = torch.from_numpy(np.array(view_matrix))
            mvp   = proj_mtx @ (mv)  #w2c
            campos = torch.linalg.inv(mv)[:3, 3]
            all_mv.append(mv[None, ...].cuda())
            all_mvp.append(mvp[None, ...].cuda())
            all_campos.append(campos[None, ...].cuda())
        for i in range(M//2):
            azimuth = 2 * np.pi * i / (M // 2)
            z = radius * np.cos(azimuth) * np.sin(elevation_1)
            x = radius * np.sin(azimuth) * np.sin(elevation_1)
            y = radius * np.cos(elevation_1)

            eye = glm.vec3(x, y, z)
            at = glm.vec3(0.0, 0.0, 0.0)
            up = glm.vec3(0.0, 1.0, 0.0)
            view_matrix = glm.lookAt(eye, at, up)
            mv = torch.from_numpy(np.array(view_matrix))
            mvp   = proj_mtx @ (mv)  #w2c
            campos = torch.linalg.inv(mv)[:3, 3]
            all_mv.append(mv[None, ...].cuda())
            all_mvp.append(mvp[None, ...].cuda())
            all_campos.append(campos[None, ...].cuda())
    else:
        # elevation = 90 - elevation
        for i in range(M):
            azimuth = 2 * np.pi * i / M
            z = radius * np.cos(azimuth) * np.sin(elevation)
            x = radius * np.sin(azimuth) * np.sin(elevation)
            y = radius * np.cos(elevation)

            eye = glm.vec3(x, y, z)
            at = glm.vec3(0.0, 0.0, 0.0)
            up = glm.vec3(0.0, 1.0, 0.0)
            view_matrix = glm.lookAt(eye, at, up)
            mv = torch.from_numpy(np.array(view_matrix))
            mvp   = proj_mtx @ (mv)  #w2c
            campos = torch.linalg.inv(mv)[:3, 3]
            all_mv.append(mv[None, ...].cuda())
            all_mvp.append(mvp[None, ...].cuda())
            all_campos.append(campos[None, ...].cuda())
    all_mv = torch.stack(all_mv, dim=0).unsqueeze(0).squeeze(2)
    all_mvp = torch.stack(all_mvp, dim=0).unsqueeze(0).squeeze(2)
    all_campos = torch.stack(all_campos, dim=0).unsqueeze(0).squeeze(2)
    return all_mv, all_mvp, all_campos

def render_frames(model, planes, render_cameras, camera_pos, env, materials, render_size=512, chunk_size=1, is_flexicubes=False):
    """
    Render frames from triplanes.
    """
    frames = []
    albedos = []
    pbr_spec_lights = []
    pbr_diffuse_lights = []
    normals = []
    alphas = []
    for i in tqdm(range(0, render_cameras.shape[1], chunk_size)):
        if is_flexicubes:
            out = model.forward_geometry(
                planes,
                render_cameras[:, i:i+chunk_size],
                camera_pos[:, i:i+chunk_size],
                [[env]*chunk_size],
                [[materials]*chunk_size],
                render_size=render_size,
            )
            frame = out['pbr_img']
            albedo = out['albedo']
            pbr_spec_light = out['pbr_spec_light']
            pbr_diffuse_light = out['pbr_diffuse_light']
            normal = out['normal']
            alpha = out['mask']
        else:
            frame = model.forward_synthesizer(
                planes,
                render_cameras[i],
                render_size=render_size,
            )['images_rgb']
        frames.append(frame)
        albedos.append(albedo)
        pbr_spec_lights.append(pbr_spec_light)
        pbr_diffuse_lights.append(pbr_diffuse_light)
        normals.append(normal)
        alphas.append(alpha)

    frames = torch.cat(frames, dim=1)[0]    # we suppose batch size is always 1
    alphas = torch.cat(alphas, dim=1)[0]    
    albedos = torch.cat(albedos, dim=1)[0]
    pbr_spec_lights = torch.cat(pbr_spec_lights, dim=1)[0]
    pbr_diffuse_lights = torch.cat(pbr_diffuse_lights, dim=1)[0]
    normals = torch.cat(normals, dim=0).permute(0,3,1,2)[:,:3]
    return frames, albedos, pbr_spec_lights, pbr_diffuse_lights, normals, alphas


###############################################################################
# Arguments.
###############################################################################

parser = argparse.ArgumentParser()
parser.add_argument('config', type=str, help='Path to config file.')
parser.add_argument('input_path', type=str, help='Path to input image or directory.')
parser.add_argument('--output_path', type=str, default='outputs/', help='Output directory.')
parser.add_argument('--model_ckpt_path', type=str, default="", help='Output directory.')
parser.add_argument('--diffusion_steps', type=int, default=100, help='Denoising Sampling steps.')
parser.add_argument('--seed', type=int, default=42, help='Random seed for sampling.')
parser.add_argument('--scale', type=float, default=1.0, help='Scale of generated object.')
parser.add_argument('--materials', type=str_to_tuple, default=(1.0, 0.1), help=' metallic and roughness')
parser.add_argument('--distance', type=float, default=4.5, help='Render distance.')
parser.add_argument('--fov', type=float, default=30, help='Render distance.')
parser.add_argument('--env_path', type=str, default='data/env_mipmap/2', help='environment map')
parser.add_argument('--view', type=int, default=6, choices=[4, 6], help='Number of input views.')
parser.add_argument('--no_rembg', action='store_true', help='Do not remove input background.')
parser.add_argument('--export_texmap', action='store_true', help='Export a mesh with texture map.')
parser.add_argument('--save_video', action='store_true', help='Save a circular-view video.')
args = parser.parse_args()
seed_everything(args.seed)

###############################################################################
# Stage 0: Configuration.
###############################################################################

config = OmegaConf.load(args.config)
config_name = os.path.basename(args.config).replace('.yaml', '')
model_config = config.model_config
infer_config = config.infer_config

IS_FLEXICUBES = True

device = torch.device('cuda')

# load diffusion model
print('Loading diffusion model ...')
pipeline = DiffusionPipeline.from_pretrained(
    "sudo-ai/zero123plus-v1.2", 
    custom_pipeline="zero123plus",
    torch_dtype=torch.float16,
)
pipeline.scheduler = EulerAncestralDiscreteScheduler.from_config(
    pipeline.scheduler.config, timestep_spacing='trailing'
)

# load custom white-background UNet
print('Loading custom white-background unet ...')
if os.path.exists(infer_config.unet_path):
    unet_ckpt_path = infer_config.unet_path
else:
    unet_ckpt_path = hf_hub_download(repo_id="LTT/PRM", filename="diffusion_pytorch_model.bin", repo_type="model")
state_dict = torch.load(unet_ckpt_path, map_location='cpu')
pipeline.unet.load_state_dict(state_dict, strict=True)

pipeline = pipeline.to(device)

# load reconstruction model
print('Loading reconstruction model ...')
model = instantiate_from_config(model_config)
if os.path.exists(infer_config.model_path):
    model_ckpt_path = infer_config.model_path
else:
    model_ckpt_path = hf_hub_download(repo_id="LTT/PRM", filename="final_ckpt.ckpt", repo_type="model")
state_dict = torch.load(model_ckpt_path, map_location='cpu')['state_dict']
state_dict = {k[14:]: v for k, v in state_dict.items() if k.startswith('lrm_generator.')}
model.load_state_dict(state_dict, strict=True)

model = model.to(device)
if IS_FLEXICUBES:
    model.init_flexicubes_geometry(device, fovy=50.0)
model = model.eval()

# make output directories
image_path = os.path.join(args.output_path, config_name, 'images')
mesh_path = os.path.join(args.output_path, config_name, 'meshes')
video_path = os.path.join(args.output_path, config_name, 'videos')
os.makedirs(image_path, exist_ok=True)
os.makedirs(mesh_path, exist_ok=True)
os.makedirs(video_path, exist_ok=True)

# process input files
if os.path.isdir(args.input_path):
    input_files = [
        os.path.join(args.input_path, file) 
        for file in os.listdir(args.input_path) 
        if file.endswith('.png') or file.endswith('.jpg') or file.endswith('.webp')
    ]
else:
    input_files = [args.input_path]
print(f'Total number of input images: {len(input_files)}')

###############################################################################
# Stage 1: Multiview generation.
###############################################################################

rembg_session = None if args.no_rembg else rembg.new_session()

outputs = []
for idx, image_file in enumerate(input_files):
    name = os.path.basename(image_file).split('.')[0]
    print(f'[{idx+1}/{len(input_files)}] Imagining {name} ...')

    # remove background optionally
    input_image = Image.open(image_file)
    if not args.no_rembg:
        input_image = remove_background(input_image, rembg_session)
        input_image = resize_foreground(input_image, 0.85)
    # sampling
    output_image = pipeline(
        input_image, 
        num_inference_steps=args.diffusion_steps, 
    ).images[0]
    print(f"Image saved to {os.path.join(image_path, f'{name}.png')}")

    images = np.asarray(output_image, dtype=np.float32) / 255.0
    images = torch.from_numpy(images).permute(2, 0, 1).contiguous().float()     # (3, 960, 640)
    images = rearrange(images, 'c (n h) (m w) -> (n m) c h w', n=3, m=2)        # (6, 3, 320, 320)
    torchvision.utils.save_image(images, os.path.join(image_path, f'{name}.png'))
    sample = {'name': name, 'images': images}

# delete pipeline to save memory
# del pipeline

###############################################################################
# Stage 2: Reconstruction.
###############################################################################

    input_cameras = get_zero123plus_input_cameras(batch_size=1, radius=3.2*args.scale, fov=30).to(device)
    chunk_size = 20 if IS_FLEXICUBES else 1

# for idx, sample in enumerate(outputs):
    name = sample['name']
    print(f'[{idx+1}/{len(outputs)}] Creating {name} ...')

    images = sample['images'].unsqueeze(0).to(device)
    images = v2.functional.resize(images, 512, interpolation=3, antialias=True).clamp(0, 1)

    with torch.no_grad():
        # get triplane
        planes = model.forward_planes(images, input_cameras)
        
        mesh_path_idx = os.path.join(mesh_path, f'{name}.obj')

        mesh_out = model.extract_mesh(
            planes,
            use_texture_map=args.export_texmap,
            **infer_config,
        )
        if args.export_texmap:
            vertices, faces, uvs, mesh_tex_idx, tex_map = mesh_out
            save_obj_with_mtl(
                vertices.data.cpu().numpy(),
                uvs.data.cpu().numpy(),
                faces.data.cpu().numpy(),
                mesh_tex_idx.data.cpu().numpy(),
                tex_map.permute(1, 2, 0).data.cpu().numpy(),
                mesh_path_idx,
            )
        else:
            vertices, faces, vertex_colors = mesh_out
            save_obj(vertices, faces, vertex_colors, mesh_path_idx)
        print(f"Mesh saved to {mesh_path_idx}")

        render_size = 512
        if args.save_video:
            video_path_idx = os.path.join(video_path, f'{name}.mp4')
            render_size = infer_config.render_resolution
            ENV = load_mipmap(args.env_path)
            materials = args.materials
            
            all_mv, all_mvp, all_campos = get_render_cameras(
                batch_size=1, 
                M=240, 
                radius=args.distance, 
                elevation=(90, 60.0),
                is_flexicubes=IS_FLEXICUBES,
                fov=args.fov
            )
            
            frames, albedos, pbr_spec_lights, pbr_diffuse_lights, normals, alphas = render_frames(
                model, 
                planes, 
                render_cameras=all_mvp,
                camera_pos=all_campos,
                env=ENV,
                materials=materials,
                render_size=render_size, 
                chunk_size=chunk_size, 
                is_flexicubes=IS_FLEXICUBES,
            )
            normals = (torch.nn.functional.normalize(normals) + 1) / 2
            normals = normals * alphas + (1-alphas)
            all_frames = torch.cat([frames, albedos, pbr_spec_lights, pbr_diffuse_lights, normals], dim=3)
                
            # breakpoint()
            save_video(
                all_frames,
                video_path_idx,
                fps=30,
            )
            print(f"Video saved to {video_path_idx}")