update config

.vscode/launch.json

@@ -8,7 +8,7 @@
             "program": "./app.py",
             "console": "integratedTerminal",
             "env": {
-                "CUDA_VISIBLE_DEVICES": "2"
+                "CUDA_VISIBLE_DEVICES": "0"
             },
             // "args": [
             //     "tiny_trf_trans_nerf",//"tiny_trf_trans_nerf" tiny_trf_trans_nerf_123plus

app.py

@@ -34,7 +34,7 @@ GRADIO_OBJ_SHADING_PATH = 'gradio_output_shading.obj'
 
 #opt = tyro.cli(AllConfigs)
 
-ckpt_path = hf_hub_download(repo_id="rgxie/LDM", filename="LDM4v01.ckpt")
+ckpt_path = hf_hub_download(repo_id="rgxie/LDM", filename="LDM6v01.ckpt")
 
 opt = Options(
     input_size=512, 
@@ -145,7 +145,6 @@ def generate_mv(condition_input_image, prompt, prompt_neg='', input_elevation=0,
     kiui.seed_everything(input_seed)
 
     os.makedirs(os.path.join(opt.workspace, "gradio"), exist_ok=True)
-    output_video_path = os.path.join(opt.workspace,"gradio", GRADIO_VIDEO_PATH)
     
     # text-conditioned
     if condition_input_image is None:
@@ -201,6 +200,8 @@ def generate_3d(input_image, condition_input_image, mv_moedl_option=None, input_
     output_obj_rgb_path = os.path.join(opt.workspace,"gradio", GRADIO_OBJ_PATH)
     output_obj_albedo_path = os.path.join(opt.workspace,"gradio", GRADIO_OBJ_ALBEDO_PATH)
     output_obj_shading_path = os.path.join(opt.workspace,"gradio", GRADIO_OBJ_SHADING_PATH)
+    
+    output_video_path = os.path.join(opt.workspace,"gradio", GRADIO_VIDEO_PATH)
     # generate gaussians
      # [4, 256, 256, 3], float32
     input_image = torch.from_numpy(input_image).permute(0, 3, 1, 2).float().to(device) # [4, 3, 256, 256]
@@ -274,8 +275,35 @@ def generate_3d(input_image, condition_input_image, mv_moedl_option=None, input_
             save_obj(vertices, faces, vertex_colors[1], output_obj_albedo_path)
             save_obj(vertices, faces, vertex_colors[2], output_obj_shading_path)
         
+        # images=[]  
+        # azimuth = np.arange(0, 360, 6, dtype=np.int32)
+        # for azi in tqdm.tqdm(azimuth):
+
+        #     cam_pose = torch.from_numpy(orbit_camera(elevation, azi, radius=opt.cam_radius, opengl=True))
+
+        #     if opt.volume_mode == 'TRF_Mesh': 
+        #         cam_view = torch.inverse(cam_pose)
+        #         cam_view=cam_view.unsqueeze(0).unsqueeze(0).to(device)
+        #         data['w2c'] = cam_view
+        #         with torch.autocast(device_type='cuda', dtype=torch.float32):
+        #             render_images=model.render_frame(data)
+        #     else:
+        #         rays_o, rays_d = get_rays(cam_pose, opt.infer_render_size, opt.infer_render_size, opt.fovy) # [h, w, 3]
+        #         rays_o=rays_o.unsqueeze(0).unsqueeze(0).to(device)# B,V,H,W,3
+        #         rays_d=rays_d.unsqueeze(0).unsqueeze(0).to(device)
+        #         data['all_rays_o']=rays_o
+        #         data['all_rays_d']=rays_d
+        #         with torch.autocast(device_type='cuda', dtype=torch.float16):
+        #             render_images=model.render_frame(data)
+        #     image=render_images['images_pred']
+
+        #     images.append((image.squeeze(1).permute(0,2,3,1).contiguous().float().cpu().numpy() * 255).astype(np.uint8))
+
+        # images = np.concatenate(images, axis=0)
+        # imageio.mimwrite(output_video_path, images, fps=30)
+        
 
-    return output_obj_rgb_path, output_obj_albedo_path, output_obj_shading_path
+    return output_obj_rgb_path, output_obj_albedo_path, output_obj_shading_path #, output_video_path
 
 
 # gradio UI
@@ -371,6 +399,8 @@ with block:
             with gr.Row():
                 # multi-view results
                 mv_image_grid = gr.Image(interactive=False, show_label=False)
+            # with gr.Row():    
+            #     output_video_path = gr.Video(label="video")
             with gr.Row():    
                 output_obj_rgb_path = gr.Model3D(
                     label="RGB Model (OBJ Format)",
@@ -393,7 +423,7 @@ with block:
                          outputs=[mv_image_grid, processed_image, input_image],).success(
                             fn=generate_3d,
                             inputs=[input_image, condition_input_image, mv_moedl_option, input_seed], 
-                            outputs=[output_obj_rgb_path, output_obj_albedo_path, output_obj_shading_path] ,
+                            outputs=[output_obj_rgb_path, output_obj_albedo_path, output_obj_shading_path] , #output_video_path
                          )
         
         

core/models.py

@@ -108,7 +108,7 @@ class LTRFM_NeRF(nn.Module):
     def extract_mesh(self, 
         planes: torch.Tensor, 
         mesh_resolution: int = 256, 
-        mesh_threshold: int = 0.009, 
+        mesh_threshold: int = 0.005, 
         use_texture_map: bool = False, 
         texture_resolution: int = 1024,):