3d visualization

app.py

@@ -2,13 +2,17 @@ import gradio as gr
 import torch
 
 from diffuserslocal.src.diffusers import UNet2DConditionModel
-import diffuserslocal.src.diffusers as diffusers
 from share_btn import community_icon_html, loading_icon_html, share_js
 from diffuserslocal.src.diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_ldm3d_inpaint import StableDiffusionLDM3DInpaintPipeline
 from PIL import Image
 import numpy as np
 import cv2
 
+from functools import partial
+import tempfile
+
+from mesh import get_mesh
+
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
 # Inpainting pipeline
@@ -64,7 +68,7 @@ def read_content(file_path: str) -> str:
 
     return content
 
-def predict(dict, depth, prompt="", negative_prompt="", guidance_scale=7.5, steps=20, strength=1.0, scheduler="EulerDiscreteScheduler"):
+def predict_images(dict, depth, prompt="", negative_prompt="", guidance_scale=7.5, steps=20, strength=1.0, scheduler="EulerDiscreteScheduler"):
     if negative_prompt == "":
         negative_prompt = None
     scheduler_class_name = scheduler.split("-")[0]
@@ -83,13 +87,7 @@ def predict(dict, depth, prompt="", negative_prompt="", guidance_scale=7.5, step
         depth_image = depth_image.astype("int32")
         depth_image = Image.fromarray(depth_image)
 
-    init_image = Image.fromarray(init_image.astype("uint8"))
-    #init_image.save("temp_image.jpg")
-    
-    #depth_image.save("temp_depth.jpg")
-    #scheduler = getattr(diffusers, scheduler_class_name)
-    #pipe.scheduler = scheduler.from_pretrained("Intel/ldm3d-4c", subfolder="scheduler")
-    
+    init_image = Image.fromarray(init_image.astype("uint8"))    
     
     depth_image = depth_image.resize((512, 512))
     
@@ -142,65 +140,147 @@ div#share-btn-container > div {flex-direction: row;background: black;align-items
 '''
 
 image_blocks = gr.Blocks(css=css, elem_id="total-container")
-with image_blocks as demo:
-    gr.HTML(read_content("header.html"))
+
+def create_vis_demo():
     with gr.Row():
-                with gr.Column():
-                    image = gr.Image(source='upload', tool='sketch', elem_id="image_upload", type="numpy", label="Upload",height=400)
-                    depth = gr.Image(source='upload', elem_id="depth_upload", type="numpy", label="Upload",height=400)
-                    
-                    with gr.Row(elem_id="prompt-container", mobile_collapse=False, equal_height=True):
-                        with gr.Row():
-                            prompt = gr.Textbox(placeholder="Your prompt (what you want in place of what is erased)", show_label=False, elem_id="prompt")
-                            btn = gr.Button("Inpaint!", elem_id="run_button")
-                    
-                    with gr.Accordion(label="Advanced Settings", open=False):
-                        with gr.Row(mobile_collapse=False, equal_height=True):
-                            guidance_scale = gr.Number(value=7.5, minimum=1.0, maximum=20.0, step=0.1, label="guidance_scale")
-                            steps = gr.Number(value=20, minimum=10, maximum=30, step=1, label="steps")
-                            strength = gr.Number(value=0.99, minimum=0.01, maximum=0.99, step=0.01, label="strength")
-                            negative_prompt = gr.Textbox(label="negative_prompt", placeholder="Your negative prompt", info="what you don't want to see in the image")
-                        with gr.Row(mobile_collapse=False, equal_height=True):
-                            schedulers = ["DEISMultistepScheduler", "HeunDiscreteScheduler", "EulerDiscreteScheduler", "DPMSolverMultistepScheduler", "DPMSolverMultistepScheduler-Karras", "DPMSolverMultistepScheduler-Karras-SDE"]
-                            scheduler = gr.Dropdown(label="Schedulers", choices=schedulers, value="EulerDiscreteScheduler")
-                        
-                with gr.Column():
-                    image_out = gr.Image(label="Output", elem_id="output-img", height=400)
-                    depth_out = gr.Image(label="Depth", elem_id="depth-img", height=400)
-
-                    with gr.Group(elem_id="share-btn-container", visible=False) as share_btn_container:
-                        community_icon = gr.HTML(community_icon_html)
-                        loading_icon = gr.HTML(loading_icon_html)
-                        share_button = gr.Button("Share to community", elem_id="share-btn",visible=True)
-
-    btn.click(fn=predict, inputs=[image, depth, prompt, negative_prompt, guidance_scale, steps, strength, scheduler], outputs=[image_out, depth_out, share_btn_container], api_name='run')
-    prompt.submit(fn=predict, inputs=[image, depth, prompt, negative_prompt, guidance_scale, steps, strength, scheduler], outputs=[image_out, depth_out, share_btn_container])
+        with gr.Column():
+            image = gr.Image(source='upload', tool='sketch', elem_id="image_upload", type="numpy", label="Upload",height=400)
+            depth = gr.Image(source='upload', elem_id="depth_upload", type="numpy", label="Upload",height=400)
+            
+            with gr.Row(elem_id="prompt-container", mobile_collapse=False, equal_height=True):
+                with gr.Row():
+                    prompt = gr.Textbox(placeholder="Your prompt (what you want in place of what is erased)", show_label=False, elem_id="prompt")
+                    btn = gr.Button("Inpaint!", elem_id="run_button")
+            
+            with gr.Accordion(label="Advanced Settings", open=False):
+                with gr.Row(mobile_collapse=False, equal_height=True):
+                    guidance_scale = gr.Number(value=7.5, minimum=1.0, maximum=20.0, step=0.1, label="guidance_scale")
+                    steps = gr.Number(value=20, minimum=10, maximum=30, step=1, label="steps")
+                    strength = gr.Number(value=0.99, minimum=0.01, maximum=0.99, step=0.01, label="strength")
+                    negative_prompt = gr.Textbox(label="negative_prompt", placeholder="Your negative prompt", info="what you don't want to see in the image")
+                with gr.Row(mobile_collapse=False, equal_height=True):
+                    schedulers = ["DEISMultistepScheduler", "HeunDiscreteScheduler", "EulerDiscreteScheduler", "DPMSolverMultistepScheduler", "DPMSolverMultistepScheduler-Karras", "DPMSolverMultistepScheduler-Karras-SDE"]
+                    scheduler = gr.Dropdown(label="Schedulers", choices=schedulers, value="EulerDiscreteScheduler")
+                
+        with gr.Column():
+            image_out = gr.Image(label="Output", elem_id="output-img", height=400)
+            depth_out = gr.Image(label="Depth", elem_id="depth-img", height=400)
+
+            with gr.Group(elem_id="share-btn-container", visible=False) as share_btn_container:
+                community_icon = gr.HTML(community_icon_html)
+                loading_icon = gr.HTML(loading_icon_html)
+                share_button = gr.Button("Share to community", elem_id="share-btn",visible=True)
+
+    btn.click(fn=predict_images, inputs=[image, depth, prompt, negative_prompt, guidance_scale, steps, strength, scheduler], outputs=[image_out, depth_out, share_btn_container], api_name='run')
+    prompt.submit(fn=predict_images, inputs=[image, depth, prompt, negative_prompt, guidance_scale, steps, strength, scheduler], outputs=[image_out, depth_out, share_btn_container])
     share_button.click(None, [], [], _js=share_js)
 
     gr.Examples(
-                examples=[
-                    ["./imgs/aaa (8).png"],
-                    ["./imgs/download (1).jpeg"],
-                    ["./imgs/0_oE0mLhfhtS_3Nfm2.png"],
-                    ["./imgs/02_HubertyBlog-1-1024x1024.jpg"],
-                    ["./imgs/jdn_jacques_de_nuce-1024x1024.jpg"],
-                    ["./imgs/c4ca473acde04280d44128ad8ee09e8a.jpg"],
-                    ["./imgs/canam-electric-motorcycles-scaled.jpg"],
-                    ["./imgs/e8717ce80b394d1b9a610d04a1decd3a.jpeg"],
-                    ["./imgs/Nature___Mountains_Big_Mountain_018453_31.jpg"],
-                    ["./imgs/Multible-sharing-room_ccexpress-2-1024x1024.jpeg"],
-                ],
-                fn=predict,
-                inputs=[image],
-                cache_examples=False,
-    )
-    gr.HTML(
-        """
-            <div class="footer">
-                <p>Model by <a href="https://huggingface.co/diffusers" style="text-decoration: underline;" target="_blank">Diffusers</a> - Gradio Demo by 🤗 Hugging Face
-                </p>
-            </div>
-        """
+        examples=[
+            ["./imgs/aaa (8).png"],
+            ["./imgs/download (1).jpeg"],
+            ["./imgs/0_oE0mLhfhtS_3Nfm2.png"],
+            ["./imgs/02_HubertyBlog-1-1024x1024.jpg"],
+            ["./imgs/jdn_jacques_de_nuce-1024x1024.jpg"],
+            ["./imgs/c4ca473acde04280d44128ad8ee09e8a.jpg"],
+            ["./imgs/canam-electric-motorcycles-scaled.jpg"],
+            ["./imgs/e8717ce80b394d1b9a610d04a1decd3a.jpeg"],
+            ["./imgs/Nature___Mountains_Big_Mountain_018453_31.jpg"],
+            ["./imgs/Multible-sharing-room_ccexpress-2-1024x1024.jpeg"],
+        ],
+        fn=predict_images,
+        inputs=[image],
+        cache_examples=False,
     )
 
+
+def predict_images_3d(dict, depth, prompt="", negative_prompt="", guidance_scale=7.5, steps=20, strength=1.0, scheduler="EulerDiscreteScheduler", keep_edges=False):
+    if negative_prompt == "":
+        negative_prompt = None
+    scheduler_class_name = scheduler.split("-")[0]
+
+    init_image = cv2.resize(dict["image"], (512, 512))
+
+    mask = Image.fromarray(cv2.resize(dict["mask"], (512, 512))[:,:,0])
+    mask.save("temp_mask.jpg")
+
+    if (depth is None):
+        depth_image = estimate_depth(init_image)
+        
+    else:
+        d_i = depth[:,:,0]
+        depth_image = 65535 * (d_i - np.min(d_i))/(np.max(d_i) - np.min(d_i))
+        depth_image = depth_image.astype("int32")
+        depth_image = Image.fromarray(depth_image)
+
+    init_image = Image.fromarray(init_image.astype("uint8"))    
+    
+    depth_image = depth_image.resize((512, 512))
+    
+    output = pipe(prompt = prompt, negative_prompt=negative_prompt, image=init_image, mask_image=mask, depth_image=depth_image, guidance_scale=guidance_scale, num_inference_steps=int(steps), strength=strength)
+    
+    depth_out = np.array(output.depth[0])
+
+    output_depth_vis = (depth_out - np.min(depth_out)) / (np.max(depth_out) - np.min(depth_out)) * 255
+    output_depth_vis = output_depth_vis.astype("uint8")
+
+    #init_image
+    #depth_image
+    output_depth = Image.fromarray(output_depth_vis)
+    output_image = output.rgb[0]
+
+    output_mesh = get_mesh(output_depth_vis, output_image, keep_edges=keep_edges)
+    input_mesh = get_mesh(np.array(depth_image),init_image, keep_edges=keep_edges)
+    
+    return input_mesh, output_mesh
+
+def create_3d_demo(model):
+
+    gr.Markdown("### Image to 3D mesh")
+
+    with gr.Column():
+            image = gr.Image(source='upload', tool='sketch', elem_id="image_upload", type="numpy", label="Upload",height=400)
+            depth = gr.Image(source='upload', elem_id="depth_upload", type="numpy", label="Upload",height=400)
+            checkbox = gr.Checkbox(label="Keep occlusion edges", value=False)
+
+            with gr.Row(elem_id="prompt-container", mobile_collapse=False, equal_height=True):
+                with gr.Row():
+                    prompt = gr.Textbox(placeholder="Your prompt (what you want in place of what is erased)", show_label=False, elem_id="prompt")
+                    btn = gr.Button("Inpaint!", elem_id="run_button")
+            
+            with gr.Accordion(label="Advanced Settings", open=False):
+                with gr.Row(mobile_collapse=False, equal_height=True):
+                    guidance_scale = gr.Number(value=7.5, minimum=1.0, maximum=20.0, step=0.1, label="guidance_scale")
+                    steps = gr.Number(value=20, minimum=10, maximum=30, step=1, label="steps")
+                    strength = gr.Number(value=0.99, minimum=0.01, maximum=0.99, step=0.01, label="strength")
+                    negative_prompt = gr.Textbox(label="negative_prompt", placeholder="Your negative prompt", info="what you don't want to see in the image")
+                with gr.Row(mobile_collapse=False, equal_height=True):
+                    schedulers = ["DEISMultistepScheduler", "HeunDiscreteScheduler", "EulerDiscreteScheduler", "DPMSolverMultistepScheduler", "DPMSolverMultistepScheduler-Karras", "DPMSolverMultistepScheduler-Karras-SDE"]
+                    scheduler = gr.Dropdown(label="Schedulers", choices=schedulers, value="EulerDiscreteScheduler")
+    
+    with gr.Column():
+        with gr.row():
+            result_og = gr.Model3D(label="original 3d reconstruction", clear_color=[
+                                                    1.0, 1.0, 1.0, 1.0])
+            
+            result_new = gr.Model3D(label="inpainted 3d reconstruction", clear_color=[
+                                                    1.0, 1.0, 1.0, 1.0])
+        
+    
+    submit = gr.Button("Submit")
+    submit.click(fn=predict_images_3d, inputs=[image, depth, prompt, negative_prompt, guidance_scale, steps, strength, scheduler, checkbox], outputs=[image_out, depth_out, share_btn_container], api_name='run')
+    examples = gr.Examples(examples=["examples/aerial_beach.jpeg", "examples/mountains.jpeg", "examples/person_1.jpeg", "examples/ancient-carved.jpeg"],
+                            inputs=[image])
+
+
+
+with image_blocks as demo:
+    with gr.Tab("Image", default=True):
+        create_vis_demo()
+    with gr.Tab("3D"):
+        create_3d_demo()
+    
+    gr.HTML(read_content("header.html"))
+    
+
 image_blocks.queue(max_size=25).launch()

mesh.py

@@ -0,0 +1,52 @@
+import gradio as gr
+import numpy as np
+import trimesh
+from geometry import depth_to_points, create_triangles
+from functools import partial
+import tempfile
+
+
+def depth_edges_mask(depth):
+    """Returns a mask of edges in the depth map.
+    Args:
+    depth: 2D numpy array of shape (H, W) with dtype float32.
+    Returns:
+    mask: 2D numpy array of shape (H, W) with dtype bool.
+    """
+    # Compute the x and y gradients of the depth map.
+    depth_dx, depth_dy = np.gradient(depth)
+    # Compute the gradient magnitude.
+    depth_grad = np.sqrt(depth_dx ** 2 + depth_dy ** 2)
+    # Compute the edge mask.
+    mask = depth_grad > 0.05
+    return mask
+
+
+def predict_depth(model, image):
+    depth = model.infer_pil(image)
+    return depth
+
+def get_mesh(depth, image, keep_edges=False):
+    # limit the size of the input image
+    pts3d = depth_to_points(depth[None])
+    pts3d = pts3d.reshape(-1, 3)
+
+    # Create a trimesh mesh from the points
+    # Each pixel is connected to its 4 neighbors
+    # colors are the RGB values of the image
+
+    verts = pts3d.reshape(-1, 3)
+    image = np.array(image)
+    if keep_edges:
+        triangles = create_triangles(image.shape[0], image.shape[1])
+    else:
+        triangles = create_triangles(image.shape[0], image.shape[1], mask=~depth_edges_mask(depth))
+    colors = image.reshape(-1, 3)
+    mesh = trimesh.Trimesh(vertices=verts, faces=triangles, vertex_colors=colors)
+
+    # Save as glb
+    glb_file = tempfile.NamedTemporaryFile(suffix='.glb', delete=False)
+    glb_path = glb_file.name
+    mesh.export(glb_path)
+    return glb_path
+

requirements.txt

@@ -9,4 +9,5 @@ numpy
 matplotlib
 uuid
 opencv-python
-timm
+timm
+trimesh