map depth attempt

app.py

@@ -159,7 +159,7 @@ with block:
 
     def on_depth_reconstruction_btn_click(inputs):
         print("depth reconstruction")
-        path = dpt.generate_obj_masks2(image=inputs[input_image],
+        path = dpt.generate_obj_rgb(image=inputs[input_image],
                                       cube_size=inputs[cube_size],
                                       n_samples=inputs[n_samples],
                                       masks=inputs[masks],

inference.py

@@ -15,7 +15,7 @@ import matplotlib.pyplot as plt
 
 
 
-def map_image_range(image, min_value, max_value):
+def map_image_range(depth, min_value, max_value):
     """
     Maps the values of a numpy image array to a specified range.
 
@@ -28,11 +28,19 @@ def map_image_range(image, min_value, max_value):
         numpy.ndarray: Image array with values mapped to the specified range.
     """
     # Ensure the input image is a numpy array
-    print(np.min(image))
-    print(np.max(image))
-
+    print(np.min(depth))
+    print(np.max(depth))
+    depth = np.array(depth)
+    # map the depth values are between 0 and 1
+    depth = (depth - depth.min()) / (depth.max() - depth.min())
+    # invert
+    depth = 1 - depth
+    print(np.min(depth))
+    print(np.max(depth))
     # Map the values to the specified range
-    mapped_image = (image - 0) * (max_value - min_value) / (1 - 0) + min_value
+    mapped_image = (depth - 0) * (max_value - min_value) / (1 - 0) + min_value
+    print(np.min(mapped_image))
+    print(np.max(mapped_image))
     return mapped_image
 
 def PCL(mask, depth):
@@ -63,6 +71,20 @@ def PCL(mask, depth):
     colors = colors[mask]
     return points, colors
 
+def PCL_rgb(rgb, depth):
+    assert rgb.shape == depth.shape
+    assert type(rgb) == np.ndarray
+    assert type(depth) == np.ndarray
+    depth_o3d = o3d.geometry.Image(depth)
+    image_o3d = o3d.geometry.Image(rgb)
+    rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(image_o3d, depth_o3d, convert_rgb_to_intensity=False)
+    # Step 3: Create a PointCloud from the RGBD image
+    pcd = o3d.geometry.PointCloud.create_from_rgbd_image(rgbd_image, o3d.camera.PinholeCameraIntrinsic(o3d.camera.PinholeCameraIntrinsicParameters.PrimeSenseDefault))
+    # Step 4: Convert PointCloud data to a NumPy array
+    points = np.asarray(pcd.points)
+    colors = np.asarray(pcd.colors)
+    return points, colors
+
 class DepthPredictor:
     def __init__(self):
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
@@ -124,11 +146,12 @@ class DepthPredictor:
         ax.scatter(points,size=0.01, c=colors, marker='o')
         return fig
     
-    def generate_obj_rgb(self, image, n_samples, cube_size):
+    def generate_obj_rgb(self, image, n_samples, cube_size, max_depth, min_depth):
         # Step 1: Create a point cloud
-        point_cloud, color_array = self.generate_pcl(image)
-        #point_cloud, color_array = point_cloud[mask.ravel()[:-1]], color_array[mask.ravel()[:-1]]
-        # sample 1000 points
+        depth = self.predict(image)
+        depth = map_image_range(depth, min_depth, max_depth)
+        point_cloud, color_array = PCL_rgb(image, depth)
+        image = np.array(image)
         idxs = np.random.choice(len(point_cloud), int(n_samples))
         point_cloud = point_cloud[idxs]
         color_array = color_array[idxs]