mean depth

handcrafted_solution.py

@@ -122,12 +122,41 @@ def get_uv_depth(vertices, depth):
     vertex_depth = depth[(uv_int[:, 1] , uv_int[:, 0])]
     return uv, vertex_depth
 
+def get_smooth_uv_depth(vertices, depth):
+    '''Get the depth of the vertices from the depth image'''
+    #print(f'max depth = {np.max(depth)}')
+    uv = []
+    for v in vertices:
+        uv.append(v['xy'])
+    uv = np.array(uv)
+    uv_int = uv.astype(np.int32)
+    H, W = depth.shape[:2]
+    a = np.clip( uv_int[:, 0], 0, W-1)
+    b = np.clip( uv_int[:, 1], 0, H-1)    
+    def get_local_depth(x,y, H, W, depth, r=5):
+      '''return a smooth version of detph in radius r'''      
+      local_depths = []
+      for i in range(max(0, x - r), min(W, x + r + 1)):
+        for j in range(max(0, y - r), min(H, y + r + 1)):
+            if np.sqrt((i - x)**2 + (j - y)**2) <= r:
+                local_depths.append(depth[j, i])      
+      return local_depths
+    
+    vertex_depth = []
+    for x,y in zip(a,b):
+      local_depths = get_local_depth(x,y, H, W, depth, 5)
+      local_mean = np.mean(local_depths)
+      vertex_depth.append(local_mean)
+    vertex_depth = np.array(vertex_depth)
+
+    return uv, vertex_depth
+
 def get_vertices_and_edges_from_two_segmentations(ade_seg_np, gest_seg_np, edge_th = 50.0):
     '''Get the vertices and edges from the gestalt segmentation mask of the house'''
     vertices = []
     connections = []
 
-    color_th = 10.0 # Cost ->2.6
+    color_th =  10.0 # Cost ->2.6
 
     #-------------------------    
     # combined map from ade
@@ -151,7 +180,7 @@ def get_vertices_and_edges_from_two_segmentations(ade_seg_np, gest_seg_np, edge_
     #print(f'apex_color= {apex_color}')
     #apex_mask = cv2.inRange(gest_seg_np,  apex_color-0.5, apex_color+0.5) 
     apex_mask = cv2.inRange(gest_seg_np,  apex_color-color_th, apex_color+color_th) # include more pts
-    # apex_mask = cv2.bitwise_and(apex_mask, ade_mask) # remove pts
+    #apex_mask = cv2.bitwise_and(apex_mask, ade_mask) # remove pts
     if apex_mask.sum() > 0:
         output = cv2.connectedComponentsWithStats(apex_mask, 8, cv2.CV_32S)
         (numLabels, labels, stats, centroids) = output
@@ -174,17 +203,6 @@ def get_vertices_and_edges_from_two_segmentations(ade_seg_np, gest_seg_np, edge_
                 apex_map_on_ade[uu+ss[0], vv+ss[1]] = (255,255,255)
                 apex_map_on_gest[uu+ss[0], vv+ss[1]] = (255,255,255)
     
-    # imsave apex
-    import random
-    rid = random.random()
-    filename_apex_ade = f'apex_map_on_ade_{rid}.jpg'
-    cv2.imwrite(filename_apex_ade, apex_map_on_ade)
-    filename_apex_gest = f'apex_map_on_gest_{rid}.jpg'
-    cv2.imwrite(filename_apex_gest, apex_map_on_gest)
-    filename_apex_map = f'apex_map_{rid}.jpg'
-    cv2.imwrite(filename_apex_map, apex_map)
-
-    
 
     eave_end_color = np.array(gestalt_color_mapping['eave_end_point'])
     #eave_end_mask = cv2.inRange(gest_seg_np,  eave_end_color-0.5, eave_end_color+0.5)
@@ -198,7 +216,29 @@ def get_vertices_and_edges_from_two_segmentations(ade_seg_np, gest_seg_np, edge_
         for i in range(numLabels-1):
             vert = {"xy": centroids[i], "type": "eave_end_point"}
             vertices.append(vert) 
-    
+
+            uu = int(centroids[i][1])
+            vv = int(centroids[i][0])
+            # plot a cross
+            apex_map_on_ade[uu, vv] = (255,0,0)
+            shift=[(1,0),(-1,0),(0,1),(0,-1), (2,0),(-2,0),(0,2),(0,-2), (3,0),(-3,0),(0,3),(0,-3)]
+            h,w,_ = apex_map_on_ade.shape
+            for ss in shift:
+              if uu+ss[0] >= 0 and uu+ss[0] < h and vv+ss[1] >= 0 and vv+ss[1] < w:
+                apex_map[uu+ss[0], vv+ss[1]] = (255,0,0)
+                apex_map_on_ade[uu+ss[0], vv+ss[1]] = (255,0,0)
+                apex_map_on_gest[uu+ss[0], vv+ss[1]] = (255,0,0)
+
+    # imsave apex and eave_end
+    import random
+    rid = random.random()
+    filename_apex_ade = f'apex_map_on_ade_{rid}.jpg'
+    cv2.imwrite(filename_apex_ade, apex_map_on_ade)
+    filename_apex_gest = f'apex_map_on_gest_{rid}.jpg'
+    cv2.imwrite(filename_apex_gest, apex_map_on_gest)
+    filename_apex_map = f'apex_map_{rid}.jpg'
+    cv2.imwrite(filename_apex_map, apex_map)
+
     print(f'{len(vertices)} vertices detected')
     # Connectivity
     apex_pts = []
@@ -360,13 +400,23 @@ def predict(entry, visualize=False) -> Tuple[np.ndarray, List[int]]:
                                                 good_entry['K'],
                                                 good_entry['R'],
                                                 good_entry['t'] 
-                                                )):
+                                                )):                                              
+        ''' entry 0 suggests:
+        depth_scale = 1
+        if i==1: 
+          depth_scale = 2.5
+        elif i==2: # only visualize view 0,1
+          continue 
+        '''
+        depth_scale = 2.5
+        
+
         ade_seg = ade.resize(depth.size)
         ade_seg_np = np.array(ade_seg).astype(np.uint8)
         gest_seg = gest.resize(depth.size)
         gest_seg_np = np.array(gest_seg).astype(np.uint8)
         # Metric3D
-        depth_np = np.array(depth) / 2.5 # 2.5 is the scale estimation coefficient
+        depth_np = np.array(depth) / depth_scale # / 2.5 # 2.5 is the scale estimation coefficient # don't use 2.5...
         #vertices, connections = get_vertices_and_edges_from_segmentation(gest_seg_np, edge_th = 20.)
         vertices, connections = get_vertices_and_edges_from_two_segmentations(ade_seg_np, gest_seg_np, edge_th = 20.)
         
@@ -374,8 +424,9 @@ def predict(entry, visualize=False) -> Tuple[np.ndarray, List[int]]:
             print (f'Not enough vertices or connections in image {i}')
             vert_edge_per_image[i] = np.empty((0, 2)), [], np.empty((0, 3))
             continue
-        uv, depth_vert = get_uv_depth(vertices, depth_np)
-        #uv, depth_vert, ade_category_vert, vertices = get_uv_dept_category(vertices, depth_np, ade_seg_np)
+        #uv, depth_vert = get_uv_depth(vertices, depth_np) 
+        uv, depth_vert = get_smooth_uv_depth(vertices, depth_np)
+        
         # Normalize the uv to the camera intrinsics
         xy_local = np.ones((len(uv), 3))
         xy_local[:, 0] = (uv[:, 0] - K[0,2]) / K[0,0]