fix: handle RGB masks correctly - use RGB channels when alpha is all 255, so white pixels become white in mask

src/core.py

@@ -444,36 +444,53 @@ def get_args_parser():
 
 def process_inpaint(image, mask, invert_mask=True):
     """
-    Process inpainting - EXACT copy from reference model.
+    Process inpainting - handles both alpha-based masks and RGB-based masks.
     Reference: https://huggingface.co/spaces/aryadytm/remove-photo-object
-    Line 444-466 in their src/core.py
     """
     image = cv2.cvtColor(image, cv2.COLOR_RGBA2RGB)
     original_shape = image.shape
     interpolation = cv2.INTER_CUBIC
 
-    #size_limit: Union[int, str] = request.form.get("sizeLimit", "1080")
-    #if size_limit == "Original":
     size_limit = max(image.shape)
-    #else:
-    #    size_limit = int(size_limit)
 
     print(f"Origin image shape: {original_shape}")
     image = resize_max_size(image, size_limit=size_limit, interpolation=interpolation)
     print(f"Resized image shape: {image.shape}")
     image = norm_img(image)
 
-    # Reference model's EXACT logic: simple alpha inversion
-    # Line 460: mask = 255-mask[:,:,3]
-    # This means: alpha=0 (transparent/drawn) → 255 (white/remove)
-    #             alpha=255 (opaque) → 0 (black/keep)
-    mask = 255 - mask[:,:,3]
+    # Handle mask: check if we should use alpha channel or RGB channels
+    alpha_channel = mask[:,:,3]
+    rgb_channels = mask[:,:,:3]
+    
+    # Check if alpha is meaningful (not all 255)
+    alpha_mean = alpha_channel.mean()
+    
+    if alpha_mean < 240:
+        # Alpha channel is meaningful (has transparent areas)
+        # Reference model logic: mask = 255-mask[:,:,3]
+        # alpha=0 (transparent) → 255 (white/remove)
+        # alpha=255 (opaque) → 0 (black/keep)
+        mask = 255 - alpha_channel
+        print(f"Using alpha channel: {int((alpha_channel < 128).sum())} transparent pixels → white")
+    else:
+        # Alpha is mostly opaque (255), use RGB channels instead
+        # RGB masks: white (255) = remove, black (0) = keep
+        gray = cv2.cvtColor(rgb_channels, cv2.COLOR_RGB2GRAY)
+        mask = (gray > 128).astype(np.uint8) * 255
+        print(f"Using RGB channels: {int((mask > 128).sum())} white pixels (to remove)")
     
     # Apply invert_mask if user wants opposite behavior
     if not invert_mask:
-        mask = 255 - mask  # double invert back
+        mask = 255 - mask  # invert: white becomes black, black becomes white
+        print(f"Applied invert_mask=False: inverted mask")
     
     mask = resize_max_size(mask, size_limit=size_limit, interpolation=interpolation)
+    
+    # Debug: log final mask statistics
+    mask_nonzero = int((mask > 128).sum())
+    mask_total = mask.shape[0] * mask.shape[1]
+    print(f"Final mask before normalization: {mask_nonzero}/{mask_total} pixels marked for removal ({100*mask_nonzero/mask_total:.2f}%)")
+    
     mask = norm_img(mask)
 
     res_np_img = run(image, mask)