Spaces:
Runtime error
Runtime error
Update utils/bubble_detect.py
Browse files- utils/bubble_detect.py +71 -125
utils/bubble_detect.py
CHANGED
|
@@ -4,109 +4,75 @@ Enhanced speech bubble detection for manga
|
|
| 4 |
import cv2
|
| 5 |
import numpy as np
|
| 6 |
from shapely.geometry import Polygon
|
|
|
|
| 7 |
|
| 8 |
|
| 9 |
def detect_speech_bubbles(img_pil, min_area=500, max_area=None, debug=False):
|
| 10 |
"""
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
Args:
|
| 14 |
-
img_pil: PIL Image
|
| 15 |
-
min_area: Minimum bubble area in pixels
|
| 16 |
-
max_area: Maximum bubble area (None = 1/4 of image)
|
| 17 |
-
debug: If True, return debug info
|
| 18 |
-
|
| 19 |
Returns:
|
| 20 |
List of bubble polygons [(x,y), ...]
|
| 21 |
"""
|
| 22 |
img = cv2.cvtColor(np.array(img_pil), cv2.COLOR_RGB2BGR)
|
| 23 |
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
|
| 24 |
-
|
| 25 |
h, w = gray.shape
|
| 26 |
if max_area is None:
|
| 27 |
-
max_area = (h * w) // 4 #
|
| 28 |
-
|
| 29 |
-
# Adaptive threshold handles varying lighting better
|
| 30 |
th = cv2.adaptiveThreshold(
|
| 31 |
-
gray,
|
|
|
|
| 32 |
cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
|
| 33 |
cv2.THRESH_BINARY,
|
| 34 |
-
35,
|
|
|
|
| 35 |
)
|
| 36 |
-
|
| 37 |
-
inv = 255 - th #
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
# Remove small noise
|
| 44 |
kernel_open = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
|
| 45 |
cleaned = cv2.morphologyEx(cleaned, cv2.MORPH_OPEN, kernel_open, iterations=1)
|
| 46 |
-
|
| 47 |
contours, _ = cv2.findContours(cleaned, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
| 48 |
-
|
| 49 |
bubbles = []
|
| 50 |
-
debug_info = []
|
| 51 |
-
|
| 52 |
for cnt in contours:
|
| 53 |
area = cv2.contourArea(cnt)
|
| 54 |
-
|
| 55 |
-
# Filter by area
|
| 56 |
if area < min_area or area > max_area:
|
| 57 |
continue
|
| 58 |
-
|
| 59 |
-
# Get bounding box
|
| 60 |
x, y, bw, bh = cv2.boundingRect(cnt)
|
| 61 |
-
|
| 62 |
-
# Filter by aspect ratio (too thin/wide = not a bubble)
|
| 63 |
aspect_ratio = max(bw, bh) / (min(bw, bh) + 1)
|
| 64 |
-
if aspect_ratio > 5:
|
| 65 |
continue
|
| 66 |
-
|
| 67 |
-
# Check if shape is reasonably bubble-like
|
| 68 |
-
# Bubbles are usually somewhat round/elliptical
|
| 69 |
perimeter = cv2.arcLength(cnt, True)
|
|
|
|
|
|
|
| 70 |
circularity = 4 * np.pi * area / (perimeter * perimeter + 1)
|
| 71 |
-
|
| 72 |
-
# Approximate polygon
|
| 73 |
epsilon = 0.01 * perimeter
|
| 74 |
approx = cv2.approxPolyDP(cnt, epsilon, True)
|
| 75 |
-
|
| 76 |
poly = [(int(p[0][0]), int(p[0][1])) for p in approx]
|
| 77 |
-
|
| 78 |
-
# Store bubble
|
| 79 |
bubbles.append(poly)
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
debug_info.append({
|
| 83 |
-
'area': area,
|
| 84 |
-
'aspect_ratio': aspect_ratio,
|
| 85 |
-
'circularity': circularity,
|
| 86 |
-
'vertices': len(poly),
|
| 87 |
-
'bbox': (x, y, bw, bh)
|
| 88 |
-
})
|
| 89 |
-
|
| 90 |
-
print(f"๐ Detected {len(bubbles)} candidate bubbles")
|
| 91 |
-
|
| 92 |
-
if debug:
|
| 93 |
-
return bubbles, debug_info
|
| 94 |
-
|
| 95 |
return bubbles
|
| 96 |
|
| 97 |
|
| 98 |
def merge_overlapping_bubbles(bubbles, iou_threshold=0.3):
|
| 99 |
"""
|
| 100 |
Merge bubbles that overlap significantly.
|
| 101 |
-
Useful when bubble detection creates multiple contours for one bubble.
|
| 102 |
"""
|
| 103 |
-
from shapely.geometry import Polygon
|
| 104 |
-
from shapely.ops import unary_union
|
| 105 |
-
|
| 106 |
if len(bubbles) <= 1:
|
| 107 |
return bubbles
|
| 108 |
-
|
| 109 |
-
# Convert to Shapely polygons
|
| 110 |
shapes = []
|
| 111 |
for b in bubbles:
|
| 112 |
try:
|
|
@@ -114,60 +80,48 @@ def merge_overlapping_bubbles(bubbles, iou_threshold=0.3):
|
|
| 114 |
if not p.is_valid:
|
| 115 |
p = p.buffer(0)
|
| 116 |
shapes.append(p)
|
| 117 |
-
except:
|
| 118 |
continue
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
merged = []
|
| 122 |
used = set()
|
| 123 |
-
|
| 124 |
-
for i,
|
| 125 |
if i in used:
|
| 126 |
continue
|
| 127 |
-
|
| 128 |
-
group = [
|
| 129 |
used.add(i)
|
| 130 |
-
|
| 131 |
-
for j,
|
| 132 |
if j in used:
|
| 133 |
continue
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
union = shape1.union(shape2).area
|
| 138 |
-
iou = intersection / union if union > 0 else 0
|
| 139 |
-
|
| 140 |
if iou > iou_threshold:
|
| 141 |
-
group.append(
|
| 142 |
used.add(j)
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
if
|
| 146 |
-
|
| 147 |
-
if merged_shape.geom_type == 'Polygon':
|
| 148 |
-
merged.append(list(merged_shape.exterior.coords)[:-1])
|
| 149 |
-
else:
|
| 150 |
-
# Multiple separate regions - add them separately
|
| 151 |
-
for geom in merged_shape.geoms:
|
| 152 |
-
if geom.geom_type == 'Polygon':
|
| 153 |
-
merged.append(list(geom.exterior.coords)[:-1])
|
| 154 |
else:
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
|
|
|
|
|
|
| 159 |
|
| 160 |
|
| 161 |
def filter_nested_bubbles(bubbles):
|
| 162 |
"""
|
| 163 |
-
Remove bubbles
|
| 164 |
-
Keeps the outer bubble.
|
| 165 |
"""
|
| 166 |
-
from shapely.geometry import Polygon
|
| 167 |
-
|
| 168 |
if len(bubbles) <= 1:
|
| 169 |
return bubbles
|
| 170 |
-
|
| 171 |
shapes = []
|
| 172 |
for b in bubbles:
|
| 173 |
try:
|
|
@@ -175,52 +129,44 @@ def filter_nested_bubbles(bubbles):
|
|
| 175 |
if not p.is_valid:
|
| 176 |
p = p.buffer(0)
|
| 177 |
shapes.append((p, b))
|
| 178 |
-
except:
|
| 179 |
continue
|
| 180 |
-
|
| 181 |
-
# Sort by area (largest first)
|
| 182 |
shapes.sort(key=lambda x: x[0].area, reverse=True)
|
| 183 |
-
|
| 184 |
filtered = []
|
| 185 |
-
for i, (
|
| 186 |
is_nested = False
|
| 187 |
-
|
| 188 |
-
for j, (shape2, poly2) in enumerate(shapes):
|
| 189 |
if i == j:
|
| 190 |
continue
|
| 191 |
-
|
| 192 |
-
# Check if shape1 is inside shape2
|
| 193 |
-
if shape2.contains(shape1):
|
| 194 |
is_nested = True
|
| 195 |
break
|
| 196 |
-
|
| 197 |
if not is_nested:
|
| 198 |
filtered.append(poly1)
|
| 199 |
-
|
| 200 |
if len(filtered) < len(bubbles):
|
| 201 |
-
print(f"๐๏ธ
|
| 202 |
-
|
| 203 |
return filtered
|
| 204 |
|
| 205 |
|
| 206 |
-
def detect_speech_bubbles_robust(img_pil, min_area=500, merge_overlaps=True,
|
| 207 |
"""
|
| 208 |
Robust bubble detection with post-processing.
|
| 209 |
-
|
| 210 |
This is the recommended function to use.
|
| 211 |
"""
|
| 212 |
-
# Initial detection
|
| 213 |
bubbles = detect_speech_bubbles(img_pil, min_area=min_area)
|
| 214 |
-
|
| 215 |
-
if
|
|
|
|
| 216 |
return []
|
| 217 |
-
|
| 218 |
-
# Post-processing
|
| 219 |
if merge_overlaps:
|
| 220 |
bubbles = merge_overlapping_bubbles(bubbles)
|
| 221 |
-
|
| 222 |
-
if
|
| 223 |
bubbles = filter_nested_bubbles(bubbles)
|
| 224 |
-
|
| 225 |
-
print(f"โ
|
| 226 |
-
return bubbles
|
|
|
|
| 4 |
import cv2
|
| 5 |
import numpy as np
|
| 6 |
from shapely.geometry import Polygon
|
| 7 |
+
from shapely.ops import unary_union
|
| 8 |
|
| 9 |
|
| 10 |
def detect_speech_bubbles(img_pil, min_area=500, max_area=None, debug=False):
|
| 11 |
"""
|
| 12 |
+
Basic speech bubble detection using adaptive threshold + morphology.
|
| 13 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
Returns:
|
| 15 |
List of bubble polygons [(x,y), ...]
|
| 16 |
"""
|
| 17 |
img = cv2.cvtColor(np.array(img_pil), cv2.COLOR_RGB2BGR)
|
| 18 |
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
|
| 19 |
+
|
| 20 |
h, w = gray.shape
|
| 21 |
if max_area is None:
|
| 22 |
+
max_area = (h * w) // 4 # bubbles should not be entire page
|
| 23 |
+
|
|
|
|
| 24 |
th = cv2.adaptiveThreshold(
|
| 25 |
+
gray,
|
| 26 |
+
255,
|
| 27 |
cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
|
| 28 |
cv2.THRESH_BINARY,
|
| 29 |
+
35,
|
| 30 |
+
10,
|
| 31 |
)
|
| 32 |
+
|
| 33 |
+
inv = 255 - th # bubbles โ white
|
| 34 |
+
|
| 35 |
+
kernel_close = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (7, 7))
|
| 36 |
+
cleaned = cv2.morphologyEx(inv, cv2.MORPH_CLOSE, kernel_close, iterations=2)
|
| 37 |
+
|
|
|
|
|
|
|
| 38 |
kernel_open = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
|
| 39 |
cleaned = cv2.morphologyEx(cleaned, cv2.MORPH_OPEN, kernel_open, iterations=1)
|
| 40 |
+
|
| 41 |
contours, _ = cv2.findContours(cleaned, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
| 42 |
+
|
| 43 |
bubbles = []
|
|
|
|
|
|
|
| 44 |
for cnt in contours:
|
| 45 |
area = cv2.contourArea(cnt)
|
|
|
|
|
|
|
| 46 |
if area < min_area or area > max_area:
|
| 47 |
continue
|
| 48 |
+
|
|
|
|
| 49 |
x, y, bw, bh = cv2.boundingRect(cnt)
|
|
|
|
|
|
|
| 50 |
aspect_ratio = max(bw, bh) / (min(bw, bh) + 1)
|
| 51 |
+
if aspect_ratio > 5:
|
| 52 |
continue
|
| 53 |
+
|
|
|
|
|
|
|
| 54 |
perimeter = cv2.arcLength(cnt, True)
|
| 55 |
+
if perimeter == 0:
|
| 56 |
+
continue
|
| 57 |
circularity = 4 * np.pi * area / (perimeter * perimeter + 1)
|
| 58 |
+
|
|
|
|
| 59 |
epsilon = 0.01 * perimeter
|
| 60 |
approx = cv2.approxPolyDP(cnt, epsilon, True)
|
|
|
|
| 61 |
poly = [(int(p[0][0]), int(p[0][1])) for p in approx]
|
| 62 |
+
|
|
|
|
| 63 |
bubbles.append(poly)
|
| 64 |
+
|
| 65 |
+
print(f"๐ detect_speech_bubbles: {len(bubbles)} candidates")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
return bubbles
|
| 67 |
|
| 68 |
|
| 69 |
def merge_overlapping_bubbles(bubbles, iou_threshold=0.3):
|
| 70 |
"""
|
| 71 |
Merge bubbles that overlap significantly.
|
|
|
|
| 72 |
"""
|
|
|
|
|
|
|
|
|
|
| 73 |
if len(bubbles) <= 1:
|
| 74 |
return bubbles
|
| 75 |
+
|
|
|
|
| 76 |
shapes = []
|
| 77 |
for b in bubbles:
|
| 78 |
try:
|
|
|
|
| 80 |
if not p.is_valid:
|
| 81 |
p = p.buffer(0)
|
| 82 |
shapes.append(p)
|
| 83 |
+
except Exception:
|
| 84 |
continue
|
| 85 |
+
|
| 86 |
+
merged_polys = []
|
|
|
|
| 87 |
used = set()
|
| 88 |
+
|
| 89 |
+
for i, s1 in enumerate(shapes):
|
| 90 |
if i in used:
|
| 91 |
continue
|
| 92 |
+
|
| 93 |
+
group = [s1]
|
| 94 |
used.add(i)
|
| 95 |
+
|
| 96 |
+
for j, s2 in enumerate(shapes[i + 1 :], start=i + 1):
|
| 97 |
if j in used:
|
| 98 |
continue
|
| 99 |
+
inter = s1.intersection(s2).area
|
| 100 |
+
union = s1.union(s2).area
|
| 101 |
+
iou = inter / union if union > 0 else 0.0
|
|
|
|
|
|
|
|
|
|
| 102 |
if iou > iou_threshold:
|
| 103 |
+
group.append(s2)
|
| 104 |
used.add(j)
|
| 105 |
+
|
| 106 |
+
merged_shape = unary_union(group)
|
| 107 |
+
if merged_shape.geom_type == "Polygon":
|
| 108 |
+
merged_polys.append([(int(x), int(y)) for x, y in merged_shape.exterior.coords[:-1]])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
else:
|
| 110 |
+
for g in merged_shape.geoms:
|
| 111 |
+
if g.geom_type == "Polygon":
|
| 112 |
+
merged_polys.append([(int(x), int(y)) for x, y in g.exterior.coords[:-1]])
|
| 113 |
+
|
| 114 |
+
print(f"๐ merge_overlapping_bubbles: {len(bubbles)} โ {len(merged_polys)}")
|
| 115 |
+
return merged_polys
|
| 116 |
|
| 117 |
|
| 118 |
def filter_nested_bubbles(bubbles):
|
| 119 |
"""
|
| 120 |
+
Remove bubbles completely inside other bubbles; keep larger ones.
|
|
|
|
| 121 |
"""
|
|
|
|
|
|
|
| 122 |
if len(bubbles) <= 1:
|
| 123 |
return bubbles
|
| 124 |
+
|
| 125 |
shapes = []
|
| 126 |
for b in bubbles:
|
| 127 |
try:
|
|
|
|
| 129 |
if not p.is_valid:
|
| 130 |
p = p.buffer(0)
|
| 131 |
shapes.append((p, b))
|
| 132 |
+
except Exception:
|
| 133 |
continue
|
| 134 |
+
|
|
|
|
| 135 |
shapes.sort(key=lambda x: x[0].area, reverse=True)
|
| 136 |
+
|
| 137 |
filtered = []
|
| 138 |
+
for i, (s1, poly1) in enumerate(shapes):
|
| 139 |
is_nested = False
|
| 140 |
+
for j, (s2, poly2) in enumerate(shapes):
|
|
|
|
| 141 |
if i == j:
|
| 142 |
continue
|
| 143 |
+
if s2.contains(s1):
|
|
|
|
|
|
|
| 144 |
is_nested = True
|
| 145 |
break
|
|
|
|
| 146 |
if not is_nested:
|
| 147 |
filtered.append(poly1)
|
| 148 |
+
|
| 149 |
if len(filtered) < len(bubbles):
|
| 150 |
+
print(f"๐๏ธ filter_nested_bubbles: removed {len(bubbles) - len(filtered)} nested")
|
|
|
|
| 151 |
return filtered
|
| 152 |
|
| 153 |
|
| 154 |
+
def detect_speech_bubbles_robust(img_pil, min_area=500, merge_overlaps=True, filter_nested_flag=True):
|
| 155 |
"""
|
| 156 |
Robust bubble detection with post-processing.
|
|
|
|
| 157 |
This is the recommended function to use.
|
| 158 |
"""
|
|
|
|
| 159 |
bubbles = detect_speech_bubbles(img_pil, min_area=min_area)
|
| 160 |
+
|
| 161 |
+
if not bubbles:
|
| 162 |
+
print("โ ๏ธ detect_speech_bubbles_robust: no initial bubbles")
|
| 163 |
return []
|
| 164 |
+
|
|
|
|
| 165 |
if merge_overlaps:
|
| 166 |
bubbles = merge_overlapping_bubbles(bubbles)
|
| 167 |
+
|
| 168 |
+
if filter_nested_flag:
|
| 169 |
bubbles = filter_nested_bubbles(bubbles)
|
| 170 |
+
|
| 171 |
+
print(f"โ
detect_speech_bubbles_robust: final {len(bubbles)} bubbles")
|
| 172 |
+
return bubbles
|