File size: 9,176 Bytes
c7159bc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
"""
Garment Segmentation Module
Uses SegFormer fine-tuned on the ATR / LIP human-parsing datasets.
Falls back to a simple colour/edge-based approach if GPU is unavailable.
"""

from __future__ import annotations

import cv2
import numpy as np
from PIL import Image
import torch
import torch.nn.functional as F


# Clothing label IDs in the mattmdjaga/segformer_b2_clothes palette.
# IMPORTANT: only true garment pixels β€” body parts (face, arms, legs, hair)
# and accessories (shoes, bag) must NOT be included, or the ironing pipeline
# will smooth skin/face along with the fabric.
CLOTHING_LABELS = {
    "upper_body": [4],        # Upper-clothes (shirt, t-shirt, jacket, coat)
    "lower_body": [6, 5],     # Pants, Skirt
    "dress":      [7],        # Dress
    "extras":     [8, 17],    # Belt, Scarf (worn fabric items)
}

# All valid "garment" ids β€” used for auto-detect
ALL_GARMENT_IDS = [4, 5, 6, 7, 8, 17]

LABEL_NAMES = {
    0:  "background",
    1:  "hat",
    2:  "hair",
    3:  "sunglasses",
    4:  "upper-clothes",
    5:  "skirt",
    6:  "pants",
    7:  "dress",
    8:  "belt",
    9:  "left-shoe",
    10: "right-shoe",
    11: "face",
    12: "left-leg",
    13: "right-leg",
    14: "left-arm",
    15: "right-arm",
    16: "bag",
    17: "scarf",
}


class GarmentSegmenter:
    """
    Wraps a HuggingFace SegFormer model for human-part segmentation.
    Model: mattmdjaga/segformer_b2_clothes
    """

    MODEL_ID = "mattmdjaga/segformer_b2_clothes"

    def __init__(self, device: str | None = None):
        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
        self._processor = None
        self._model      = None
        self._load()

    # ── private ──────────────────────────────────────────────────────────────
    def _load(self):
        try:
            from transformers import SegformerImageProcessor, SegformerForSemanticSegmentation
            print(f"[Segmenter] Loading {self.MODEL_ID} on {self.device} …")
            self._processor = SegformerImageProcessor.from_pretrained(self.MODEL_ID)
            self._model     = SegformerForSemanticSegmentation.from_pretrained(self.MODEL_ID)
            self._model.to(self.device).eval()
            print("[Segmenter] Model ready.")
        except Exception as exc:
            print(f"[Segmenter] Could not load transformer model: {exc}")
            print("[Segmenter] Falling back to simple masking.")
            self._model = None

    # ── public ────────────────────────────────────────────────────────────────
    def segment(
        self,
        image: Image.Image,
        clothing_type: str = "Auto-detect",
    ) -> dict:
        """
        Parameters
        ----------
        image        : PIL RGB image
        clothing_type: hint for which garment to focus on

        Returns
        -------
        dict with keys:
          garment_mask  – HΓ—W bool ndarray (True = garment pixel)
          labels        – list of detected garment label names
          label_mask    – HΓ—W int ndarray with per-pixel label ids
          confidence    – float 0-1
        """
        if self._model is not None:
            return self._segment_segformer(image, clothing_type)
        return self._segment_fallback(image, clothing_type)

    # ── SegFormer path ────────────────────────────────────────────────────────
    def _segment_segformer(self, image: Image.Image, clothing_type: str) -> dict:
        inputs = self._processor(images=image, return_tensors="pt")
        inputs = {k: v.to(self.device) for k, v in inputs.items()}

        with torch.no_grad():
            logits = self._model(**inputs).logits          # 1 Γ— C Γ— H' Γ— W'

        # upsample to original size
        h, w = image.size[1], image.size[0]
        upsampled = F.interpolate(
            logits, size=(h, w), mode="bilinear", align_corners=False
        )
        probabilities = torch.softmax(upsampled, dim=1)
        pred = probabilities.argmax(dim=1).squeeze(0).cpu().numpy()  # H Γ— W

        # decide which labels count as "garment" given the hint
        target_ids = self._target_ids(clothing_type)
        garment_mask = np.isin(pred, target_ids)

        # morphological clean-up (remove tiny islands)
        garment_mask = _clean_mask(garment_mask)

        detected_ids = [int(i) for i in np.unique(pred) if i in target_ids]
        labels       = [LABEL_NAMES.get(i, str(i)) for i in detected_ids]

        pixel_confidence = probabilities.max(dim=1).values.squeeze(0).cpu().numpy()
        confidence = (
            float(pixel_confidence[garment_mask].mean()) if garment_mask.any() else 0.0
        )

        return {
            "garment_mask": garment_mask,
            "labels":       labels,
            "label_mask":   pred,
            "confidence":   confidence,
        }

    # ── Fallback: simple skin-exclusion mask ─────────────────────────────────
    def _segment_fallback(self, image: Image.Image, clothing_type: str) -> dict:
        arr  = np.array(image.convert("RGB")).astype(float)
        h, w = arr.shape[:2]
        r, g, b = arr[..., 0], arr[..., 1], arr[..., 2]

        # exclude skin-coloured pixels (very rough heuristic)
        skin = (r > 95) & (g > 40) & (b > 20) & \
               (r > g) & (r > b) & \
               (np.abs(r - g) > 15) & \
               (r > 100)

        # Estimate background color from image borders and remove pixels close
        # to it. This is still a fallback, but is substantially safer than
        # treating every non-skin, non-white pixel as clothing.
        border = np.concatenate(
            [arr[0], arr[-1], arr[:, 0], arr[:, -1]], axis=0
        )
        background_color = np.median(border, axis=0)
        background_distance = np.linalg.norm(arr - background_color, axis=2)
        foreground = background_distance > 24.0

        garment_mask = ~skin & foreground
        garment_mask = _clean_mask(garment_mask)

        key = clothing_type.lower()
        vertical_filter = np.ones((h, w), dtype=bool)
        label = "clothing item"
        if any(name in key for name in ("shirt", "t-shirt", "jacket", "blazer")):
            vertical_filter[int(h * 0.76):, :] = False
            label = "upper-clothes"
        elif "trousers" in key:
            vertical_filter[:int(h * 0.34), :] = False
            label = "pants"
        elif "dress" in key:
            label = "dress"
        garment_mask &= vertical_filter
        garment_mask = _clean_mask(garment_mask)

        return {
            "garment_mask": garment_mask,
            "labels":       [label],
            "label_mask":   garment_mask.astype(int),
            "confidence":   0.35,
        }

    # ── helpers ───────────────────────────────────────────────────────────────
    @staticmethod
    def _target_ids(clothing_type: str) -> list[int]:
        mapping = {
            "shirt / blouse":   [4],
            "t-shirt":          [4],
            "trousers":         [6],
            "dress":            [7],
            "jacket / blazer":  [4],
            "traditional wear": [4, 6, 7],
            "saree / kurta":    [4, 6, 7],
        }
        key = clothing_type.lower()
        for k, v in mapping.items():
            if k in key:
                return v
        # auto-detect: return only true garment labels (no body parts)
        return ALL_GARMENT_IDS


# ── utility ───────────────────────────────────────────────────────────────────
def _clean_mask(mask: np.ndarray, min_area: int | None = None) -> np.ndarray:
    """Close small gaps, fill holes, and remove tiny connected components."""
    min_area = min_area or max(64, int(mask.size * 0.0008))
    mask_u8 = mask.astype(np.uint8)
    close_kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (7, 7))
    open_kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
    mask_u8 = cv2.morphologyEx(mask_u8, cv2.MORPH_CLOSE, close_kernel)
    mask_u8 = cv2.morphologyEx(mask_u8, cv2.MORPH_OPEN, open_kernel)

    try:
        from scipy import ndimage
        mask_u8 = ndimage.binary_fill_holes(mask_u8).astype(np.uint8)
        labeled, n = ndimage.label(mask_u8)
        sizes = ndimage.sum(mask_u8, labeled, range(1, n + 1))
        clean = np.zeros_like(mask_u8, dtype=bool)
        for i, s in enumerate(sizes, start=1):
            if s >= min_area:
                clean |= labeled == i
        return clean
    except ImportError:
        return mask_u8.astype(bool)