Wojtekb30 commited on
Commit
8301504
·
verified ·
1 Parent(s): 77e7525

Upload 16 files

Browse files
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ image1.jpg filter=lfs diff=lfs merge=lfs -text
37
+ result_0_boxes_and_masks.png filter=lfs diff=lfs merge=lfs -text
LIS1-9B-LoraAdapter/adapter_config.json ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alora_invocation_tokens": null,
3
+ "alpha_pattern": {},
4
+ "arrow_config": null,
5
+ "auto_mapping": {
6
+ "base_model_class": "Qwen3_5TextModel",
7
+ "parent_library": "transformers.models.qwen3_5.modeling_qwen3_5"
8
+ },
9
+ "base_model_name_or_path": null,
10
+ "bias": "none",
11
+ "corda_config": null,
12
+ "ensure_weight_tying": false,
13
+ "eva_config": null,
14
+ "exclude_modules": null,
15
+ "fan_in_fan_out": false,
16
+ "inference_mode": true,
17
+ "init_lora_weights": true,
18
+ "kasa_config": null,
19
+ "layer_replication": null,
20
+ "layers_pattern": null,
21
+ "layers_to_transform": null,
22
+ "loftq_config": {},
23
+ "lora_alpha": 16,
24
+ "lora_bias": false,
25
+ "lora_dropout": 0.05,
26
+ "lora_ga_config": null,
27
+ "megatron_config": null,
28
+ "megatron_core": "megatron.core",
29
+ "modules_to_save": null,
30
+ "monteclora_config": null,
31
+ "peft_type": "LORA",
32
+ "peft_version": "0.21.2",
33
+ "qalora_group_size": 16,
34
+ "r": 8,
35
+ "rank_pattern": {},
36
+ "revision": null,
37
+ "target_modules": [
38
+ "layers.31.self_attn.v_proj",
39
+ "layers.31.self_attn.o_proj",
40
+ "layers.28.linear_attn.out_proj",
41
+ "layers.31.self_attn.k_proj",
42
+ "layers.28.linear_attn.in_proj_qkv",
43
+ "layers.31.self_attn.q_proj",
44
+ "layers.29.linear_attn.out_proj",
45
+ "layers.29.linear_attn.in_proj_qkv",
46
+ "layers.30.linear_attn.in_proj_qkv",
47
+ "layers.30.linear_attn.out_proj"
48
+ ],
49
+ "target_parameters": null,
50
+ "task_type": null,
51
+ "trainable_token_indices": null,
52
+ "use_bdlora": null,
53
+ "use_dora": false,
54
+ "use_qalora": false,
55
+ "use_rslora": false,
56
+ "velora_config": null
57
+ }
LIS1-9B-LoraAdapter/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e113ecf5c50b23446537f6ae9b06ba16a3900ba21554c0863b3245056690074e
3
+ size 2951736
LIS1-9B-forLora.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:57b16aa7c92b7f40f7293ed5c93928f7c4d0a6bf4f24c112af1065f771a8b6fc
3
+ size 27216895
LIS1-9B-forLora.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:33ac1e881ceebfacba209c3e96cec1edc17f581f380b167a137d19581b8aaf7a
3
+ size 27200852
LIS1-9B-noLora.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ad62dddaeaf26a8c588fcf320d5aca23772b631353490bc0ebee85319352d158
3
+ size 27219743
LIS1-9B-noLora.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:365d2e214c8023d0066518d705aa68c9d4f6c3d9c4a82b0cd9f9c096bbb79969
3
+ size 27200812
LIS1ModelCode.py ADDED
@@ -0,0 +1,1152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import importlib.metadata as _im
2
+ import inspect
3
+ import json
4
+ from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
5
+
6
+ import cv2
7
+ import numpy as np
8
+ import torch
9
+ import torch.nn as nn
10
+ import torch.nn.functional as F
11
+
12
+ # Guard against huggingface-hub>=2.0 version check in transformers while keeping full functionality
13
+ _orig_version = _im.version
14
+
15
+
16
+ def _safe_version(pkg: str) -> str:
17
+ if pkg == "huggingface-hub":
18
+ try:
19
+ ver = _orig_version(pkg)
20
+ if ver.startswith("2."):
21
+ return "1.99.0"
22
+ return ver
23
+ except Exception:
24
+ return "1.99.0"
25
+ return _orig_version(pkg)
26
+
27
+
28
+ _im.version = _safe_version
29
+
30
+ from transformers import AutoModel, AutoProcessor
31
+
32
+
33
+ OBJECT_START_TOKEN = "<|object_ref_start|>"
34
+ OBJECT_QUERY_TOKEN = "<|object_ref_end|>"
35
+ DEFAULT_SYSTEM_PROMPT = (
36
+ "You are a fast single-pass visual reasoning and segmentation model. "
37
+ "For each query or riddle in the input JSON, infer the intended target object in the image and ground it precisely."
38
+ )
39
+
40
+
41
+ def normalize_input_json(
42
+ query_input: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]],
43
+ ) -> List[Dict[str, str]]:
44
+ """
45
+ Normalizes user input (JSON string, dict with 'objects', or list of strings/dicts)
46
+ into a canonical list of [{'id': str, 'description': str}, ...].
47
+ """
48
+ if isinstance(query_input, str):
49
+ parsed = json.loads(query_input)
50
+ else:
51
+ parsed = query_input
52
+
53
+ if isinstance(parsed, dict):
54
+ raw_objects = parsed.get("objects", parsed.get("queries", [parsed]))
55
+ elif isinstance(parsed, (list, tuple)):
56
+ raw_objects = list(parsed)
57
+ else:
58
+ raise ValueError(f"Unsupported query_input format: {type(query_input)}")
59
+
60
+ normalized: List[Dict[str, str]] = []
61
+ for idx, item in enumerate(raw_objects):
62
+ if isinstance(item, str):
63
+ normalized.append({"id": str(idx), "description": item.strip()})
64
+ elif isinstance(item, dict):
65
+ obj_id = str(item.get("id", idx))
66
+ desc = str(
67
+ item.get("description")
68
+ or item.get("prompt")
69
+ or item.get("text")
70
+ or item.get("sent")
71
+ or ""
72
+ ).strip()
73
+ if not desc:
74
+ raise ValueError(f"Object at index {idx} has an empty description: {item}")
75
+ normalized.append({"id": obj_id, "description": desc})
76
+ else:
77
+ raise ValueError(f"Unsupported object entry at index {idx}: {item}")
78
+
79
+ if not normalized:
80
+ raise ValueError("Input JSON must contain at least one object description.")
81
+ return normalized
82
+
83
+
84
+ def build_segmentation_prompt(
85
+ query_input: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]],
86
+ query_token: str = OBJECT_QUERY_TOKEN,
87
+ include_query_token: bool = False,
88
+ ) -> Tuple[str, List[Dict[str, str]]]:
89
+ """
90
+ Builds the user-side JSON prompt describing all objects/riddles to segment.
91
+ """
92
+ objects = normalize_input_json(query_input)
93
+ entries = []
94
+ for obj in objects:
95
+ entry_json = json.dumps(
96
+ {"id": obj["id"], "description": obj["description"]},
97
+ ensure_ascii=False,
98
+ )
99
+ suffix = query_token if include_query_token else ""
100
+ entries.append(f" {entry_json}{suffix}")
101
+ json_block = "{\n \"objects\": [\n" + ",\n".join(entries) + "\n ]\n}"
102
+ prompt = (
103
+ "Reason about each description or riddle in the following JSON "
104
+ "and segment the corresponding object in the image:\n"
105
+ f"{json_block}"
106
+ )
107
+ return prompt, objects
108
+
109
+
110
+ def build_assistant_grounding_prefill(
111
+ objects: Sequence[Dict[str, str]],
112
+ start_token: str = OBJECT_START_TOKEN,
113
+ query_token: str = OBJECT_QUERY_TOKEN,
114
+ ) -> str:
115
+ """
116
+ Builds the prefilled assistant response containing `<|object_ref_start|>{desc}<|object_ref_end|>`
117
+ for each object. Because this is passed as part of the single forward pass:
118
+ 1. The LLM has already read the user's riddle AND processed all `<|image_pad|>` tokens.
119
+ 2. Crossing `<|im_start|>assistant` switches Qwen into answering/grounding mode.
120
+ 3. `<|object_ref_start|>...<|object_ref_end|>` activates Qwen's native localization circuit.
121
+ """
122
+ entries = []
123
+ for obj in objects:
124
+ target_str = f"{start_token}{obj['description']}{query_token}"
125
+ entry_json = json.dumps(
126
+ {"id": obj["id"], "target": target_str},
127
+ ensure_ascii=False,
128
+ )
129
+ entries.append(f" {entry_json}")
130
+ return "{\n \"objects\": [\n" + ",\n".join(entries) + "\n ]\n}"
131
+
132
+
133
+ def build_segmentation_messages(
134
+ image: Any,
135
+ query_input: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]],
136
+ system_prompt: str = DEFAULT_SYSTEM_PROMPT,
137
+ query_token: str = OBJECT_QUERY_TOKEN,
138
+ ) -> Tuple[List[Dict[str, Any]], List[Dict[str, str]]]:
139
+ """
140
+ Constructs the single-pass multimodal chat messages in causal reasoning order:
141
+ [System] -> [User: JSON Riddles -> Image] -> [Assistant: <|object_ref_start|>...<|object_ref_end|>]
142
+ Putting the user JSON before the image allows every `<|image_pad|>` token in the LLM
143
+ to attend to the riddle across all 36 layers before `<|object_ref_end|>` reads them out.
144
+ """
145
+ user_prompt, objects = build_segmentation_prompt(
146
+ query_input, query_token=query_token, include_query_token=False
147
+ )
148
+ assistant_prefill = build_assistant_grounding_prefill(objects, query_token=query_token)
149
+ messages = [
150
+ {"role": "system", "content": system_prompt},
151
+ {
152
+ "role": "user",
153
+ "content": [
154
+ {"type": "text", "text": user_prompt + "\n"},
155
+ {"type": "image", "image": image},
156
+ ],
157
+ },
158
+ {
159
+ "role": "assistant",
160
+ "content": assistant_prefill,
161
+ },
162
+ ]
163
+ return messages, objects
164
+
165
+
166
+ def polygons_to_mask(
167
+ segmentation: Union[List[List[float]], Dict[str, Any]],
168
+ height: int,
169
+ width: int,
170
+ ) -> np.ndarray:
171
+ """
172
+ Rasterizes a COCO/RefCOCO annotation `segmentation` (polygon list or uncompressed RLE)
173
+ into a binary numpy mask of shape (height, width) with values in {0, 1}.
174
+ """
175
+ mask = np.zeros((height, width), dtype=np.uint8)
176
+ if isinstance(segmentation, list):
177
+ for poly in segmentation:
178
+ if len(poly) < 6:
179
+ continue
180
+ pts = np.asarray(poly, dtype=np.float32).reshape(-1, 2)
181
+ pts = np.round(pts).astype(np.int32)
182
+ cv2.fillPoly(mask, [pts], 1)
183
+ elif isinstance(segmentation, dict) and "counts" in segmentation:
184
+ counts = segmentation["counts"]
185
+ if isinstance(counts, list):
186
+ h, w = segmentation.get("size", [height, width])
187
+ flat = np.zeros(h * w, dtype=np.uint8)
188
+ idx = 0
189
+ val = 0
190
+ for c in counts:
191
+ flat[idx : idx + c] = val
192
+ idx += c
193
+ val = 1 - val
194
+ mask = flat.reshape((w, h)).T
195
+ if mask.shape != (height, width):
196
+ mask = cv2.resize(mask, (width, height), interpolation=cv2.INTER_NEAREST)
197
+ else:
198
+ raise ValueError("Compressed string RLE requires pycocotools; RefCOCO uses polygon lists.")
199
+ else:
200
+ raise ValueError(f"Unsupported segmentation format: {type(segmentation)}")
201
+ return mask
202
+
203
+
204
+ def mask_to_polygons(
205
+ binary_mask: Union[np.ndarray, torch.Tensor],
206
+ epsilon_ratio: float = 0.0015,
207
+ min_area: float = 4.0,
208
+ ) -> Tuple[List[List[float]], List[float], float]:
209
+ """
210
+ Converts a 2D binary mask (H, W) into RefCOCO/COCO dataset format:
211
+ - segmentation: List[List[float]] of polygon coordinates [x1, y1, x2, y2, ...]
212
+ - bbox: [x, y, width, height]
213
+ - area: float pixel area
214
+ """
215
+ if isinstance(binary_mask, torch.Tensor):
216
+ mask_np = binary_mask.detach().cpu().numpy().astype(np.uint8)
217
+ else:
218
+ mask_np = np.asarray(binary_mask, dtype=np.uint8)
219
+
220
+ if mask_np.ndim > 2:
221
+ mask_np = mask_np.squeeze()
222
+
223
+ area = float(mask_np.sum())
224
+ if area < min_area:
225
+ return [], [0.0, 0.0, 0.0, 0.0], 0.0
226
+
227
+ ys, xs = np.where(mask_np > 0)
228
+ x_min, x_max = float(xs.min()), float(xs.max())
229
+ y_min, y_max = float(ys.min()), float(ys.max())
230
+ bbox = [
231
+ round(x_min, 2),
232
+ round(y_min, 2),
233
+ round(x_max - x_min + 1.0, 2),
234
+ round(y_max - y_min + 1.0, 2),
235
+ ]
236
+
237
+ contours, _ = cv2.findContours(mask_np, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
238
+ polygons: List[List[float]] = []
239
+ for cnt in contours:
240
+ if cv2.contourArea(cnt) < min_area and len(contours) > 1:
241
+ continue
242
+ peri = cv2.arcLength(cnt, True)
243
+ approx = cv2.approxPolyDP(cnt, epsilon_ratio * peri, True) if epsilon_ratio > 0 else cnt
244
+ pts = approx.reshape(-1, 2)
245
+ if pts.shape[0] < 3:
246
+ pts = cnt.reshape(-1, 2)
247
+ if pts.shape[0] >= 3:
248
+ poly = [round(float(coord), 2) for coord in pts.flatten()]
249
+ if len(poly) >= 6:
250
+ polygons.append(poly)
251
+
252
+ if not polygons and area >= min_area:
253
+ # Fallback box polygon if contour is degenerate (e.g. 1-pixel thin line)
254
+ polygons.append(
255
+ [
256
+ round(x_min, 2),
257
+ round(y_min, 2),
258
+ round(x_max, 2),
259
+ round(y_min, 2),
260
+ round(x_max, 2),
261
+ round(y_max, 2),
262
+ round(x_min, 2),
263
+ round(y_max, 2),
264
+ ]
265
+ )
266
+
267
+ return polygons, bbox, round(area, 2)
268
+
269
+
270
+ class ConvGNAct(nn.Module):
271
+ """Residual-friendly 3x3 Conv + GroupNorm + GELU block."""
272
+
273
+ def __init__(self, in_ch: int, out_ch: int, kernel_size: int = 3, groups: int = 8):
274
+ super().__init__()
275
+ padding = kernel_size // 2
276
+ eff_groups = min(groups, out_ch)
277
+ while out_ch % eff_groups != 0 and eff_groups > 1:
278
+ eff_groups -= 1
279
+ self.block = nn.Sequential(
280
+ nn.Conv2d(in_ch, out_ch, kernel_size=kernel_size, padding=padding, bias=False),
281
+ nn.GroupNorm(eff_groups, out_ch),
282
+ nn.GELU(),
283
+ )
284
+
285
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
286
+ return self.block(x)
287
+
288
+
289
+ class ResidualRefinementBlock(nn.Module):
290
+ """Residual spatial refinement block for crisp mask boundaries."""
291
+
292
+ def __init__(self, channels: int):
293
+ super().__init__()
294
+ self.conv1 = ConvGNAct(channels, channels, kernel_size=3)
295
+ self.conv2 = ConvGNAct(channels, channels, kernel_size=3)
296
+
297
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
298
+ return x + self.conv2(self.conv1(x))
299
+
300
+
301
+ class MultiScaleMaskHead(nn.Module):
302
+ """
303
+ Lightweight System-1 mask head that fuses:
304
+ 1. LLM object decision vectors Q [K, llm_dim]
305
+ 2. Coarse merged ViT features F_28 [C_out, H_m, W_m] (1/28 stride)
306
+ 3. Fine unmerged ViT skip features F_14 [C_in, 2*H_m, 2*W_m] (1/14 stride)
307
+ and outputs full-resolution mask logits [K, orig_H, orig_W] + IoU confidence scores [K].
308
+ """
309
+
310
+ def __init__(
311
+ self,
312
+ llm_hidden_size: int = 2048,
313
+ vision_merged_dim: int = 2048,
314
+ vision_unmerged_dim: int = 1280,
315
+ hidden_dim: int = 256,
316
+ fine_dim: int = 128,
317
+ num_heads: int = 4,
318
+ num_fused_layers: int = 4,
319
+ ):
320
+ super().__init__()
321
+ self.hidden_dim = hidden_dim
322
+ self.fine_dim = fine_dim
323
+ self.num_fused_layers = num_fused_layers
324
+
325
+ # Learnable fusion across the last N LLM layers (preserves frozen LLM reasoning + mid-late visual grounding)
326
+ self.layer_weights = nn.Parameter(torch.zeros(num_fused_layers))
327
+ self.layer_norms = nn.ModuleList([nn.LayerNorm(llm_hidden_size) for _ in range(num_fused_layers)])
328
+
329
+ # Visual feature projections (computed once per image, shared across K object queries)
330
+ self.coarse_vis_proj = ConvGNAct(vision_merged_dim, hidden_dim, kernel_size=1)
331
+ self.llm_vis_proj = ConvGNAct(llm_hidden_size, hidden_dim, kernel_size=1)
332
+ self.llm_vis_gate = nn.Parameter(torch.tensor(0.5))
333
+ self.fine_vis_proj = ConvGNAct(vision_unmerged_dim, fine_dim, kernel_size=1)
334
+
335
+ # Query projection + cross-attention over coarse 2D visual tokens
336
+ self.query_in_proj = nn.Sequential(
337
+ nn.Linear(llm_hidden_size, hidden_dim),
338
+ nn.LayerNorm(hidden_dim),
339
+ nn.GELU(),
340
+ nn.Linear(hidden_dim, hidden_dim),
341
+ )
342
+ self.cross_attn = nn.MultiheadAttention(
343
+ embed_dim=hidden_dim,
344
+ num_heads=num_heads,
345
+ batch_first=True,
346
+ )
347
+ self.cross_norm = nn.LayerNorm(hidden_dim)
348
+
349
+ # Query-to-coarse and query-to-fine dynamic modulators
350
+ self.coarse_query_proj = nn.Linear(hidden_dim, hidden_dim)
351
+ self.llm_query_proj = nn.Linear(hidden_dim, hidden_dim)
352
+ self.coarse_gate_proj = nn.Linear(hidden_dim, hidden_dim)
353
+ self.fine_gate_proj = nn.Linear(hidden_dim, fine_dim)
354
+
355
+ # Coarse decoder at 1/28 scale (concatenates modulated features + dot-product similarity map)
356
+ self.coarse_fuse = ConvGNAct(hidden_dim + 1, hidden_dim, kernel_size=3)
357
+ self.coarse_res = ResidualRefinementBlock(hidden_dim)
358
+
359
+ # Fine decoder at 1/14 scale (fuses upsampled coarse features + fine 1/14 ViT skip features)
360
+ self.fine_fuse = ConvGNAct(hidden_dim + fine_dim, fine_dim, kernel_size=3)
361
+ self.fine_res1 = ResidualRefinementBlock(fine_dim)
362
+ self.fine_res2 = ResidualRefinementBlock(fine_dim)
363
+
364
+ # 2x learned upsampler to 1/7 stride before final bilinear interpolation
365
+ self.up_conv = nn.Sequential(
366
+ nn.ConvTranspose2d(fine_dim, fine_dim // 2, kernel_size=2, stride=2),
367
+ nn.GroupNorm(min(8, fine_dim // 2), fine_dim // 2),
368
+ nn.GELU(),
369
+ ConvGNAct(fine_dim // 2, fine_dim // 2, kernel_size=3),
370
+ )
371
+ self.mask_out = nn.Conv2d(fine_dim // 2, 1, kernel_size=1)
372
+
373
+ # Predicts calibrated mask confidence / IoU score per object query
374
+ self.score_head = nn.Sequential(
375
+ nn.Linear(hidden_dim, hidden_dim // 2),
376
+ nn.GELU(),
377
+ nn.Linear(hidden_dim // 2, 1),
378
+ )
379
+
380
+ def _build_2d_sincos_pos_embed(
381
+ self, h: int, w: int, dim: int, device: torch.device, dtype: torch.dtype
382
+ ) -> torch.Tensor:
383
+ """Generates parameter-free 2D sine-cosine positional embeddings [1, h*w, dim]."""
384
+ y = torch.linspace(-1.0, 1.0, steps=h, device=device, dtype=torch.float32)
385
+ x = torch.linspace(-1.0, 1.0, steps=w, device=device, dtype=torch.float32)
386
+ grid_y, grid_x = torch.meshgrid(y, x, indexing="ij")
387
+ half = dim // 4
388
+ if half == 0:
389
+ return torch.zeros((1, h * w, dim), device=device, dtype=dtype)
390
+ omega = torch.arange(half, device=device, dtype=torch.float32) / max(half, 1)
391
+ omega = 1.0 / (10000.0**omega)
392
+ out_y = grid_y.reshape(-1, 1) * omega.reshape(1, -1)
393
+ out_x = grid_x.reshape(-1, 1) * omega.reshape(1, -1)
394
+ pos = torch.cat([torch.sin(out_y), torch.cos(out_y), torch.sin(out_x), torch.cos(out_x)], dim=-1)
395
+ if pos.shape[-1] < dim:
396
+ pos = F.pad(pos, (0, dim - pos.shape[-1]))
397
+ return pos.unsqueeze(0).to(dtype=dtype)
398
+
399
+ def forward_single_image(
400
+ self,
401
+ queries: torch.Tensor,
402
+ f_coarse: torch.Tensor,
403
+ f_fine: torch.Tensor,
404
+ target_size: Tuple[int, int],
405
+ f_llm_coarse: Optional[torch.Tensor] = None,
406
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
407
+ """
408
+ Args:
409
+ queries: [K, llm_hidden_size] or [K, L, llm_hidden_size] decision vectors for K objects
410
+ f_coarse: [1, vision_merged_dim, H_m, W_m] (1/28 resolution from ViT)
411
+ f_fine: [1, vision_unmerged_dim, H_f, W_f] (1/14 resolution from ViT)
412
+ target_size: (orig_H, orig_W)
413
+ f_llm_coarse: optional [1, llm_hidden_size, H_m, W_m] riddle-conditioned `<|image_pad|>` grid from LLM
414
+
415
+ Returns:
416
+ mask_logits: [K, orig_H, orig_W]
417
+ pred_scores: [K] (in [0, 1])
418
+ """
419
+ num_queries = queries.shape[0]
420
+ _, _, h_m, w_m = f_coarse.shape
421
+ _, _, h_f, w_f = f_fine.shape
422
+
423
+ # Fuse multi-layer LLM hidden states [K, L, llm_hidden_size] -> [K, llm_hidden_size]
424
+ if queries.ndim == 3:
425
+ num_layers = queries.shape[1]
426
+ normed_layers = [
427
+ self.layer_norms[min(i, len(self.layer_norms) - 1)](queries[:, i, :])
428
+ for i in range(num_layers)
429
+ ]
430
+ stacked = torch.stack(normed_layers, dim=1) # [K, L, llm_hidden_size]
431
+ weights = F.softmax(self.layer_weights[:num_layers], dim=0).view(1, num_layers, 1)
432
+ queries = (stacked * weights).sum(dim=1)
433
+
434
+ # Project visual features once for the image
435
+ vis_coarse = self.coarse_vis_proj(f_coarse) # [1, hidden_dim, H_m, W_m]
436
+ vis_llm = None
437
+ if f_llm_coarse is not None:
438
+ vis_llm = self.llm_vis_proj(f_llm_coarse) # [1, hidden_dim, H_m, W_m]
439
+ vis_coarse = vis_coarse + torch.tanh(self.llm_vis_gate) * vis_llm
440
+ vis_fine = self.fine_vis_proj(f_fine) # [1, fine_dim, H_f, W_f]
441
+
442
+ # Project queries & refine with cross-attention over 2D visual tokens
443
+ q = self.query_in_proj(queries).unsqueeze(0) # [1, K, hidden_dim]
444
+ vis_tokens = vis_coarse.flatten(2).transpose(1, 2) # [1, H_m*W_m, hidden_dim]
445
+ pos_2d = self._build_2d_sincos_pos_embed(
446
+ h_m, w_m, self.hidden_dim, vis_tokens.device, vis_tokens.dtype
447
+ )
448
+ attn_out, _ = self.cross_attn(
449
+ query=q,
450
+ key=vis_tokens + pos_2d,
451
+ value=vis_tokens,
452
+ )
453
+ q = self.cross_norm(q + attn_out).squeeze(0) # [K, hidden_dim]
454
+
455
+ # Predict confidence score per object query
456
+ pred_scores = torch.sigmoid(self.score_head(q).squeeze(-1)) # [K]
457
+
458
+ # 1. Coarse (1/28) dot-product similarity + channel gating
459
+ q_dot = F.normalize(self.coarse_query_proj(q), dim=-1) # [K, hidden_dim]
460
+ v_norm = F.normalize(vis_coarse.squeeze(0), dim=0) # [hidden_dim, H_m, W_m]
461
+ sim_map = torch.einsum("kd,dhw->khw", q_dot, v_norm).unsqueeze(1) # [K, 1, H_m, W_m]
462
+ if vis_llm is not None:
463
+ q_llm_dot = F.normalize(self.llm_query_proj(q), dim=-1)
464
+ v_llm_norm = F.normalize(vis_llm.squeeze(0), dim=0)
465
+ sim_map_llm = torch.einsum("kd,dhw->khw", q_llm_dot, v_llm_norm).unsqueeze(1)
466
+ sim_map = 0.5 * (sim_map + sim_map_llm)
467
+
468
+ coarse_gate = torch.sigmoid(self.coarse_gate_proj(q)).unsqueeze(-1).unsqueeze(-1) # [K, hidden_dim, 1, 1]
469
+ vis_coarse_exp = vis_coarse.expand(num_queries, -1, -1, -1)
470
+ coarse_mod = vis_coarse_exp * coarse_gate # [K, hidden_dim, H_m, W_m]
471
+
472
+ x = torch.cat([coarse_mod, sim_map], dim=1) # [K, hidden_dim + 1, H_m, W_m]
473
+ x = self.coarse_res(self.coarse_fuse(x))
474
+
475
+ # 2. Upsample 2x to 1/14 resolution & fuse with unmerged ViT skip features
476
+ x = F.interpolate(x, size=(h_f, w_f), mode="bilinear", align_corners=False)
477
+ fine_gate = torch.sigmoid(self.fine_gate_proj(q)).unsqueeze(-1).unsqueeze(-1) # [K, fine_dim, 1, 1]
478
+ vis_fine_exp = vis_fine.expand(num_queries, -1, -1, -1) * fine_gate
479
+
480
+ x = torch.cat([x, vis_fine_exp], dim=1) # [K, hidden_dim + fine_dim, H_f, W_f]
481
+ x = self.fine_fuse(x)
482
+ x = self.fine_res2(self.fine_res1(x))
483
+
484
+ # 3. Learned 2x upsample (to 1/7 stride) + bilinear resize to original image resolution
485
+ x = self.up_conv(x)
486
+ raw_logits = self.mask_out(x) # [K, 1, 2*H_f, 2*W_f]
487
+ upsampled_logits = F.interpolate(
488
+ raw_logits,
489
+ size=target_size,
490
+ mode="bilinear",
491
+ align_corners=False,
492
+ ).squeeze(1) # [K, orig_H, orig_W]
493
+
494
+ return upsampled_logits, pred_scores
495
+
496
+
497
+ class Qwen2_5_VL_SinglePassSegmenter(nn.Module):
498
+ """
499
+ System-1 Single-Pass Multi-Object Segmenter built on Qwen2.5-VL.
500
+ - Drops the autoregressive LM head completely.
501
+ - Runs the ViT encoder and LLM backbone once.
502
+ - Captures both 1/14 (unmerged) and 1/28 (merged) ViT spatial grids from the single pass.
503
+ - Extracts K decision vectors from `<|object_ref_end|>` tokens in the input JSON.
504
+ - Decodes all K object masks in parallel.
505
+ """
506
+
507
+ def __init__(
508
+ self,
509
+ qwen_model_name: Optional[str] = "./Qwen3.5-4B",
510
+ backbone: Optional[nn.Module] = None,
511
+ processor: Optional[Any] = None,
512
+ torch_dtype: torch.dtype = torch.bfloat16,
513
+ device_map: Optional[Union[str, Dict[str, Any]]] = None,
514
+ quantization_config: Optional[Any] = None,
515
+ freeze_backbone: bool = True,
516
+ head_hidden_dim: int = 256,
517
+ head_fine_dim: int = 128,
518
+ num_fused_layers: int = 4,
519
+ ):
520
+ super().__init__()
521
+ self.num_fused_layers = num_fused_layers
522
+
523
+ if backbone is not None:
524
+ self.qwen = backbone
525
+ self.processor = processor
526
+ else:
527
+ if qwen_model_name is None:
528
+ raise ValueError("Provide either `qwen_model_name` or a pre-built `backbone`.")
529
+ self.processor = processor or AutoProcessor.from_pretrained(qwen_model_name)
530
+ load_kwargs: Dict[str, Any] = {"torch_dtype": torch_dtype}
531
+ if device_map is not None:
532
+ load_kwargs["device_map"] = device_map
533
+ if quantization_config is not None:
534
+ load_kwargs["quantization_config"] = quantization_config
535
+ self.qwen = AutoModel.from_pretrained(qwen_model_name, **load_kwargs)
536
+
537
+ # Support both Qwen2_5_VLModel and Qwen2_5_VLForConditionalGeneration
538
+ if hasattr(self.qwen, "model") and hasattr(self.qwen.model, "visual"):
539
+ self.backbone = self.qwen.model
540
+ else:
541
+ self.backbone = self.qwen
542
+
543
+ self.freeze_backbone = freeze_backbone
544
+ if freeze_backbone:
545
+ for param in self.backbone.parameters():
546
+ param.requires_grad = False
547
+
548
+ # Read dimensions safely across transformers versions
549
+ cfg = self.backbone.config
550
+ text_cfg = getattr(cfg, "text_config", cfg)
551
+ vis_cfg = cfg.vision_config
552
+
553
+ self.llm_hidden_size = getattr(text_cfg, "hidden_size", 2560)
554
+ self.vision_merged_dim = getattr(vis_cfg, "out_hidden_size", self.llm_hidden_size)
555
+ self.vision_unmerged_dim = getattr(vis_cfg, "hidden_size", 1024)
556
+ self.spatial_merge_size = getattr(vis_cfg, "spatial_merge_size", 2)
557
+ self.patch_size = getattr(vis_cfg, "patch_size", 16)
558
+
559
+ if self.processor is not None and hasattr(self.processor, "tokenizer"):
560
+ self.query_token_id = self.processor.tokenizer.convert_tokens_to_ids(OBJECT_QUERY_TOKEN)
561
+ self.im_end_token_id = self.processor.tokenizer.convert_tokens_to_ids("<|im_end|>")
562
+ self.image_token_id = self.processor.tokenizer.convert_tokens_to_ids("<|image_pad|>")
563
+ else:
564
+ self.query_token_id = getattr(cfg, "object_ref_end_token_id", 248048)
565
+ self.im_end_token_id = getattr(text_cfg, "eos_token_id", 248046)
566
+ self.image_token_id = getattr(cfg, "image_token_id", 248056)
567
+
568
+ self.mask_head = MultiScaleMaskHead(
569
+ llm_hidden_size=self.llm_hidden_size,
570
+ vision_merged_dim=self.vision_merged_dim,
571
+ vision_unmerged_dim=self.vision_unmerged_dim,
572
+ hidden_dim=head_hidden_dim,
573
+ fine_dim=head_fine_dim,
574
+ num_fused_layers=num_fused_layers,
575
+ )
576
+
577
+ # Hook buffers to capture single-pass visual outputs & last N LLM layers without extra memory
578
+ self._captured_visual: Dict[str, Any] = {}
579
+ self._captured_llm_layers: List[torch.Tensor] = []
580
+ self._visual_hook = self.backbone.visual.register_forward_hook(self._capture_visual_hook)
581
+ self._llm_hooks: List[Any] = []
582
+ self._register_llm_layer_hooks()
583
+ self._needs_mm_token_type_ids = self._check_needs_mm_token_type_ids()
584
+
585
+ def _check_needs_mm_token_type_ids(self) -> bool:
586
+ return "mm_token_type_ids" in inspect.signature(
587
+ self.backbone.forward
588
+ ).parameters or (
589
+ hasattr(self.backbone, "compute_3d_position_ids")
590
+ and "mm_token_type_ids" in inspect.signature(self.backbone.compute_3d_position_ids).parameters
591
+ )
592
+
593
+ def _get_llm_decoder_layers(self) -> Optional[nn.ModuleList]:
594
+ lm = getattr(self.backbone, "language_model", None)
595
+ if lm is None:
596
+ return None
597
+ if hasattr(lm, "layers"):
598
+ return lm.layers
599
+ if hasattr(lm, "model") and hasattr(lm.model, "layers"):
600
+ return lm.model.layers
601
+ return None
602
+
603
+ def _register_llm_layer_hooks(self) -> None:
604
+ for h in self._llm_hooks:
605
+ h.remove()
606
+ self._llm_hooks.clear()
607
+ layers = self._get_llm_decoder_layers()
608
+ if layers is not None and len(layers) > 0:
609
+ k = min(self.num_fused_layers, len(layers))
610
+ for layer in list(layers)[-k:]:
611
+ self._llm_hooks.append(layer.register_forward_hook(self._capture_llm_layer_hook))
612
+
613
+ def _capture_llm_layer_hook(self, module: nn.Module, inputs: Any, output: Any) -> None:
614
+ hidden = output[0] if isinstance(output, tuple) else output
615
+ if isinstance(hidden, torch.Tensor):
616
+ self._captured_llm_layers.append(hidden)
617
+
618
+ def _capture_visual_hook(self, module: nn.Module, inputs: Any, output: Any) -> None:
619
+ if hasattr(output, "last_hidden_state") and hasattr(output, "pooler_output"):
620
+ self._captured_visual["unmerged"] = output.last_hidden_state
621
+ # Clone reference before get_image_features mutates output.pooler_output into a tuple
622
+ self._captured_visual["merged"] = output.pooler_output
623
+ elif isinstance(output, tuple) and len(output) >= 2:
624
+ self._captured_visual["unmerged"] = output[0]
625
+ self._captured_visual["merged"] = output[1]
626
+ else:
627
+ self._captured_visual["merged"] = output
628
+ self._captured_visual["unmerged"] = None
629
+
630
+ def enable_lora(
631
+ self,
632
+ r: int = 8,
633
+ lora_alpha: int = 16,
634
+ lora_dropout: float = 0.05,
635
+ top_k_layers: int = 4,
636
+ target_modules: Tuple[str, ...] = (
637
+ "q_proj",
638
+ "k_proj",
639
+ "v_proj",
640
+ "o_proj",
641
+ "in_proj_qkv",
642
+ "out_proj",
643
+ ),
644
+ ) -> None:
645
+ """
646
+ Safe Stage 2 training helper:
647
+ Applies low-rank LoRA ONLY to the top `top_k_layers` (e.g., last 4 of 36 layers),
648
+ leaving the first ~32 layers 100% frozen so Qwen retains all world knowledge and
649
+ riddle-solving ability while the top layers adapt to spatial token compression.
650
+ """
651
+ from peft import LoraConfig, get_peft_model
652
+
653
+ layers = self._get_llm_decoder_layers()
654
+ resolved_targets: List[str] = list(target_modules)
655
+ if layers is not None and top_k_layers > 0 and len(layers) > top_k_layers:
656
+ total_l = len(layers)
657
+ allowed_layer_indices = set(range(total_l - top_k_layers, total_l))
658
+ exact_targets: List[str] = []
659
+ for name, _ in self.backbone.language_model.named_modules():
660
+ leaf = name.split(".")[-1]
661
+ if leaf not in target_modules:
662
+ continue
663
+ if any(
664
+ f"layers.{idx}." in name or name.startswith(f"layers.{idx}.")
665
+ for idx in allowed_layer_indices
666
+ ):
667
+ exact_targets.append(name)
668
+ if exact_targets:
669
+ resolved_targets = exact_targets
670
+
671
+ lora_cfg = LoraConfig(
672
+ r=r,
673
+ lora_alpha=lora_alpha,
674
+ lora_dropout=lora_dropout,
675
+ target_modules=resolved_targets,
676
+ bias="none",
677
+ )
678
+ self.backbone.language_model = get_peft_model(self.backbone.language_model, lora_cfg)
679
+ self._register_llm_layer_hooks()
680
+ self._needs_mm_token_type_ids = self._check_needs_mm_token_type_ids()
681
+ self.freeze_backbone = False
682
+
683
+ def _compute_window_index(
684
+ self,
685
+ grid_thw: torch.Tensor,
686
+ grid_thw_list: Optional[Sequence[Sequence[int]]] = None,
687
+ ) -> torch.Tensor:
688
+ if hasattr(self.backbone.visual, "get_window_index"):
689
+ window_index, _ = self.backbone.visual.get_window_index(grid_thw)
690
+ return window_index
691
+ vis_cfg = getattr(self.backbone.config, "vision_config", None)
692
+ window_size = getattr(vis_cfg, "window_size", 112)
693
+ m = self.spatial_merge_size
694
+ vit_merger_window_size = max(window_size // m // self.patch_size, 1)
695
+ window_index: List[torch.Tensor] = []
696
+ window_index_id = 0
697
+ rows = grid_thw_list if grid_thw_list is not None else grid_thw.tolist()
698
+ for grid_t, grid_h, grid_w in rows:
699
+ llm_grid_h, llm_grid_w = grid_h // m, grid_w // m
700
+ index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape(grid_t, llm_grid_h, llm_grid_w)
701
+ pad_h = vit_merger_window_size - llm_grid_h % vit_merger_window_size
702
+ pad_w = vit_merger_window_size - llm_grid_w % vit_merger_window_size
703
+ num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size
704
+ num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size
705
+ index_padded = F.pad(index, (0, pad_w, 0, pad_h), "constant", -100)
706
+ index_padded = index_padded.reshape(
707
+ grid_t, num_windows_h, vit_merger_window_size, num_windows_w, vit_merger_window_size
708
+ )
709
+ index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape(
710
+ grid_t, num_windows_h * num_windows_w, vit_merger_window_size, vit_merger_window_size
711
+ )
712
+ index_padded = index_padded.reshape(-1)
713
+ index_new = index_padded[index_padded != -100]
714
+ window_index.append(index_new + window_index_id)
715
+ window_index_id += grid_t * llm_grid_h * llm_grid_w
716
+ return torch.cat(window_index, dim=0)
717
+
718
+ def _reconstruct_2d_vision_features(
719
+ self,
720
+ merged_tokens: Union[torch.Tensor, Sequence[torch.Tensor]],
721
+ unmerged_tokens: Optional[torch.Tensor],
722
+ image_grid_thw: torch.Tensor,
723
+ grid_thw_list: Optional[Sequence[Sequence[int]]] = None,
724
+ ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
725
+ """
726
+ Reconstructs per-image 2D spatial grids from Qwen2.5-VL's window-permuted ViT outputs:
727
+ - coarse_grids: list of [1, C_merged, H_m, W_m] at 1/28 resolution
728
+ - fine_grids: list of [1, C_unmerged, 2*H_m, 2*W_m] at 1/14 resolution
729
+ """
730
+ m = self.spatial_merge_size
731
+ m_sq = m * m
732
+ rows = grid_thw_list if grid_thw_list is not None else image_grid_thw.tolist()
733
+
734
+ if isinstance(merged_tokens, (tuple, list)):
735
+ merged_cat = torch.cat(list(merged_tokens), dim=0)
736
+ else:
737
+ merged_cat = merged_tokens
738
+
739
+ # Un-permute the unmerged tokens from window attention order if the ViT uses window attention (Qwen2.5-VL);
740
+ # Qwen3.5-4B uses full ViT attention so its unmerged tokens are already in natural block-raster order.
741
+ vis_cfg = getattr(self.backbone.config, "vision_config", None)
742
+ has_window_attn = hasattr(vis_cfg, "window_size") and getattr(vis_cfg, "window_size", None) is not None
743
+ if unmerged_tokens is not None:
744
+ num_merged = merged_cat.shape[0]
745
+ if has_window_attn:
746
+ window_index = self._compute_window_index(image_grid_thw, grid_thw_list=rows)
747
+ reverse_indices = torch.argsort(window_index.to(unmerged_tokens.device))
748
+ unmerged_blocks = unmerged_tokens.view(num_merged, m_sq, -1)[reverse_indices]
749
+ else:
750
+ unmerged_blocks = unmerged_tokens.view(num_merged, m_sq, -1)
751
+ else:
752
+ unmerged_blocks = None
753
+
754
+ coarse_grids: List[torch.Tensor] = []
755
+ fine_grids: List[torch.Tensor] = []
756
+
757
+ offset = 0
758
+ for grid_t, grid_h, grid_w in rows:
759
+ h_m = grid_h // m
760
+ w_m = grid_w // m
761
+ n_tokens = grid_t * h_m * w_m
762
+
763
+ img_merged = merged_cat[offset : offset + n_tokens]
764
+ # Average across temporal dimension if grid_t > 1 (for static images grid_t == 1)
765
+ img_merged = img_merged.view(grid_t, h_m, w_m, -1).mean(dim=0)
766
+ f_coarse = img_merged.permute(2, 0, 1).unsqueeze(0).contiguous() # [1, C_out, H_m, W_m]
767
+ coarse_grids.append(f_coarse)
768
+
769
+ if unmerged_blocks is not None:
770
+ img_unmerged = unmerged_blocks[offset : offset + n_tokens]
771
+ # Matches Qwen2VLImageProcessor patch ordering: (T, H_m, W_m, m_h, m_w, C_in)
772
+ img_unmerged = img_unmerged.view(grid_t, h_m, w_m, m, m, -1).mean(dim=0)
773
+ f_fine = (
774
+ img_unmerged.permute(4, 0, 2, 1, 3)
775
+ .contiguous()
776
+ .view(1, self.vision_unmerged_dim, grid_h, grid_w)
777
+ )
778
+ fine_grids.append(f_fine)
779
+ else:
780
+ # Fallback if unmerged tokens are unavailable
781
+ f_fine = F.interpolate(f_coarse, size=(grid_h, grid_w), mode="bilinear", align_corners=False)
782
+ fine_grids.append(f_fine)
783
+
784
+ offset += n_tokens
785
+
786
+ return coarse_grids, fine_grids
787
+
788
+ def _extract_llm_visual_grids(
789
+ self,
790
+ last_hidden_states: torch.Tensor,
791
+ input_ids: torch.Tensor,
792
+ image_grid_thw: torch.Tensor,
793
+ layer_states: Optional[Sequence[torch.Tensor]] = None,
794
+ grid_thw_list: Optional[Sequence[Sequence[int]]] = None,
795
+ input_ids_cpu: Optional[torch.Tensor] = None,
796
+ ) -> List[Optional[torch.Tensor]]:
797
+ """
798
+ Extracts the LLM-contextualized `<|image_pad|>` visual tokens from the top LLM layers
799
+ and reshapes them into 2D grids [1, llm_dim, H_m, W_m] per image.
800
+ Because the user's JSON riddle precedes `<|image_pad|>`, these tokens carry
801
+ 36 layers of riddle-to-image self-attention context.
802
+ """
803
+ m = self.spatial_merge_size
804
+ base_dev = last_hidden_states.device
805
+ source_state = (
806
+ torch.stack([st.to(base_dev) for st in layer_states], dim=0).mean(dim=0)
807
+ if layer_states and len(layer_states) > 0
808
+ else last_hidden_states
809
+ )
810
+ rows = grid_thw_list if grid_thw_list is not None else image_grid_thw.tolist()
811
+ ids_src = input_ids_cpu if input_ids_cpu is not None else input_ids
812
+ target_dev = source_state.device
813
+ llm_grids: List[Optional[torch.Tensor]] = []
814
+ for b in range(ids_src.shape[0]):
815
+ grid_t, grid_h, grid_w = rows[b]
816
+ h_m, w_m = grid_h // m, grid_w // m
817
+ n_tokens = grid_t * h_m * w_m
818
+ img_pos = (ids_src[b] == self.image_token_id).nonzero(as_tuple=True)[0]
819
+ if img_pos.numel() == n_tokens:
820
+ if img_pos.device != target_dev:
821
+ img_pos = img_pos.to(device=target_dev, non_blocking=True)
822
+ vis_seq = source_state[b, img_pos, :] # [n_tokens, llm_dim]
823
+ vis_2d = (
824
+ vis_seq.view(grid_t, h_m, w_m, -1)
825
+ .mean(dim=0)
826
+ .permute(2, 0, 1)
827
+ .unsqueeze(0)
828
+ .contiguous()
829
+ )
830
+ llm_grids.append(vis_2d)
831
+ else:
832
+ llm_grids.append(None)
833
+ return llm_grids
834
+
835
+ def _extract_query_vectors(
836
+ self,
837
+ last_hidden_states: torch.Tensor,
838
+ input_ids: torch.Tensor,
839
+ layer_states: Optional[Sequence[torch.Tensor]] = None,
840
+ input_ids_cpu: Optional[torch.Tensor] = None,
841
+ ) -> List[torch.Tensor]:
842
+ """
843
+ Extracts the K `<|object_ref_end|>` decision vectors per batch item.
844
+ If multi-layer states are captured, returns [K, L, llm_dim] per batch item;
845
+ otherwise returns [K, llm_dim].
846
+ """
847
+ states_to_index = list(layer_states) if layer_states else [last_hidden_states]
848
+ ids_src = input_ids_cpu if input_ids_cpu is not None else input_ids
849
+ target_dev = states_to_index[0].device
850
+ batch_queries: List[torch.Tensor] = []
851
+ for b in range(ids_src.shape[0]):
852
+ pos = (ids_src[b] == self.query_token_id).nonzero(as_tuple=True)[0]
853
+ if pos.numel() == 0:
854
+ pos = (ids_src[b] == self.im_end_token_id).nonzero(as_tuple=True)[0]
855
+ if pos.numel() > 0:
856
+ pos = pos[-1:]
857
+ else:
858
+ pos = torch.tensor([ids_src.shape[1] - 1], dtype=torch.long, device=ids_src.device)
859
+ if pos.device != target_dev:
860
+ pos = pos.to(device=target_dev, non_blocking=True)
861
+ if len(states_to_index) > 1:
862
+ per_layer = [
863
+ st[b, (pos.to(st.device) if pos.device != st.device else pos), :].to(target_dev)
864
+ for st in states_to_index
865
+ ]
866
+ batch_queries.append(torch.stack(per_layer, dim=1)) # [K, L, llm_dim]
867
+ else:
868
+ batch_queries.append(states_to_index[0][b, pos, :]) # [K, llm_dim]
869
+ return batch_queries
870
+
871
+ def forward(
872
+ self,
873
+ input_ids: torch.LongTensor,
874
+ attention_mask: torch.Tensor,
875
+ pixel_values: torch.Tensor,
876
+ image_grid_thw: torch.LongTensor,
877
+ target_sizes: Optional[Sequence[Tuple[int, int]]] = None,
878
+ mm_token_type_ids: Optional[torch.Tensor] = None,
879
+ grid_thw_list: Optional[Sequence[Sequence[int]]] = None,
880
+ input_ids_cpu: Optional[torch.Tensor] = None,
881
+ ) -> Dict[str, Any]:
882
+ """
883
+ Single forward pass from tokenized JSON + image patches to pixel masks.
884
+
885
+ Args:
886
+ input_ids: [B, seq_len]
887
+ attention_mask: [B, seq_len]
888
+ pixel_values: [sum_patches, patch_dim]
889
+ image_grid_thw: [B, 3] (T, grid_h, grid_w)
890
+ target_sizes: optional list of (orig_H, orig_W) per image in batch.
891
+ Defaults to (grid_h * patch_size, grid_w * patch_size).
892
+ mm_token_type_ids: optional [B, seq_len] multimodal token type IDs (auto-computed if needed).
893
+ grid_thw_list: optional pre-extracted Python list of [T, grid_h, grid_w] per image to avoid CUDA syncs.
894
+ input_ids_cpu: optional CPU copy of `input_ids` to extract token positions without CUDA `.nonzero()` syncs.
895
+
896
+ Returns:
897
+ dict with:
898
+ - "mask_logits": List[Tensor] of shape [K_b, orig_H, orig_W] per image
899
+ - "mask_probs": List[Tensor] of shape [K_b, orig_H, orig_W] per image
900
+ - "pred_scores": List[Tensor] of shape [K_b] per image
901
+ """
902
+ self._captured_visual.clear()
903
+ self._captured_llm_layers.clear()
904
+
905
+ resolved_grid_thw = list(grid_thw_list) if grid_thw_list is not None else image_grid_thw.tolist()
906
+
907
+ backbone_kwargs: Dict[str, Any] = {
908
+ "input_ids": input_ids,
909
+ "attention_mask": attention_mask,
910
+ "pixel_values": pixel_values,
911
+ "image_grid_thw": image_grid_thw,
912
+ "use_cache": False,
913
+ "output_hidden_states": False,
914
+ "return_dict": True,
915
+ }
916
+ if self._needs_mm_token_type_ids:
917
+ if mm_token_type_ids is None:
918
+ mm_token_type_ids = (input_ids == self.image_token_id).to(
919
+ dtype=torch.long, device=input_ids.device
920
+ )
921
+ backbone_kwargs["mm_token_type_ids"] = mm_token_type_ids
922
+
923
+ grad_ctx = torch.no_grad() if self.freeze_backbone else torch.enable_grad()
924
+ with grad_ctx:
925
+ outputs = self.backbone(**backbone_kwargs)
926
+ last_hidden_states = outputs.last_hidden_state
927
+
928
+ merged_vis = self._captured_visual.get("merged")
929
+ unmerged_vis = self._captured_visual.get("unmerged")
930
+ if merged_vis is None:
931
+ raise RuntimeError("Visual forward hook did not capture features; check pixel_values input.")
932
+
933
+ coarse_grids, fine_grids = self._reconstruct_2d_vision_features(
934
+ merged_tokens=merged_vis,
935
+ unmerged_tokens=unmerged_vis,
936
+ image_grid_thw=image_grid_thw,
937
+ grid_thw_list=resolved_grid_thw,
938
+ )
939
+ llm_vis_grids = self._extract_llm_visual_grids(
940
+ last_hidden_states=last_hidden_states,
941
+ input_ids=input_ids,
942
+ image_grid_thw=image_grid_thw,
943
+ layer_states=self._captured_llm_layers,
944
+ grid_thw_list=resolved_grid_thw,
945
+ input_ids_cpu=input_ids_cpu,
946
+ )
947
+ batch_queries = self._extract_query_vectors(
948
+ last_hidden_states=last_hidden_states,
949
+ input_ids=input_ids,
950
+ layer_states=self._captured_llm_layers,
951
+ input_ids_cpu=input_ids_cpu,
952
+ )
953
+
954
+ head_param = next(self.mask_head.parameters())
955
+ head_dtype = head_param.dtype
956
+ head_device = head_param.device
957
+ mask_logits_list: List[torch.Tensor] = []
958
+ mask_probs_list: List[torch.Tensor] = []
959
+ pred_scores_list: List[torch.Tensor] = []
960
+
961
+ for b in range(len(batch_queries)):
962
+ if target_sizes is not None and b < len(target_sizes):
963
+ tgt_size = (int(target_sizes[b][0]), int(target_sizes[b][1]))
964
+ else:
965
+ tgt_size = (
966
+ int(resolved_grid_thw[b][1] * self.patch_size),
967
+ int(resolved_grid_thw[b][2] * self.patch_size),
968
+ )
969
+
970
+ f_llm_b = (
971
+ llm_vis_grids[b].to(device=head_device, dtype=head_dtype)
972
+ if llm_vis_grids[b] is not None
973
+ else None
974
+ )
975
+ logits_b, scores_b = self.mask_head.forward_single_image(
976
+ queries=batch_queries[b].to(device=head_device, dtype=head_dtype),
977
+ f_coarse=coarse_grids[b].to(device=head_device, dtype=head_dtype),
978
+ f_fine=fine_grids[b].to(device=head_device, dtype=head_dtype),
979
+ target_size=tgt_size,
980
+ f_llm_coarse=f_llm_b,
981
+ )
982
+ mask_logits_list.append(logits_b)
983
+ mask_probs_list.append(torch.sigmoid(logits_b))
984
+ pred_scores_list.append(scores_b)
985
+
986
+ return {
987
+ "mask_logits": mask_logits_list,
988
+ "mask_probs": mask_probs_list,
989
+ "pred_scores": pred_scores_list,
990
+ }
991
+
992
+ @torch.no_grad()
993
+ def predict_json(
994
+ self,
995
+ image: Any,
996
+ query_json: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]],
997
+ threshold: float = 0.5,
998
+ ) -> Dict[str, Any]:
999
+ """
1000
+ End-to-end System 1 inference:
1001
+ Takes a PIL image + JSON object descriptions, runs 1 forward pass, and returns
1002
+ a JSON dict with COCO polygon segmentations (`segmentation`), `bbox`, `area`, and `score`.
1003
+ """
1004
+ if self.processor is None:
1005
+ raise RuntimeError("`processor` is required for `predict_json`.")
1006
+
1007
+ self.eval()
1008
+ orig_w, orig_h = image.size
1009
+ messages, objects = build_segmentation_messages(image=image, query_input=query_json)
1010
+
1011
+ formatted_text = self.processor.apply_chat_template(
1012
+ messages, tokenize=False, add_generation_prompt=False
1013
+ )
1014
+ inputs = self.processor(
1015
+ text=[formatted_text],
1016
+ images=[image],
1017
+ padding=True,
1018
+ return_tensors="pt",
1019
+ )
1020
+ input_ids_cpu = inputs["input_ids"]
1021
+ grid_thw_list = inputs["image_grid_thw"].tolist()
1022
+ device = next(self.mask_head.parameters()).device
1023
+ inputs = {k: (v.to(device) if isinstance(v, torch.Tensor) else v) for k, v in inputs.items()}
1024
+
1025
+ outputs = self.forward(
1026
+ input_ids=inputs["input_ids"],
1027
+ attention_mask=inputs["attention_mask"],
1028
+ pixel_values=inputs["pixel_values"],
1029
+ image_grid_thw=inputs["image_grid_thw"],
1030
+ target_sizes=[(orig_h, orig_w)],
1031
+ grid_thw_list=grid_thw_list,
1032
+ input_ids_cpu=input_ids_cpu,
1033
+ )
1034
+
1035
+ probs = outputs["mask_probs"][0] # [K, orig_H, orig_W]
1036
+ scores = outputs["pred_scores"][0] # [K]
1037
+
1038
+ results = []
1039
+ for idx, obj in enumerate(objects):
1040
+ if idx >= probs.shape[0]:
1041
+ break
1042
+ prob_map = probs[idx]
1043
+ binary_mask = (prob_map >= threshold).to(torch.uint8)
1044
+ polygons, bbox, area = mask_to_polygons(binary_mask)
1045
+
1046
+ # Combine IoU head score with mean foreground probability
1047
+ fg_prob = float(prob_map[binary_mask > 0].mean().item()) if area > 0 else 0.0
1048
+ score = round(0.5 * float(scores[idx].item()) + 0.5 * fg_prob, 4)
1049
+
1050
+ results.append(
1051
+ {
1052
+ "id": obj["id"],
1053
+ "description": obj["description"],
1054
+ "score": score,
1055
+ "bbox": bbox,
1056
+ "area": area,
1057
+ "segmentation": polygons,
1058
+ }
1059
+ )
1060
+
1061
+ return {
1062
+ "image_size": {"height": orig_h, "width": orig_w},
1063
+ "objects": results,
1064
+ }
1065
+
1066
+
1067
+ class SegmentationLoss(nn.Module):
1068
+ """
1069
+ Combined Focal + Soft Dice + BCE + IoU Score Calibration loss for training the mask head.
1070
+ """
1071
+
1072
+ def __init__(
1073
+ self,
1074
+ bce_weight: float = 1.0,
1075
+ focal_weight: float = 2.0,
1076
+ dice_weight: float = 2.0,
1077
+ score_weight: float = 0.5,
1078
+ focal_alpha: float = 0.25,
1079
+ focal_gamma: float = 2.0,
1080
+ ):
1081
+ super().__init__()
1082
+ self.bce_weight = bce_weight
1083
+ self.focal_weight = focal_weight
1084
+ self.dice_weight = dice_weight
1085
+ self.score_weight = score_weight
1086
+ self.focal_alpha = focal_alpha
1087
+ self.focal_gamma = focal_gamma
1088
+
1089
+ def _dice_loss(self, logits: torch.Tensor, targets: torch.Tensor, eps: float = 1.0) -> torch.Tensor:
1090
+ probs = torch.sigmoid(logits).flatten(1)
1091
+ tgt = targets.flatten(1)
1092
+ intersection = (probs * tgt).sum(dim=1)
1093
+ union = probs.sum(dim=1) + tgt.sum(dim=1)
1094
+ dice = (2.0 * intersection + eps) / (union + eps)
1095
+ return (1.0 - dice).mean()
1096
+
1097
+ def _sigmoid_focal_loss(self, logits: torch.Tensor, targets: torch.Tensor) -> torch.Tensor:
1098
+ bce = F.binary_cross_entropy_with_logits(logits, targets, reduction="none")
1099
+ probs = torch.sigmoid(logits)
1100
+ p_t = probs * targets + (1.0 - probs) * (1.0 - targets)
1101
+ alpha_t = self.focal_alpha * targets + (1.0 - self.focal_alpha) * (1.0 - targets)
1102
+ focal = alpha_t * ((1.0 - p_t) ** self.focal_gamma) * bce
1103
+ return focal.mean()
1104
+
1105
+ def forward(
1106
+ self,
1107
+ mask_logits_list: Sequence[torch.Tensor],
1108
+ pred_scores_list: Sequence[torch.Tensor],
1109
+ gt_masks_list: Sequence[torch.Tensor],
1110
+ ) -> Dict[str, torch.Tensor]:
1111
+ total_bce = torch.tensor(0.0, device=mask_logits_list[0].device)
1112
+ total_focal = torch.tensor(0.0, device=mask_logits_list[0].device)
1113
+ total_dice = torch.tensor(0.0, device=mask_logits_list[0].device)
1114
+ total_score = torch.tensor(0.0, device=mask_logits_list[0].device)
1115
+
1116
+ n_batches = max(len(mask_logits_list), 1)
1117
+ for logits, scores, gt in zip(mask_logits_list, pred_scores_list, gt_masks_list):
1118
+ gt = gt.to(device=logits.device, dtype=logits.dtype)
1119
+ if gt.shape[-2:] != logits.shape[-2:]:
1120
+ gt = F.interpolate(gt.unsqueeze(1), size=logits.shape[-2:], mode="nearest").squeeze(1)
1121
+
1122
+ total_bce = total_bce + F.binary_cross_entropy_with_logits(logits, gt)
1123
+ total_focal = total_focal + self._sigmoid_focal_loss(logits, gt)
1124
+ total_dice = total_dice + self._dice_loss(logits, gt)
1125
+
1126
+ with torch.no_grad():
1127
+ pred_bin = (torch.sigmoid(logits) >= 0.5).to(gt.dtype).flatten(1)
1128
+ gt_flat = gt.flatten(1)
1129
+ inter = (pred_bin * gt_flat).sum(dim=1)
1130
+ union = (pred_bin + gt_flat - pred_bin * gt_flat).sum(dim=1).clamp(min=1e-6)
1131
+ true_iou = inter / union
1132
+
1133
+ total_score = total_score + F.mse_loss(scores, true_iou.to(scores.dtype))
1134
+
1135
+ total_bce = total_bce / n_batches
1136
+ total_focal = total_focal / n_batches
1137
+ total_dice = total_dice / n_batches
1138
+ total_score = total_score / n_batches
1139
+
1140
+ loss = (
1141
+ self.bce_weight * total_bce
1142
+ + self.focal_weight * total_focal
1143
+ + self.dice_weight * total_dice
1144
+ + self.score_weight * total_score
1145
+ )
1146
+ return {
1147
+ "loss": loss,
1148
+ "bce_loss": total_bce.detach(),
1149
+ "focal_loss": total_focal.detach(),
1150
+ "dice_loss": total_dice.detach(),
1151
+ "score_loss": total_score.detach(),
1152
+ }
README.md ADDED
@@ -0,0 +1,233 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ datasets:
4
+ - Wojtekb30/ref_coco_lichengunc
5
+ language:
6
+ - en
7
+ pipeline_tag: mask-generation
8
+ tags:
9
+ - mask-generation
10
+ - image-segmentation
11
+ - System1
12
+ ---
13
+
14
+ <p align="center">
15
+ <img src="logo.png" alt="LIS1 Logo" width="320"/>
16
+ </p>
17
+
18
+ <h1 align="center">LIS1: Single-Pass Visual Reasoning & Referring Image Segmentation</h1>
19
+
20
+ **LIS1** (Linguistic Image Segmenter) is a fast, non-autoregressive visual reasoning and referring image segmentation model family built on **Qwen3.5** (`0.8B`, `2B`, `4B`, `9B`, and `27B`).
21
+
22
+ Unlike traditional reasoning segmentation models (such as LISA) that generate text tokens autoregressively and rely on a separate heavyweight SAM backbone, **LIS1** performs both complex commonsense/riddle reasoning and crisp pixel-level mask decoding in a **single forward pass** using Qwen3.5's native Vision Transformer and LLM hidden states—with **zero autoregressive token generation** and **no external SAM encoder**.
23
+
24
+ ---
25
+
26
+ ## Example Output
27
+
28
+ Below is a single-image inference example (`image1.jpg` to `result_0_boxes_and_masks.png`) generated by **LIS1-0.8B** in 4-bit quantization (`example.py`) for three simultaneous queries combining direct object names and functional riddles:
29
+ - `"toast"` (Red — score `0.75`)
30
+ - `"red sauce"` (Green — score `0.80`)
31
+ - `"something you can drink"` (Blue — score `0.88`)
32
+
33
+ | Input Image (`image1.jpg`) | Rendered Output (`result_0_boxes_and_masks.png`) |
34
+ | :---: | :---: |
35
+ | <img src="image1.jpg" alt="Input Image" width="460"/> | <img src="result_0_boxes_and_masks.png" alt="Segmented Output" width="460"/> |
36
+
37
+ ---
38
+
39
+ ## ReasonSeg Benchmark Results
40
+
41
+ All **LIS1** models were trained **exclusively on the [RefCOCO dataset by lichengunc](https://huggingface.co/datasets/Wojtekb30/ref_coco_lichengunc)**, which includes RefCOCO, RefCOCO+, and RefCOCOg (with 35% synthetic functional riddle augmentation). They were evaluated **zero-shot on the ReasonSeg benchmark**, without seeing ReasonSeg training images, using the [ReasonSeg validation split](https://huggingface.co/datasets/Ricky06662/ReasonSeg_val) and [ReasonSeg test split](https://huggingface.co/datasets/Ricky06662/ReasonSeg_test).
42
+
43
+ Each repository provides two checkpoint variants (in both `.safetensors` and `.pt` formats):
44
+ - **`noLora` (Stage 1)**: Trained mask head with the base Qwen3.5 model kept fully frozen.
45
+ - **`forLora` (Stage 2)**: Trained mask head + lightweight PEFT LoRA adapter (`r=8`, `alpha=16`) applied only to the top 4 LLM layers.
46
+
47
+ | Method | Val overall gIoU | Val overall cIoU | Test short query gIoU | Test short query cIoU | Test long query gIoU | Test long query cIoU | Test overall gIoU | Test overall cIoU |
48
+ |---|---:|---:|---:|---:|---:|---:|---:|---:|
49
+ | OVSeg (Liang et al., 2023) | 28.5 | 18.6 | 18.0 | 15.5 | 28.7 | 22.5 | 26.1 | 20.8 |
50
+ | GRES (Liu et al., 2023b) | 22.4 | 19.9 | 17.6 | 15.0 | 22.6 | 23.8 | 21.3 | 22.0 |
51
+ | X-Decoder (Zou et al., 2023) | 22.6 | 17.9 | 20.4 | 11.6 | 22.2 | 17.5 | 21.7 | 16.3 |
52
+ | SEEM (Zou et al., 2023b) | 25.5 | 21.2 | 20.1 | 11.5 | 25.6 | 20.8 | 24.3 | 18.7 |
53
+ | LISA-7B | 44.4 | 46.0 | 37.6 | 34.4 | 36.6 | 34.7 | 36.8 | 34.1 |
54
+ | LISA-7B (ft) | 52.9 | 54.0 | 40.6 | 40.6 | 49.4 | 51.0 | 47.3 | 48.4 |
55
+ | LISA-13B | 48.9 | 46.9 | 39.9 | 43.3 | 46.4 | 46.5 | 44.8 | 45.8 |
56
+ | LISA-13B (ft) | 56.2 | 62.9 | 44.3 | 42.0 | 54.0 | 54.3 | 51.7 | 51.1 |
57
+ | LISA-Llama2-13B (ft) | 60.0 | 67.8 | 43.9 | 45.8 | 54.0 | 53.8 | 51.5 | 51.3 |
58
+ | LIS1-0.8B noLora | 31.7 | 32.5 | 28.6 | 23.7 | 27.3 | 30.6 | 27.5 | 29.6 |
59
+ | LIS1-0.8B forLora | 32.4 | 31.7 | 30.4 | 23.9 | 29.1 | 29.6 | 29.4 | 28.6 |
60
+ | LIS1-2B noLora | 37.4 | 33.1 | 35.1 | 27.2 | 34.5 | 33.6 | 34.6 | 32.6 |
61
+ | LIS1-2B forLora | 37.5 | 33.1 | 35.5 | 27.1 | 34.6 | 34.3 | 34.8 | 33.1 |
62
+ | LIS1-4B noLora | 42.3 | 38.3 | 39.9 | 29.9 | 43.5 | 40.5 | 42.8 | 38.7 |
63
+ | LIS1-4B forLora | 43.5 | 39.6 | 40.1 | 29.7 | 42.9 | 39.2 | 42.4 | 37.7 |
64
+ | LIS1-9B noLora | 45.2 | 38.3 | 43.1 | 35.7 | 42.7 | 37.3 | 42.8 | 37.0 |
65
+ | LIS1-9B forLora | 45.1 | 40.0 | 40.8 | 33.3 | 44.7 | 40.9 | 43.9 | 39.6 |
66
+ | LIS1-27B noLora | 54.1 | 54.3 | 46.9 | 44.0 | 49.9 | 49.8 | 49.3 | 49.0 |
67
+ | LIS1-27B forLora | 54.0 | 54.8 | 47.1 | 44.6 | 50.7 | 51.4 | 50.0 | 50.4 |
68
+
69
+ Other results come from [evaluation done by the team behind LISA](https://github.com/JIA-Lab-research/LISA/blob/main/imgs/table1.jpg)
70
+
71
+ LIS1 4B and bigger had better results than Ultralytics MobileSAM and YOLOE Open Vocabulary in iternal evaluation.
72
+
73
+ ---
74
+
75
+ ## Installation & Quick Start
76
+
77
+ ### 1. Install Dependencies
78
+
79
+ ```bash
80
+ pip install transformers peft safetensors pillow opencv-python numpy pyarrow torch torchvision
81
+ ```
82
+
83
+ *(Optional)* For 4-bit or 8-bit quantization and faster Qwen3.5 linear-attention kernels on CUDA GPUs:
84
+
85
+ ```bash
86
+ pip install bitsandbytes accelerate causal-conv1d flash-linear-attention
87
+ ```
88
+
89
+ ### 2. Minimal Inference Code
90
+
91
+ See **`example.py`** for the complete minimal script that produces the rendered image above. Make sure `LIS1ModelCode.py` and `lis1_infer.py` are in your working directory:
92
+
93
+ ```python
94
+ from PIL import Image
95
+ from lis1_infer import LIS1Inferencer
96
+
97
+ # 1. Initialize the inferencer
98
+ inferencer = LIS1Inferencer(debug=True)
99
+
100
+ # 2. Load the base Qwen3.5 model, LIS1 mask head (.safetensors or .pt), and optional LoRA adapter
101
+ # Swap "0.8B" with "2B", "4B", "9B", or "27B" depending on the model size you use.
102
+ inferencer.load_model(
103
+ qwen_path="./Qwen3.5-9B",
104
+ lis_model_path="./LIS1-9B-forLora.safetensors",
105
+ lora_adapter="./LIS1-9B-LoraAdapter", # Pass None if using a *-noLora checkpoint
106
+ device="cuda",
107
+ max_pixels=524288,
108
+ load_in_4bit=True, # Set False for full bfloat16, or load_in_8bit=True for 8-bit
109
+ )
110
+
111
+ # 3. Prepare images and natural language / riddle queries
112
+ images = [Image.open("image1.jpg").convert("RGB")]
113
+ queries = [["toast", "red sauce", "something you can drink"]]
114
+
115
+ # 4. Run single-pass segmentation
116
+ outputs = inferencer.run_model(
117
+ images=images,
118
+ queries=queries,
119
+ threshold=0.7,
120
+ isolate_queries=True,
121
+ cache_vision=True,
122
+ exclusive_masks=True,
123
+ )
124
+
125
+ # 5. Render masks, contours, bounding boxes, and non-overlapping labels
126
+ rendered = inferencer.render_image(images=images, outputs=outputs)
127
+ rendered[0].save("result_0_boxes_and_masks.png")
128
+
129
+ # 6. Free GPU/CPU memory when finished
130
+ inferencer.offload_model()
131
+ ```
132
+
133
+ ---
134
+
135
+ ## How the LIS1 Architecture Works
136
+
137
+ LIS1 turns a pretrained Qwen3.5 vision-language model into a single-pass, multi-object pixel segmenter through six sequential stages:
138
+
139
+ ### Step 1: Causal Riddle-Before-Image Prompt Construction
140
+ In causal language models, a token can only attend to tokens that appear earlier in the sequence. LIS1 formats the multimodal chat messages in a specific causal order:
141
+ 1. **System Prompt**: Instructs the model to act as a single-pass visual reasoning and segmentation model.
142
+ 2. **User Turn (JSON Queries First, Image Second)**: The target descriptions or riddles are serialized into a JSON block *before* the `<|image_pad|>` visual tokens. Because the text precedes the image, every visual patch token inside the LLM can attend directly to the user's riddles across all 24–64 layers.
143
+ 3. **Prefilled Assistant Turn**: Instead of generating tokens one by one, LIS1 prefills the assistant response with a JSON structure wrapping each query inside Qwen's native grounding delimiters: `<|object_ref_start|>{description}<|object_ref_end|>`. Crossing into the assistant turn and reading `<|object_ref_end|>` activates Qwen's internal spatial localization circuit in a single forward pass.
144
+
145
+ ### Step 2: Single-Pass Dual-Scale Vision Extraction (With LRU Caching)
146
+ When an image enters Qwen3.5's Vision Transformer (`backbone.visual`), a forward hook captures two spatial feature representations from the same pass without running an external segmentation backbone:
147
+ - **Fine Unmerged ViT Features (`f_fine`)**: Captured at 1/16 resolution before spatial patch merging, preserving sharp object edges and thin structures.
148
+ - **Coarse Merged ViT Features (`f_coarse`)**: Captured at 1/32 resolution after 2x2 spatial merging (`spatial_merge_size = 2`).
149
+ - **Vision Caching**: When segmenting multiple isolated queries on the same image (`isolate_queries=True, cache_vision=True`), LIS1 runs the ViT encoder only **once**, caches the reconstructed 2D spatial grids (`f_coarse`, `f_fine`) and merged visual embeddings in an LRU cache, and injects the cached embeddings directly into the LLM token stream via `masked_scatter`.
150
+
151
+ ### Step 3: LLM Forward Pass & Multi-Layer Feature Readout
152
+ The sequence of text and visual tokens passes once through the Qwen3.5 hybrid backbone (which interleaves Gated DeltaNet linear-attention layers with full self-attention layers; the `lm_head` is dropped completely). Forward hooks on the **last 4 decoder layers** extract two sets of representations:
153
+ - **Riddle-Conditioned Visual Grid (`f_llm_coarse`)**: The hidden states at all `<|image_pad|>` positions are averaged across the top 4 layers and reshaped back into a 2D spatial grid that now carries full multimodal reasoning context.
154
+ - **Multi-Layer Object Decision Vectors (`queries`)**: For each of the K target objects, the hidden states at the `<|object_ref_end|>` token position are extracted from the last 4 layers (`[K, 4, llm_dim]`).
155
+
156
+ *(Note on Quantization)*: When 4-bit or 8-bit `bitsandbytes` quantization is enabled, LIS1 automatically excludes the ViT encoder (`visual`) and Qwen3.5's tiny Gated DeltaNet decay gates (`in_proj_a`, `in_proj_b`) from quantization so spatial accuracy and linear-attention stability are preserved.
157
+
158
+ ### Step 4: Cross-Attention & Dynamic Feature Modulation
159
+ Inside `MultiScaleMaskHead` (kept in `float32` for crisp boundary decoding):
160
+ 1. **Layer Fusion**: The 4 captured LLM layers for each query vector are normalized via per-layer `LayerNorm` and combined using learned softmax weights into a single decision vector per object (`[K, llm_dim]`).
161
+ 2. **Visual Context Fusion**: The coarse ViT grid (`f_coarse`) and the LLM-contextualized visual grid (`f_llm_coarse`) are projected to `hidden_dim = 256` using 1x1 `Conv2d + GroupNorm + GELU` blocks and combined via a learned `tanh`-gated residual connection.
162
+ 3. **2D Cross-Attention**: The K object queries are projected to `256` dimensions and refined via 4-head cross-attention over the coarse 2D visual tokens, which are augmented with parameter-free 2D sine-cosine positional embeddings.
163
+ 4. **Calibrated IoU Score Prediction**: A 2-layer MLP (`score_head`) reads the refined query vector to predict an intrinsic mask confidence / IoU score in [0, 1].
164
+
165
+ ### Step 5: Coarse-to-Fine Spatial Mask Decoding
166
+ All K object masks are decoded in parallel through a three-stage hierarchy:
167
+ 1. **Coarse Stage (1/32 Scale)**: Each query computes a spatial dot-product cosine similarity map against both the ViT coarse features and the LLM visual features. Simultaneously, the query generates a channel-wise sigmoid gate to modulate the 2D coarse feature map. The modulated features and similarity map (`256 + 1` channels) are fused with a 3x3 convolution and a residual refinement block.
168
+ 2. **Fine Skip-Connection Stage (1/16 Scale)**: The coarse features are bilinearly upsampled 2x and concatenated with the fine unmerged ViT skip features (`f_fine`, projected to `128` channels and gated by the query vector). Two consecutive 3x3 residual refinement blocks sharpen the object boundaries.
169
+ 3. **Full-Resolution Output Stage**: A learned 2x transposed convolution (`ConvTranspose2d` to 1/8 scale) followed by a 1x1 convolution produces mask logits, which are bilinearly interpolated to the exact original image dimensions `(H, W)`.
170
+
171
+ ### Step 6: Post-Processing & Two-Stage Training
172
+ - **Inference Post-Processing**: Pixel probabilities are thresholded, cleaned with connected-component filtering (`filter_mask_components` to drop tiny stray noise islands while preserving small single-component objects), optionally disambiguated across queries via pixel-wise winner-takes-all (`exclusive_masks=True`), and converted into both binary masks and COCO polygons/bounding boxes. The final confidence score blends the IoU score-head output (50%) with the mean foreground pixel probability (50%).
173
+ - **Two-Stage Training**:
174
+ - **Stage 1 (`noLora`)**: Freezes all of the Qwen3.5 backbone and trains only `MultiScaleMaskHead` for 5 epochs using a combination of Binary Cross-Entropy (1.0), Sigmoid Focal Loss (2.0), Soft Dice Loss (2.0), and MSE IoU Score Calibration Loss (0.5).
175
+ - **Stage 2 (`forLora`)**: Keeps the first 85-95% of the LLM layers completely frozen (preserving pretrained world knowledge and riddle-solving capabilities) and attaches low-rank LoRA adapters (`r=8`) **only to the top 4 LLM layers** (`q_proj, k_proj, v_proj, o_proj, in_proj_qkv, out_proj`) for 1–3 epochs of joint fine-tuning.
176
+
177
+ ---
178
+
179
+ ## Training and Benchmark Commands
180
+
181
+ The commands below assume the project scripts are in the current directory and the datasets are available locally. Replace `{SIZE}` with a Qwen3.5 model size such as `0.8B`, `2B`, `4B`, `9B`, or `27B`; update the model and dataset paths for your setup.
182
+
183
+ ### Stage 1: Train the segmentation head
184
+
185
+ ```bash
186
+ python3 train.py \
187
+ --stage 1 \
188
+ --model-path ./Qwen3.5-{SIZE} \
189
+ --data-root ./ref_coco_lichengunc \
190
+ --dataset all \
191
+ --riddle-prob 0.35 \
192
+ --epochs 5 \
193
+ --batch-size 4 \
194
+ --max-objects 4 \
195
+ --lr-head 2e-4 \
196
+ --output-dir ./checkpoints_{SIZE}
197
+ ```
198
+
199
+ ### Stage 2: Fine-tune with LoRA
200
+
201
+ Resume from the Stage 1 checkpoint you want to continue from (the example uses epoch 5):
202
+
203
+ ```bash
204
+ python3 train.py \
205
+ --stage 2 \
206
+ --model-path ./Qwen3.5-{SIZE} \
207
+ --data-root ./ref_coco_lichengunc \
208
+ --dataset all \
209
+ --riddle-prob 0.35 \
210
+ --resume-ckpt ./checkpoints_{SIZE}/segmenter_stage1_epoch5.pt \
211
+ --lora-top-layers 4 \
212
+ --lora-r 8 \
213
+ --lora-alpha 16 \
214
+ --epochs 3 \
215
+ --batch-size 4 \
216
+ --lr-head 1e-4 \
217
+ --lr-lora 2e-5 \
218
+ --output-dir ./checkpoints_{SIZE}
219
+ ```
220
+
221
+ ### Evaluate on ReasonSeg
222
+
223
+ ```bash
224
+ python3 benchmark_reasonseg.py \
225
+ --model-path ./Qwen3.5-{SIZE} \
226
+ --data-dir ./ReasonSeg_val \
227
+ --test-dir ./ReasonSeg_test \
228
+ --resume-ckpt ./checkpoints_{SIZE}/segmenter_stage2_epoch3.pt \
229
+ --method-name "LIS1-{SIZE} (Stage 2)" \
230
+ --out-md ./reasonseg_benchmark-{SIZE}.md
231
+ ```
232
+
233
+ Use the Stage 1 checkpoint path instead to benchmark a `noLora` model. Stage 2 checkpoints require the corresponding LoRA adapter directory saved alongside the checkpoint; the path recorded in the checkpoint must resolve when running the benchmark. `train.py` defaults to CUDA when available, while the benchmark command can be run on CPU by adding `--device cpu`.
benchmark_reasonseg.py ADDED
@@ -0,0 +1,791 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ from collections import deque
3
+ from concurrent.futures import Future, ThreadPoolExecutor
4
+ import copy
5
+ import io
6
+ import json
7
+ import pathlib
8
+ import threading
9
+ import time
10
+ from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple
11
+
12
+ import cv2
13
+ import numpy as np
14
+ import torch
15
+ import torch.nn.functional as F
16
+ from PIL import Image
17
+
18
+ from LIS1ModelCode import (
19
+ DEFAULT_SYSTEM_PROMPT,
20
+ OBJECT_QUERY_TOKEN,
21
+ Qwen2_5_VL_SinglePassSegmenter,
22
+ build_segmentation_messages,
23
+ )
24
+ from train import load_checkpoint
25
+
26
+
27
+ BASELINE_ROWS = [
28
+ ("OVSeg (Liang et al., 2022)", 28.5, 18.6, 18.0, 15.5, 28.7, 22.5, 26.1, 20.8),
29
+ ("GRES (Liu et al., 2023b)", 22.4, 19.9, 17.6, 15.0, 22.6, 23.8, 21.3, 22.0),
30
+ ("X-Decoder (Zou et al., 2023)", 22.6, 17.9, 20.4, 11.6, 22.2, 17.5, 21.7, 16.3),
31
+ ("SEEM (Zou et al., 2023b)", 25.5, 21.2, 20.1, 11.5, 25.6, 20.8, 24.3, 18.7),
32
+ ("LISA-7B", 44.4, 46.0, 37.6, 34.4, 36.6, 34.7, 36.8, 34.1),
33
+ ("LISA-7B (ft)", 52.9, 54.0, 40.6, 40.6, 49.4, 51.0, 47.3, 48.4),
34
+ ("LISA-13B", 48.9, 46.9, 39.9, 43.3, 46.4, 46.5, 44.8, 45.8),
35
+ ("LISA-13B (ft)", 56.2, 62.9, 44.3, 42.0, 54.0, 54.3, 51.7, 51.1),
36
+ ("LISA-Llama2-13B (ft)", 60.0, 67.8, 43.9, 45.8, 54.0, 53.8, 51.5, 51.3),
37
+ ]
38
+
39
+
40
+ def is_long_query(text: str, explicit_flag: Optional[bool] = None) -> bool:
41
+ """
42
+ Determines whether a ReasonSeg query belongs to the 'long query' (sentence/riddle)
43
+ or 'short query' (phrase) subset.
44
+ Uses `is_sentence` if present in raw ReasonSeg JSON; otherwise checks sentence structure.
45
+ """
46
+ if explicit_flag is not None:
47
+ return bool(explicit_flag)
48
+ cleaned = text.strip()
49
+ if "?" in cleaned or cleaned.endswith(".") or len(cleaned.split()) >= 10:
50
+ return True
51
+ return False
52
+
53
+
54
+ def _decode_parquet_image(img_entry: Any) -> Image.Image:
55
+ if isinstance(img_entry, dict) and img_entry.get("bytes"):
56
+ return Image.open(io.BytesIO(img_entry["bytes"])).convert("RGB")
57
+ if isinstance(img_entry, dict) and img_entry.get("path"):
58
+ return Image.open(img_entry["path"]).convert("RGB")
59
+ if isinstance(img_entry, (bytes, bytearray)):
60
+ return Image.open(io.BytesIO(img_entry)).convert("RGB")
61
+ if isinstance(img_entry, Image.Image):
62
+ return img_entry.convert("RGB")
63
+ raise ValueError(f"Unsupported parquet image entry format: {type(img_entry)}")
64
+
65
+
66
+ def _unpack_parquet_mask_slice(mask_slice: Any) -> np.ndarray:
67
+ """
68
+ Unpacks a 1-row Arrow `mask` slice (`list<list<bool>>` / `list<list<uint8>>`) in C++
69
+ without materializing millions of nested Python `bool` objects via `to_pydict()`.
70
+ """
71
+ import pyarrow as pa
72
+ import pyarrow.compute as pc
73
+
74
+ if pa.types.is_nested(mask_slice.type):
75
+ row_inner = pc.list_flatten(mask_slice)
76
+ h = len(row_inner)
77
+ if h == 0:
78
+ return np.zeros((0, 0), dtype=np.uint8)
79
+ if pa.types.is_nested(row_inner.type):
80
+ flat_arr = pc.list_flatten(row_inner)
81
+ flat_np = flat_arr.to_numpy(zero_copy_only=False)
82
+ if flat_np.dtype == np.bool_:
83
+ flat_u8 = flat_np.view(np.uint8)
84
+ else:
85
+ flat_u8 = (flat_np > 0).astype(np.uint8, copy=False)
86
+ w = flat_u8.size // h
87
+ if h * w == flat_u8.size:
88
+ out = flat_u8.reshape(h, w)
89
+ return out if out.flags.writeable else out.copy()
90
+
91
+ raw_val = mask_slice[0].as_py() if hasattr(mask_slice[0], "as_py") else mask_slice[0]
92
+ if isinstance(raw_val, dict) and raw_val.get("bytes"):
93
+ mask_img = Image.open(io.BytesIO(raw_val["bytes"])).convert("L")
94
+ return (np.asarray(mask_img, dtype=np.uint8) > 0).astype(np.uint8)
95
+ if isinstance(raw_val, (bytes, bytearray)):
96
+ mask_img = Image.open(io.BytesIO(raw_val)).convert("L")
97
+ return (np.asarray(mask_img, dtype=np.uint8) > 0).astype(np.uint8)
98
+ out = np.asarray(raw_val, dtype=np.uint8)
99
+ return out if out.flags.writeable else out.copy()
100
+
101
+
102
+ def _rasterize_json_shapes(shapes: List[Dict[str, Any]], height: int, width: int) -> np.ndarray:
103
+ mask = np.zeros((height, width), dtype=np.uint8)
104
+ for shape in shapes:
105
+ pts = np.asarray(shape.get("points", []), dtype=np.float32)
106
+ if pts.ndim == 2 and pts.shape[0] >= 3:
107
+ cv2.fillPoly(mask, [np.round(pts).astype(np.int32)], 1)
108
+ return mask
109
+
110
+
111
+ def _iter_raw_reasonseg_entries(
112
+ data_dir: pathlib.Path,
113
+ max_samples: int = 0,
114
+ ) -> Iterator[Dict[str, Any]]:
115
+ """
116
+ Streams lightweight, undecoded ReasonSeg entries so worker threads can decode JPEGs,
117
+ unpack Arrow bitmasks, and run `processor` in parallel.
118
+ """
119
+ parquet_files = sorted(data_dir.rglob("*.parquet"))
120
+ yielded = 0
121
+
122
+ if parquet_files:
123
+ try:
124
+ import pyarrow.parquet as pq
125
+ except ImportError as exc:
126
+ raise ImportError(
127
+ "Reading ReasonSeg parquet files requires `pyarrow`. "
128
+ "Install it on your server with: `pip install pyarrow`"
129
+ ) from exc
130
+
131
+ for pfile in parquet_files:
132
+ pf = pq.ParquetFile(pfile)
133
+ for batch in pf.iter_batches(batch_size=16):
134
+ col_names = set(batch.schema.names)
135
+ meta_cols = [c for c in ("image", "text", "image_id", "is_sentence") if c in col_names]
136
+ rows = batch.select(meta_cols).to_pydict()
137
+ mask_col = batch.column("mask")
138
+ n_rows = len(rows["text"])
139
+ explicit_flags = rows.get("is_sentence")
140
+ image_ids = rows.get("image_id")
141
+
142
+ for i in range(n_rows):
143
+ text = str(rows["text"][i]).strip()
144
+ explicit_flag = explicit_flags[i] if explicit_flags is not None else None
145
+ yield {
146
+ "kind": "parquet",
147
+ "image_id": str(image_ids[i]) if image_ids is not None else str(i),
148
+ "img_entry": rows["image"][i],
149
+ "mask_slice": mask_col.slice(i, 1),
150
+ "text": text,
151
+ "is_long": is_long_query(text, explicit_flag),
152
+ }
153
+ yielded += 1
154
+ if max_samples > 0 and yielded >= max_samples:
155
+ return
156
+ return
157
+
158
+ json_files = sorted(data_dir.rglob("*.json"))
159
+ for jpath in json_files:
160
+ data = json.loads(jpath.read_text(encoding="utf-8"))
161
+ if "text" not in data or "shapes" not in data:
162
+ continue
163
+ img_path = None
164
+ for ext in (".jpg", ".jpeg", ".png", ".JPG", ".PNG"):
165
+ cand = jpath.with_suffix(ext)
166
+ if cand.exists():
167
+ img_path = cand
168
+ break
169
+ if img_path is None:
170
+ continue
171
+
172
+ shapes = data.get("shapes", [])
173
+ texts = data["text"] if isinstance(data["text"], list) else [data["text"]]
174
+ explicit_flag = data.get("is_sentence", None)
175
+ for t in texts:
176
+ t_str = str(t).strip()
177
+ if not t_str:
178
+ continue
179
+ yield {
180
+ "kind": "json",
181
+ "image_id": jpath.stem,
182
+ "img_path": img_path,
183
+ "shapes": shapes,
184
+ "text": t_str,
185
+ "is_long": is_long_query(t_str, explicit_flag),
186
+ }
187
+ yielded += 1
188
+ if max_samples > 0 and yielded >= max_samples:
189
+ return
190
+
191
+
192
+ def _decode_raw_entry(entry: Dict[str, Any]) -> Dict[str, Any]:
193
+ if entry["kind"] == "parquet":
194
+ pil_img = _decode_parquet_image(entry["img_entry"])
195
+ gt_mask = _unpack_parquet_mask_slice(entry["mask_slice"])
196
+ else:
197
+ pil_img = Image.open(entry["img_path"]).convert("RGB")
198
+ w, h = pil_img.size
199
+ gt_mask = _rasterize_json_shapes(entry["shapes"], height=h, width=w)
200
+
201
+ return {
202
+ "image_id": entry["image_id"],
203
+ "image": pil_img,
204
+ "text": entry["text"],
205
+ "gt_mask": gt_mask,
206
+ "is_long": entry["is_long"],
207
+ }
208
+
209
+
210
+ def _iter_parquet_samples(parquet_files: List[pathlib.Path]) -> Iterator[Dict[str, Any]]:
211
+ """Streams ReasonSeg samples from HuggingFace parquet shards."""
212
+ try:
213
+ import pyarrow.parquet as pq
214
+ except ImportError as exc:
215
+ raise ImportError(
216
+ "Reading ReasonSeg parquet files requires `pyarrow`. "
217
+ "Install it on your server with: `pip install pyarrow`"
218
+ ) from exc
219
+
220
+ for pfile in sorted(parquet_files):
221
+ pf = pq.ParquetFile(pfile)
222
+ for batch in pf.iter_batches(batch_size=16):
223
+ col_names = set(batch.schema.names)
224
+ meta_cols = [c for c in ("image", "text", "image_id", "is_sentence") if c in col_names]
225
+ rows = batch.select(meta_cols).to_pydict()
226
+ mask_col = batch.column("mask")
227
+ n_rows = len(rows["text"])
228
+ explicit_flags = rows.get("is_sentence")
229
+ image_ids = rows.get("image_id")
230
+
231
+ for i in range(n_rows):
232
+ pil_img = _decode_parquet_image(rows["image"][i])
233
+ gt_mask = _unpack_parquet_mask_slice(mask_col.slice(i, 1))
234
+ text = str(rows["text"][i]).strip()
235
+ explicit_flag = explicit_flags[i] if explicit_flags is not None else None
236
+ yield {
237
+ "image_id": str(image_ids[i]) if image_ids is not None else str(i),
238
+ "image": pil_img,
239
+ "text": text,
240
+ "gt_mask": gt_mask,
241
+ "is_long": is_long_query(text, explicit_flag),
242
+ }
243
+
244
+
245
+ def _iter_json_folder_samples(folder: pathlib.Path) -> Iterator[Dict[str, Any]]:
246
+ """Streams ReasonSeg samples from a raw directory of .jpg + .json pairs (original LISA format)."""
247
+ json_files = sorted(folder.rglob("*.json"))
248
+ for jpath in json_files:
249
+ data = json.loads(jpath.read_text(encoding="utf-8"))
250
+ if "text" not in data or "shapes" not in data:
251
+ continue
252
+ img_path = None
253
+ for ext in (".jpg", ".jpeg", ".png", ".JPG", ".PNG"):
254
+ cand = jpath.with_suffix(ext)
255
+ if cand.exists():
256
+ img_path = cand
257
+ break
258
+ if img_path is None:
259
+ continue
260
+
261
+ pil_img = Image.open(img_path).convert("RGB")
262
+ w, h = pil_img.size
263
+ mask = _rasterize_json_shapes(data.get("shapes", []), height=h, width=w)
264
+
265
+ texts = data["text"] if isinstance(data["text"], list) else [data["text"]]
266
+ explicit_flag = data.get("is_sentence", None)
267
+ for t in texts:
268
+ t_str = str(t).strip()
269
+ if not t_str:
270
+ continue
271
+ yield {
272
+ "image_id": jpath.stem,
273
+ "image": pil_img,
274
+ "text": t_str,
275
+ "gt_mask": mask,
276
+ "is_long": is_long_query(t_str, explicit_flag),
277
+ }
278
+
279
+
280
+ def iter_reasonseg_dataset(data_dir: pathlib.Path) -> Iterator[Dict[str, Any]]:
281
+ parquet_files = sorted(data_dir.rglob("*.parquet"))
282
+ if parquet_files:
283
+ yield from _iter_parquet_samples(parquet_files)
284
+ return
285
+ yield from _iter_json_folder_samples(data_dir)
286
+
287
+
288
+ def _apply_max_pixels(processor: Any, max_pixels: int) -> None:
289
+ if max_pixels <= 0 or processor is None:
290
+ return
291
+ ip = getattr(processor, "image_processor", None)
292
+ if ip is None:
293
+ return
294
+ if hasattr(ip, "max_pixels"):
295
+ ip.max_pixels = int(max_pixels)
296
+ if hasattr(ip, "min_pixels") and getattr(ip, "min_pixels", 0) > max_pixels:
297
+ ip.min_pixels = int(max_pixels)
298
+ size = getattr(ip, "size", None)
299
+ if isinstance(size, dict):
300
+ if "longest_edge" in size:
301
+ size["longest_edge"] = int(max_pixels)
302
+ if "shortest_edge" in size and size["shortest_edge"] > max_pixels:
303
+ size["shortest_edge"] = int(max_pixels)
304
+ if "max_pixels" in size:
305
+ size["max_pixels"] = int(max_pixels)
306
+ if "min_pixels" in size and size["min_pixels"] > max_pixels:
307
+ size["min_pixels"] = int(max_pixels)
308
+
309
+
310
+ def _make_processor_getter(
311
+ base_processor: Any,
312
+ use_threads: bool,
313
+ ) -> Tuple[Callable[[], Any], Optional[threading.Lock]]:
314
+ if not use_threads:
315
+ return (lambda: base_processor), None
316
+
317
+ tls = threading.local()
318
+ fallback_lock = threading.Lock()
319
+ state = {"deepcopy_ok": True}
320
+
321
+ def _get() -> Any:
322
+ proc = getattr(tls, "processor", None)
323
+ if proc is not None:
324
+ return proc
325
+ if state["deepcopy_ok"]:
326
+ try:
327
+ proc = copy.deepcopy(base_processor)
328
+ tls.processor = proc
329
+ return proc
330
+ except Exception:
331
+ state["deepcopy_ok"] = False
332
+ return base_processor
333
+
334
+ return _get, fallback_lock
335
+
336
+
337
+ def _prepare_sample_for_eval(
338
+ entry: Dict[str, Any],
339
+ base_processor: Any,
340
+ get_processor: Callable[[], Any],
341
+ fallback_lock: Optional[threading.Lock],
342
+ needs_mm_token_type_ids: bool,
343
+ image_token_id: int,
344
+ ) -> Dict[str, Any]:
345
+ decoded = _decode_raw_entry(entry)
346
+ pil_img: Image.Image = decoded["image"]
347
+ orig_w, orig_h = pil_img.size
348
+
349
+ messages, _ = build_segmentation_messages(
350
+ image=pil_img,
351
+ query_input={"objects": [{"id": "0", "description": decoded["text"]}]},
352
+ system_prompt=DEFAULT_SYSTEM_PROMPT,
353
+ query_token=OBJECT_QUERY_TOKEN,
354
+ )
355
+
356
+ proc = get_processor()
357
+ if proc is base_processor and fallback_lock is not None:
358
+ with fallback_lock:
359
+ formatted_text = proc.apply_chat_template(
360
+ messages, tokenize=False, add_generation_prompt=False
361
+ )
362
+ inputs = proc(
363
+ text=[formatted_text],
364
+ images=[pil_img],
365
+ padding=True,
366
+ return_tensors="pt",
367
+ )
368
+ else:
369
+ formatted_text = proc.apply_chat_template(
370
+ messages, tokenize=False, add_generation_prompt=False
371
+ )
372
+ inputs = proc(
373
+ text=[formatted_text],
374
+ images=[pil_img],
375
+ padding=True,
376
+ return_tensors="pt",
377
+ )
378
+
379
+ input_ids_cpu = inputs["input_ids"]
380
+ grid_thw_list = inputs["image_grid_thw"].tolist()
381
+ if needs_mm_token_type_ids:
382
+ inputs["mm_token_type_ids"] = (input_ids_cpu == image_token_id).to(dtype=torch.long)
383
+
384
+ gt_mask_np = np.ascontiguousarray(decoded["gt_mask"])
385
+ if not gt_mask_np.flags.writeable:
386
+ gt_mask_np = gt_mask_np.copy()
387
+ gt_mask_t = torch.from_numpy(gt_mask_np)
388
+
389
+ return {
390
+ "inputs": inputs,
391
+ "input_ids_cpu": input_ids_cpu,
392
+ "grid_thw_list": grid_thw_list,
393
+ "orig_size": (orig_h, orig_w),
394
+ "gt_mask": gt_mask_t,
395
+ "is_long": decoded["is_long"],
396
+ }
397
+
398
+
399
+ def _iter_prepared_samples(
400
+ data_dir: pathlib.Path,
401
+ model: Qwen2_5_VL_SinglePassSegmenter,
402
+ max_samples: int = 0,
403
+ num_workers: int = 4,
404
+ ) -> Iterator[Dict[str, Any]]:
405
+ raw_stream = _iter_raw_reasonseg_entries(data_dir, max_samples=max_samples)
406
+ needs_mm = getattr(model, "_needs_mm_token_type_ids", False)
407
+ image_token_id = model.image_token_id
408
+ base_processor = model.processor
409
+ use_threads = num_workers > 0
410
+ get_processor, fallback_lock = _make_processor_getter(base_processor, use_threads=use_threads)
411
+
412
+ if not use_threads:
413
+ for entry in raw_stream:
414
+ yield _prepare_sample_for_eval(
415
+ entry=entry,
416
+ base_processor=base_processor,
417
+ get_processor=get_processor,
418
+ fallback_lock=fallback_lock,
419
+ needs_mm_token_type_ids=needs_mm,
420
+ image_token_id=image_token_id,
421
+ )
422
+ return
423
+
424
+ max_inflight = max(num_workers * 2, 2)
425
+ inflight: deque[Future] = deque()
426
+ raw_it = iter(raw_stream)
427
+ exhausted = False
428
+
429
+ with ThreadPoolExecutor(max_workers=num_workers) as pool:
430
+ def _fill_queue() -> None:
431
+ nonlocal exhausted
432
+ while not exhausted and len(inflight) < max_inflight:
433
+ try:
434
+ item = next(raw_it)
435
+ except StopIteration:
436
+ exhausted = True
437
+ break
438
+ inflight.append(
439
+ pool.submit(
440
+ _prepare_sample_for_eval,
441
+ item,
442
+ base_processor,
443
+ get_processor,
444
+ fallback_lock,
445
+ needs_mm,
446
+ image_token_id,
447
+ )
448
+ )
449
+
450
+ try:
451
+ _fill_queue()
452
+ while inflight:
453
+ fut = inflight.popleft()
454
+ _fill_queue()
455
+ yield fut.result()
456
+ finally:
457
+ for fut in inflight:
458
+ fut.cancel()
459
+
460
+
461
+ @torch.no_grad()
462
+ def predict_single_mask(
463
+ model: Qwen2_5_VL_SinglePassSegmenter,
464
+ image: Image.Image,
465
+ description: str,
466
+ device: torch.device,
467
+ threshold: float = 0.5,
468
+ ) -> np.ndarray:
469
+ """Runs 1 forward pass and returns the binary numpy mask (H, W) in {0, 1}."""
470
+ orig_w, orig_h = image.size
471
+ messages, _ = build_segmentation_messages(
472
+ image=image,
473
+ query_input={"objects": [{"id": "0", "description": description}]},
474
+ system_prompt=DEFAULT_SYSTEM_PROMPT,
475
+ query_token=OBJECT_QUERY_TOKEN,
476
+ )
477
+ formatted_text = model.processor.apply_chat_template(
478
+ messages, tokenize=False, add_generation_prompt=False
479
+ )
480
+ inputs = model.processor(
481
+ text=[formatted_text],
482
+ images=[image],
483
+ padding=True,
484
+ return_tensors="pt",
485
+ )
486
+ input_ids_cpu = inputs["input_ids"]
487
+ grid_thw_list = inputs["image_grid_thw"].tolist()
488
+ inputs = {k: (v.to(device) if isinstance(v, torch.Tensor) else v) for k, v in inputs.items()}
489
+
490
+ outputs = model(
491
+ input_ids=inputs["input_ids"],
492
+ attention_mask=inputs["attention_mask"],
493
+ pixel_values=inputs["pixel_values"],
494
+ image_grid_thw=inputs["image_grid_thw"],
495
+ target_sizes=[(orig_h, orig_w)],
496
+ grid_thw_list=grid_thw_list,
497
+ input_ids_cpu=input_ids_cpu,
498
+ )
499
+ prob = outputs["mask_probs"][0][0]
500
+ return (prob >= threshold).to(torch.uint8).cpu().numpy()
501
+
502
+
503
+ class MetricBucket:
504
+ def __init__(self) -> None:
505
+ self.ious: List[float] = []
506
+ self.total_inter: float = 0.0
507
+ self.total_union: float = 0.0
508
+
509
+ def update_counts(self, inter: float, union: float) -> float:
510
+ iou = 1.0 if union == 0 else (inter / union)
511
+ self.ious.append(iou)
512
+ self.total_inter += inter
513
+ self.total_union += union
514
+ return iou
515
+
516
+ def update(self, pred_mask: np.ndarray, gt_mask: np.ndarray) -> float:
517
+ pred_b = pred_mask > 0
518
+ gt_b = gt_mask > 0
519
+ inter = float(np.logical_and(pred_b, gt_b).sum())
520
+ union = float(np.logical_or(pred_b, gt_b).sum())
521
+ return self.update_counts(inter, union)
522
+
523
+ @property
524
+ def giou(self) -> float:
525
+ return (sum(self.ious) / len(self.ious) * 100.0) if self.ious else 0.0
526
+
527
+ @property
528
+ def ciou(self) -> float:
529
+ return (self.total_inter / self.total_union * 100.0) if self.total_union > 0 else 0.0
530
+
531
+
532
+ @torch.no_grad()
533
+ def evaluate_split(
534
+ model: Qwen2_5_VL_SinglePassSegmenter,
535
+ data_dir: pathlib.Path,
536
+ device: torch.device,
537
+ threshold: float = 0.5,
538
+ max_samples: int = 0,
539
+ log_every: int = 25,
540
+ num_workers: int = 4,
541
+ ) -> Dict[str, MetricBucket]:
542
+ buckets = {
543
+ "short": MetricBucket(),
544
+ "long": MetricBucket(),
545
+ "overall": MetricBucket(),
546
+ }
547
+ is_cuda = device.type == "cuda"
548
+ pending: List[Tuple[torch.Tensor, bool, Optional[Tuple[Any, Any]]]] = []
549
+ gpu_ms_total = 0.0
550
+
551
+ def _flush_pending() -> None:
552
+ nonlocal gpu_ms_total
553
+ if not pending:
554
+ return
555
+ pairs_cpu = torch.stack([p for p, _, _ in pending], dim=0).cpu().tolist()
556
+ if is_cuda:
557
+ torch.cuda.synchronize(device)
558
+ for (inter_i, union_i), (_, is_long_i, ev_pair) in zip(pairs_cpu, pending):
559
+ inter_f = float(inter_i)
560
+ union_f = float(union_i)
561
+ buckets["overall"].update_counts(inter_f, union_f)
562
+ if is_long_i:
563
+ buckets["long"].update_counts(inter_f, union_f)
564
+ else:
565
+ buckets["short"].update_counts(inter_f, union_f)
566
+ if ev_pair is not None:
567
+ gpu_ms_total += float(ev_pair[0].elapsed_time(ev_pair[1]))
568
+ pending.clear()
569
+
570
+ t0 = time.perf_counter()
571
+ stream = _iter_prepared_samples(
572
+ data_dir=data_dir,
573
+ model=model,
574
+ max_samples=max_samples,
575
+ num_workers=num_workers,
576
+ )
577
+ for idx, prepared in enumerate(stream, start=1):
578
+ dev_inputs = {
579
+ k: (v.to(device) if isinstance(v, torch.Tensor) else v)
580
+ for k, v in prepared["inputs"].items()
581
+ }
582
+ gt_t = prepared["gt_mask"].to(device=device)
583
+
584
+ ev_pair: Optional[Tuple[Any, Any]] = None
585
+ if is_cuda:
586
+ ev_start = torch.cuda.Event(enable_timing=True)
587
+ ev_end = torch.cuda.Event(enable_timing=True)
588
+ ev_start.record()
589
+ ev_pair = (ev_start, ev_end)
590
+
591
+ outputs = model(
592
+ input_ids=dev_inputs["input_ids"],
593
+ attention_mask=dev_inputs["attention_mask"],
594
+ pixel_values=dev_inputs["pixel_values"],
595
+ image_grid_thw=dev_inputs["image_grid_thw"],
596
+ target_sizes=[prepared["orig_size"]],
597
+ mm_token_type_ids=dev_inputs.get("mm_token_type_ids"),
598
+ grid_thw_list=prepared["grid_thw_list"],
599
+ input_ids_cpu=prepared["input_ids_cpu"],
600
+ )
601
+ prob = outputs["mask_probs"][0][0]
602
+ pred_b = prob >= threshold
603
+ if pred_b.shape != gt_t.shape:
604
+ pred_b = (
605
+ F.interpolate(
606
+ pred_b.to(torch.float32).unsqueeze(0).unsqueeze(0),
607
+ size=gt_t.shape,
608
+ mode="nearest",
609
+ )
610
+ .squeeze(0)
611
+ .squeeze(0)
612
+ > 0.5
613
+ )
614
+ gt_b = gt_t > 0
615
+ inter_t = torch.logical_and(pred_b, gt_b).sum(dtype=torch.int64)
616
+ union_t = torch.logical_or(pred_b, gt_b).sum(dtype=torch.int64)
617
+ pair_t = torch.stack((inter_t, union_t))
618
+
619
+ if ev_pair is not None:
620
+ ev_pair[1].record()
621
+
622
+ pending.append((pair_t, prepared["is_long"], ev_pair))
623
+
624
+ if log_every > 0 and idx % log_every == 0:
625
+ _flush_pending()
626
+ elapsed = time.perf_counter() - t0
627
+ timing_str = f"{elapsed / idx * 1000:.1f} ms/sample"
628
+ if is_cuda and gpu_ms_total > 0.0:
629
+ timing_str += f" (gpu {gpu_ms_total / idx:.1f} ms)"
630
+ print(
631
+ f"[{data_dir.name} | {idx} samples | {timing_str}] "
632
+ f"overall gIoU={buckets['overall'].giou:.1f} cIoU={buckets['overall'].ciou:.1f} | "
633
+ f"short gIoU={buckets['short'].giou:.1f} | long gIoU={buckets['long'].giou:.1f}",
634
+ flush=True,
635
+ )
636
+
637
+ _flush_pending()
638
+ return buckets
639
+
640
+
641
+ def format_markdown_table(
642
+ method_name: str,
643
+ val_buckets: Dict[str, MetricBucket],
644
+ test_buckets: Dict[str, MetricBucket],
645
+ ) -> str:
646
+ lines = [
647
+ "| Method | Val overall gIoU | Val overall cIoU | Test short query gIoU | Test short query cIoU | Test long query gIoU | Test long query cIoU | Test overall gIoU | Test overall cIoU |",
648
+ "|---|---:|---:|---:|---:|---:|---:|---:|---:|",
649
+ ]
650
+ for row in BASELINE_ROWS:
651
+ name, v_g, v_c, ts_g, ts_c, tl_g, tl_c, to_g, to_c = row
652
+ lines.append(
653
+ f"| {name} | {v_g:.1f} | {v_c:.1f} | {ts_g:.1f} | {ts_c:.1f} | {tl_g:.1f} | {tl_c:.1f} | {to_g:.1f} | {to_c:.1f} |"
654
+ )
655
+
656
+ lines.append(
657
+ f"| {method_name} | "
658
+ f"{val_buckets['overall'].giou:.1f} | {val_buckets['overall'].ciou:.1f} | "
659
+ f"{test_buckets['short'].giou:.1f} | {test_buckets['short'].ciou:.1f} | "
660
+ f"{test_buckets['long'].giou:.1f} | {test_buckets['long'].ciou:.1f} | "
661
+ f"{test_buckets['overall'].giou:.1f} | {test_buckets['overall'].ciou:.1f} |"
662
+ )
663
+ lines.append("")
664
+ lines.append("*Table 1: Reasoning segmentation results among LISA (ours) and previous related works.*")
665
+ return "\n".join(lines)
666
+
667
+
668
+ def parse_args() -> argparse.Namespace:
669
+ parser = argparse.ArgumentParser(description="Benchmark Single-Pass Segmenter on ReasonSeg")
670
+ parser.add_argument(
671
+ "--data-dir",
672
+ default="./ReasonSeg_val",
673
+ help="Default ReasonSeg directory (used for both Val and Test columns unless --val-dir / --test-dir are set)",
674
+ )
675
+ parser.add_argument("--val-dir", default="", help="Optional separate ReasonSeg val split directory")
676
+ parser.add_argument("--test-dir", default="", help="Optional separate ReasonSeg test split directory")
677
+ parser.add_argument("--resume-ckpt", default="./checkpoints/segmenter_stage1_epoch5.pt")
678
+ parser.add_argument("--model-path", default="./Qwen3.5-4B")
679
+ parser.add_argument("--method-name", default="TurboSmartSegment-Qwen3.5-4B (Ours)")
680
+ parser.add_argument("--fused-layers", type=int, default=4)
681
+ parser.add_argument("--threshold", type=float, default=0.5)
682
+ parser.add_argument("--max-samples", type=int, default=0, help="0 = evaluate all samples")
683
+ parser.add_argument(
684
+ "--num-workers",
685
+ type=int,
686
+ default=4,
687
+ help="Background CPU threads for parallel Parquet/image decoding and preprocessing (0 = main thread)",
688
+ )
689
+ parser.add_argument(
690
+ "--max-pixels",
691
+ type=int,
692
+ default=0,
693
+ help="Optional max image pixels for processor (0 = use model default without capping)",
694
+ )
695
+ parser.add_argument("--log-every", type=int, default=25)
696
+ parser.add_argument("--out-md", default="./reasonseg_benchmark.md")
697
+ parser.add_argument("--device", default="cuda")
698
+ return parser.parse_args()
699
+
700
+
701
+ def main() -> None:
702
+ args = parse_args()
703
+ device = torch.device(args.device if torch.cuda.is_available() else "cpu")
704
+ dtype = (
705
+ torch.bfloat16
706
+ if (device.type == "cuda" and torch.cuda.is_bf16_supported())
707
+ else torch.float32
708
+ )
709
+
710
+ model = Qwen2_5_VL_SinglePassSegmenter(
711
+ qwen_model_name=args.model_path,
712
+ torch_dtype=dtype,
713
+ freeze_backbone=True,
714
+ num_fused_layers=args.fused_layers,
715
+ )
716
+ _apply_max_pixels(model.processor, args.max_pixels)
717
+ model.to(device)
718
+ model.mask_head.to(device=device, dtype=torch.float32)
719
+ model.eval()
720
+
721
+ if args.resume_ckpt:
722
+ ckpt_path = pathlib.Path(args.resume_ckpt)
723
+ load_checkpoint(model, ckpt_path, device)
724
+ state = torch.load(ckpt_path, map_location=device)
725
+ if "lora_dir" not in state or not state["lora_dir"]:
726
+ print(
727
+ "LoRA adapter: checkpoint has no adapter path; evaluating without a Stage 2 adapter.",
728
+ flush=True,
729
+ )
730
+ else:
731
+ lora_dir = pathlib.Path(state["lora_dir"])
732
+ if not lora_dir.exists():
733
+ print(
734
+ f"LoRA adapter: not found at {lora_dir.resolve()} "
735
+ "(path is resolved from the current working directory); evaluating without it.",
736
+ flush=True,
737
+ )
738
+ else:
739
+ print(f"LoRA adapter: found at {lora_dir.resolve()}; loading...", flush=True)
740
+ from peft import PeftModel
741
+
742
+ model.backbone.language_model = PeftModel.from_pretrained(
743
+ model.backbone.language_model, str(lora_dir)
744
+ ).to(device=device, dtype=dtype)
745
+ model._register_llm_layer_hooks()
746
+ model.eval()
747
+ print(
748
+ f"LoRA adapter: successfully applied from {lora_dir.resolve()}.",
749
+ flush=True,
750
+ )
751
+
752
+ val_dir = pathlib.Path(args.val_dir) if args.val_dir else pathlib.Path(args.data_dir)
753
+ test_dir = pathlib.Path(args.test_dir) if args.test_dir else pathlib.Path(args.data_dir)
754
+
755
+ print(f"Evaluating on: {val_dir} ...", flush=True)
756
+ val_buckets = evaluate_split(
757
+ model=model,
758
+ data_dir=val_dir,
759
+ device=device,
760
+ threshold=args.threshold,
761
+ max_samples=args.max_samples,
762
+ log_every=args.log_every,
763
+ num_workers=args.num_workers,
764
+ )
765
+
766
+ if test_dir.resolve() == val_dir.resolve():
767
+ test_buckets = val_buckets
768
+ else:
769
+ print(f"Evaluating on: {test_dir} ...", flush=True)
770
+ test_buckets = evaluate_split(
771
+ model=model,
772
+ data_dir=test_dir,
773
+ device=device,
774
+ threshold=args.threshold,
775
+ max_samples=args.max_samples,
776
+ log_every=args.log_every,
777
+ num_workers=args.num_workers,
778
+ )
779
+
780
+ table_md = format_markdown_table(args.method_name, val_buckets, test_buckets)
781
+ print("\n" + table_md + "\n", flush=True)
782
+
783
+ if args.out_md:
784
+ out_path = pathlib.Path(args.out_md)
785
+ out_path.parent.mkdir(parents=True, exist_ok=True)
786
+ out_path.write_text(table_md + "\n", encoding="utf-8")
787
+ print(f"Saved Markdown benchmark table to: {out_path}", flush=True)
788
+
789
+
790
+ if __name__ == "__main__":
791
+ main()
dataset.py ADDED
@@ -0,0 +1,329 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import pathlib
3
+ import pickle
4
+ import random
5
+ from collections import defaultdict
6
+ from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
7
+
8
+ import torch
9
+ from PIL import Image
10
+ from torch.utils.data import Dataset
11
+
12
+ from LIS1ModelCode import (
13
+ DEFAULT_SYSTEM_PROMPT,
14
+ OBJECT_QUERY_TOKEN,
15
+ build_segmentation_messages,
16
+ build_segmentation_prompt,
17
+ polygons_to_mask,
18
+ )
19
+
20
+
21
+ # Functional & commonsense descriptions for COCO categories to bridge RefCOCO -> ReasonSeg riddles
22
+ COCO_FUNCTIONAL_CLUES: Dict[str, List[str]] = {
23
+ "person": ["someone who is present in the scene", "the human individual shown here"],
24
+ "bicycle": ["a two-wheeled vehicle propelled by pedaling", "what you can ride for eco-friendly commuting"],
25
+ "car": ["a motor vehicle used for driving on roads", "the automobile parked or moving in the scene"],
26
+ "motorcycle": ["a two-wheeled motor vehicle for riding", "the motorbike in the picture"],
27
+ "airplane": ["an aircraft built for flying through the sky", "what transports passengers by air"],
28
+ "bus": ["a large road vehicle designed to carry many passengers", "public transit vehicle"],
29
+ "train": ["a rail vehicle that travels along tracks", "locomotive or rail car"],
30
+ "truck": ["a heavy motor vehicle built for hauling cargo", "the utility or freight vehicle"],
31
+ "boat": ["a watercraft used for traveling across water", "vessel floating on the water"],
32
+ "traffic light": ["the signaling device that controls road traffic flow", "where drivers look to know whether to stop or go"],
33
+ "bench": ["a long seat where multiple people can sit outdoors or indoors", "place to sit down and rest"],
34
+ "backpack": ["a bag carried on the back to hold personal belongings", "what someone wears on their shoulders to carry gear"],
35
+ "umbrella": ["what you should hold overhead to stay dry when it rains", "item used for shade or protection from rain"],
36
+ "handbag": ["a personal bag held in the hand or over the shoulder", "accessory used to carry daily essentials"],
37
+ "tie": ["a formal cloth accessory worn around the neck with a collared shirt", "neckwear piece"],
38
+ "suitcase": ["luggage used to pack clothes when traveling on a trip", "travel baggage"],
39
+ "bottle": ["a narrow-necked container used to hold liquids or drinks", "vessel you can pour a beverage from"],
40
+ "wine glass": ["a stemmed glass vessel used for drinking wine", "glassware for a formal beverage"],
41
+ "cup": ["what you can drink from if you are thirsty", "vessel used to hold a hot or cold beverage"],
42
+ "fork": ["utensil with prongs used to spear and eat solid food", "cutlery item for dining"],
43
+ "knife": ["bladed utensil used for cutting food at the table or kitchen", "sharp tool for slicing"],
44
+ "spoon": ["curved utensil used for eating soup or stirring liquids", "utensil for scooping food"],
45
+ "bowl": ["a round deep dish used to serve soup, cereal, or salad", "container holding food"],
46
+ "banana": ["a curved yellow fruit you peel before eating", "potassium-rich snack fruit"],
47
+ "apple": ["a crisp round fruit that keeps the doctor away", "fresh fruit snack"],
48
+ "sandwich": ["a meal made of fillings placed between slices of bread", "prepared food you can eat for lunch"],
49
+ "orange": ["a citrus fruit rich in vitamin C", "round citrus snack"],
50
+ "broccoli": ["a green tree-shaped vegetable rich in nutrients", "green vegetable side"],
51
+ "carrot": ["an crunchy orange root vegetable", "root vegetable"],
52
+ "pizza": ["a flat baked dough topped with tomato sauce and melted cheese", "sliced Italian dish"],
53
+ "donut": ["a sweet fried ring-shaped pastry", "sugary dessert treat"],
54
+ "cake": ["a sweet baked dessert often served at celebrations or birthdays", "baked dessert"],
55
+ "chair": ["a piece of furniture designed for one person to sit on", "where someone can sit down"],
56
+ "couch": ["an upholstered sofa where several people can sit or lounge comfortably", "living room seating furniture"],
57
+ "potted plant": ["indoor or outdoor greenery growing inside a container", "decorative living plant in a pot"],
58
+ "bed": ["furniture with a mattress where someone can lie down and sleep", "place to rest and sleep at night"],
59
+ "dining table": ["the surface where meals are served and people gather to eat", "table used for dining"],
60
+ "toilet": ["bathroom fixture used for sanitation", "plumbing fixture in the restroom"],
61
+ "tv": ["the screen used to watch broadcasts, movies, or visual presentations", "display screen for viewing video"],
62
+ "laptop": ["a portable personal computer with a clamshell screen and keyboard", "device used for mobile work or browsing"],
63
+ "mouse": ["handheld pointing device used to move a cursor on a computer screen", "computer input peripheral"],
64
+ "remote": ["handheld controller used to operate a television or device from a distance", "what you press to change channels"],
65
+ "keyboard": ["panel of keys used for typing text into a computer", "input device for typing"],
66
+ "cell phone": ["portable handheld device used to make calls, text, or take photos", "smartphone or mobile device"],
67
+ "microwave": ["kitchen appliance that quickly heats or reheats food using electromagnetic waves", "appliance for warming up leftovers"],
68
+ "oven": ["heated chamber used for baking or roasting food in the kitchen", "cooking appliance"],
69
+ "toaster": ["small electric kitchen appliance used to brown slices of bread", "breakfast appliance"],
70
+ "sink": ["basin with a faucet where you can wash your hands or dishes", "where to wash hands"],
71
+ "refrigerator": ["insulated kitchen appliance that keeps food and beverages cold and fresh", "where perishable food is stored"],
72
+ "book": ["printed work of pages bound together that you can read for knowledge or leisure", "something to read"],
73
+ "clock": ["the instrument you look at to see what time it is", "timepiece showing the hour"],
74
+ "vase": ["decorative container used to hold cut flowers", "vessel holding stems or flowers"],
75
+ "scissors": ["handheld tool with two pivoted blades used for cutting paper or fabric", "cutting tool"],
76
+ "teddy bear": ["a soft stuffed toy bear made for comfort or play", "plush toy"],
77
+ "hair drier": ["electromechanical device that blows warm air to dry wet hair", "grooming appliance"],
78
+ "toothbrush": ["small brush used with toothpaste to clean teeth", "oral hygiene tool"],
79
+ }
80
+
81
+ SHORT_RIDDLE_TEMPLATES: List[str] = [
82
+ "{clue} ({desc})",
83
+ "the {cat} that is {clue}, specifically {desc}",
84
+ "where to find {clue}: {desc}",
85
+ ]
86
+
87
+ LONG_RIDDLE_TEMPLATES: List[str] = [
88
+ "In this scene, suppose you are looking for {clue}. Which specific object—described as '{desc}'—matches this need?",
89
+ "Sometimes we identify an object by its function or role, such as {clue}. What part of the picture corresponds to '{desc}'?",
90
+ "If someone asked you to point out {clue} in this image, specifically the one that looks like '{desc}', which object would you select?",
91
+ ]
92
+
93
+
94
+ class RefCOCOJsonDataset(Dataset):
95
+ """
96
+ Loads RefCOCO / RefCOCO+ / RefCOCOg entries (or all combined when dataset_name='all')
97
+ and groups referring expressions by image_id for single-pass multi-object JSON training.
98
+ Includes optional functional/riddle prompt augmentation to generalize to ReasonSeg.
99
+ """
100
+
101
+ def __init__(
102
+ self,
103
+ data_root: Union[str, pathlib.Path] = "./ref_coco_lichengunc",
104
+ dataset_name: str = "refcoco",
105
+ split_by: str = "unc",
106
+ split: str = "train",
107
+ max_objects_per_image: int = 4,
108
+ min_objects_per_image: int = 1,
109
+ group_by_image: bool = True,
110
+ riddle_prob: float = 0.35,
111
+ seed: int = 42,
112
+ ):
113
+ super().__init__()
114
+ self.data_root = pathlib.Path(data_root)
115
+ self.dataset_name = dataset_name
116
+ self.split_by = split_by
117
+ self.split = split
118
+ self.max_objects_per_image = max_objects_per_image
119
+ self.min_objects_per_image = min_objects_per_image
120
+ self.group_by_image = group_by_image
121
+ self.riddle_prob = riddle_prob if split == "train" else 0.0
122
+ self.rng = random.Random(seed)
123
+
124
+ candidate_dirs = [
125
+ self.data_root / "images" / "train2014",
126
+ self.data_root / "images" / "train2014" / "train2014",
127
+ self.data_root / "train2014",
128
+ self.data_root / "images",
129
+ ]
130
+ self.image_dir = candidate_dirs[0]
131
+ for cand in candidate_dirs:
132
+ if cand.exists() and any(cand.glob("*.jpg")):
133
+ self.image_dir = cand
134
+ break
135
+
136
+ if dataset_name == "all":
137
+ specs = [("refcoco", "unc"), ("refcoco+", "unc"), ("refcocog", "umd")]
138
+ else:
139
+ default_split_by = "umd" if (dataset_name == "refcocog" and split_by == "unc") else split_by
140
+ specs = [(dataset_name, default_split_by)]
141
+
142
+ self.images_by_id: Dict[int, Dict[str, Any]] = {}
143
+ self.anns_by_id: Dict[Union[int, str], Dict[str, Any]] = {}
144
+ self.cats_by_id: Dict[int, str] = {}
145
+ filtered_refs: List[Dict[str, Any]] = []
146
+
147
+ for ds_name, sp_by in specs:
148
+ sub_dir = self.data_root / ds_name
149
+ refs_path = sub_dir / f"refs({sp_by}).p"
150
+ instances_path = sub_dir / "instances.json"
151
+ if not refs_path.exists() or not instances_path.exists():
152
+ if dataset_name != "all":
153
+ raise FileNotFoundError(f"Could not find {refs_path} or {instances_path}")
154
+ continue
155
+
156
+ with open(refs_path, "rb") as f:
157
+ refs = pickle.load(f)
158
+ with open(instances_path, "r", encoding="utf-8") as f:
159
+ instances = json.load(f)
160
+
161
+ for img in instances.get("images", []):
162
+ self.images_by_id[img["id"]] = img
163
+ for ann in instances.get("annotations", []):
164
+ self.anns_by_id[ann["id"]] = ann
165
+ for cat in instances.get("categories", []):
166
+ self.cats_by_id[cat["id"]] = cat["name"]
167
+
168
+ for ref in refs:
169
+ if ref.get("split") != split:
170
+ continue
171
+ img_id = ref["image_id"]
172
+ ann_id = ref["ann_id"]
173
+ if img_id not in self.images_by_id or ann_id not in self.anns_by_id:
174
+ continue
175
+ img_info = self.images_by_id[img_id]
176
+ img_path = self.image_dir / img_info["file_name"]
177
+ if not img_path.exists():
178
+ continue
179
+ sentences = [s["sent"].strip() for s in ref.get("sentences", []) if s.get("sent", "").strip()]
180
+ if not sentences:
181
+ continue
182
+ filtered_refs.append(
183
+ {
184
+ "ref_id": f"{ds_name}_{ref['ref_id']}",
185
+ "image_id": img_id,
186
+ "ann_id": ann_id,
187
+ "category_id": ref.get("category_id"),
188
+ "sentences": sentences,
189
+ }
190
+ )
191
+
192
+ if group_by_image:
193
+ by_img: Dict[int, List[Dict[str, Any]]] = defaultdict(list)
194
+ for item in filtered_refs:
195
+ by_img[item["image_id"]].append(item)
196
+ self.samples: List[List[Dict[str, Any]]] = list(by_img.values())
197
+ else:
198
+ self.samples = [[item] for item in filtered_refs]
199
+
200
+ if not self.samples:
201
+ raise FileNotFoundError(
202
+ f"No valid samples found for dataset='{dataset_name}' split='{split}'. "
203
+ f"Checked image directory: {self.image_dir} (ensure COCO_train2014_*.jpg files are present)."
204
+ )
205
+
206
+ def __len__(self) -> int:
207
+ return len(self.samples)
208
+
209
+ def _augment_as_riddle(self, desc: str, category_id: Optional[int]) -> str:
210
+ cat_name = self.cats_by_id.get(category_id or -1, "object")
211
+ clues = COCO_FUNCTIONAL_CLUES.get(cat_name, [f"an instance of {cat_name} in the scene"])
212
+ clue = self.rng.choice(clues)
213
+ if self.rng.random() < 0.5:
214
+ tpl = self.rng.choice(SHORT_RIDDLE_TEMPLATES)
215
+ else:
216
+ tpl = self.rng.choice(LONG_RIDDLE_TEMPLATES)
217
+ return tpl.format(clue=clue, desc=desc, cat=cat_name)
218
+
219
+ def __getitem__(self, idx: int) -> Dict[str, Any]:
220
+ ref_group = self.samples[idx]
221
+ image_id = ref_group[0]["image_id"]
222
+ img_info = self.images_by_id[image_id]
223
+ img_path = self.image_dir / img_info["file_name"]
224
+
225
+ image = Image.open(img_path).convert("RGB")
226
+ width, height = image.size
227
+
228
+ if self.split == "train" and len(ref_group) > 1:
229
+ k_max = min(len(ref_group), self.max_objects_per_image)
230
+ k_min = min(k_max, self.min_objects_per_image)
231
+ k = self.rng.randint(k_min, k_max)
232
+ chosen_refs = self.rng.sample(ref_group, k=k)
233
+ else:
234
+ chosen_refs = ref_group[: self.max_objects_per_image]
235
+
236
+ query_objects: List[Dict[str, str]] = []
237
+ gt_masks = []
238
+ gt_polygons = []
239
+ gt_bboxes = []
240
+
241
+ for obj_idx, ref in enumerate(chosen_refs):
242
+ ann = self.anns_by_id[ref["ann_id"]]
243
+ if self.split == "train":
244
+ desc = self.rng.choice(ref["sentences"])
245
+ if self.riddle_prob > 0.0 and self.rng.random() < self.riddle_prob:
246
+ desc = self._augment_as_riddle(desc, ref.get("category_id"))
247
+ else:
248
+ desc = ref["sentences"][0]
249
+
250
+ obj_id = f"obj_{obj_idx}"
251
+ query_objects.append({"id": obj_id, "description": desc})
252
+
253
+ mask_np = polygons_to_mask(ann["segmentation"], height=height, width=width)
254
+ gt_masks.append(torch.from_numpy(mask_np).to(torch.float32))
255
+ gt_polygons.append(ann["segmentation"])
256
+ gt_bboxes.append(ann.get("bbox", [0.0, 0.0, 0.0, 0.0]))
257
+
258
+ prompt_text, normalized_objects = build_segmentation_prompt(
259
+ {"objects": query_objects},
260
+ query_token=OBJECT_QUERY_TOKEN,
261
+ )
262
+
263
+ return {
264
+ "image_id": image_id,
265
+ "image_path": str(img_path),
266
+ "image": image,
267
+ "target_size": (height, width),
268
+ "query_json": {"objects": normalized_objects},
269
+ "prompt_text": prompt_text,
270
+ "gt_masks": torch.stack(gt_masks, dim=0), # [K, H, W]
271
+ "gt_polygons": gt_polygons,
272
+ "gt_bboxes": gt_bboxes,
273
+ }
274
+
275
+
276
+ class RefCOCOJsonCollator:
277
+ """
278
+ Collates a batch of (PIL Image, multi-object JSON prompt, ground-truth masks)
279
+ into Qwen2.5-VL processor tensors using the causal reasoning message order:
280
+ [System] -> [User: JSON Riddles -> Image] -> [Assistant: <|object_ref_start|>...<|object_ref_end|>]
281
+ """
282
+
283
+ def __init__(self, processor: Any, system_prompt: str = DEFAULT_SYSTEM_PROMPT):
284
+ self.processor = processor
285
+ self.system_prompt = system_prompt
286
+
287
+ def __call__(self, batch: Sequence[Dict[str, Any]]) -> Dict[str, Any]:
288
+ texts: List[str] = []
289
+ images: List[Image.Image] = []
290
+ target_sizes: List[Tuple[int, int]] = []
291
+ gt_masks_list: List[torch.Tensor] = []
292
+ query_jsons: List[Dict[str, Any]] = []
293
+ gt_polygons_list: List[List[Any]] = []
294
+
295
+ for item in batch:
296
+ messages, _ = build_segmentation_messages(
297
+ image=item["image"],
298
+ query_input=item["query_json"],
299
+ system_prompt=self.system_prompt,
300
+ )
301
+ chat_text = self.processor.apply_chat_template(
302
+ messages,
303
+ tokenize=False,
304
+ add_generation_prompt=False,
305
+ )
306
+ texts.append(chat_text)
307
+ images.append(item["image"])
308
+ target_sizes.append(item["target_size"])
309
+ gt_masks_list.append(item["gt_masks"])
310
+ query_jsons.append(item["query_json"])
311
+ gt_polygons_list.append(item["gt_polygons"])
312
+
313
+ proc_inputs = self.processor(
314
+ text=texts,
315
+ images=images,
316
+ padding=True,
317
+ return_tensors="pt",
318
+ )
319
+
320
+ return {
321
+ "input_ids": proc_inputs["input_ids"],
322
+ "attention_mask": proc_inputs["attention_mask"],
323
+ "pixel_values": proc_inputs["pixel_values"],
324
+ "image_grid_thw": proc_inputs["image_grid_thw"],
325
+ "target_sizes": target_sizes,
326
+ "gt_masks_list": gt_masks_list,
327
+ "query_jsons": query_jsons,
328
+ "gt_polygons_list": gt_polygons_list,
329
+ }
example.py ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pathlib
2
+ from PIL import Image
3
+ from lis1_infer import LIS1Inferencer
4
+
5
+
6
+ def main() -> None:
7
+ # 1. Instantiate the inference class and enable debug logging
8
+ inferencer = LIS1Inferencer()
9
+ inferencer.debug = True
10
+
11
+ # 2. Load Qwen3.5 backbone + LIS1 decoder + LoRA adapter (with 4-bit bitsandbytes quantization)
12
+ inferencer.load_model(
13
+ qwen_path="./Qwen3.5-9B",
14
+ lis_model_path="./LIS1-9B-forLora.safetensors",
15
+ lora_adapter="./LIS1-9B-LoraAdapter",
16
+ device="cuda",
17
+ max_pixels=524288,
18
+ load_in_4bit=True, # Or set load_in_8bit=True for 8-bit quantization
19
+ )
20
+
21
+ # 3. Load "image1.jpg" and define target queries
22
+ pil_images = [Image.open("image1.jpg").convert("RGB")]
23
+ queries_per_image = [["toast", "red sauce", "something you can drink"]]
24
+
25
+ # 4. Run inference (isolate_queries=True avoids multi-query cross-bleed while reusing cached ViT features)
26
+ outputs = inferencer.run_model(
27
+ images=pil_images,
28
+ queries=queries_per_image,
29
+ threshold=0.7,
30
+ batch_size=4,
31
+ isolate_queries=True, # Compute queries separately
32
+ cache_vision=True, # Cache vision when above is True for better performance
33
+ exclusive_masks=True, # Make masks not overlap, choose the best one
34
+ )
35
+
36
+ for img_idx, out in enumerate(outputs):
37
+ print(f"\nImage {img_idx} ({out['image_size']['width']}x{out['image_size']['height']}):")
38
+ for obj in out["objects"]:
39
+ print(
40
+ f" - [{obj['id']}] '{obj['description']}': "
41
+ f"score={obj['score']:.4f}, bbox={obj['bbox']}, area={obj['area']}"
42
+ )
43
+
44
+ # 5. Render masks and bounding boxes onto the image and save
45
+ rendered_images = inferencer.render_image(
46
+ images=pil_images,
47
+ outputs=outputs,
48
+ draw_masks=True,
49
+ draw_boxes=True,
50
+ draw_labels=True,
51
+ draw_scores=True,
52
+ draw_contours=True,
53
+ )
54
+
55
+ out_dir = pathlib.Path(".")
56
+ out_dir.mkdir(parents=True, exist_ok=True)
57
+ for i, img_full in enumerate(rendered_images):
58
+ full_path = out_dir / f"result_{i}_boxes_and_masks.png"
59
+ img_full.save(full_path)
60
+ print(f"Saved rendered output to: {full_path}")
61
+
62
+ # 6. Offload model from GPU/CPU memory when done
63
+ inferencer.offload_model()
64
+
65
+
66
+ if __name__ == "__main__":
67
+ main()
image1.jpg ADDED

Git LFS Details

  • SHA256: 4416872f4908f45785e5f988aaf5afa90f28cf4b029d0ebb8045be63cc3afa70
  • Pointer size: 131 Bytes
  • Size of remote file: 854 kB
lis1_infer.py ADDED
@@ -0,0 +1,1461 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from concurrent.futures import Future, ThreadPoolExecutor
2
+ from collections import OrderedDict, deque
3
+ import copy
4
+ import gc
5
+ import hashlib
6
+ import pathlib
7
+ import threading
8
+ import time
9
+ import warnings
10
+ from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple, Union
11
+
12
+ import cv2
13
+ import numpy as np
14
+ import torch
15
+ from PIL import Image, ImageDraw, ImageFont
16
+ from safetensors import safe_open
17
+ from safetensors.torch import load_file
18
+
19
+ from LIS1ModelCode import (
20
+ DEFAULT_SYSTEM_PROMPT,
21
+ OBJECT_QUERY_TOKEN,
22
+ Qwen2_5_VL_SinglePassSegmenter,
23
+ build_segmentation_messages,
24
+ mask_to_polygons,
25
+ )
26
+
27
+
28
+ # Distinct vibrant RGB palette for multi-object mask & box rendering
29
+ DEFAULT_PALETTE: List[Tuple[int, int, int]] = [
30
+ (255, 56, 56), # Red
31
+ (44, 160, 44), # Green
32
+ (31, 119, 180), # Blue
33
+ (255, 157, 151), # Coral
34
+ (255, 112, 31), # Orange
35
+ (148, 103, 189), # Purple
36
+ (23, 190, 207), # Cyan
37
+ (188, 189, 34), # Olive
38
+ (227, 119, 194), # Pink
39
+ (140, 86, 75), # Brown
40
+ ]
41
+
42
+
43
+ def inspect_lis_checkpoint(
44
+ lis_model_path: Union[str, pathlib.Path],
45
+ ) -> Tuple[Dict[str, torch.Tensor], Optional[str], Dict[str, Any]]:
46
+ """
47
+ Loads a LIS1 mask_head checkpoint from either `.pt` or `.safetensors` format.
48
+
49
+ Returns:
50
+ mask_head_state: state_dict for `model.mask_head`
51
+ lora_dir: optional LoRA directory string stored in checkpoint metadata (if `forLora`)
52
+ raw_metadata: dictionary of raw metadata from the file
53
+ """
54
+ ckpt_path = pathlib.Path(lis_model_path)
55
+ if not ckpt_path.exists():
56
+ raise FileNotFoundError(f"LIS model checkpoint not found: {ckpt_path}")
57
+
58
+ suffix = ckpt_path.suffix.lower()
59
+ if suffix == ".safetensors":
60
+ with safe_open(str(ckpt_path), framework="pt", device="cpu") as f:
61
+ metadata = dict(f.metadata() or {})
62
+ tensors = load_file(str(ckpt_path), device="cpu")
63
+ lora_dir = metadata.get("lora_dir")
64
+ if any(k.startswith("mask_head.") for k in tensors):
65
+ mask_head_state = {
66
+ (k[len("mask_head.") :] if k.startswith("mask_head.") else k): v
67
+ for k, v in tensors.items()
68
+ }
69
+ else:
70
+ mask_head_state = tensors
71
+ return mask_head_state, (lora_dir if lora_dir else None), metadata
72
+
73
+ # PyTorch .pt / .pth checkpoint
74
+ try:
75
+ state = torch.load(ckpt_path, map_location="cpu", weights_only=True)
76
+ except TypeError:
77
+ state = torch.load(ckpt_path, map_location="cpu")
78
+
79
+ if isinstance(state, dict) and "mask_head" in state:
80
+ mask_head_state = state["mask_head"]
81
+ lora_dir = state.get("lora_dir")
82
+ metadata = {k: v for k, v in state.items() if k != "mask_head"}
83
+ elif isinstance(state, dict):
84
+ mask_head_state = state
85
+ lora_dir = state.get("lora_dir") if isinstance(state.get("lora_dir"), str) else None
86
+ metadata = {"lora_dir": lora_dir} if lora_dir else {}
87
+ else:
88
+ raise ValueError(f"Unsupported checkpoint structure in {ckpt_path}")
89
+
90
+ return mask_head_state, (str(lora_dir) if lora_dir else None), metadata
91
+
92
+
93
+ def _apply_max_pixels(processor: Any, max_pixels: int) -> None:
94
+ if max_pixels <= 0 or processor is None:
95
+ return
96
+ ip = getattr(processor, "image_processor", None)
97
+ if ip is None:
98
+ return
99
+ if hasattr(ip, "max_pixels") and getattr(ip, "max_pixels", None) is not None:
100
+ ip.max_pixels = int(max_pixels)
101
+ if hasattr(ip, "min_pixels") and getattr(ip, "min_pixels", None) is not None:
102
+ if ip.min_pixels > max_pixels:
103
+ ip.min_pixels = int(max_pixels)
104
+ size = getattr(ip, "size", None)
105
+ if size is not None:
106
+ if isinstance(size, dict) or hasattr(size, "get"):
107
+ if size.get("longest_edge") is not None:
108
+ size["longest_edge"] = int(max_pixels)
109
+ if size.get("shortest_edge") is not None and size["shortest_edge"] > max_pixels:
110
+ size["shortest_edge"] = int(max_pixels)
111
+ if size.get("max_pixels") is not None:
112
+ size["max_pixels"] = int(max_pixels)
113
+ if size.get("min_pixels") is not None and size["min_pixels"] > max_pixels:
114
+ size["min_pixels"] = int(max_pixels)
115
+
116
+
117
+ def filter_mask_components(
118
+ mask_np: np.ndarray,
119
+ min_component_ratio: float = 0.05,
120
+ min_component_area: int = 16,
121
+ ) -> np.ndarray:
122
+ """
123
+ Removes tiny disconnected foreground islands whose area is smaller than
124
+ `max(min_component_area, min_component_ratio * largest_component_area)`.
125
+ Single-component masks are returned unchanged so genuine small objects are preserved.
126
+ """
127
+ if min_component_ratio <= 0.0 and min_component_area <= 0:
128
+ return mask_np
129
+ if not np.any(mask_np):
130
+ return mask_np
131
+
132
+ num_labels, labels, stats, _ = cv2.connectedComponentsWithStats(mask_np, connectivity=8)
133
+ if num_labels <= 2:
134
+ return mask_np
135
+
136
+ fg_areas = stats[1:, cv2.CC_STAT_AREA]
137
+ max_area = int(fg_areas.max())
138
+ area_cutoff = max(float(min_component_area), float(min_component_ratio) * float(max_area))
139
+
140
+ keep_lut = np.zeros(num_labels, dtype=np.uint8)
141
+ for comp_idx in range(1, num_labels):
142
+ if stats[comp_idx, cv2.CC_STAT_AREA] >= area_cutoff:
143
+ keep_lut[comp_idx] = 1
144
+ return keep_lut[labels]
145
+
146
+
147
+ def _rects_overlap(
148
+ a: Tuple[float, float, float, float],
149
+ b: Tuple[float, float, float, float],
150
+ margin: float = 2.0,
151
+ ) -> bool:
152
+ return not (
153
+ a[2] + margin <= b[0]
154
+ or b[2] + margin <= a[0]
155
+ or a[3] + margin <= b[1]
156
+ or b[3] + margin <= a[1]
157
+ )
158
+
159
+
160
+ def _place_non_overlapping_label(
161
+ x1: float,
162
+ y1: float,
163
+ label_w: float,
164
+ label_h: float,
165
+ img_w: int,
166
+ img_h: int,
167
+ occupied: List[Tuple[float, float, float, float]],
168
+ ) -> Tuple[float, float, float, float]:
169
+ """
170
+ Finds a non-overlapping rectangle (tx1, ty1, tx2, ty2) for a bounding-box label,
171
+ preferring above top-left of the box, then stacking upward or downward on collision.
172
+ """
173
+ tx1 = max(0.0, min(x1, max(0.0, float(img_w) - label_w)))
174
+ tx2 = min(float(img_w), tx1 + label_w)
175
+
176
+ pref_ty1 = (y1 - label_h) if (y1 - label_h >= 0.0) else y1
177
+ pref_ty1 = max(0.0, min(pref_ty1, max(0.0, float(img_h) - label_h)))
178
+ cand = (tx1, pref_ty1, tx2, pref_ty1 + label_h)
179
+
180
+ if not any(_rects_overlap(cand, occ) for occ in occupied):
181
+ return cand
182
+
183
+ # 1. Try stacking upward above colliding labels
184
+ ty1 = pref_ty1
185
+ for _ in range(len(occupied) + 2):
186
+ hits = [occ for occ in occupied if _rects_overlap((tx1, ty1, tx2, ty1 + label_h), occ)]
187
+ if not hits:
188
+ return (tx1, ty1, tx2, ty1 + label_h)
189
+ min_occ_top = min(occ[1] for occ in hits)
190
+ ty1 = min_occ_top - label_h - 2.0
191
+ if ty1 < 0.0:
192
+ break
193
+
194
+ # 2. Otherwise stack downward starting just inside/below the top of the box
195
+ ty1 = max(0.0, y1 + 2.0)
196
+ for _ in range(len(occupied) + 4):
197
+ if ty1 + label_h > float(img_h):
198
+ ty1 = max(0.0, float(img_h) - label_h)
199
+ break
200
+ hits = [occ for occ in occupied if _rects_overlap((tx1, ty1, tx2, ty1 + label_h), occ)]
201
+ if not hits:
202
+ return (tx1, ty1, tx2, ty1 + label_h)
203
+ max_occ_bot = max(occ[3] for occ in hits)
204
+ ty1 = max_occ_bot + 2.0
205
+
206
+ return (tx1, ty1, tx2, min(float(img_h), ty1 + label_h))
207
+
208
+
209
+ def _compute_image_cache_key(pil_rgb: Image.Image, max_pixels: int) -> Tuple[int, int, int, bytes]:
210
+ arr = np.ascontiguousarray(np.asarray(pil_rgb, dtype=np.uint8))
211
+ digest = hashlib.blake2b(memoryview(arr), digest_size=16).digest()
212
+ w, h = pil_rgb.size
213
+ return (h, w, int(max_pixels), digest)
214
+
215
+
216
+ class LIS1Inferencer:
217
+ """
218
+ High-efficiency single-pass inference wrapper for Qwen3.5 + LIS1 segmentation models.
219
+
220
+ Provides:
221
+ 1. `load_model(qwen_path, lis_model_path, lora_adapter=None, device="cuda", ...)`
222
+ 2. `offload_model()`
223
+ 3. `run_model(images, queries, isolate_queries=False, cache_vision=True, ...)`
224
+ 4. `render_image(images, outputs, ...)`
225
+ 5. `debug` property + setter (`inferencer.debug = True` or `inferencer.set_debug(True)`)
226
+ """
227
+
228
+ def __init__(self, debug: bool = False, vision_cache_size: int = 8) -> None:
229
+ self._debug: bool = bool(debug)
230
+ self.model: Optional[Qwen2_5_VL_SinglePassSegmenter] = None
231
+ self.device: torch.device = torch.device("cpu")
232
+ self.dtype: torch.dtype = torch.float32
233
+ self.quantization_mode: str = "none"
234
+ self.max_pixels: int = 0
235
+ self.vision_cache_size: int = max(0, int(vision_cache_size))
236
+ self._vision_cache: "OrderedDict[Tuple[int, int, int, bytes], Dict[str, Any]]" = OrderedDict()
237
+ self.qwen_path: Optional[pathlib.Path] = None
238
+ self.lis_model_path: Optional[pathlib.Path] = None
239
+ self.lora_adapter_path: Optional[pathlib.Path] = None
240
+
241
+ def clear_vision_cache(self) -> None:
242
+ """Clears cached ViT visual features from memory."""
243
+ self._vision_cache.clear()
244
+
245
+ @property
246
+ def debug(self) -> bool:
247
+ return self._debug
248
+
249
+ @debug.setter
250
+ def debug(self, value: bool) -> None:
251
+ self._debug = bool(value)
252
+ if self._debug:
253
+ print("[LIS1 Debug] Debug mode enabled.", flush=True)
254
+
255
+ def set_debug(self, value: bool) -> None:
256
+ """Setter method for enabling or disabling debug prints."""
257
+ self.debug = value
258
+
259
+ def _log(self, msg: str) -> None:
260
+ if self._debug:
261
+ print(f"[LIS1 Debug] {msg}", flush=True)
262
+
263
+ def load_model(
264
+ self,
265
+ qwen_path: Union[str, pathlib.Path],
266
+ lis_model_path: Union[str, pathlib.Path],
267
+ lora_adapter: Optional[Union[str, pathlib.Path]] = None,
268
+ device: Union[str, torch.device] = "cuda",
269
+ fused_layers: int = 4,
270
+ max_pixels: int = 0,
271
+ load_in_4bit: bool = False,
272
+ load_in_8bit: bool = False,
273
+ bnb_4bit_quant_type: str = "nf4",
274
+ bnb_4bit_use_double_quant: bool = True,
275
+ bnb_4bit_compute_dtype: Optional[torch.dtype] = None,
276
+ bnb_skip_vision: bool = True,
277
+ quantization_config: Optional[Any] = None,
278
+ ) -> "LIS1Inferencer":
279
+ """
280
+ Loads the Qwen3.5 backbone, the LIS1 mask_head weights (.pt or .safetensors),
281
+ and optionally the Stage-2 PEFT LoRA adapter and bitsandbytes quantization.
282
+
283
+ Args:
284
+ qwen_path: Path or HuggingFace ID of the base Qwen3.5 model.
285
+ lis_model_path: Path to the LIS1 mask_head checkpoint (.pt or .safetensors).
286
+ lora_adapter: Optional path to Stage-2 PEFT LoRA adapter directory.
287
+ If None, treats the LIS model as a noLora model, but prints a warning
288
+ if checkpoint metadata indicates it was trained for LoRA (`forLora`).
289
+ device: Target Torch device (e.g. "cuda", "cuda:0", "cpu", or "auto").
290
+ fused_layers: Number of top LLM layers fused into the mask head.
291
+ max_pixels: Optional cap on image pixels for faster ViT processing (0 = uncapped).
292
+ load_in_4bit: Enable bitsandbytes 4-bit quantization (NF4/FP4).
293
+ load_in_8bit: Enable bitsandbytes 8-bit quantization (LLM.int8()).
294
+ bnb_4bit_quant_type: 4-bit quantization data type ("nf4" or "fp4").
295
+ bnb_4bit_use_double_quant: Enable nested double quantization in 4-bit mode.
296
+ bnb_4bit_compute_dtype: Optional compute dtype override for 4-bit matmuls.
297
+ bnb_skip_vision: Keep the ViT `visual` encoder unquantized to preserve spatial mask precision.
298
+ quantization_config: Optional custom `transformers.BitsAndBytesConfig` instance.
299
+ """
300
+ if load_in_4bit and load_in_8bit:
301
+ raise ValueError("Cannot enable both `load_in_4bit=True` and `load_in_8bit=True`. Choose one.")
302
+
303
+ if self.model is not None:
304
+ self._log("Existing model detected in memory; offloading before loading new model.")
305
+ self.offload_model()
306
+
307
+ t0 = time.perf_counter()
308
+
309
+ use_auto_device_map = isinstance(device, str) and device.strip().lower() == "auto"
310
+ if use_auto_device_map:
311
+ self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
312
+ else:
313
+ req_device = torch.device(device)
314
+ if req_device.type == "cuda" and not torch.cuda.is_available():
315
+ print(
316
+ f"WARNING: CUDA device '{device}' requested but CUDA is not available. Falling back to CPU.",
317
+ flush=True,
318
+ )
319
+ req_device = torch.device("cpu")
320
+ self.device = req_device
321
+
322
+ use_bnb = bool(load_in_4bit or load_in_8bit or quantization_config is not None)
323
+ if quantization_config is not None:
324
+ resolved_bnb_cfg = quantization_config
325
+ is_4bit = bool(getattr(resolved_bnb_cfg, "load_in_4bit", False))
326
+ is_8bit = bool(getattr(resolved_bnb_cfg, "load_in_8bit", False))
327
+ self.quantization_mode = "4bit" if is_4bit else ("8bit" if is_8bit else "custom_bnb")
328
+ elif load_in_4bit or load_in_8bit:
329
+ try:
330
+ import accelerate # noqa: F401
331
+ import bitsandbytes # noqa: F401
332
+ from transformers import BitsAndBytesConfig
333
+ except ImportError as exc:
334
+ raise ImportError(
335
+ "bitsandbytes quantization requires `bitsandbytes` and `accelerate`. "
336
+ "Install them with: `pip install -U bitsandbytes accelerate`"
337
+ ) from exc
338
+ resolved_bnb_cfg = None
339
+ is_4bit = bool(load_in_4bit)
340
+ is_8bit = bool(load_in_8bit)
341
+ self.quantization_mode = "4bit" if is_4bit else "8bit"
342
+ else:
343
+ resolved_bnb_cfg = None
344
+ is_4bit = False
345
+ is_8bit = False
346
+ self.quantization_mode = "none"
347
+
348
+ if self.device.type == "cuda":
349
+ if is_8bit:
350
+ self.dtype = torch.float16
351
+ elif is_4bit:
352
+ self.dtype = bnb_4bit_compute_dtype or (
353
+ torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
354
+ )
355
+ else:
356
+ self.dtype = (
357
+ torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
358
+ )
359
+ else:
360
+ # On CPU, float32 avoids 100x+ software-emulated fp16/bf16 matmul slowdowns on x86 CPUs
361
+ self.dtype = bnb_4bit_compute_dtype or torch.float32
362
+
363
+ if resolved_bnb_cfg is None and (load_in_4bit or load_in_8bit):
364
+ from transformers import BitsAndBytesConfig
365
+
366
+ # Protect Qwen3.5 tiny 16xD Gated DeltaNet decay gates (in_proj_a/b) and optionally ViT
367
+ skip_modules = ["in_proj_a", "in_proj_b"]
368
+ if bnb_skip_vision:
369
+ skip_modules.insert(0, "visual")
370
+ if load_in_4bit:
371
+ resolved_bnb_cfg = BitsAndBytesConfig(
372
+ load_in_4bit=True,
373
+ bnb_4bit_quant_type=bnb_4bit_quant_type,
374
+ bnb_4bit_use_double_quant=bnb_4bit_use_double_quant,
375
+ bnb_4bit_compute_dtype=self.dtype,
376
+ llm_int8_skip_modules=skip_modules,
377
+ )
378
+ else:
379
+ resolved_bnb_cfg = BitsAndBytesConfig(
380
+ load_in_8bit=True,
381
+ llm_int8_skip_modules=skip_modules,
382
+ )
383
+
384
+ resolved_device_map: Optional[Union[str, Dict[str, Any]]]
385
+ if use_auto_device_map:
386
+ resolved_device_map = "auto"
387
+ elif use_bnb:
388
+ if self.device.type == "cuda":
389
+ dev_idx = (
390
+ self.device.index
391
+ if self.device.index is not None
392
+ else torch.cuda.current_device()
393
+ )
394
+ resolved_device_map = {"": f"cuda:{dev_idx}"}
395
+ else:
396
+ resolved_device_map = {"": str(self.device)}
397
+ else:
398
+ resolved_device_map = None
399
+
400
+ self.qwen_path = pathlib.Path(qwen_path)
401
+ self.lis_model_path = pathlib.Path(lis_model_path)
402
+ self.max_pixels = int(max_pixels)
403
+ self.clear_vision_cache()
404
+
405
+ # 1. Inspect LIS checkpoint (.pt or .safetensors) first to validate paths & metadata
406
+ mask_head_state, ckpt_lora_dir, _ = inspect_lis_checkpoint(self.lis_model_path)
407
+ has_lora_arg = lora_adapter is not None and str(lora_adapter).strip() != ""
408
+
409
+ if not has_lora_arg and ckpt_lora_dir:
410
+ warn_msg = (
411
+ f"WARNING: LIS model '{self.lis_model_path.name}' is a 'forLora' checkpoint "
412
+ f"(metadata contains lora_dir='{ckpt_lora_dir}'), but `lora_adapter` was set to None. "
413
+ "Proceeding without a LoRA adapter as a noLora model, which may reduce segmentation accuracy."
414
+ )
415
+ print(warn_msg, flush=True)
416
+ warnings.warn(warn_msg, UserWarning, stacklevel=2)
417
+
418
+ self._log(
419
+ f"Loading Qwen3.5 backbone from '{self.qwen_path}' on device={self.device} "
420
+ f"(dtype={self.dtype}, quantization={self.quantization_mode})..."
421
+ )
422
+ self.model = Qwen2_5_VL_SinglePassSegmenter(
423
+ qwen_model_name=str(self.qwen_path),
424
+ torch_dtype=self.dtype,
425
+ device_map=resolved_device_map,
426
+ quantization_config=resolved_bnb_cfg,
427
+ freeze_backbone=True,
428
+ num_fused_layers=fused_layers,
429
+ )
430
+ _apply_max_pixels(self.model.processor, max_pixels)
431
+
432
+ # 2. Load LIS mask_head weights (kept in float32 for crisp spatial decoding)
433
+ self._log(
434
+ f"Loading LIS mask head from '{self.lis_model_path}' ({self.lis_model_path.suffix})..."
435
+ )
436
+ self.model.mask_head.load_state_dict(mask_head_state, strict=True)
437
+ if resolved_device_map is None:
438
+ self.model.to(self.device)
439
+ self.model.mask_head.to(device=self.device, dtype=torch.float32)
440
+
441
+ # 3. Load LoRA adapter if provided
442
+ if has_lora_arg:
443
+ lora_cand = pathlib.Path(lora_adapter) # type: ignore[arg-type]
444
+ if not lora_cand.exists():
445
+ rel_cand = self.lis_model_path.parent / lora_cand
446
+ if rel_cand.exists():
447
+ lora_cand = rel_cand
448
+ else:
449
+ raise FileNotFoundError(
450
+ f"LoRA adapter directory not found at '{lora_cand}' or '{rel_cand}'."
451
+ )
452
+
453
+ self._log(f"Loading LoRA adapter from '{lora_cand.resolve()}'...")
454
+ from peft import PeftModel
455
+
456
+ if use_bnb or resolved_device_map is not None:
457
+ # Propagate top-level quantization flags to language_model so PEFT uses bnb LoRA layers
458
+ lm_sub = self.model.backbone.language_model
459
+ for q_attr in ("is_loaded_in_4bit", "is_loaded_in_8bit", "hf_quantizer", "quantization_method"):
460
+ if hasattr(self.model.qwen, q_attr) and not hasattr(lm_sub, q_attr):
461
+ setattr(lm_sub, q_attr, getattr(self.model.qwen, q_attr))
462
+ if hasattr(self.model.qwen.config, "quantization_config") and hasattr(lm_sub, "config"):
463
+ lm_sub.config.quantization_config = self.model.qwen.config.quantization_config
464
+
465
+ self.model.backbone.language_model = PeftModel.from_pretrained(
466
+ lm_sub, str(lora_cand)
467
+ )
468
+ else:
469
+ self.model.backbone.language_model = PeftModel.from_pretrained(
470
+ self.model.backbone.language_model, str(lora_cand)
471
+ ).to(device=self.device, dtype=self.dtype)
472
+ self.model._register_llm_layer_hooks()
473
+ self.model._needs_mm_token_type_ids = self.model._check_needs_mm_token_type_ids()
474
+ self.lora_adapter_path = lora_cand
475
+ else:
476
+ self.lora_adapter_path = None
477
+
478
+ self.model.eval()
479
+
480
+ elapsed = (time.perf_counter() - t0) * 1000.0
481
+ self._log(
482
+ f"Model loaded successfully in {elapsed:.1f} ms "
483
+ f"(quantization={self.quantization_mode}, LoRA={'enabled' if self.lora_adapter_path else 'disabled'})."
484
+ )
485
+ return self
486
+
487
+ def offload_model(self) -> None:
488
+ """
489
+ Unloads the Qwen3.5 backbone, LoRA adapter, and LIS mask head from memory
490
+ and frees GPU/CPU caches.
491
+ """
492
+ t0 = time.perf_counter()
493
+ if self.model is not None:
494
+ # Remove forward hooks to break circular references before GC
495
+ if getattr(self.model, "_visual_hook", None) is not None:
496
+ try:
497
+ self.model._visual_hook.remove()
498
+ except Exception:
499
+ pass
500
+ for h in getattr(self.model, "_llm_hooks", []):
501
+ try:
502
+ h.remove()
503
+ except Exception:
504
+ pass
505
+ if hasattr(self.model, "_llm_hooks"):
506
+ self.model._llm_hooks.clear()
507
+ if hasattr(self.model, "_captured_visual"):
508
+ self.model._captured_visual.clear()
509
+ if hasattr(self.model, "_captured_llm_layers"):
510
+ self.model._captured_llm_layers.clear()
511
+
512
+ if self.quantization_mode == "none":
513
+ try:
514
+ self.model.to("cpu")
515
+ except Exception:
516
+ pass
517
+ del self.model
518
+ self.model = None
519
+ self.quantization_mode = "none"
520
+
521
+ self.clear_vision_cache()
522
+ gc.collect()
523
+ if torch.cuda.is_available():
524
+ torch.cuda.empty_cache()
525
+ torch.cuda.ipc_collect()
526
+
527
+ elapsed = (time.perf_counter() - t0) * 1000.0
528
+ self._log(f"Model offloaded from memory in {elapsed:.1f} ms.")
529
+
530
+ def _encode_image_vision(
531
+ self,
532
+ pil_rgb: Image.Image,
533
+ ) -> Tuple[Dict[str, Any], bool, float]:
534
+ """
535
+ Runs `image_processor` and the ViT `visual` encoder once for `pil_rgb` (or retrieves
536
+ from the LRU `_vision_cache`), returning `(cache_entry, cache_hit, elapsed_ms)`.
537
+ """
538
+ assert self.model is not None
539
+ t0 = time.perf_counter()
540
+
541
+ cache_key: Optional[Tuple[int, int, int, bytes]] = None
542
+ if self.vision_cache_size > 0:
543
+ cache_key = _compute_image_cache_key(pil_rgb, self.max_pixels)
544
+ cached = self._vision_cache.get(cache_key)
545
+ if cached is not None:
546
+ self._vision_cache.move_to_end(cache_key)
547
+ return cached, True, (time.perf_counter() - t0) * 1000.0
548
+
549
+ proc = self.model.processor
550
+ vis_mod = self.model.backbone.visual
551
+ vis_param = next(vis_mod.parameters(), None)
552
+ vis_dtype = getattr(vis_mod, "dtype", None) or (
553
+ vis_param.dtype if vis_param is not None else self.dtype
554
+ )
555
+ vis_device = vis_param.device if vis_param is not None else self.device
556
+
557
+ ip_out = proc.image_processor(images=[pil_rgb], return_tensors="pt")
558
+ pv_1 = ip_out["pixel_values"].to(device=vis_device, dtype=vis_dtype)
559
+ gthw_1 = ip_out["image_grid_thw"].to(device=vis_device)
560
+ gthw_row = ip_out["image_grid_thw"][0].tolist()
561
+
562
+ self.model._captured_visual.clear()
563
+ _ = self.model.backbone.get_image_features(pv_1, gthw_1, return_dict=True)
564
+ merged_vis = self.model._captured_visual.get("merged")
565
+ unmerged_vis = self.model._captured_visual.get("unmerged")
566
+ if merged_vis is None:
567
+ raise RuntimeError("Visual forward hook did not capture features during vision encoding.")
568
+
569
+ merged_1 = (
570
+ torch.cat(list(merged_vis), dim=0)
571
+ if isinstance(merged_vis, (tuple, list))
572
+ else merged_vis
573
+ )
574
+ coarse_grids, fine_grids = self.model._reconstruct_2d_vision_features(
575
+ merged_tokens=merged_1,
576
+ unmerged_tokens=unmerged_vis,
577
+ image_grid_thw=gthw_1,
578
+ grid_thw_list=[gthw_row],
579
+ )
580
+
581
+ head_param = next(self.model.mask_head.parameters())
582
+ head_device = head_param.device
583
+ head_dtype = head_param.dtype
584
+
585
+ orig_w, orig_h = pil_rgb.size
586
+ entry: Dict[str, Any] = {
587
+ "merged": merged_1.detach(),
588
+ "f_coarse": coarse_grids[0].detach().to(device=head_device, dtype=head_dtype),
589
+ "f_fine": fine_grids[0].detach().to(device=head_device, dtype=head_dtype),
590
+ "grid_thw_1": gthw_1.detach(),
591
+ "grid_thw_row": gthw_row,
592
+ "num_image_tokens": int(merged_1.shape[0]),
593
+ "orig_size": (orig_h, orig_w),
594
+ }
595
+
596
+ if cache_key is not None and self.vision_cache_size > 0:
597
+ self._vision_cache[cache_key] = entry
598
+ while len(self._vision_cache) > self.vision_cache_size:
599
+ self._vision_cache.popitem(last=False)
600
+
601
+ return entry, False, (time.perf_counter() - t0) * 1000.0
602
+
603
+ def _prepare_batch(
604
+ self,
605
+ batch_images: Sequence[Image.Image],
606
+ batch_prompt_objs: Sequence[Sequence[Dict[str, str]]],
607
+ processor: Any,
608
+ lock: Optional[threading.Lock],
609
+ needs_mm_token_type_ids: bool,
610
+ image_token_id: int,
611
+ ) -> Dict[str, Any]:
612
+ texts: List[str] = []
613
+ rgb_images: List[Image.Image] = []
614
+ orig_sizes: List[Tuple[int, int]] = []
615
+ batch_objects: List[List[Dict[str, str]]] = []
616
+
617
+ for img, query_objs in zip(batch_images, batch_prompt_objs):
618
+ pil_rgb = img.convert("RGB") if img.mode != "RGB" else img
619
+ orig_w, orig_h = pil_rgb.size
620
+ orig_sizes.append((orig_h, orig_w))
621
+ rgb_images.append(pil_rgb)
622
+
623
+ messages, norm_objs = build_segmentation_messages(
624
+ image=pil_rgb,
625
+ query_input={"objects": list(query_objs)},
626
+ system_prompt=DEFAULT_SYSTEM_PROMPT,
627
+ query_token=OBJECT_QUERY_TOKEN,
628
+ )
629
+ batch_objects.append(norm_objs)
630
+
631
+ if lock is not None:
632
+ with lock:
633
+ chat_text = processor.apply_chat_template(
634
+ messages, tokenize=False, add_generation_prompt=False
635
+ )
636
+ else:
637
+ chat_text = processor.apply_chat_template(
638
+ messages, tokenize=False, add_generation_prompt=False
639
+ )
640
+ texts.append(chat_text)
641
+
642
+ if lock is not None:
643
+ with lock:
644
+ inputs = processor(
645
+ text=texts,
646
+ images=rgb_images,
647
+ padding=True,
648
+ return_tensors="pt",
649
+ )
650
+ else:
651
+ inputs = processor(
652
+ text=texts,
653
+ images=rgb_images,
654
+ padding=True,
655
+ return_tensors="pt",
656
+ )
657
+
658
+ input_ids_cpu = inputs["input_ids"]
659
+ grid_thw_list = inputs["image_grid_thw"].tolist()
660
+ if needs_mm_token_type_ids:
661
+ inputs["mm_token_type_ids"] = (input_ids_cpu == image_token_id).to(dtype=torch.long)
662
+
663
+ return {
664
+ "inputs": inputs,
665
+ "input_ids_cpu": input_ids_cpu,
666
+ "grid_thw_list": grid_thw_list,
667
+ "orig_sizes": orig_sizes,
668
+ "batch_objects": batch_objects,
669
+ }
670
+
671
+ def _postprocess_task_outputs(
672
+ self,
673
+ probs_b: torch.Tensor,
674
+ scores_b: torch.Tensor,
675
+ output_meta_objs: Sequence[Dict[str, str]],
676
+ threshold: float,
677
+ min_component_ratio: float,
678
+ min_component_area: int,
679
+ ) -> List[Dict[str, Any]]:
680
+ bin_masks_gpu = (probs_b >= threshold).to(torch.uint8)
681
+ bin_masks_np = bin_masks_gpu.cpu().numpy()
682
+ probs_np: Optional[np.ndarray] = None
683
+ scores_cpu = scores_b.detach().cpu().tolist()
684
+
685
+ task_results: List[Dict[str, Any]] = []
686
+ for q_idx, meta_obj in enumerate(output_meta_objs):
687
+ if q_idx >= bin_masks_np.shape[0]:
688
+ break
689
+ raw_mask_np = bin_masks_np[q_idx]
690
+ mask_np = filter_mask_components(
691
+ raw_mask_np,
692
+ min_component_ratio=min_component_ratio,
693
+ min_component_area=min_component_area,
694
+ )
695
+ polygons, bbox, area = mask_to_polygons(mask_np)
696
+
697
+ if area > 0:
698
+ if mask_np is raw_mask_np:
699
+ fg_prob = float(probs_b[q_idx][bin_masks_gpu[q_idx] > 0].mean().item())
700
+ else:
701
+ if probs_np is None:
702
+ probs_np = probs_b.detach().cpu().to(torch.float32).numpy()
703
+ fg_prob = float(probs_np[q_idx][mask_np > 0].mean())
704
+ else:
705
+ fg_prob = 0.0
706
+
707
+ pred_s = (
708
+ float(scores_cpu[q_idx])
709
+ if isinstance(scores_cpu, list)
710
+ else float(scores_b[q_idx].item())
711
+ )
712
+ score = round(0.5 * pred_s + 0.5 * fg_prob, 4)
713
+
714
+ x, y, w, h = bbox
715
+ bbox_xyxy = (
716
+ [round(x, 2), round(y, 2), round(x + w, 2), round(y + h, 2)]
717
+ if area > 0
718
+ else [0.0, 0.0, 0.0, 0.0]
719
+ )
720
+
721
+ task_results.append(
722
+ {
723
+ "id": meta_obj["id"],
724
+ "description": meta_obj["description"],
725
+ "score": score,
726
+ "bbox": bbox,
727
+ "bbox_xyxy": bbox_xyxy,
728
+ "area": area,
729
+ "mask": mask_np,
730
+ "segmentation": polygons,
731
+ }
732
+ )
733
+ return task_results
734
+
735
+ def _finalize_exclusive_image_outputs(
736
+ self,
737
+ items: Sequence[Tuple[np.ndarray, float, Dict[str, str]]],
738
+ threshold: float,
739
+ min_component_ratio: float,
740
+ min_component_area: int,
741
+ ) -> List[Dict[str, Any]]:
742
+ """
743
+ Resolves pixel-level mask overlaps across all queries on the same image via
744
+ winner-takes-all (`argmax` over pixel probabilities among queries >= `threshold`).
745
+ """
746
+ num_q = len(items)
747
+ if num_q == 0:
748
+ return []
749
+
750
+ probs_stack = np.stack([it[0] for it in items], axis=0) # [K, H, W] float32
751
+ eff_probs = np.full_like(probs_stack, -1.0)
752
+ for k in range(num_q):
753
+ raw_bin = (probs_stack[k] >= threshold).astype(np.uint8)
754
+ clean_bin = filter_mask_components(
755
+ raw_bin,
756
+ min_component_ratio=min_component_ratio,
757
+ min_component_area=min_component_area,
758
+ )
759
+ eff_probs[k] = np.where(clean_bin > 0, probs_stack[k], -1.0)
760
+
761
+ winner_idx = np.argmax(eff_probs, axis=0)
762
+ max_eff_prob = np.max(eff_probs, axis=0)
763
+ valid_fg = max_eff_prob >= threshold
764
+
765
+ img_results: List[Dict[str, Any]] = []
766
+ for k, (prob_np, pred_s, meta_obj) in enumerate(items):
767
+ win_mask = ((winner_idx == k) & valid_fg).astype(np.uint8)
768
+ mask_np = filter_mask_components(
769
+ win_mask,
770
+ min_component_ratio=min_component_ratio,
771
+ min_component_area=min_component_area,
772
+ )
773
+ polygons, bbox, area = mask_to_polygons(mask_np)
774
+
775
+ if area > 0:
776
+ fg_prob = float(prob_np[mask_np > 0].mean())
777
+ else:
778
+ fg_prob = 0.0
779
+ score = round(0.5 * float(pred_s) + 0.5 * fg_prob, 4)
780
+
781
+ x, y, w, h = bbox
782
+ bbox_xyxy = (
783
+ [round(x, 2), round(y, 2), round(x + w, 2), round(y + h, 2)]
784
+ if area > 0
785
+ else [0.0, 0.0, 0.0, 0.0]
786
+ )
787
+
788
+ img_results.append(
789
+ {
790
+ "id": meta_obj["id"],
791
+ "description": meta_obj["description"],
792
+ "score": score,
793
+ "bbox": bbox,
794
+ "bbox_xyxy": bbox_xyxy,
795
+ "area": area,
796
+ "mask": mask_np,
797
+ "segmentation": polygons,
798
+ }
799
+ )
800
+ return img_results
801
+
802
+ @torch.no_grad()
803
+ def run_model(
804
+ self,
805
+ images: Sequence[Image.Image],
806
+ queries: Sequence[Sequence[str]],
807
+ threshold: float = 0.5,
808
+ batch_size: int = 4,
809
+ num_workers: int = 2,
810
+ isolate_queries: bool = False,
811
+ cache_vision: bool = True,
812
+ exclusive_masks: bool = False,
813
+ min_component_ratio: float = 0.05,
814
+ min_component_area: int = 16,
815
+ ) -> List[Dict[str, Any]]:
816
+ """
817
+ Runs single-pass segmentation over a list of PIL images and a list of query string lists.
818
+
819
+ Args:
820
+ images: List of PIL.Image.Image (length N).
821
+ queries: List of lists of query/riddle strings for each image (length N).
822
+ threshold: Binary mask probability threshold in [0, 1] (default 0.5).
823
+ batch_size: Maximum number of prompts processed simultaneously in one LLM batch.
824
+ num_workers: Background CPU threads used for uncached batch preprocessing.
825
+ isolate_queries: Optional flag. When True, runs each query in its own isolated prompt
826
+ (matching ReasonSeg benchmark evaluation and preventing multi-query cross-bleed),
827
+ while sharing a single ViT vision pass per image when `cache_vision=True`.
828
+ cache_vision: Reuse `image_processor` and ViT visual features across queries on the
829
+ same image (and across consecutive `run_model` calls via an LRU cache).
830
+ exclusive_masks: Optional flag. When True, assigns each pixel to at most one query
831
+ (pixel-wise winner-takes-all `argmax` probability among queries >= `threshold`)
832
+ so predicted masks on the same image never overlap.
833
+ min_component_ratio: Drops disconnected stray mask islands smaller than this fraction
834
+ of the largest component's area (default 0.05; set 0.0 to disable).
835
+ min_component_area: Minimum pixel area floor when filtering multi-component masks.
836
+
837
+ Returns:
838
+ List of dicts (one per image), where each dict contains:
839
+ - "image_size": {"height": H, "width": W}
840
+ - "objects": List of dicts with keys:
841
+ "id", "description", "score", "bbox" ([x, y, w, h]),
842
+ "bbox_xyxy" ([x1, y1, x2, y2]), "area",
843
+ "mask" (np.ndarray of shape (H, W) uint8 in {0, 1}),
844
+ "segmentation" (COCO polygons)
845
+ """
846
+ if self.model is None:
847
+ raise RuntimeError("Model is not loaded. Call `load_model(...)` before `run_model(...)`.")
848
+
849
+ if isinstance(images, Image.Image):
850
+ images = [images]
851
+ if queries and isinstance(queries[0], str):
852
+ queries = [queries] # type: ignore[list-item]
853
+
854
+ if len(images) != len(queries):
855
+ raise ValueError(
856
+ f"Length mismatch: got {len(images)} images and {len(queries)} query lists."
857
+ )
858
+
859
+ num_images = len(images)
860
+ if num_images == 0:
861
+ return []
862
+
863
+ self.model.eval()
864
+ t_start = time.perf_counter()
865
+ is_cuda = self.device.type == "cuda"
866
+
867
+ rgb_images_all: List[Image.Image] = [
868
+ (img.convert("RGB") if img.mode != "RGB" else img) for img in images
869
+ ]
870
+ results: List[Dict[str, Any]] = []
871
+ tasks: List[Dict[str, Any]] = []
872
+ valid_img_indices: List[int] = []
873
+ expected_query_counts: Dict[int, int] = {}
874
+
875
+ for idx, (pil_rgb, q_list) in enumerate(zip(rgb_images_all, queries)):
876
+ w, h = pil_rgb.size
877
+ results.append({"image_size": {"height": h, "width": w}, "objects": []})
878
+
879
+ cleaned_q = [str(q).strip() for q in q_list if str(q).strip()]
880
+ if not cleaned_q:
881
+ continue
882
+ valid_img_indices.append(idx)
883
+ expected_query_counts[idx] = len(cleaned_q)
884
+
885
+ if isolate_queries:
886
+ for q_i, desc in enumerate(cleaned_q):
887
+ tasks.append(
888
+ {
889
+ "img_idx": idx,
890
+ "prompt_objs": [{"id": "0", "description": desc}],
891
+ "meta_objs": [{"id": str(q_i), "description": desc}],
892
+ }
893
+ )
894
+ else:
895
+ obj_specs = [
896
+ {"id": str(q_i), "description": desc}
897
+ for q_i, desc in enumerate(cleaned_q)
898
+ ]
899
+ tasks.append(
900
+ {
901
+ "img_idx": idx,
902
+ "prompt_objs": obj_specs,
903
+ "meta_objs": obj_specs,
904
+ }
905
+ )
906
+
907
+ if not tasks:
908
+ return results
909
+
910
+ pending_exclusive: Dict[int, List[Tuple[np.ndarray, float, Dict[str, str]]]] = {}
911
+
912
+ def _record_task_output(
913
+ task: Dict[str, Any],
914
+ probs_b: torch.Tensor,
915
+ scores_b: torch.Tensor,
916
+ ) -> None:
917
+ img_idx = task["img_idx"]
918
+ meta_objs = task["meta_objs"]
919
+ if exclusive_masks and expected_query_counts.get(img_idx, 0) > 1:
920
+ probs_np = probs_b.detach().cpu().to(torch.float32).numpy()
921
+ scores_cpu = scores_b.detach().cpu().tolist()
922
+ buf = pending_exclusive.setdefault(img_idx, [])
923
+ for q_idx, meta_obj in enumerate(meta_objs):
924
+ if q_idx >= probs_np.shape[0]:
925
+ break
926
+ pred_s = (
927
+ float(scores_cpu[q_idx])
928
+ if isinstance(scores_cpu, list)
929
+ else float(scores_b[q_idx].item())
930
+ )
931
+ buf.append((probs_np[q_idx], pred_s, meta_obj))
932
+ if len(buf) >= expected_query_counts[img_idx]:
933
+ results[img_idx]["objects"] = self._finalize_exclusive_image_outputs(
934
+ items=buf,
935
+ threshold=threshold,
936
+ min_component_ratio=min_component_ratio,
937
+ min_component_area=min_component_area,
938
+ )
939
+ del pending_exclusive[img_idx]
940
+ else:
941
+ task_objs = self._postprocess_task_outputs(
942
+ probs_b=probs_b,
943
+ scores_b=scores_b,
944
+ output_meta_objs=meta_objs,
945
+ threshold=threshold,
946
+ min_component_ratio=min_component_ratio,
947
+ min_component_area=min_component_area,
948
+ )
949
+ results[img_idx]["objects"].extend(task_objs)
950
+
951
+ eff_batch_size = max(1, int(batch_size))
952
+ task_chunks: List[List[Dict[str, Any]]] = [
953
+ tasks[i : i + eff_batch_size] for i in range(0, len(tasks), eff_batch_size)
954
+ ]
955
+
956
+ needs_mm = getattr(self.model, "_needs_mm_token_type_ids", False)
957
+ image_token_id = self.model.image_token_id
958
+ base_processor = self.model.processor
959
+
960
+ can_use_vision_cache = bool(
961
+ cache_vision
962
+ and base_processor is not None
963
+ and hasattr(base_processor, "image_processor")
964
+ and hasattr(base_processor, "tokenizer")
965
+ and hasattr(self.model.backbone, "get_image_features")
966
+ and hasattr(self.model.backbone, "get_input_embeddings")
967
+ )
968
+
969
+ total_queries = sum(len(t["meta_objs"]) for t in tasks)
970
+ self._log(
971
+ f"Starting inference on {len(valid_img_indices)} image(s) with {total_queries} query(ies) "
972
+ f"-> {len(tasks)} prompt(s) across {len(task_chunks)} batch(es) "
973
+ f"(batch_size={eff_batch_size}, isolate_queries={isolate_queries}, "
974
+ f"cache_vision={can_use_vision_cache}, exclusive_masks={exclusive_masks})."
975
+ )
976
+
977
+ total_prep_ms = 0.0
978
+ total_vit_ms = 0.0
979
+ total_forward_ms = 0.0
980
+ total_gpu_ms = 0.0
981
+ total_post_ms = 0.0
982
+
983
+ # =====================================================================
984
+ # Path A: Vision-Cached Execution (1x ViT per unique image + batched LLM)
985
+ # =====================================================================
986
+ if can_use_vision_cache:
987
+ vis_entries_by_img: Dict[int, Dict[str, Any]] = {}
988
+ cache_hits = 0
989
+ cache_misses = 0
990
+ for img_idx in valid_img_indices:
991
+ entry, hit, vit_ms = self._encode_image_vision(rgb_images_all[img_idx])
992
+ vis_entries_by_img[img_idx] = entry
993
+ total_vit_ms += vit_ms
994
+ if hit:
995
+ cache_hits += 1
996
+ else:
997
+ cache_misses += 1
998
+
999
+ if self._debug:
1000
+ self._log(
1001
+ f"Vision encoding: {total_vit_ms:.1f} ms across {len(valid_img_indices)} image(s) "
1002
+ f"(cache_hits={cache_hits}, cache_misses={cache_misses})."
1003
+ )
1004
+
1005
+ img_tok_str = getattr(base_processor, "image_token", "<|image_pad|>")
1006
+ head_param = next(self.model.mask_head.parameters())
1007
+ head_device = head_param.device
1008
+ head_dtype = head_param.dtype
1009
+
1010
+ for batch_num, chunk in enumerate(task_chunks, start=1):
1011
+ tp0 = time.perf_counter()
1012
+ expanded_texts: List[str] = []
1013
+ chunk_vis: List[Dict[str, Any]] = []
1014
+
1015
+ for task in chunk:
1016
+ img_idx = task["img_idx"]
1017
+ vis_entry = vis_entries_by_img[img_idx]
1018
+ chunk_vis.append(vis_entry)
1019
+ messages, _ = build_segmentation_messages(
1020
+ image=rgb_images_all[img_idx],
1021
+ query_input={"objects": task["prompt_objs"]},
1022
+ system_prompt=DEFAULT_SYSTEM_PROMPT,
1023
+ query_token=OBJECT_QUERY_TOKEN,
1024
+ )
1025
+ chat_text = base_processor.apply_chat_template(
1026
+ messages, tokenize=False, add_generation_prompt=False
1027
+ )
1028
+ expanded_texts.append(
1029
+ chat_text.replace(
1030
+ img_tok_str, img_tok_str * vis_entry["num_image_tokens"], 1
1031
+ )
1032
+ )
1033
+
1034
+ tok_out = base_processor.tokenizer(
1035
+ expanded_texts, padding=True, return_tensors="pt"
1036
+ )
1037
+ input_ids_cpu = tok_out["input_ids"]
1038
+ attention_mask_cpu = tok_out["attention_mask"]
1039
+ mm_ids_cpu = (
1040
+ (input_ids_cpu == image_token_id).to(dtype=torch.long)
1041
+ if needs_mm
1042
+ else None
1043
+ )
1044
+ prep_ms = (time.perf_counter() - tp0) * 1000.0
1045
+ total_prep_ms += prep_ms
1046
+
1047
+ t_fwd0 = time.perf_counter()
1048
+ ev_start = None
1049
+ ev_end = None
1050
+ if is_cuda and self._debug:
1051
+ ev_start = torch.cuda.Event(enable_timing=True)
1052
+ ev_end = torch.cuda.Event(enable_timing=True)
1053
+ ev_start.record()
1054
+
1055
+ input_ids = input_ids_cpu.to(self.device, non_blocking=is_cuda)
1056
+ attention_mask = attention_mask_cpu.to(self.device, non_blocking=is_cuda)
1057
+ batch_grid_thw = torch.cat(
1058
+ [v["grid_thw_1"] for v in chunk_vis], dim=0
1059
+ ).to(device=self.device)
1060
+ grid_thw_list = [v["grid_thw_row"] for v in chunk_vis]
1061
+
1062
+ inputs_embeds = self.model.backbone.get_input_embeddings()(input_ids)
1063
+ merged_cat = torch.cat(
1064
+ [
1065
+ v["merged"].to(device=inputs_embeds.device, dtype=inputs_embeds.dtype)
1066
+ for v in chunk_vis
1067
+ ],
1068
+ dim=0,
1069
+ )
1070
+ img_mask = (input_ids == image_token_id).unsqueeze(-1).expand_as(inputs_embeds)
1071
+ inputs_embeds = inputs_embeds.masked_scatter(img_mask, merged_cat)
1072
+
1073
+ bb_kwargs: Dict[str, Any] = {
1074
+ "input_ids": input_ids,
1075
+ "inputs_embeds": inputs_embeds,
1076
+ "attention_mask": attention_mask,
1077
+ "pixel_values": None,
1078
+ "image_grid_thw": batch_grid_thw,
1079
+ "use_cache": False,
1080
+ "output_hidden_states": False,
1081
+ "return_dict": True,
1082
+ }
1083
+ if needs_mm and mm_ids_cpu is not None:
1084
+ bb_kwargs["mm_token_type_ids"] = mm_ids_cpu.to(
1085
+ self.device, non_blocking=is_cuda
1086
+ )
1087
+
1088
+ self.model._captured_llm_layers.clear()
1089
+ llm_outputs = self.model.backbone(**bb_kwargs)
1090
+ last_hs = llm_outputs.last_hidden_state
1091
+
1092
+ llm_vis_grids = self.model._extract_llm_visual_grids(
1093
+ last_hidden_states=last_hs,
1094
+ input_ids=input_ids,
1095
+ image_grid_thw=batch_grid_thw,
1096
+ layer_states=self.model._captured_llm_layers,
1097
+ grid_thw_list=grid_thw_list,
1098
+ input_ids_cpu=input_ids_cpu,
1099
+ )
1100
+ batch_queries = self.model._extract_query_vectors(
1101
+ last_hidden_states=last_hs,
1102
+ input_ids=input_ids,
1103
+ layer_states=self.model._captured_llm_layers,
1104
+ input_ids_cpu=input_ids_cpu,
1105
+ )
1106
+
1107
+ probs_list: List[torch.Tensor] = []
1108
+ scores_list: List[torch.Tensor] = []
1109
+ for b_pos, vis_entry in enumerate(chunk_vis):
1110
+ f_llm_b = (
1111
+ llm_vis_grids[b_pos].to(device=head_device, dtype=head_dtype)
1112
+ if llm_vis_grids[b_pos] is not None
1113
+ else None
1114
+ )
1115
+ logits_b, scores_b = self.model.mask_head.forward_single_image(
1116
+ queries=batch_queries[b_pos].to(device=head_device, dtype=head_dtype),
1117
+ f_coarse=vis_entry["f_coarse"],
1118
+ f_fine=vis_entry["f_fine"],
1119
+ target_size=vis_entry["orig_size"],
1120
+ f_llm_coarse=f_llm_b,
1121
+ )
1122
+ probs_list.append(torch.sigmoid(logits_b))
1123
+ scores_list.append(scores_b)
1124
+
1125
+ if ev_end is not None and ev_start is not None:
1126
+ ev_end.record()
1127
+ torch.cuda.synchronize(self.device)
1128
+ batch_gpu_ms = float(ev_start.elapsed_time(ev_end))
1129
+ total_gpu_ms += batch_gpu_ms
1130
+ else:
1131
+ batch_gpu_ms = 0.0
1132
+
1133
+ batch_fwd_ms = (time.perf_counter() - t_fwd0) * 1000.0
1134
+ total_forward_ms += batch_fwd_ms
1135
+
1136
+ t_post0 = time.perf_counter()
1137
+ for b_pos, task in enumerate(chunk):
1138
+ _record_task_output(
1139
+ task=task,
1140
+ probs_b=probs_list[b_pos],
1141
+ scores_b=scores_list[b_pos],
1142
+ )
1143
+
1144
+ batch_post_ms = (time.perf_counter() - t_post0) * 1000.0
1145
+ total_post_ms += batch_post_ms
1146
+
1147
+ if self._debug:
1148
+ gpu_info = f", gpu_forward={batch_gpu_ms:.1f} ms" if is_cuda else ""
1149
+ self._log(
1150
+ f"Batch {batch_num}/{len(task_chunks)} ({len(chunk)} prompt(s)): "
1151
+ f"prep={prep_ms:.1f} ms, llm+head={batch_fwd_ms:.1f} ms{gpu_info}, "
1152
+ f"post={batch_post_ms:.1f} ms"
1153
+ )
1154
+
1155
+ total_ms = (time.perf_counter() - t_start) * 1000.0
1156
+ if self._debug:
1157
+ per_img_ms = total_ms / max(len(valid_img_indices), 1)
1158
+ self._log(
1159
+ f"Inference finished in {total_ms:.1f} ms ({per_img_ms:.1f} ms/img) | "
1160
+ f"vision={total_vit_ms:.1f} ms, prep={total_prep_ms:.1f} ms, "
1161
+ f"llm+head={total_forward_ms:.1f} ms, post={total_post_ms:.1f} ms."
1162
+ )
1163
+ return results
1164
+
1165
+ # =====================================================================
1166
+ # Path B: Uncached Standard Execution (`cache_vision=False` or custom processor)
1167
+ # =====================================================================
1168
+ use_threads = num_workers > 0 and len(task_chunks) > 1
1169
+ tls = threading.local()
1170
+ fallback_lock = threading.Lock()
1171
+ deepcopy_ok = {"ok": True}
1172
+
1173
+ def _get_proc() -> Tuple[Any, Optional[threading.Lock]]:
1174
+ if not use_threads:
1175
+ return base_processor, None
1176
+ proc = getattr(tls, "proc", None)
1177
+ if proc is not None:
1178
+ return proc, None
1179
+ if deepcopy_ok["ok"]:
1180
+ try:
1181
+ proc = copy.deepcopy(base_processor)
1182
+ tls.proc = proc
1183
+ return proc, None
1184
+ except Exception:
1185
+ deepcopy_ok["ok"] = False
1186
+ return base_processor, fallback_lock
1187
+
1188
+ def _prep_chunk(chunk: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], Dict[str, Any], float]:
1189
+ tp0 = time.perf_counter()
1190
+ proc, lck = _get_proc()
1191
+ b_imgs = [rgb_images_all[t["img_idx"]] for t in chunk]
1192
+ b_prompt_objs = [t["prompt_objs"] for t in chunk]
1193
+ prepared = self._prepare_batch(
1194
+ batch_images=b_imgs,
1195
+ batch_prompt_objs=b_prompt_objs,
1196
+ processor=proc,
1197
+ lock=lck,
1198
+ needs_mm_token_type_ids=needs_mm,
1199
+ image_token_id=image_token_id,
1200
+ )
1201
+ prep_ms = (time.perf_counter() - tp0) * 1000.0
1202
+ return chunk, prepared, prep_ms
1203
+
1204
+ def _iter_batches() -> Iterator[Tuple[List[Dict[str, Any]], Dict[str, Any], float]]:
1205
+ if not use_threads:
1206
+ for c in task_chunks:
1207
+ yield _prep_chunk(c)
1208
+ return
1209
+
1210
+ max_inflight = max(num_workers * 2, 2)
1211
+ inflight: deque[Future] = deque()
1212
+ chunk_it = iter(task_chunks)
1213
+ exhausted = False
1214
+
1215
+ with ThreadPoolExecutor(max_workers=num_workers) as pool:
1216
+ def _fill() -> None:
1217
+ nonlocal exhausted
1218
+ while not exhausted and len(inflight) < max_inflight:
1219
+ try:
1220
+ c = next(chunk_it)
1221
+ except StopIteration:
1222
+ exhausted = True
1223
+ break
1224
+ inflight.append(pool.submit(_prep_chunk, c))
1225
+
1226
+ _fill()
1227
+ while inflight:
1228
+ fut = inflight.popleft()
1229
+ _fill()
1230
+ yield fut.result()
1231
+
1232
+ for batch_num, (chunk, prepared, prep_ms) in enumerate(_iter_batches(), start=1):
1233
+ total_prep_ms += prep_ms
1234
+
1235
+ t_fwd0 = time.perf_counter()
1236
+ dev_inputs = {
1237
+ k: (v.to(self.device, non_blocking=is_cuda) if isinstance(v, torch.Tensor) else v)
1238
+ for k, v in prepared["inputs"].items()
1239
+ }
1240
+
1241
+ ev_start = None
1242
+ ev_end = None
1243
+ if is_cuda and self._debug:
1244
+ ev_start = torch.cuda.Event(enable_timing=True)
1245
+ ev_end = torch.cuda.Event(enable_timing=True)
1246
+ ev_start.record()
1247
+
1248
+ outputs = self.model(
1249
+ input_ids=dev_inputs["input_ids"],
1250
+ attention_mask=dev_inputs["attention_mask"],
1251
+ pixel_values=dev_inputs["pixel_values"],
1252
+ image_grid_thw=dev_inputs["image_grid_thw"],
1253
+ target_sizes=prepared["orig_sizes"],
1254
+ mm_token_type_ids=dev_inputs.get("mm_token_type_ids"),
1255
+ grid_thw_list=prepared["grid_thw_list"],
1256
+ input_ids_cpu=prepared["input_ids_cpu"],
1257
+ )
1258
+
1259
+ if ev_end is not None and ev_start is not None:
1260
+ ev_end.record()
1261
+ torch.cuda.synchronize(self.device)
1262
+ batch_gpu_ms = float(ev_start.elapsed_time(ev_end))
1263
+ total_gpu_ms += batch_gpu_ms
1264
+ else:
1265
+ batch_gpu_ms = 0.0
1266
+
1267
+ batch_fwd_ms = (time.perf_counter() - t_fwd0) * 1000.0
1268
+ total_forward_ms += batch_fwd_ms
1269
+
1270
+ t_post0 = time.perf_counter()
1271
+ for b_pos, task in enumerate(chunk):
1272
+ _record_task_output(
1273
+ task=task,
1274
+ probs_b=outputs["mask_probs"][b_pos],
1275
+ scores_b=outputs["pred_scores"][b_pos],
1276
+ )
1277
+
1278
+ batch_post_ms = (time.perf_counter() - t_post0) * 1000.0
1279
+ total_post_ms += batch_post_ms
1280
+
1281
+ if self._debug:
1282
+ gpu_info = f", gpu_forward={batch_gpu_ms:.1f} ms" if is_cuda else ""
1283
+ self._log(
1284
+ f"Batch {batch_num}/{len(task_chunks)} ({len(chunk)} prompt(s)): "
1285
+ f"prep={prep_ms:.1f} ms, forward={batch_fwd_ms:.1f} ms{gpu_info}, "
1286
+ f"post={batch_post_ms:.1f} ms"
1287
+ )
1288
+
1289
+ total_ms = (time.perf_counter() - t_start) * 1000.0
1290
+ if self._debug:
1291
+ per_img_ms = total_ms / max(len(valid_img_indices), 1)
1292
+ gpu_summary = (
1293
+ f", total_gpu={total_gpu_ms:.1f} ms ({total_gpu_ms / max(len(valid_img_indices), 1):.1f} ms/img)"
1294
+ if is_cuda
1295
+ else ""
1296
+ )
1297
+ self._log(
1298
+ f"Inference finished in {total_ms:.1f} ms ({per_img_ms:.1f} ms/img{gpu_summary}) | "
1299
+ f"prep={total_prep_ms:.1f} ms, forward={total_forward_ms:.1f} ms, post={total_post_ms:.1f} ms."
1300
+ )
1301
+
1302
+ return results
1303
+
1304
+ def render_image(
1305
+ self,
1306
+ images: Union[Image.Image, Sequence[Image.Image]],
1307
+ outputs: Union[Dict[str, Any], Sequence[Dict[str, Any]]],
1308
+ draw_masks: bool = True,
1309
+ draw_boxes: bool = True,
1310
+ draw_labels: bool = True,
1311
+ draw_scores: bool = True,
1312
+ draw_contours: bool = True,
1313
+ mask_alpha: float = 0.45,
1314
+ palette: Sequence[Tuple[int, int, int]] = DEFAULT_PALETTE,
1315
+ ) -> List[Image.Image]:
1316
+ """
1317
+ Renders predicted segmentation masks, bounding boxes, and labels onto PIL images.
1318
+
1319
+ Args:
1320
+ images: List of PIL.Image.Image (or a single PIL.Image.Image).
1321
+ outputs: Output list of dicts returned by `run_model(...)` (or a single output dict).
1322
+ draw_masks: Whether to overlay semi-transparent segmentation masks.
1323
+ draw_boxes: Whether to draw 2D bounding boxes around segmented objects.
1324
+ draw_labels: Whether to draw object description text labels.
1325
+ draw_scores: Whether to include confidence scores in the text labels (when `draw_labels=True`).
1326
+ draw_contours: Whether to draw crisp boundary contours around masks (when `draw_masks=True`).
1327
+ mask_alpha: Opacity of the mask overlay in [0, 1].
1328
+ palette: Sequence of (R, G, B) color tuples cycled across objects.
1329
+
1330
+ Returns:
1331
+ List of rendered PIL.Image.Image objects (RGB).
1332
+ """
1333
+ t0 = time.perf_counter()
1334
+
1335
+ img_list: List[Image.Image] = (
1336
+ [images] if isinstance(images, Image.Image) else list(images)
1337
+ )
1338
+ out_list: List[Dict[str, Any]] = (
1339
+ [outputs] if isinstance(outputs, dict) else list(outputs)
1340
+ )
1341
+
1342
+ if len(img_list) != len(out_list):
1343
+ raise ValueError(
1344
+ f"Length mismatch in `render_image`: got {len(img_list)} images and {len(out_list)} outputs."
1345
+ )
1346
+
1347
+ rendered_images: List[Image.Image] = []
1348
+ alpha = float(np.clip(mask_alpha, 0.0, 1.0))
1349
+
1350
+ try:
1351
+ font = ImageFont.load_default()
1352
+ except Exception:
1353
+ font = None
1354
+
1355
+ for img, out_dict in zip(img_list, out_list):
1356
+ canvas_np = np.array(img.convert("RGB"), dtype=np.uint8, copy=True)
1357
+ h, w = canvas_np.shape[:2]
1358
+ objects = out_dict.get("objects", [])
1359
+
1360
+ # 1. Render masks and contours on numpy canvas
1361
+ if draw_masks:
1362
+ for obj_idx, obj in enumerate(objects):
1363
+ mask = obj.get("mask")
1364
+ if mask is None:
1365
+ continue
1366
+ mask_u8 = np.asarray(mask, dtype=np.uint8)
1367
+ if mask_u8.ndim > 2:
1368
+ mask_u8 = mask_u8.squeeze()
1369
+ if mask_u8.shape != (h, w):
1370
+ mask_u8 = cv2.resize(mask_u8, (w, h), interpolation=cv2.INTER_NEAREST)
1371
+
1372
+ fg = mask_u8 > 0
1373
+ if not np.any(fg):
1374
+ continue
1375
+
1376
+ color = palette[obj_idx % len(palette)]
1377
+ color_arr = np.array(color, dtype=np.float32)
1378
+ canvas_np[fg] = np.clip(
1379
+ (1.0 - alpha) * canvas_np[fg].astype(np.float32) + alpha * color_arr,
1380
+ 0,
1381
+ 255,
1382
+ ).astype(np.uint8)
1383
+
1384
+ if draw_contours:
1385
+ contours, _ = cv2.findContours(
1386
+ mask_u8, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE
1387
+ )
1388
+ cv2.drawContours(canvas_np, contours, -1, color, thickness=2)
1389
+
1390
+ pil_canvas = Image.fromarray(canvas_np)
1391
+ draw = ImageDraw.Draw(pil_canvas)
1392
+
1393
+ # 2. Render bounding boxes first, then non-overlapping labels on top
1394
+ if draw_boxes or draw_labels:
1395
+ valid_box_entries: List[Tuple[int, Dict[str, Any], float, float, float, float]] = []
1396
+ for obj_idx, obj in enumerate(objects):
1397
+ area = float(obj.get("area", 0.0))
1398
+ bbox = obj.get("bbox", [0.0, 0.0, 0.0, 0.0])
1399
+ if area <= 0 or len(bbox) != 4 or bbox[2] <= 0 or bbox[3] <= 0:
1400
+ continue
1401
+
1402
+ x, y, bw, bh = [float(v) for v in bbox]
1403
+ x1 = max(0.0, min(x, w - 1.0))
1404
+ y1 = max(0.0, min(y, h - 1.0))
1405
+ x2 = max(0.0, min(x + bw, float(w)))
1406
+ y2 = max(0.0, min(y + bh, float(h)))
1407
+ valid_box_entries.append((obj_idx, obj, x1, y1, x2, y2))
1408
+
1409
+ if draw_boxes:
1410
+ for obj_idx, _, x1, y1, x2, y2 in valid_box_entries:
1411
+ color = palette[obj_idx % len(palette)]
1412
+ draw.rectangle([x1, y1, x2, y2], outline=color, width=2)
1413
+
1414
+ if draw_labels:
1415
+ occupied_labels: List[Tuple[float, float, float, float]] = []
1416
+ for obj_idx, obj, x1, y1, _, _ in valid_box_entries:
1417
+ color = palette[obj_idx % len(palette)]
1418
+ desc = str(obj.get("description", f"obj_{obj_idx}")).strip()
1419
+ if len(desc) > 48:
1420
+ desc = desc[:45] + "..."
1421
+ if draw_scores and "score" in obj:
1422
+ label_text = f"{desc} ({float(obj['score']):.2f})"
1423
+ else:
1424
+ label_text = desc
1425
+
1426
+ if hasattr(draw, "textbbox"):
1427
+ tb = draw.textbbox((0, 0), label_text, font=font)
1428
+ tw, th = float(tb[2] - tb[0]), float(tb[3] - tb[1])
1429
+ else:
1430
+ tw, th = float(len(label_text) * 6), 11.0
1431
+
1432
+ pad = 3.0
1433
+ label_w = tw + 2.0 * pad
1434
+ label_h = th + 2.0 * pad
1435
+ tx1, ty1, tx2, ty2 = _place_non_overlapping_label(
1436
+ x1=x1,
1437
+ y1=y1,
1438
+ label_w=label_w,
1439
+ label_h=label_h,
1440
+ img_w=w,
1441
+ img_h=h,
1442
+ occupied=occupied_labels,
1443
+ )
1444
+ occupied_labels.append((tx1, ty1, tx2, ty2))
1445
+
1446
+ draw.rectangle([tx1, ty1, tx2, ty2], fill=color)
1447
+ draw.text(
1448
+ (tx1 + pad, ty1 + pad),
1449
+ label_text,
1450
+ fill=(255, 255, 255),
1451
+ font=font,
1452
+ )
1453
+
1454
+ rendered_images.append(pil_canvas)
1455
+
1456
+ elapsed = (time.perf_counter() - t0) * 1000.0
1457
+ self._log(
1458
+ f"Rendered {len(rendered_images)} image(s) in {elapsed:.1f} ms "
1459
+ f"(draw_masks={draw_masks}, draw_boxes={draw_boxes}, draw_labels={draw_labels})."
1460
+ )
1461
+ return rendered_images
logo.png ADDED
result_0_boxes_and_masks.png ADDED

Git LFS Details

  • SHA256: 256826408c7ab3105a1e922d92d3c625fa59fa123d7848c1c21c22f6de2b9859
  • Pointer size: 132 Bytes
  • Size of remote file: 1.53 MB
train.py ADDED
@@ -0,0 +1,224 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import pathlib
3
+ from typing import Dict, Tuple
4
+
5
+ import torch
6
+ from torch.optim import AdamW
7
+ from torch.utils.data import DataLoader
8
+
9
+ from LIS1ModelCode import Qwen2_5_VL_SinglePassSegmenter, SegmentationLoss
10
+ from dataset import RefCOCOJsonCollator, RefCOCOJsonDataset
11
+
12
+
13
+ @torch.no_grad()
14
+ def evaluate_miou(
15
+ model: Qwen2_5_VL_SinglePassSegmenter,
16
+ val_loader: DataLoader,
17
+ device: torch.device,
18
+ max_batches: int = 100,
19
+ ) -> Dict[str, float]:
20
+ model.eval()
21
+ total_iou = 0.0
22
+ total_inter = 0.0
23
+ total_union = 0.0
24
+ total_objects = 0
25
+
26
+ for step, batch in enumerate(val_loader):
27
+ if max_batches > 0 and step >= max_batches:
28
+ break
29
+
30
+ outputs = model(
31
+ input_ids=batch["input_ids"].to(device),
32
+ attention_mask=batch["attention_mask"].to(device),
33
+ pixel_values=batch["pixel_values"].to(device),
34
+ image_grid_thw=batch["image_grid_thw"].to(device),
35
+ target_sizes=batch["target_sizes"],
36
+ )
37
+
38
+ for probs, gt in zip(outputs["mask_probs"], batch["gt_masks_list"]):
39
+ pred_bin = (probs >= 0.5).to(torch.float32).cpu().flatten(1)
40
+ gt_flat = gt.to(torch.float32).cpu().flatten(1)
41
+ inter = (pred_bin * gt_flat).sum(dim=1)
42
+ union = (pred_bin + gt_flat - pred_bin * gt_flat).sum(dim=1)
43
+
44
+ iou = inter / union.clamp(min=1e-6)
45
+ total_iou += float(iou.sum().item())
46
+ total_inter += float(inter.sum().item())
47
+ total_union += float(union.sum().item())
48
+ total_objects += int(gt.shape[0])
49
+
50
+ miou = total_iou / max(total_objects, 1)
51
+ ciou = total_inter / max(total_union, 1e-6)
52
+ return {"mIoU": round(miou, 4), "cIoU": round(ciou, 4), "objects": total_objects}
53
+
54
+
55
+ def save_checkpoint(
56
+ model: Qwen2_5_VL_SinglePassSegmenter,
57
+ output_dir: pathlib.Path,
58
+ tag: str,
59
+ ) -> pathlib.Path:
60
+ output_dir.mkdir(parents=True, exist_ok=True)
61
+ ckpt_path = output_dir / f"segmenter_{tag}.pt"
62
+ state = {"mask_head": model.mask_head.state_dict()}
63
+ if not model.freeze_backbone and hasattr(model.backbone.language_model, "peft_config"):
64
+ lora_dir = output_dir / f"lora_{tag}"
65
+ model.backbone.language_model.save_pretrained(str(lora_dir))
66
+ state["lora_dir"] = str(lora_dir)
67
+ torch.save(state, ckpt_path)
68
+ return ckpt_path
69
+
70
+
71
+ def load_checkpoint(
72
+ model: Qwen2_5_VL_SinglePassSegmenter,
73
+ ckpt_path: pathlib.Path,
74
+ device: torch.device,
75
+ ) -> None:
76
+ state = torch.load(ckpt_path, map_location=device)
77
+ model.mask_head.load_state_dict(state["mask_head"], strict=False)
78
+
79
+
80
+ def run_training(args: argparse.Namespace) -> None:
81
+ device = torch.device(args.device if torch.cuda.is_available() else "cpu")
82
+ dtype = torch.bfloat16 if (device.type == "cuda" and torch.cuda.is_bf16_supported()) else torch.float32
83
+
84
+ model = Qwen2_5_VL_SinglePassSegmenter(
85
+ qwen_model_name=args.model_path,
86
+ torch_dtype=dtype,
87
+ freeze_backbone=True,
88
+ num_fused_layers=args.fused_layers,
89
+ )
90
+
91
+ if args.stage == 2:
92
+ model.enable_lora(
93
+ r=args.lora_r,
94
+ lora_alpha=args.lora_alpha,
95
+ top_k_layers=args.lora_top_layers,
96
+ )
97
+
98
+ model.to(device)
99
+ model.mask_head.to(device=device, dtype=torch.float32)
100
+
101
+ if args.resume_ckpt:
102
+ load_checkpoint(model, pathlib.Path(args.resume_ckpt), device)
103
+
104
+ train_ds = RefCOCOJsonDataset(
105
+ data_root=args.data_root,
106
+ dataset_name=args.dataset,
107
+ split_by=args.split_by,
108
+ split="train",
109
+ max_objects_per_image=args.max_objects,
110
+ riddle_prob=args.riddle_prob,
111
+ )
112
+ val_ds = RefCOCOJsonDataset(
113
+ data_root=args.data_root,
114
+ dataset_name="refcoco" if args.dataset == "all" else args.dataset,
115
+ split_by=args.split_by,
116
+ split="val",
117
+ max_objects_per_image=args.max_objects,
118
+ riddle_prob=0.0,
119
+ )
120
+
121
+ collator = RefCOCOJsonCollator(model.processor)
122
+ train_loader = DataLoader(
123
+ train_ds,
124
+ batch_size=args.batch_size,
125
+ shuffle=True,
126
+ num_workers=args.num_workers,
127
+ collate_fn=collator,
128
+ )
129
+ val_loader = DataLoader(
130
+ val_ds,
131
+ batch_size=args.batch_size,
132
+ shuffle=False,
133
+ num_workers=args.num_workers,
134
+ collate_fn=collator,
135
+ )
136
+
137
+ param_groups = [{"params": list(model.mask_head.parameters()), "lr": args.lr_head}]
138
+ if args.stage == 2:
139
+ lora_params = [p for p in model.backbone.parameters() if p.requires_grad]
140
+ param_groups.append({"params": lora_params, "lr": args.lr_lora})
141
+
142
+ optimizer = AdamW(param_groups, weight_decay=args.weight_decay)
143
+ criterion = SegmentationLoss()
144
+ output_dir = pathlib.Path(args.output_dir)
145
+
146
+ for epoch in range(1, args.epochs + 1):
147
+ model.train()
148
+ running_loss = 0.0
149
+ for step, batch in enumerate(train_loader, start=1):
150
+ optimizer.zero_grad(set_to_none=True)
151
+
152
+ outputs = model(
153
+ input_ids=batch["input_ids"].to(device),
154
+ attention_mask=batch["attention_mask"].to(device),
155
+ pixel_values=batch["pixel_values"].to(device),
156
+ image_grid_thw=batch["image_grid_thw"].to(device),
157
+ target_sizes=batch["target_sizes"],
158
+ )
159
+ loss_dict = criterion(
160
+ mask_logits_list=outputs["mask_logits"],
161
+ pred_scores_list=outputs["pred_scores"],
162
+ gt_masks_list=batch["gt_masks_list"],
163
+ )
164
+ loss = loss_dict["loss"]
165
+ loss.backward()
166
+ torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
167
+ optimizer.step()
168
+
169
+ running_loss += float(loss.item())
170
+ if step % args.log_every == 0:
171
+ avg_loss = running_loss / args.log_every
172
+ running_loss = 0.0
173
+ print(
174
+ f"[Epoch {epoch}/{args.epochs} | Step {step}/{len(train_loader)}] "
175
+ f"loss={avg_loss:.4f} dice={loss_dict['dice_loss'].item():.4f} "
176
+ f"focal={loss_dict['focal_loss'].item():.4f}"
177
+ )
178
+
179
+ metrics = evaluate_miou(model, val_loader, device=device, max_batches=args.val_batches)
180
+ print(f"[Epoch {epoch} Validation] mIoU={metrics['mIoU']} cIoU={metrics['cIoU']}")
181
+ ckpt = save_checkpoint(model, output_dir, tag=f"stage{args.stage}_epoch{epoch}")
182
+ print(f"Saved checkpoint: {ckpt}")
183
+
184
+
185
+ def parse_args() -> argparse.Namespace:
186
+ parser = argparse.ArgumentParser(description="Single-Pass Qwen2.5-VL JSON Segmenter")
187
+ parser.add_argument("--model-path", default="./Qwen3.5-4B")
188
+ parser.add_argument("--data-root", default="./ref_coco_lichengunc")
189
+ parser.add_argument("--dataset", default="refcoco", choices=["refcoco", "refcoco+", "refcocog", "all"])
190
+ parser.add_argument("--split-by", default="unc")
191
+ parser.add_argument(
192
+ "--riddle-prob",
193
+ type=float,
194
+ default=0.35,
195
+ help="Probability of augmenting training descriptions into functional/riddle prompts",
196
+ )
197
+ parser.add_argument("--output-dir", default="./checkpoints")
198
+ parser.add_argument("--resume-ckpt", default="")
199
+ parser.add_argument("--stage", type=int, choices=[1, 2], default=1)
200
+ parser.add_argument("--epochs", type=int, default=5)
201
+ parser.add_argument("--batch-size", type=int, default=2)
202
+ parser.add_argument("--max-objects", type=int, default=4)
203
+ parser.add_argument("--lr-head", type=float, default=2e-4)
204
+ parser.add_argument("--lr-lora", type=float, default=2e-5)
205
+ parser.add_argument("--weight-decay", type=float, default=1e-4)
206
+ parser.add_argument("--fused-layers", type=int, default=4, help="Number of top LLM layers fused into mask head")
207
+ parser.add_argument("--lora-r", type=int, default=8)
208
+ parser.add_argument("--lora-alpha", type=int, default=16)
209
+ parser.add_argument(
210
+ "--lora-top-layers",
211
+ type=int,
212
+ default=4,
213
+ help="Only apply LoRA to the last N LLM layers so earlier reasoning layers stay 100%% frozen",
214
+ )
215
+ parser.add_argument("--num-workers", type=int, default=2)
216
+ parser.add_argument("--log-every", type=int, default=50)
217
+ parser.add_argument("--val-batches", type=int, default=100)
218
+ parser.add_argument("--device", default="cuda")
219
+ return parser.parse_args()
220
+
221
+
222
+ if __name__ == "__main__":
223
+ cli_args = parse_args()
224
+ run_training(cli_args)