LIS1-9B / LIS1ModelCode.py
Wojtekb30's picture
Upload 16 files
8301504 verified
Raw History Blame Contribute Delete
49.2 kB
import importlib.metadata as _im import inspect import json from typing import Any, Dict, List, Optional, Sequence, Tuple, Union import cv2 import numpy as np import torch import torch.nn as nn import torch.nn.functional as F # Guard against huggingface-hub>=2.0 version check in transformers while keeping full functionality _orig_version = _im.version def _safe_version(pkg: str) -> str: if pkg == "huggingface-hub": try: ver = _orig_version(pkg) if ver.startswith("2."): return "1.99.0" return ver except Exception: return "1.99.0" return _orig_version(pkg) _im.version = _safe_version from transformers import AutoModel, AutoProcessor OBJECT_START_TOKEN = "<|object_ref_start|>" OBJECT_QUERY_TOKEN = "<|object_ref_end|>" DEFAULT_SYSTEM_PROMPT = ( "You are a fast single-pass visual reasoning and segmentation model. " "For each query or riddle in the input JSON, infer the intended target object in the image and ground it precisely." ) def normalize_input_json( query_input: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]], ) -> List[Dict[str, str]]: """ Normalizes user input (JSON string, dict with 'objects', or list of strings/dicts) into a canonical list of [{'id': str, 'description': str}, ...]. """ if isinstance(query_input, str): parsed = json.loads(query_input) else: parsed = query_input if isinstance(parsed, dict): raw_objects = parsed.get("objects", parsed.get("queries", [parsed])) elif isinstance(parsed, (list, tuple)): raw_objects = list(parsed) else: raise ValueError(f"Unsupported query_input format: {type(query_input)}") normalized: List[Dict[str, str]] = [] for idx, item in enumerate(raw_objects): if isinstance(item, str): normalized.append({"id": str(idx), "description": item.strip()}) elif isinstance(item, dict): obj_id = str(item.get("id", idx)) desc = str( item.get("description") or item.get("prompt") or item.get("text") or item.get("sent") or "" ).strip() if not desc: raise ValueError(f"Object at index {idx} has an empty description: {item}") normalized.append({"id": obj_id, "description": desc}) else: raise ValueError(f"Unsupported object entry at index {idx}: {item}") if not normalized: raise ValueError("Input JSON must contain at least one object description.") return normalized def build_segmentation_prompt( query_input: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]], query_token: str = OBJECT_QUERY_TOKEN, include_query_token: bool = False, ) -> Tuple[str, List[Dict[str, str]]]: """ Builds the user-side JSON prompt describing all objects/riddles to segment. """ objects = normalize_input_json(query_input) entries = [] for obj in objects: entry_json = json.dumps( {"id": obj["id"], "description": obj["description"]}, ensure_ascii=False, ) suffix = query_token if include_query_token else "" entries.append(f" {entry_json}{suffix}") json_block = "{\n \"objects\": [\n" + ",\n".join(entries) + "\n ]\n}" prompt = ( "Reason about each description or riddle in the following JSON " "and segment the corresponding object in the image:\n" f"{json_block}" ) return prompt, objects def build_assistant_grounding_prefill( objects: Sequence[Dict[str, str]], start_token: str = OBJECT_START_TOKEN, query_token: str = OBJECT_QUERY_TOKEN, ) -> str: """ Builds the prefilled assistant response containing `<|object_ref_start|>{desc}<|object_ref_end|>` for each object. Because this is passed as part of the single forward pass: 1. The LLM has already read the user's riddle AND processed all `<|image_pad|>` tokens. 2. Crossing `<|im_start|>assistant` switches Qwen into answering/grounding mode. 3. `<|object_ref_start|>...<|object_ref_end|>` activates Qwen's native localization circuit. """ entries = [] for obj in objects: target_str = f"{start_token}{obj['description']}{query_token}" entry_json = json.dumps( {"id": obj["id"], "target": target_str}, ensure_ascii=False, ) entries.append(f" {entry_json}") return "{\n \"objects\": [\n" + ",\n".join(entries) + "\n ]\n}" def build_segmentation_messages( image: Any, query_input: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]], system_prompt: str = DEFAULT_SYSTEM_PROMPT, query_token: str = OBJECT_QUERY_TOKEN, ) -> Tuple[List[Dict[str, Any]], List[Dict[str, str]]]: """ Constructs the single-pass multimodal chat messages in causal reasoning order: [System] -> [User: JSON Riddles -> Image] -> [Assistant: <|object_ref_start|>...<|object_ref_end|>] Putting the user JSON before the image allows every `<|image_pad|>` token in the LLM to attend to the riddle across all 36 layers before `<|object_ref_end|>` reads them out. """ user_prompt, objects = build_segmentation_prompt( query_input, query_token=query_token, include_query_token=False ) assistant_prefill = build_assistant_grounding_prefill(objects, query_token=query_token) messages = [ {"role": "system", "content": system_prompt}, { "role": "user", "content": [ {"type": "text", "text": user_prompt + "\n"}, {"type": "image", "image": image}, ], }, { "role": "assistant", "content": assistant_prefill, }, ] return messages, objects def polygons_to_mask( segmentation: Union[List[List[float]], Dict[str, Any]], height: int, width: int, ) -> np.ndarray: """ Rasterizes a COCO/RefCOCO annotation `segmentation` (polygon list or uncompressed RLE) into a binary numpy mask of shape (height, width) with values in {0, 1}. """ mask = np.zeros((height, width), dtype=np.uint8) if isinstance(segmentation, list): for poly in segmentation: if len(poly) < 6: continue pts = np.asarray(poly, dtype=np.float32).reshape(-1, 2) pts = np.round(pts).astype(np.int32) cv2.fillPoly(mask, [pts], 1) elif isinstance(segmentation, dict) and "counts" in segmentation: counts = segmentation["counts"] if isinstance(counts, list): h, w = segmentation.get("size", [height, width]) flat = np.zeros(h * w, dtype=np.uint8) idx = 0 val = 0 for c in counts: flat[idx : idx + c] = val idx += c val = 1 - val mask = flat.reshape((w, h)).T if mask.shape != (height, width): mask = cv2.resize(mask, (width, height), interpolation=cv2.INTER_NEAREST) else: raise ValueError("Compressed string RLE requires pycocotools; RefCOCO uses polygon lists.") else: raise ValueError(f"Unsupported segmentation format: {type(segmentation)}") return mask def mask_to_polygons( binary_mask: Union[np.ndarray, torch.Tensor], epsilon_ratio: float = 0.0015, min_area: float = 4.0, ) -> Tuple[List[List[float]], List[float], float]: """ Converts a 2D binary mask (H, W) into RefCOCO/COCO dataset format: - segmentation: List[List[float]] of polygon coordinates [x1, y1, x2, y2, ...] - bbox: [x, y, width, height] - area: float pixel area """ if isinstance(binary_mask, torch.Tensor): mask_np = binary_mask.detach().cpu().numpy().astype(np.uint8) else: mask_np = np.asarray(binary_mask, dtype=np.uint8) if mask_np.ndim > 2: mask_np = mask_np.squeeze() area = float(mask_np.sum()) if area < min_area: return [], [0.0, 0.0, 0.0, 0.0], 0.0 ys, xs = np.where(mask_np > 0) x_min, x_max = float(xs.min()), float(xs.max()) y_min, y_max = float(ys.min()), float(ys.max()) bbox = [ round(x_min, 2), round(y_min, 2), round(x_max - x_min + 1.0, 2), round(y_max - y_min + 1.0, 2), ] contours, _ = cv2.findContours(mask_np, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) polygons: List[List[float]] = [] for cnt in contours: if cv2.contourArea(cnt) < min_area and len(contours) > 1: continue peri = cv2.arcLength(cnt, True) approx = cv2.approxPolyDP(cnt, epsilon_ratio * peri, True) if epsilon_ratio > 0 else cnt pts = approx.reshape(-1, 2) if pts.shape[0] < 3: pts = cnt.reshape(-1, 2) if pts.shape[0] >= 3: poly = [round(float(coord), 2) for coord in pts.flatten()] if len(poly) >= 6: polygons.append(poly) if not polygons and area >= min_area: # Fallback box polygon if contour is degenerate (e.g. 1-pixel thin line) polygons.append( [ round(x_min, 2), round(y_min, 2), round(x_max, 2), round(y_min, 2), round(x_max, 2), round(y_max, 2), round(x_min, 2), round(y_max, 2), ] ) return polygons, bbox, round(area, 2) class ConvGNAct(nn.Module): """Residual-friendly 3x3 Conv + GroupNorm + GELU block.""" def __init__(self, in_ch: int, out_ch: int, kernel_size: int = 3, groups: int = 8): super().__init__() padding = kernel_size // 2 eff_groups = min(groups, out_ch) while out_ch % eff_groups != 0 and eff_groups > 1: eff_groups -= 1 self.block = nn.Sequential( nn.Conv2d(in_ch, out_ch, kernel_size=kernel_size, padding=padding, bias=False), nn.GroupNorm(eff_groups, out_ch), nn.GELU(), ) def forward(self, x: torch.Tensor) -> torch.Tensor: return self.block(x) class ResidualRefinementBlock(nn.Module): """Residual spatial refinement block for crisp mask boundaries.""" def __init__(self, channels: int): super().__init__() self.conv1 = ConvGNAct(channels, channels, kernel_size=3) self.conv2 = ConvGNAct(channels, channels, kernel_size=3) def forward(self, x: torch.Tensor) -> torch.Tensor: return x + self.conv2(self.conv1(x)) class MultiScaleMaskHead(nn.Module): """ Lightweight System-1 mask head that fuses: 1. LLM object decision vectors Q [K, llm_dim] 2. Coarse merged ViT features F_28 [C_out, H_m, W_m] (1/28 stride) 3. Fine unmerged ViT skip features F_14 [C_in, 2*H_m, 2*W_m] (1/14 stride) and outputs full-resolution mask logits [K, orig_H, orig_W] + IoU confidence scores [K]. """ def __init__( self, llm_hidden_size: int = 2048, vision_merged_dim: int = 2048, vision_unmerged_dim: int = 1280, hidden_dim: int = 256, fine_dim: int = 128, num_heads: int = 4, num_fused_layers: int = 4, ): super().__init__() self.hidden_dim = hidden_dim self.fine_dim = fine_dim self.num_fused_layers = num_fused_layers # Learnable fusion across the last N LLM layers (preserves frozen LLM reasoning + mid-late visual grounding) self.layer_weights = nn.Parameter(torch.zeros(num_fused_layers)) self.layer_norms = nn.ModuleList([nn.LayerNorm(llm_hidden_size) for _ in range(num_fused_layers)]) # Visual feature projections (computed once per image, shared across K object queries) self.coarse_vis_proj = ConvGNAct(vision_merged_dim, hidden_dim, kernel_size=1) self.llm_vis_proj = ConvGNAct(llm_hidden_size, hidden_dim, kernel_size=1) self.llm_vis_gate = nn.Parameter(torch.tensor(0.5)) self.fine_vis_proj = ConvGNAct(vision_unmerged_dim, fine_dim, kernel_size=1) # Query projection + cross-attention over coarse 2D visual tokens self.query_in_proj = nn.Sequential( nn.Linear(llm_hidden_size, hidden_dim), nn.LayerNorm(hidden_dim), nn.GELU(), nn.Linear(hidden_dim, hidden_dim), ) self.cross_attn = nn.MultiheadAttention( embed_dim=hidden_dim, num_heads=num_heads, batch_first=True, ) self.cross_norm = nn.LayerNorm(hidden_dim) # Query-to-coarse and query-to-fine dynamic modulators self.coarse_query_proj = nn.Linear(hidden_dim, hidden_dim) self.llm_query_proj = nn.Linear(hidden_dim, hidden_dim) self.coarse_gate_proj = nn.Linear(hidden_dim, hidden_dim) self.fine_gate_proj = nn.Linear(hidden_dim, fine_dim) # Coarse decoder at 1/28 scale (concatenates modulated features + dot-product similarity map) self.coarse_fuse = ConvGNAct(hidden_dim + 1, hidden_dim, kernel_size=3) self.coarse_res = ResidualRefinementBlock(hidden_dim) # Fine decoder at 1/14 scale (fuses upsampled coarse features + fine 1/14 ViT skip features) self.fine_fuse = ConvGNAct(hidden_dim + fine_dim, fine_dim, kernel_size=3) self.fine_res1 = ResidualRefinementBlock(fine_dim) self.fine_res2 = ResidualRefinementBlock(fine_dim) # 2x learned upsampler to 1/7 stride before final bilinear interpolation self.up_conv = nn.Sequential( nn.ConvTranspose2d(fine_dim, fine_dim // 2, kernel_size=2, stride=2), nn.GroupNorm(min(8, fine_dim // 2), fine_dim // 2), nn.GELU(), ConvGNAct(fine_dim // 2, fine_dim // 2, kernel_size=3), ) self.mask_out = nn.Conv2d(fine_dim // 2, 1, kernel_size=1) # Predicts calibrated mask confidence / IoU score per object query self.score_head = nn.Sequential( nn.Linear(hidden_dim, hidden_dim // 2), nn.GELU(), nn.Linear(hidden_dim // 2, 1), ) def _build_2d_sincos_pos_embed( self, h: int, w: int, dim: int, device: torch.device, dtype: torch.dtype ) -> torch.Tensor: """Generates parameter-free 2D sine-cosine positional embeddings [1, h*w, dim].""" y = torch.linspace(-1.0, 1.0, steps=h, device=device, dtype=torch.float32) x = torch.linspace(-1.0, 1.0, steps=w, device=device, dtype=torch.float32) grid_y, grid_x = torch.meshgrid(y, x, indexing="ij") half = dim // 4 if half == 0: return torch.zeros((1, h * w, dim), device=device, dtype=dtype) omega = torch.arange(half, device=device, dtype=torch.float32) / max(half, 1) omega = 1.0 / (10000.0**omega) out_y = grid_y.reshape(-1, 1) * omega.reshape(1, -1) out_x = grid_x.reshape(-1, 1) * omega.reshape(1, -1) pos = torch.cat([torch.sin(out_y), torch.cos(out_y), torch.sin(out_x), torch.cos(out_x)], dim=-1) if pos.shape[-1] < dim: pos = F.pad(pos, (0, dim - pos.shape[-1])) return pos.unsqueeze(0).to(dtype=dtype) def forward_single_image( self, queries: torch.Tensor, f_coarse: torch.Tensor, f_fine: torch.Tensor, target_size: Tuple[int, int], f_llm_coarse: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: """ Args: queries: [K, llm_hidden_size] or [K, L, llm_hidden_size] decision vectors for K objects f_coarse: [1, vision_merged_dim, H_m, W_m] (1/28 resolution from ViT) f_fine: [1, vision_unmerged_dim, H_f, W_f] (1/14 resolution from ViT) target_size: (orig_H, orig_W) f_llm_coarse: optional [1, llm_hidden_size, H_m, W_m] riddle-conditioned `<|image_pad|>` grid from LLM Returns: mask_logits: [K, orig_H, orig_W] pred_scores: [K] (in [0, 1]) """ num_queries = queries.shape[0] _, _, h_m, w_m = f_coarse.shape _, _, h_f, w_f = f_fine.shape # Fuse multi-layer LLM hidden states [K, L, llm_hidden_size] -> [K, llm_hidden_size] if queries.ndim == 3: num_layers = queries.shape[1] normed_layers = [ self.layer_norms[min(i, len(self.layer_norms) - 1)](queries[:, i, :]) for i in range(num_layers) ] stacked = torch.stack(normed_layers, dim=1) # [K, L, llm_hidden_size] weights = F.softmax(self.layer_weights[:num_layers], dim=0).view(1, num_layers, 1) queries = (stacked * weights).sum(dim=1) # Project visual features once for the image vis_coarse = self.coarse_vis_proj(f_coarse) # [1, hidden_dim, H_m, W_m] vis_llm = None if f_llm_coarse is not None: vis_llm = self.llm_vis_proj(f_llm_coarse) # [1, hidden_dim, H_m, W_m] vis_coarse = vis_coarse + torch.tanh(self.llm_vis_gate) * vis_llm vis_fine = self.fine_vis_proj(f_fine) # [1, fine_dim, H_f, W_f] # Project queries & refine with cross-attention over 2D visual tokens q = self.query_in_proj(queries).unsqueeze(0) # [1, K, hidden_dim] vis_tokens = vis_coarse.flatten(2).transpose(1, 2) # [1, H_m*W_m, hidden_dim] pos_2d = self._build_2d_sincos_pos_embed( h_m, w_m, self.hidden_dim, vis_tokens.device, vis_tokens.dtype ) attn_out, _ = self.cross_attn( query=q, key=vis_tokens + pos_2d, value=vis_tokens, ) q = self.cross_norm(q + attn_out).squeeze(0) # [K, hidden_dim] # Predict confidence score per object query pred_scores = torch.sigmoid(self.score_head(q).squeeze(-1)) # [K] # 1. Coarse (1/28) dot-product similarity + channel gating q_dot = F.normalize(self.coarse_query_proj(q), dim=-1) # [K, hidden_dim] v_norm = F.normalize(vis_coarse.squeeze(0), dim=0) # [hidden_dim, H_m, W_m] sim_map = torch.einsum("kd,dhw->khw", q_dot, v_norm).unsqueeze(1) # [K, 1, H_m, W_m] if vis_llm is not None: q_llm_dot = F.normalize(self.llm_query_proj(q), dim=-1) v_llm_norm = F.normalize(vis_llm.squeeze(0), dim=0) sim_map_llm = torch.einsum("kd,dhw->khw", q_llm_dot, v_llm_norm).unsqueeze(1) sim_map = 0.5 * (sim_map + sim_map_llm) coarse_gate = torch.sigmoid(self.coarse_gate_proj(q)).unsqueeze(-1).unsqueeze(-1) # [K, hidden_dim, 1, 1] vis_coarse_exp = vis_coarse.expand(num_queries, -1, -1, -1) coarse_mod = vis_coarse_exp * coarse_gate # [K, hidden_dim, H_m, W_m] x = torch.cat([coarse_mod, sim_map], dim=1) # [K, hidden_dim + 1, H_m, W_m] x = self.coarse_res(self.coarse_fuse(x)) # 2. Upsample 2x to 1/14 resolution & fuse with unmerged ViT skip features x = F.interpolate(x, size=(h_f, w_f), mode="bilinear", align_corners=False) fine_gate = torch.sigmoid(self.fine_gate_proj(q)).unsqueeze(-1).unsqueeze(-1) # [K, fine_dim, 1, 1] vis_fine_exp = vis_fine.expand(num_queries, -1, -1, -1) * fine_gate x = torch.cat([x, vis_fine_exp], dim=1) # [K, hidden_dim + fine_dim, H_f, W_f] x = self.fine_fuse(x) x = self.fine_res2(self.fine_res1(x)) # 3. Learned 2x upsample (to 1/7 stride) + bilinear resize to original image resolution x = self.up_conv(x) raw_logits = self.mask_out(x) # [K, 1, 2*H_f, 2*W_f] upsampled_logits = F.interpolate( raw_logits, size=target_size, mode="bilinear", align_corners=False, ).squeeze(1) # [K, orig_H, orig_W] return upsampled_logits, pred_scores class Qwen2_5_VL_SinglePassSegmenter(nn.Module): """ System-1 Single-Pass Multi-Object Segmenter built on Qwen2.5-VL. - Drops the autoregressive LM head completely. - Runs the ViT encoder and LLM backbone once. - Captures both 1/14 (unmerged) and 1/28 (merged) ViT spatial grids from the single pass. - Extracts K decision vectors from `<|object_ref_end|>` tokens in the input JSON. - Decodes all K object masks in parallel. """ def __init__( self, qwen_model_name: Optional[str] = "./Qwen3.5-4B", backbone: Optional[nn.Module] = None, processor: Optional[Any] = None, torch_dtype: torch.dtype = torch.bfloat16, device_map: Optional[Union[str, Dict[str, Any]]] = None, quantization_config: Optional[Any] = None, freeze_backbone: bool = True, head_hidden_dim: int = 256, head_fine_dim: int = 128, num_fused_layers: int = 4, ): super().__init__() self.num_fused_layers = num_fused_layers if backbone is not None: self.qwen = backbone self.processor = processor else: if qwen_model_name is None: raise ValueError("Provide either `qwen_model_name` or a pre-built `backbone`.") self.processor = processor or AutoProcessor.from_pretrained(qwen_model_name) load_kwargs: Dict[str, Any] = {"torch_dtype": torch_dtype} if device_map is not None: load_kwargs["device_map"] = device_map if quantization_config is not None: load_kwargs["quantization_config"] = quantization_config self.qwen = AutoModel.from_pretrained(qwen_model_name, **load_kwargs) # Support both Qwen2_5_VLModel and Qwen2_5_VLForConditionalGeneration if hasattr(self.qwen, "model") and hasattr(self.qwen.model, "visual"): self.backbone = self.qwen.model else: self.backbone = self.qwen self.freeze_backbone = freeze_backbone if freeze_backbone: for param in self.backbone.parameters(): param.requires_grad = False # Read dimensions safely across transformers versions cfg = self.backbone.config text_cfg = getattr(cfg, "text_config", cfg) vis_cfg = cfg.vision_config self.llm_hidden_size = getattr(text_cfg, "hidden_size", 2560) self.vision_merged_dim = getattr(vis_cfg, "out_hidden_size", self.llm_hidden_size) self.vision_unmerged_dim = getattr(vis_cfg, "hidden_size", 1024) self.spatial_merge_size = getattr(vis_cfg, "spatial_merge_size", 2) self.patch_size = getattr(vis_cfg, "patch_size", 16) if self.processor is not None and hasattr(self.processor, "tokenizer"): self.query_token_id = self.processor.tokenizer.convert_tokens_to_ids(OBJECT_QUERY_TOKEN) self.im_end_token_id = self.processor.tokenizer.convert_tokens_to_ids("<|im_end|>") self.image_token_id = self.processor.tokenizer.convert_tokens_to_ids("<|image_pad|>") else: self.query_token_id = getattr(cfg, "object_ref_end_token_id", 248048) self.im_end_token_id = getattr(text_cfg, "eos_token_id", 248046) self.image_token_id = getattr(cfg, "image_token_id", 248056) self.mask_head = MultiScaleMaskHead( llm_hidden_size=self.llm_hidden_size, vision_merged_dim=self.vision_merged_dim, vision_unmerged_dim=self.vision_unmerged_dim, hidden_dim=head_hidden_dim, fine_dim=head_fine_dim, num_fused_layers=num_fused_layers, ) # Hook buffers to capture single-pass visual outputs & last N LLM layers without extra memory self._captured_visual: Dict[str, Any] = {} self._captured_llm_layers: List[torch.Tensor] = [] self._visual_hook = self.backbone.visual.register_forward_hook(self._capture_visual_hook) self._llm_hooks: List[Any] = [] self._register_llm_layer_hooks() self._needs_mm_token_type_ids = self._check_needs_mm_token_type_ids() def _check_needs_mm_token_type_ids(self) -> bool: return "mm_token_type_ids" in inspect.signature( self.backbone.forward ).parameters or ( hasattr(self.backbone, "compute_3d_position_ids") and "mm_token_type_ids" in inspect.signature(self.backbone.compute_3d_position_ids).parameters ) def _get_llm_decoder_layers(self) -> Optional[nn.ModuleList]: lm = getattr(self.backbone, "language_model", None) if lm is None: return None if hasattr(lm, "layers"): return lm.layers if hasattr(lm, "model") and hasattr(lm.model, "layers"): return lm.model.layers return None def _register_llm_layer_hooks(self) -> None: for h in self._llm_hooks: h.remove() self._llm_hooks.clear() layers = self._get_llm_decoder_layers() if layers is not None and len(layers) > 0: k = min(self.num_fused_layers, len(layers)) for layer in list(layers)[-k:]: self._llm_hooks.append(layer.register_forward_hook(self._capture_llm_layer_hook)) def _capture_llm_layer_hook(self, module: nn.Module, inputs: Any, output: Any) -> None: hidden = output[0] if isinstance(output, tuple) else output if isinstance(hidden, torch.Tensor): self._captured_llm_layers.append(hidden) def _capture_visual_hook(self, module: nn.Module, inputs: Any, output: Any) -> None: if hasattr(output, "last_hidden_state") and hasattr(output, "pooler_output"): self._captured_visual["unmerged"] = output.last_hidden_state # Clone reference before get_image_features mutates output.pooler_output into a tuple self._captured_visual["merged"] = output.pooler_output elif isinstance(output, tuple) and len(output) >= 2: self._captured_visual["unmerged"] = output[0] self._captured_visual["merged"] = output[1] else: self._captured_visual["merged"] = output self._captured_visual["unmerged"] = None def enable_lora( self, r: int = 8, lora_alpha: int = 16, lora_dropout: float = 0.05, top_k_layers: int = 4, target_modules: Tuple[str, ...] = ( "q_proj", "k_proj", "v_proj", "o_proj", "in_proj_qkv", "out_proj", ), ) -> None: """ Safe Stage 2 training helper: Applies low-rank LoRA ONLY to the top `top_k_layers` (e.g., last 4 of 36 layers), leaving the first ~32 layers 100% frozen so Qwen retains all world knowledge and riddle-solving ability while the top layers adapt to spatial token compression. """ from peft import LoraConfig, get_peft_model layers = self._get_llm_decoder_layers() resolved_targets: List[str] = list(target_modules) if layers is not None and top_k_layers > 0 and len(layers) > top_k_layers: total_l = len(layers) allowed_layer_indices = set(range(total_l - top_k_layers, total_l)) exact_targets: List[str] = [] for name, _ in self.backbone.language_model.named_modules(): leaf = name.split(".")[-1] if leaf not in target_modules: continue if any( f"layers.{idx}." in name or name.startswith(f"layers.{idx}.") for idx in allowed_layer_indices ): exact_targets.append(name) if exact_targets: resolved_targets = exact_targets lora_cfg = LoraConfig( r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, target_modules=resolved_targets, bias="none", ) self.backbone.language_model = get_peft_model(self.backbone.language_model, lora_cfg) self._register_llm_layer_hooks() self._needs_mm_token_type_ids = self._check_needs_mm_token_type_ids() self.freeze_backbone = False def _compute_window_index( self, grid_thw: torch.Tensor, grid_thw_list: Optional[Sequence[Sequence[int]]] = None, ) -> torch.Tensor: if hasattr(self.backbone.visual, "get_window_index"): window_index, _ = self.backbone.visual.get_window_index(grid_thw) return window_index vis_cfg = getattr(self.backbone.config, "vision_config", None) window_size = getattr(vis_cfg, "window_size", 112) m = self.spatial_merge_size vit_merger_window_size = max(window_size // m // self.patch_size, 1) window_index: List[torch.Tensor] = [] window_index_id = 0 rows = grid_thw_list if grid_thw_list is not None else grid_thw.tolist() for grid_t, grid_h, grid_w in rows: llm_grid_h, llm_grid_w = grid_h // m, grid_w // m index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape(grid_t, llm_grid_h, llm_grid_w) pad_h = vit_merger_window_size - llm_grid_h % vit_merger_window_size pad_w = vit_merger_window_size - llm_grid_w % vit_merger_window_size num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size index_padded = F.pad(index, (0, pad_w, 0, pad_h), "constant", -100) index_padded = index_padded.reshape( grid_t, num_windows_h, vit_merger_window_size, num_windows_w, vit_merger_window_size ) index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape( grid_t, num_windows_h * num_windows_w, vit_merger_window_size, vit_merger_window_size ) index_padded = index_padded.reshape(-1) index_new = index_padded[index_padded != -100] window_index.append(index_new + window_index_id) window_index_id += grid_t * llm_grid_h * llm_grid_w return torch.cat(window_index, dim=0) def _reconstruct_2d_vision_features( self, merged_tokens: Union[torch.Tensor, Sequence[torch.Tensor]], unmerged_tokens: Optional[torch.Tensor], image_grid_thw: torch.Tensor, grid_thw_list: Optional[Sequence[Sequence[int]]] = None, ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]: """ Reconstructs per-image 2D spatial grids from Qwen2.5-VL's window-permuted ViT outputs: - coarse_grids: list of [1, C_merged, H_m, W_m] at 1/28 resolution - fine_grids: list of [1, C_unmerged, 2*H_m, 2*W_m] at 1/14 resolution """ m = self.spatial_merge_size m_sq = m * m rows = grid_thw_list if grid_thw_list is not None else image_grid_thw.tolist() if isinstance(merged_tokens, (tuple, list)): merged_cat = torch.cat(list(merged_tokens), dim=0) else: merged_cat = merged_tokens # Un-permute the unmerged tokens from window attention order if the ViT uses window attention (Qwen2.5-VL); # Qwen3.5-4B uses full ViT attention so its unmerged tokens are already in natural block-raster order. vis_cfg = getattr(self.backbone.config, "vision_config", None) has_window_attn = hasattr(vis_cfg, "window_size") and getattr(vis_cfg, "window_size", None) is not None if unmerged_tokens is not None: num_merged = merged_cat.shape[0] if has_window_attn: window_index = self._compute_window_index(image_grid_thw, grid_thw_list=rows) reverse_indices = torch.argsort(window_index.to(unmerged_tokens.device)) unmerged_blocks = unmerged_tokens.view(num_merged, m_sq, -1)[reverse_indices] else: unmerged_blocks = unmerged_tokens.view(num_merged, m_sq, -1) else: unmerged_blocks = None coarse_grids: List[torch.Tensor] = [] fine_grids: List[torch.Tensor] = [] offset = 0 for grid_t, grid_h, grid_w in rows: h_m = grid_h // m w_m = grid_w // m n_tokens = grid_t * h_m * w_m img_merged = merged_cat[offset : offset + n_tokens] # Average across temporal dimension if grid_t > 1 (for static images grid_t == 1) img_merged = img_merged.view(grid_t, h_m, w_m, -1).mean(dim=0) f_coarse = img_merged.permute(2, 0, 1).unsqueeze(0).contiguous() # [1, C_out, H_m, W_m] coarse_grids.append(f_coarse) if unmerged_blocks is not None: img_unmerged = unmerged_blocks[offset : offset + n_tokens] # Matches Qwen2VLImageProcessor patch ordering: (T, H_m, W_m, m_h, m_w, C_in) img_unmerged = img_unmerged.view(grid_t, h_m, w_m, m, m, -1).mean(dim=0) f_fine = ( img_unmerged.permute(4, 0, 2, 1, 3) .contiguous() .view(1, self.vision_unmerged_dim, grid_h, grid_w) ) fine_grids.append(f_fine) else: # Fallback if unmerged tokens are unavailable f_fine = F.interpolate(f_coarse, size=(grid_h, grid_w), mode="bilinear", align_corners=False) fine_grids.append(f_fine) offset += n_tokens return coarse_grids, fine_grids def _extract_llm_visual_grids( self, last_hidden_states: torch.Tensor, input_ids: torch.Tensor, image_grid_thw: torch.Tensor, layer_states: Optional[Sequence[torch.Tensor]] = None, grid_thw_list: Optional[Sequence[Sequence[int]]] = None, input_ids_cpu: Optional[torch.Tensor] = None, ) -> List[Optional[torch.Tensor]]: """ Extracts the LLM-contextualized `<|image_pad|>` visual tokens from the top LLM layers and reshapes them into 2D grids [1, llm_dim, H_m, W_m] per image. Because the user's JSON riddle precedes `<|image_pad|>`, these tokens carry 36 layers of riddle-to-image self-attention context. """ m = self.spatial_merge_size base_dev = last_hidden_states.device source_state = ( torch.stack([st.to(base_dev) for st in layer_states], dim=0).mean(dim=0) if layer_states and len(layer_states) > 0 else last_hidden_states ) rows = grid_thw_list if grid_thw_list is not None else image_grid_thw.tolist() ids_src = input_ids_cpu if input_ids_cpu is not None else input_ids target_dev = source_state.device llm_grids: List[Optional[torch.Tensor]] = [] for b in range(ids_src.shape[0]): grid_t, grid_h, grid_w = rows[b] h_m, w_m = grid_h // m, grid_w // m n_tokens = grid_t * h_m * w_m img_pos = (ids_src[b] == self.image_token_id).nonzero(as_tuple=True)[0] if img_pos.numel() == n_tokens: if img_pos.device != target_dev: img_pos = img_pos.to(device=target_dev, non_blocking=True) vis_seq = source_state[b, img_pos, :] # [n_tokens, llm_dim] vis_2d = ( vis_seq.view(grid_t, h_m, w_m, -1) .mean(dim=0) .permute(2, 0, 1) .unsqueeze(0) .contiguous() ) llm_grids.append(vis_2d) else: llm_grids.append(None) return llm_grids def _extract_query_vectors( self, last_hidden_states: torch.Tensor, input_ids: torch.Tensor, layer_states: Optional[Sequence[torch.Tensor]] = None, input_ids_cpu: Optional[torch.Tensor] = None, ) -> List[torch.Tensor]: """ Extracts the K `<|object_ref_end|>` decision vectors per batch item. If multi-layer states are captured, returns [K, L, llm_dim] per batch item; otherwise returns [K, llm_dim]. """ states_to_index = list(layer_states) if layer_states else [last_hidden_states] ids_src = input_ids_cpu if input_ids_cpu is not None else input_ids target_dev = states_to_index[0].device batch_queries: List[torch.Tensor] = [] for b in range(ids_src.shape[0]): pos = (ids_src[b] == self.query_token_id).nonzero(as_tuple=True)[0] if pos.numel() == 0: pos = (ids_src[b] == self.im_end_token_id).nonzero(as_tuple=True)[0] if pos.numel() > 0: pos = pos[-1:] else: pos = torch.tensor([ids_src.shape[1] - 1], dtype=torch.long, device=ids_src.device) if pos.device != target_dev: pos = pos.to(device=target_dev, non_blocking=True) if len(states_to_index) > 1: per_layer = [ st[b, (pos.to(st.device) if pos.device != st.device else pos), :].to(target_dev) for st in states_to_index ] batch_queries.append(torch.stack(per_layer, dim=1)) # [K, L, llm_dim] else: batch_queries.append(states_to_index[0][b, pos, :]) # [K, llm_dim] return batch_queries def forward( self, input_ids: torch.LongTensor, attention_mask: torch.Tensor, pixel_values: torch.Tensor, image_grid_thw: torch.LongTensor, target_sizes: Optional[Sequence[Tuple[int, int]]] = None, mm_token_type_ids: Optional[torch.Tensor] = None, grid_thw_list: Optional[Sequence[Sequence[int]]] = None, input_ids_cpu: Optional[torch.Tensor] = None, ) -> Dict[str, Any]: """ Single forward pass from tokenized JSON + image patches to pixel masks. Args: input_ids: [B, seq_len] attention_mask: [B, seq_len] pixel_values: [sum_patches, patch_dim] image_grid_thw: [B, 3] (T, grid_h, grid_w) target_sizes: optional list of (orig_H, orig_W) per image in batch. Defaults to (grid_h * patch_size, grid_w * patch_size). mm_token_type_ids: optional [B, seq_len] multimodal token type IDs (auto-computed if needed). grid_thw_list: optional pre-extracted Python list of [T, grid_h, grid_w] per image to avoid CUDA syncs. input_ids_cpu: optional CPU copy of `input_ids` to extract token positions without CUDA `.nonzero()` syncs. Returns: dict with: - "mask_logits": List[Tensor] of shape [K_b, orig_H, orig_W] per image - "mask_probs": List[Tensor] of shape [K_b, orig_H, orig_W] per image - "pred_scores": List[Tensor] of shape [K_b] per image """ self._captured_visual.clear() self._captured_llm_layers.clear() resolved_grid_thw = list(grid_thw_list) if grid_thw_list is not None else image_grid_thw.tolist() backbone_kwargs: Dict[str, Any] = { "input_ids": input_ids, "attention_mask": attention_mask, "pixel_values": pixel_values, "image_grid_thw": image_grid_thw, "use_cache": False, "output_hidden_states": False, "return_dict": True, } if self._needs_mm_token_type_ids: if mm_token_type_ids is None: mm_token_type_ids = (input_ids == self.image_token_id).to( dtype=torch.long, device=input_ids.device ) backbone_kwargs["mm_token_type_ids"] = mm_token_type_ids grad_ctx = torch.no_grad() if self.freeze_backbone else torch.enable_grad() with grad_ctx: outputs = self.backbone(**backbone_kwargs) last_hidden_states = outputs.last_hidden_state merged_vis = self._captured_visual.get("merged") unmerged_vis = self._captured_visual.get("unmerged") if merged_vis is None: raise RuntimeError("Visual forward hook did not capture features; check pixel_values input.") coarse_grids, fine_grids = self._reconstruct_2d_vision_features( merged_tokens=merged_vis, unmerged_tokens=unmerged_vis, image_grid_thw=image_grid_thw, grid_thw_list=resolved_grid_thw, ) llm_vis_grids = self._extract_llm_visual_grids( last_hidden_states=last_hidden_states, input_ids=input_ids, image_grid_thw=image_grid_thw, layer_states=self._captured_llm_layers, grid_thw_list=resolved_grid_thw, input_ids_cpu=input_ids_cpu, ) batch_queries = self._extract_query_vectors( last_hidden_states=last_hidden_states, input_ids=input_ids, layer_states=self._captured_llm_layers, input_ids_cpu=input_ids_cpu, ) head_param = next(self.mask_head.parameters()) head_dtype = head_param.dtype head_device = head_param.device mask_logits_list: List[torch.Tensor] = [] mask_probs_list: List[torch.Tensor] = [] pred_scores_list: List[torch.Tensor] = [] for b in range(len(batch_queries)): if target_sizes is not None and b < len(target_sizes): tgt_size = (int(target_sizes[b][0]), int(target_sizes[b][1])) else: tgt_size = ( int(resolved_grid_thw[b][1] * self.patch_size), int(resolved_grid_thw[b][2] * self.patch_size), ) f_llm_b = ( llm_vis_grids[b].to(device=head_device, dtype=head_dtype) if llm_vis_grids[b] is not None else None ) logits_b, scores_b = self.mask_head.forward_single_image( queries=batch_queries[b].to(device=head_device, dtype=head_dtype), f_coarse=coarse_grids[b].to(device=head_device, dtype=head_dtype), f_fine=fine_grids[b].to(device=head_device, dtype=head_dtype), target_size=tgt_size, f_llm_coarse=f_llm_b, ) mask_logits_list.append(logits_b) mask_probs_list.append(torch.sigmoid(logits_b)) pred_scores_list.append(scores_b) return { "mask_logits": mask_logits_list, "mask_probs": mask_probs_list, "pred_scores": pred_scores_list, } @torch.no_grad() def predict_json( self, image: Any, query_json: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]], threshold: float = 0.5, ) -> Dict[str, Any]: """ End-to-end System 1 inference: Takes a PIL image + JSON object descriptions, runs 1 forward pass, and returns a JSON dict with COCO polygon segmentations (`segmentation`), `bbox`, `area`, and `score`. """ if self.processor is None: raise RuntimeError("`processor` is required for `predict_json`.") self.eval() orig_w, orig_h = image.size messages, objects = build_segmentation_messages(image=image, query_input=query_json) formatted_text = self.processor.apply_chat_template( messages, tokenize=False, add_generation_prompt=False ) inputs = self.processor( text=[formatted_text], images=[image], padding=True, return_tensors="pt", ) input_ids_cpu = inputs["input_ids"] grid_thw_list = inputs["image_grid_thw"].tolist() device = next(self.mask_head.parameters()).device inputs = {k: (v.to(device) if isinstance(v, torch.Tensor) else v) for k, v in inputs.items()} outputs = self.forward( input_ids=inputs["input_ids"], attention_mask=inputs["attention_mask"], pixel_values=inputs["pixel_values"], image_grid_thw=inputs["image_grid_thw"], target_sizes=[(orig_h, orig_w)], grid_thw_list=grid_thw_list, input_ids_cpu=input_ids_cpu, ) probs = outputs["mask_probs"][0] # [K, orig_H, orig_W] scores = outputs["pred_scores"][0] # [K] results = [] for idx, obj in enumerate(objects): if idx >= probs.shape[0]: break prob_map = probs[idx] binary_mask = (prob_map >= threshold).to(torch.uint8) polygons, bbox, area = mask_to_polygons(binary_mask) # Combine IoU head score with mean foreground probability fg_prob = float(prob_map[binary_mask > 0].mean().item()) if area > 0 else 0.0 score = round(0.5 * float(scores[idx].item()) + 0.5 * fg_prob, 4) results.append( { "id": obj["id"], "description": obj["description"], "score": score, "bbox": bbox, "area": area, "segmentation": polygons, } ) return { "image_size": {"height": orig_h, "width": orig_w}, "objects": results, } class SegmentationLoss(nn.Module): """ Combined Focal + Soft Dice + BCE + IoU Score Calibration loss for training the mask head. """ def __init__( self, bce_weight: float = 1.0, focal_weight: float = 2.0, dice_weight: float = 2.0, score_weight: float = 0.5, focal_alpha: float = 0.25, focal_gamma: float = 2.0, ): super().__init__() self.bce_weight = bce_weight self.focal_weight = focal_weight self.dice_weight = dice_weight self.score_weight = score_weight self.focal_alpha = focal_alpha self.focal_gamma = focal_gamma def _dice_loss(self, logits: torch.Tensor, targets: torch.Tensor, eps: float = 1.0) -> torch.Tensor: probs = torch.sigmoid(logits).flatten(1) tgt = targets.flatten(1) intersection = (probs * tgt).sum(dim=1) union = probs.sum(dim=1) + tgt.sum(dim=1) dice = (2.0 * intersection + eps) / (union + eps) return (1.0 - dice).mean() def _sigmoid_focal_loss(self, logits: torch.Tensor, targets: torch.Tensor) -> torch.Tensor: bce = F.binary_cross_entropy_with_logits(logits, targets, reduction="none") probs = torch.sigmoid(logits) p_t = probs * targets + (1.0 - probs) * (1.0 - targets) alpha_t = self.focal_alpha * targets + (1.0 - self.focal_alpha) * (1.0 - targets) focal = alpha_t * ((1.0 - p_t) ** self.focal_gamma) * bce return focal.mean() def forward( self, mask_logits_list: Sequence[torch.Tensor], pred_scores_list: Sequence[torch.Tensor], gt_masks_list: Sequence[torch.Tensor], ) -> Dict[str, torch.Tensor]: total_bce = torch.tensor(0.0, device=mask_logits_list[0].device) total_focal = torch.tensor(0.0, device=mask_logits_list[0].device) total_dice = torch.tensor(0.0, device=mask_logits_list[0].device) total_score = torch.tensor(0.0, device=mask_logits_list[0].device) n_batches = max(len(mask_logits_list), 1) for logits, scores, gt in zip(mask_logits_list, pred_scores_list, gt_masks_list): gt = gt.to(device=logits.device, dtype=logits.dtype) if gt.shape[-2:] != logits.shape[-2:]: gt = F.interpolate(gt.unsqueeze(1), size=logits.shape[-2:], mode="nearest").squeeze(1) total_bce = total_bce + F.binary_cross_entropy_with_logits(logits, gt) total_focal = total_focal + self._sigmoid_focal_loss(logits, gt) total_dice = total_dice + self._dice_loss(logits, gt) with torch.no_grad(): pred_bin = (torch.sigmoid(logits) >= 0.5).to(gt.dtype).flatten(1) gt_flat = gt.flatten(1) inter = (pred_bin * gt_flat).sum(dim=1) union = (pred_bin + gt_flat - pred_bin * gt_flat).sum(dim=1).clamp(min=1e-6) true_iou = inter / union total_score = total_score + F.mse_loss(scores, true_iou.to(scores.dtype)) total_bce = total_bce / n_batches total_focal = total_focal / n_batches total_dice = total_dice / n_batches total_score = total_score / n_batches loss = ( self.bce_weight * total_bce + self.focal_weight * total_focal + self.dice_weight * total_dice + self.score_weight * total_score ) return { "loss": loss, "bce_loss": total_bce.detach(), "focal_loss": total_focal.detach(), "dice_loss": total_dice.detach(), "score_loss": total_score.detach(), }
import importlib.metadata as _im
import inspect
import json
from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
import cv2
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
# Guard against huggingface-hub>=2.0 version check in transformers while keeping full functionality
_orig_version = _im.version
def _safe_version(pkg: str) -> str:
if pkg == "huggingface-hub":
try:
ver = _orig_version(pkg)
if ver.startswith("2."):
return "1.99.0"
return ver
except Exception:
return "1.99.0"
return _orig_version(pkg)
_im.version = _safe_version
from transformers import AutoModel, AutoProcessor
OBJECT_START_TOKEN = "<|object_ref_start|>"
OBJECT_QUERY_TOKEN = "<|object_ref_end|>"
DEFAULT_SYSTEM_PROMPT = (
"You are a fast single-pass visual reasoning and segmentation model. "
"For each query or riddle in the input JSON, infer the intended target object in the image and ground it precisely."
)
def normalize_input_json(
query_input: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]],
) -> List[Dict[str, str]]:
"""
Normalizes user input (JSON string, dict with 'objects', or list of strings/dicts)
into a canonical list of [{'id': str, 'description': str}, ...].
"""
if isinstance(query_input, str):
parsed = json.loads(query_input)
else:
parsed = query_input
if isinstance(parsed, dict):
raw_objects = parsed.get("objects", parsed.get("queries", [parsed]))
elif isinstance(parsed, (list, tuple)):
raw_objects = list(parsed)
else:
raise ValueError(f"Unsupported query_input format: {type(query_input)}")
normalized: List[Dict[str, str]] = []
for idx, item in enumerate(raw_objects):
if isinstance(item, str):
normalized.append({"id": str(idx), "description": item.strip()})
elif isinstance(item, dict):
obj_id = str(item.get("id", idx))
desc = str(
item.get("description")
or item.get("prompt")
or item.get("text")
or item.get("sent")
or ""
).strip()
if not desc:
raise ValueError(f"Object at index {idx} has an empty description: {item}")
normalized.append({"id": obj_id, "description": desc})
else:
raise ValueError(f"Unsupported object entry at index {idx}: {item}")
if not normalized:
raise ValueError("Input JSON must contain at least one object description.")
return normalized
def build_segmentation_prompt(
query_input: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]],
query_token: str = OBJECT_QUERY_TOKEN,
include_query_token: bool = False,
) -> Tuple[str, List[Dict[str, str]]]:
"""
Builds the user-side JSON prompt describing all objects/riddles to segment.
"""
objects = normalize_input_json(query_input)
entries = []
for obj in objects:
entry_json = json.dumps(
{"id": obj["id"], "description": obj["description"]},
ensure_ascii=False,
)
suffix = query_token if include_query_token else ""
entries.append(f" {entry_json}{suffix}")