Download LIS1ModelCode.py from Wojtekb30/LIS1-9B: direct link, hf CLI and curl.
- Browser
- Download file 49.2 kB
-
https://huggingface.co/Wojtekb30/LIS1-9B/resolve/main/LIS1ModelCode.py
- Command line
-
hf download hf://Wojtekb30/LIS1-9B/LIS1ModelCode.py
-
curl -L -o LIS1ModelCode.py https://huggingface.co/Wojtekb30/LIS1-9B/resolve/main/LIS1ModelCode.py
49.2 kB
import importlib.metadata as _im
import inspect
import json
from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
import cv2
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
# Guard against huggingface-hub>=2.0 version check in transformers while keeping full functionality
_orig_version = _im.version
def _safe_version(pkg: str) -> str:
if pkg == "huggingface-hub":
try:
ver = _orig_version(pkg)
if ver.startswith("2."):
return "1.99.0"
return ver
except Exception:
return "1.99.0"
return _orig_version(pkg)
_im.version = _safe_version
from transformers import AutoModel, AutoProcessor
OBJECT_START_TOKEN = "<|object_ref_start|>"
OBJECT_QUERY_TOKEN = "<|object_ref_end|>"
DEFAULT_SYSTEM_PROMPT = (
"You are a fast single-pass visual reasoning and segmentation model. "
"For each query or riddle in the input JSON, infer the intended target object in the image and ground it precisely."
)
def normalize_input_json(
query_input: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]],
) -> List[Dict[str, str]]:
"""
Normalizes user input (JSON string, dict with 'objects', or list of strings/dicts)
into a canonical list of [{'id': str, 'description': str}, ...].
"""
if isinstance(query_input, str):
parsed = json.loads(query_input)
else:
parsed = query_input
if isinstance(parsed, dict):
raw_objects = parsed.get("objects", parsed.get("queries", [parsed]))
elif isinstance(parsed, (list, tuple)):
raw_objects = list(parsed)
else:
raise ValueError(f"Unsupported query_input format: {type(query_input)}")
normalized: List[Dict[str, str]] = []
for idx, item in enumerate(raw_objects):
if isinstance(item, str):
normalized.append({"id": str(idx), "description": item.strip()})
elif isinstance(item, dict):
obj_id = str(item.get("id", idx))
desc = str(
item.get("description")
or item.get("prompt")
or item.get("text")
or item.get("sent")
or ""
).strip()
if not desc:
raise ValueError(f"Object at index {idx} has an empty description: {item}")
normalized.append({"id": obj_id, "description": desc})
else:
raise ValueError(f"Unsupported object entry at index {idx}: {item}")
if not normalized:
raise ValueError("Input JSON must contain at least one object description.")
return normalized
def build_segmentation_prompt(
query_input: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]],
query_token: str = OBJECT_QUERY_TOKEN,
include_query_token: bool = False,
) -> Tuple[str, List[Dict[str, str]]]:
"""
Builds the user-side JSON prompt describing all objects/riddles to segment.
"""
objects = normalize_input_json(query_input)
entries = []
for obj in objects:
entry_json = json.dumps(
{"id": obj["id"], "description": obj["description"]},
ensure_ascii=False,
)
suffix = query_token if include_query_token else ""
entries.append(f" {entry_json}{suffix}")
json_block = "{\n \"objects\": [\n" + ",\n".join(entries) + "\n ]\n}"
prompt = (
"Reason about each description or riddle in the following JSON "
"and segment the corresponding object in the image:\n"
f"{json_block}"
)
return prompt, objects
def build_assistant_grounding_prefill(
objects: Sequence[Dict[str, str]],
start_token: str = OBJECT_START_TOKEN,
query_token: str = OBJECT_QUERY_TOKEN,
) -> str:
"""
Builds the prefilled assistant response containing `<|object_ref_start|>{desc}<|object_ref_end|>`
for each object. Because this is passed as part of the single forward pass:
1. The LLM has already read the user's riddle AND processed all `<|image_pad|>` tokens.
2. Crossing `<|im_start|>assistant` switches Qwen into answering/grounding mode.
3. `<|object_ref_start|>...<|object_ref_end|>` activates Qwen's native localization circuit.
"""
entries = []
for obj in objects:
target_str = f"{start_token}{obj['description']}{query_token}"
entry_json = json.dumps(
{"id": obj["id"], "target": target_str},
ensure_ascii=False,
)
entries.append(f" {entry_json}")
return "{\n \"objects\": [\n" + ",\n".join(entries) + "\n ]\n}"
def build_segmentation_messages(
image: Any,
query_input: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]],
system_prompt: str = DEFAULT_SYSTEM_PROMPT,
query_token: str = OBJECT_QUERY_TOKEN,
) -> Tuple[List[Dict[str, Any]], List[Dict[str, str]]]:
"""
Constructs the single-pass multimodal chat messages in causal reasoning order:
[System] -> [User: JSON Riddles -> Image] -> [Assistant: <|object_ref_start|>...<|object_ref_end|>]
Putting the user JSON before the image allows every `<|image_pad|>` token in the LLM
to attend to the riddle across all 36 layers before `<|object_ref_end|>` reads them out.
"""
user_prompt, objects = build_segmentation_prompt(
query_input, query_token=query_token, include_query_token=False
)
assistant_prefill = build_assistant_grounding_prefill(objects, query_token=query_token)
messages = [
{"role": "system", "content": system_prompt},
{
"role": "user",
"content": [
{"type": "text", "text": user_prompt + "\n"},
{"type": "image", "image": image},
],
},
{
"role": "assistant",
"content": assistant_prefill,
},
]
return messages, objects
def polygons_to_mask(
segmentation: Union[List[List[float]], Dict[str, Any]],
height: int,
width: int,
) -> np.ndarray:
"""
Rasterizes a COCO/RefCOCO annotation `segmentation` (polygon list or uncompressed RLE)
into a binary numpy mask of shape (height, width) with values in {0, 1}.
"""
mask = np.zeros((height, width), dtype=np.uint8)
if isinstance(segmentation, list):
for poly in segmentation:
if len(poly) < 6:
continue
pts = np.asarray(poly, dtype=np.float32).reshape(-1, 2)
pts = np.round(pts).astype(np.int32)
cv2.fillPoly(mask, [pts], 1)
elif isinstance(segmentation, dict) and "counts" in segmentation:
counts = segmentation["counts"]
if isinstance(counts, list):
h, w = segmentation.get("size", [height, width])
flat = np.zeros(h * w, dtype=np.uint8)
idx = 0
val = 0
for c in counts:
flat[idx : idx + c] = val
idx += c
val = 1 - val
mask = flat.reshape((w, h)).T
if mask.shape != (height, width):
mask = cv2.resize(mask, (width, height), interpolation=cv2.INTER_NEAREST)
else:
raise ValueError("Compressed string RLE requires pycocotools; RefCOCO uses polygon lists.")
else:
raise ValueError(f"Unsupported segmentation format: {type(segmentation)}")
return mask
def mask_to_polygons(
binary_mask: Union[np.ndarray, torch.Tensor],
epsilon_ratio: float = 0.0015,
min_area: float = 4.0,
) -> Tuple[List[List[float]], List[float], float]:
"""
Converts a 2D binary mask (H, W) into RefCOCO/COCO dataset format:
- segmentation: List[List[float]] of polygon coordinates [x1, y1, x2, y2, ...]
- bbox: [x, y, width, height]
- area: float pixel area
"""
if isinstance(binary_mask, torch.Tensor):
mask_np = binary_mask.detach().cpu().numpy().astype(np.uint8)
else:
mask_np = np.asarray(binary_mask, dtype=np.uint8)
if mask_np.ndim > 2:
mask_np = mask_np.squeeze()
area = float(mask_np.sum())
if area < min_area:
return [], [0.0, 0.0, 0.0, 0.0], 0.0
ys, xs = np.where(mask_np > 0)
x_min, x_max = float(xs.min()), float(xs.max())
y_min, y_max = float(ys.min()), float(ys.max())
bbox = [
round(x_min, 2),
round(y_min, 2),
round(x_max - x_min + 1.0, 2),
round(y_max - y_min + 1.0, 2),
]
contours, _ = cv2.findContours(mask_np, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
polygons: List[List[float]] = []
for cnt in contours:
if cv2.contourArea(cnt) < min_area and len(contours) > 1:
continue
peri = cv2.arcLength(cnt, True)
approx = cv2.approxPolyDP(cnt, epsilon_ratio * peri, True) if epsilon_ratio > 0 else cnt
pts = approx.reshape(-1, 2)
if pts.shape[0] < 3:
pts = cnt.reshape(-1, 2)
if pts.shape[0] >= 3:
poly = [round(float(coord), 2) for coord in pts.flatten()]
if len(poly) >= 6:
polygons.append(poly)
if not polygons and area >= min_area:
# Fallback box polygon if contour is degenerate (e.g. 1-pixel thin line)
polygons.append(
[
round(x_min, 2),
round(y_min, 2),
round(x_max, 2),
round(y_min, 2),
round(x_max, 2),
round(y_max, 2),
round(x_min, 2),
round(y_max, 2),
]
)
return polygons, bbox, round(area, 2)
class ConvGNAct(nn.Module):
"""Residual-friendly 3x3 Conv + GroupNorm + GELU block."""
def __init__(self, in_ch: int, out_ch: int, kernel_size: int = 3, groups: int = 8):
super().__init__()
padding = kernel_size // 2
eff_groups = min(groups, out_ch)
while out_ch % eff_groups != 0 and eff_groups > 1:
eff_groups -= 1
self.block = nn.Sequential(
nn.Conv2d(in_ch, out_ch, kernel_size=kernel_size, padding=padding, bias=False),
nn.GroupNorm(eff_groups, out_ch),
nn.GELU(),
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.block(x)
class ResidualRefinementBlock(nn.Module):
"""Residual spatial refinement block for crisp mask boundaries."""
def __init__(self, channels: int):
super().__init__()
self.conv1 = ConvGNAct(channels, channels, kernel_size=3)
self.conv2 = ConvGNAct(channels, channels, kernel_size=3)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return x + self.conv2(self.conv1(x))
class MultiScaleMaskHead(nn.Module):
"""
Lightweight System-1 mask head that fuses:
1. LLM object decision vectors Q [K, llm_dim]
2. Coarse merged ViT features F_28 [C_out, H_m, W_m] (1/28 stride)
3. Fine unmerged ViT skip features F_14 [C_in, 2*H_m, 2*W_m] (1/14 stride)
and outputs full-resolution mask logits [K, orig_H, orig_W] + IoU confidence scores [K].
"""
def __init__(
self,
llm_hidden_size: int = 2048,
vision_merged_dim: int = 2048,
vision_unmerged_dim: int = 1280,
hidden_dim: int = 256,
fine_dim: int = 128,
num_heads: int = 4,
num_fused_layers: int = 4,
):
super().__init__()
self.hidden_dim = hidden_dim
self.fine_dim = fine_dim
self.num_fused_layers = num_fused_layers
# Learnable fusion across the last N LLM layers (preserves frozen LLM reasoning + mid-late visual grounding)
self.layer_weights = nn.Parameter(torch.zeros(num_fused_layers))
self.layer_norms = nn.ModuleList([nn.LayerNorm(llm_hidden_size) for _ in range(num_fused_layers)])
# Visual feature projections (computed once per image, shared across K object queries)
self.coarse_vis_proj = ConvGNAct(vision_merged_dim, hidden_dim, kernel_size=1)
self.llm_vis_proj = ConvGNAct(llm_hidden_size, hidden_dim, kernel_size=1)
self.llm_vis_gate = nn.Parameter(torch.tensor(0.5))
self.fine_vis_proj = ConvGNAct(vision_unmerged_dim, fine_dim, kernel_size=1)
# Query projection + cross-attention over coarse 2D visual tokens
self.query_in_proj = nn.Sequential(
nn.Linear(llm_hidden_size, hidden_dim),
nn.LayerNorm(hidden_dim),
nn.GELU(),
nn.Linear(hidden_dim, hidden_dim),
)
self.cross_attn = nn.MultiheadAttention(
embed_dim=hidden_dim,
num_heads=num_heads,
batch_first=True,
)
self.cross_norm = nn.LayerNorm(hidden_dim)
# Query-to-coarse and query-to-fine dynamic modulators
self.coarse_query_proj = nn.Linear(hidden_dim, hidden_dim)
self.llm_query_proj = nn.Linear(hidden_dim, hidden_dim)
self.coarse_gate_proj = nn.Linear(hidden_dim, hidden_dim)
self.fine_gate_proj = nn.Linear(hidden_dim, fine_dim)
# Coarse decoder at 1/28 scale (concatenates modulated features + dot-product similarity map)
self.coarse_fuse = ConvGNAct(hidden_dim + 1, hidden_dim, kernel_size=3)
self.coarse_res = ResidualRefinementBlock(hidden_dim)
# Fine decoder at 1/14 scale (fuses upsampled coarse features + fine 1/14 ViT skip features)
self.fine_fuse = ConvGNAct(hidden_dim + fine_dim, fine_dim, kernel_size=3)
self.fine_res1 = ResidualRefinementBlock(fine_dim)
self.fine_res2 = ResidualRefinementBlock(fine_dim)
# 2x learned upsampler to 1/7 stride before final bilinear interpolation
self.up_conv = nn.Sequential(
nn.ConvTranspose2d(fine_dim, fine_dim // 2, kernel_size=2, stride=2),
nn.GroupNorm(min(8, fine_dim // 2), fine_dim // 2),
nn.GELU(),
ConvGNAct(fine_dim // 2, fine_dim // 2, kernel_size=3),
)
self.mask_out = nn.Conv2d(fine_dim // 2, 1, kernel_size=1)
# Predicts calibrated mask confidence / IoU score per object query
self.score_head = nn.Sequential(
nn.Linear(hidden_dim, hidden_dim // 2),
nn.GELU(),
nn.Linear(hidden_dim // 2, 1),
)
def _build_2d_sincos_pos_embed(
self, h: int, w: int, dim: int, device: torch.device, dtype: torch.dtype
) -> torch.Tensor:
"""Generates parameter-free 2D sine-cosine positional embeddings [1, h*w, dim]."""
y = torch.linspace(-1.0, 1.0, steps=h, device=device, dtype=torch.float32)
x = torch.linspace(-1.0, 1.0, steps=w, device=device, dtype=torch.float32)
grid_y, grid_x = torch.meshgrid(y, x, indexing="ij")
half = dim // 4
if half == 0:
return torch.zeros((1, h * w, dim), device=device, dtype=dtype)
omega = torch.arange(half, device=device, dtype=torch.float32) / max(half, 1)
omega = 1.0 / (10000.0**omega)
out_y = grid_y.reshape(-1, 1) * omega.reshape(1, -1)
out_x = grid_x.reshape(-1, 1) * omega.reshape(1, -1)
pos = torch.cat([torch.sin(out_y), torch.cos(out_y), torch.sin(out_x), torch.cos(out_x)], dim=-1)
if pos.shape[-1] < dim:
pos = F.pad(pos, (0, dim - pos.shape[-1]))
return pos.unsqueeze(0).to(dtype=dtype)
def forward_single_image(
self,
queries: torch.Tensor,
f_coarse: torch.Tensor,
f_fine: torch.Tensor,
target_size: Tuple[int, int],
f_llm_coarse: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""
Args:
queries: [K, llm_hidden_size] or [K, L, llm_hidden_size] decision vectors for K objects
f_coarse: [1, vision_merged_dim, H_m, W_m] (1/28 resolution from ViT)
f_fine: [1, vision_unmerged_dim, H_f, W_f] (1/14 resolution from ViT)
target_size: (orig_H, orig_W)
f_llm_coarse: optional [1, llm_hidden_size, H_m, W_m] riddle-conditioned `<|image_pad|>` grid from LLM
Returns:
mask_logits: [K, orig_H, orig_W]
pred_scores: [K] (in [0, 1])
"""
num_queries = queries.shape[0]
_, _, h_m, w_m = f_coarse.shape
_, _, h_f, w_f = f_fine.shape
# Fuse multi-layer LLM hidden states [K, L, llm_hidden_size] -> [K, llm_hidden_size]
if queries.ndim == 3:
num_layers = queries.shape[1]
normed_layers = [
self.layer_norms[min(i, len(self.layer_norms) - 1)](queries[:, i, :])
for i in range(num_layers)
]
stacked = torch.stack(normed_layers, dim=1) # [K, L, llm_hidden_size]
weights = F.softmax(self.layer_weights[:num_layers], dim=0).view(1, num_layers, 1)
queries = (stacked * weights).sum(dim=1)
# Project visual features once for the image
vis_coarse = self.coarse_vis_proj(f_coarse) # [1, hidden_dim, H_m, W_m]
vis_llm = None
if f_llm_coarse is not None:
vis_llm = self.llm_vis_proj(f_llm_coarse) # [1, hidden_dim, H_m, W_m]
vis_coarse = vis_coarse + torch.tanh(self.llm_vis_gate) * vis_llm
vis_fine = self.fine_vis_proj(f_fine) # [1, fine_dim, H_f, W_f]
# Project queries & refine with cross-attention over 2D visual tokens
q = self.query_in_proj(queries).unsqueeze(0) # [1, K, hidden_dim]
vis_tokens = vis_coarse.flatten(2).transpose(1, 2) # [1, H_m*W_m, hidden_dim]
pos_2d = self._build_2d_sincos_pos_embed(
h_m, w_m, self.hidden_dim, vis_tokens.device, vis_tokens.dtype
)
attn_out, _ = self.cross_attn(
query=q,
key=vis_tokens + pos_2d,
value=vis_tokens,
)
q = self.cross_norm(q + attn_out).squeeze(0) # [K, hidden_dim]
# Predict confidence score per object query
pred_scores = torch.sigmoid(self.score_head(q).squeeze(-1)) # [K]
# 1. Coarse (1/28) dot-product similarity + channel gating
q_dot = F.normalize(self.coarse_query_proj(q), dim=-1) # [K, hidden_dim]
v_norm = F.normalize(vis_coarse.squeeze(0), dim=0) # [hidden_dim, H_m, W_m]
sim_map = torch.einsum("kd,dhw->khw", q_dot, v_norm).unsqueeze(1) # [K, 1, H_m, W_m]
if vis_llm is not None:
q_llm_dot = F.normalize(self.llm_query_proj(q), dim=-1)
v_llm_norm = F.normalize(vis_llm.squeeze(0), dim=0)
sim_map_llm = torch.einsum("kd,dhw->khw", q_llm_dot, v_llm_norm).unsqueeze(1)
sim_map = 0.5 * (sim_map + sim_map_llm)
coarse_gate = torch.sigmoid(self.coarse_gate_proj(q)).unsqueeze(-1).unsqueeze(-1) # [K, hidden_dim, 1, 1]
vis_coarse_exp = vis_coarse.expand(num_queries, -1, -1, -1)
coarse_mod = vis_coarse_exp * coarse_gate # [K, hidden_dim, H_m, W_m]
x = torch.cat([coarse_mod, sim_map], dim=1) # [K, hidden_dim + 1, H_m, W_m]
x = self.coarse_res(self.coarse_fuse(x))
# 2. Upsample 2x to 1/14 resolution & fuse with unmerged ViT skip features
x = F.interpolate(x, size=(h_f, w_f), mode="bilinear", align_corners=False)
fine_gate = torch.sigmoid(self.fine_gate_proj(q)).unsqueeze(-1).unsqueeze(-1) # [K, fine_dim, 1, 1]
vis_fine_exp = vis_fine.expand(num_queries, -1, -1, -1) * fine_gate
x = torch.cat([x, vis_fine_exp], dim=1) # [K, hidden_dim + fine_dim, H_f, W_f]
x = self.fine_fuse(x)
x = self.fine_res2(self.fine_res1(x))
# 3. Learned 2x upsample (to 1/7 stride) + bilinear resize to original image resolution
x = self.up_conv(x)
raw_logits = self.mask_out(x) # [K, 1, 2*H_f, 2*W_f]
upsampled_logits = F.interpolate(
raw_logits,
size=target_size,
mode="bilinear",
align_corners=False,
).squeeze(1) # [K, orig_H, orig_W]
return upsampled_logits, pred_scores
class Qwen2_5_VL_SinglePassSegmenter(nn.Module):
"""
System-1 Single-Pass Multi-Object Segmenter built on Qwen2.5-VL.
- Drops the autoregressive LM head completely.
- Runs the ViT encoder and LLM backbone once.
- Captures both 1/14 (unmerged) and 1/28 (merged) ViT spatial grids from the single pass.
- Extracts K decision vectors from `<|object_ref_end|>` tokens in the input JSON.
- Decodes all K object masks in parallel.
"""
def __init__(
self,
qwen_model_name: Optional[str] = "./Qwen3.5-4B",
backbone: Optional[nn.Module] = None,
processor: Optional[Any] = None,
torch_dtype: torch.dtype = torch.bfloat16,
device_map: Optional[Union[str, Dict[str, Any]]] = None,
quantization_config: Optional[Any] = None,
freeze_backbone: bool = True,
head_hidden_dim: int = 256,
head_fine_dim: int = 128,
num_fused_layers: int = 4,
):
super().__init__()
self.num_fused_layers = num_fused_layers
if backbone is not None:
self.qwen = backbone
self.processor = processor
else:
if qwen_model_name is None:
raise ValueError("Provide either `qwen_model_name` or a pre-built `backbone`.")
self.processor = processor or AutoProcessor.from_pretrained(qwen_model_name)
load_kwargs: Dict[str, Any] = {"torch_dtype": torch_dtype}
if device_map is not None:
load_kwargs["device_map"] = device_map
if quantization_config is not None:
load_kwargs["quantization_config"] = quantization_config
self.qwen = AutoModel.from_pretrained(qwen_model_name, **load_kwargs)
# Support both Qwen2_5_VLModel and Qwen2_5_VLForConditionalGeneration
if hasattr(self.qwen, "model") and hasattr(self.qwen.model, "visual"):
self.backbone = self.qwen.model
else:
self.backbone = self.qwen
self.freeze_backbone = freeze_backbone
if freeze_backbone:
for param in self.backbone.parameters():
param.requires_grad = False
# Read dimensions safely across transformers versions
cfg = self.backbone.config
text_cfg = getattr(cfg, "text_config", cfg)
vis_cfg = cfg.vision_config
self.llm_hidden_size = getattr(text_cfg, "hidden_size", 2560)
self.vision_merged_dim = getattr(vis_cfg, "out_hidden_size", self.llm_hidden_size)
self.vision_unmerged_dim = getattr(vis_cfg, "hidden_size", 1024)
self.spatial_merge_size = getattr(vis_cfg, "spatial_merge_size", 2)
self.patch_size = getattr(vis_cfg, "patch_size", 16)
if self.processor is not None and hasattr(self.processor, "tokenizer"):
self.query_token_id = self.processor.tokenizer.convert_tokens_to_ids(OBJECT_QUERY_TOKEN)
self.im_end_token_id = self.processor.tokenizer.convert_tokens_to_ids("<|im_end|>")
self.image_token_id = self.processor.tokenizer.convert_tokens_to_ids("<|image_pad|>")
else:
self.query_token_id = getattr(cfg, "object_ref_end_token_id", 248048)
self.im_end_token_id = getattr(text_cfg, "eos_token_id", 248046)
self.image_token_id = getattr(cfg, "image_token_id", 248056)
self.mask_head = MultiScaleMaskHead(
llm_hidden_size=self.llm_hidden_size,
vision_merged_dim=self.vision_merged_dim,
vision_unmerged_dim=self.vision_unmerged_dim,
hidden_dim=head_hidden_dim,
fine_dim=head_fine_dim,
num_fused_layers=num_fused_layers,
)
# Hook buffers to capture single-pass visual outputs & last N LLM layers without extra memory
self._captured_visual: Dict[str, Any] = {}
self._captured_llm_layers: List[torch.Tensor] = []
self._visual_hook = self.backbone.visual.register_forward_hook(self._capture_visual_hook)
self._llm_hooks: List[Any] = []
self._register_llm_layer_hooks()
self._needs_mm_token_type_ids = self._check_needs_mm_token_type_ids()
def _check_needs_mm_token_type_ids(self) -> bool:
return "mm_token_type_ids" in inspect.signature(
self.backbone.forward
).parameters or (
hasattr(self.backbone, "compute_3d_position_ids")
and "mm_token_type_ids" in inspect.signature(self.backbone.compute_3d_position_ids).parameters
)
def _get_llm_decoder_layers(self) -> Optional[nn.ModuleList]:
lm = getattr(self.backbone, "language_model", None)
if lm is None:
return None
if hasattr(lm, "layers"):
return lm.layers
if hasattr(lm, "model") and hasattr(lm.model, "layers"):
return lm.model.layers
return None
def _register_llm_layer_hooks(self) -> None:
for h in self._llm_hooks:
h.remove()
self._llm_hooks.clear()
layers = self._get_llm_decoder_layers()
if layers is not None and len(layers) > 0:
k = min(self.num_fused_layers, len(layers))
for layer in list(layers)[-k:]:
self._llm_hooks.append(layer.register_forward_hook(self._capture_llm_layer_hook))
def _capture_llm_layer_hook(self, module: nn.Module, inputs: Any, output: Any) -> None:
hidden = output[0] if isinstance(output, tuple) else output
if isinstance(hidden, torch.Tensor):
self._captured_llm_layers.append(hidden)
def _capture_visual_hook(self, module: nn.Module, inputs: Any, output: Any) -> None:
if hasattr(output, "last_hidden_state") and hasattr(output, "pooler_output"):
self._captured_visual["unmerged"] = output.last_hidden_state
# Clone reference before get_image_features mutates output.pooler_output into a tuple
self._captured_visual["merged"] = output.pooler_output
elif isinstance(output, tuple) and len(output) >= 2:
self._captured_visual["unmerged"] = output[0]
self._captured_visual["merged"] = output[1]
else:
self._captured_visual["merged"] = output
self._captured_visual["unmerged"] = None
def enable_lora(
self,
r: int = 8,
lora_alpha: int = 16,
lora_dropout: float = 0.05,
top_k_layers: int = 4,
target_modules: Tuple[str, ...] = (
"q_proj",
"k_proj",
"v_proj",
"o_proj",
"in_proj_qkv",
"out_proj",
),
) -> None:
"""
Safe Stage 2 training helper:
Applies low-rank LoRA ONLY to the top `top_k_layers` (e.g., last 4 of 36 layers),
leaving the first ~32 layers 100% frozen so Qwen retains all world knowledge and
riddle-solving ability while the top layers adapt to spatial token compression.
"""
from peft import LoraConfig, get_peft_model
layers = self._get_llm_decoder_layers()
resolved_targets: List[str] = list(target_modules)
if layers is not None and top_k_layers > 0 and len(layers) > top_k_layers:
total_l = len(layers)
allowed_layer_indices = set(range(total_l - top_k_layers, total_l))
exact_targets: List[str] = []
for name, _ in self.backbone.language_model.named_modules():
leaf = name.split(".")[-1]
if leaf not in target_modules:
continue
if any(
f"layers.{idx}." in name or name.startswith(f"layers.{idx}.")
for idx in allowed_layer_indices
):
exact_targets.append(name)
if exact_targets:
resolved_targets = exact_targets
lora_cfg = LoraConfig(
r=r,
lora_alpha=lora_alpha,
lora_dropout=lora_dropout,
target_modules=resolved_targets,
bias="none",
)
self.backbone.language_model = get_peft_model(self.backbone.language_model, lora_cfg)
self._register_llm_layer_hooks()
self._needs_mm_token_type_ids = self._check_needs_mm_token_type_ids()
self.freeze_backbone = False
def _compute_window_index(
self,
grid_thw: torch.Tensor,
grid_thw_list: Optional[Sequence[Sequence[int]]] = None,
) -> torch.Tensor:
if hasattr(self.backbone.visual, "get_window_index"):
window_index, _ = self.backbone.visual.get_window_index(grid_thw)
return window_index
vis_cfg = getattr(self.backbone.config, "vision_config", None)
window_size = getattr(vis_cfg, "window_size", 112)
m = self.spatial_merge_size
vit_merger_window_size = max(window_size // m // self.patch_size, 1)
window_index: List[torch.Tensor] = []
window_index_id = 0
rows = grid_thw_list if grid_thw_list is not None else grid_thw.tolist()
for grid_t, grid_h, grid_w in rows:
llm_grid_h, llm_grid_w = grid_h // m, grid_w // m
index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape(grid_t, llm_grid_h, llm_grid_w)
pad_h = vit_merger_window_size - llm_grid_h % vit_merger_window_size
pad_w = vit_merger_window_size - llm_grid_w % vit_merger_window_size
num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size
num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size
index_padded = F.pad(index, (0, pad_w, 0, pad_h), "constant", -100)
index_padded = index_padded.reshape(
grid_t, num_windows_h, vit_merger_window_size, num_windows_w, vit_merger_window_size
)
index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape(
grid_t, num_windows_h * num_windows_w, vit_merger_window_size, vit_merger_window_size
)
index_padded = index_padded.reshape(-1)
index_new = index_padded[index_padded != -100]
window_index.append(index_new + window_index_id)
window_index_id += grid_t * llm_grid_h * llm_grid_w
return torch.cat(window_index, dim=0)
def _reconstruct_2d_vision_features(
self,
merged_tokens: Union[torch.Tensor, Sequence[torch.Tensor]],
unmerged_tokens: Optional[torch.Tensor],
image_grid_thw: torch.Tensor,
grid_thw_list: Optional[Sequence[Sequence[int]]] = None,
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
"""
Reconstructs per-image 2D spatial grids from Qwen2.5-VL's window-permuted ViT outputs:
- coarse_grids: list of [1, C_merged, H_m, W_m] at 1/28 resolution
- fine_grids: list of [1, C_unmerged, 2*H_m, 2*W_m] at 1/14 resolution
"""
m = self.spatial_merge_size
m_sq = m * m
rows = grid_thw_list if grid_thw_list is not None else image_grid_thw.tolist()
if isinstance(merged_tokens, (tuple, list)):
merged_cat = torch.cat(list(merged_tokens), dim=0)
else:
merged_cat = merged_tokens
# Un-permute the unmerged tokens from window attention order if the ViT uses window attention (Qwen2.5-VL);
# Qwen3.5-4B uses full ViT attention so its unmerged tokens are already in natural block-raster order.
vis_cfg = getattr(self.backbone.config, "vision_config", None)
has_window_attn = hasattr(vis_cfg, "window_size") and getattr(vis_cfg, "window_size", None) is not None
if unmerged_tokens is not None:
num_merged = merged_cat.shape[0]
if has_window_attn:
window_index = self._compute_window_index(image_grid_thw, grid_thw_list=rows)
reverse_indices = torch.argsort(window_index.to(unmerged_tokens.device))
unmerged_blocks = unmerged_tokens.view(num_merged, m_sq, -1)[reverse_indices]
else:
unmerged_blocks = unmerged_tokens.view(num_merged, m_sq, -1)
else:
unmerged_blocks = None
coarse_grids: List[torch.Tensor] = []
fine_grids: List[torch.Tensor] = []
offset = 0
for grid_t, grid_h, grid_w in rows:
h_m = grid_h // m
w_m = grid_w // m
n_tokens = grid_t * h_m * w_m
img_merged = merged_cat[offset : offset + n_tokens]
# Average across temporal dimension if grid_t > 1 (for static images grid_t == 1)
img_merged = img_merged.view(grid_t, h_m, w_m, -1).mean(dim=0)
f_coarse = img_merged.permute(2, 0, 1).unsqueeze(0).contiguous() # [1, C_out, H_m, W_m]
coarse_grids.append(f_coarse)
if unmerged_blocks is not None:
img_unmerged = unmerged_blocks[offset : offset + n_tokens]
# Matches Qwen2VLImageProcessor patch ordering: (T, H_m, W_m, m_h, m_w, C_in)
img_unmerged = img_unmerged.view(grid_t, h_m, w_m, m, m, -1).mean(dim=0)
f_fine = (
img_unmerged.permute(4, 0, 2, 1, 3)
.contiguous()
.view(1, self.vision_unmerged_dim, grid_h, grid_w)
)
fine_grids.append(f_fine)
else:
# Fallback if unmerged tokens are unavailable
f_fine = F.interpolate(f_coarse, size=(grid_h, grid_w), mode="bilinear", align_corners=False)
fine_grids.append(f_fine)
offset += n_tokens
return coarse_grids, fine_grids
def _extract_llm_visual_grids(
self,
last_hidden_states: torch.Tensor,
input_ids: torch.Tensor,
image_grid_thw: torch.Tensor,
layer_states: Optional[Sequence[torch.Tensor]] = None,
grid_thw_list: Optional[Sequence[Sequence[int]]] = None,
input_ids_cpu: Optional[torch.Tensor] = None,
) -> List[Optional[torch.Tensor]]:
"""
Extracts the LLM-contextualized `<|image_pad|>` visual tokens from the top LLM layers
and reshapes them into 2D grids [1, llm_dim, H_m, W_m] per image.
Because the user's JSON riddle precedes `<|image_pad|>`, these tokens carry
36 layers of riddle-to-image self-attention context.
"""
m = self.spatial_merge_size
base_dev = last_hidden_states.device
source_state = (
torch.stack([st.to(base_dev) for st in layer_states], dim=0).mean(dim=0)
if layer_states and len(layer_states) > 0
else last_hidden_states
)
rows = grid_thw_list if grid_thw_list is not None else image_grid_thw.tolist()
ids_src = input_ids_cpu if input_ids_cpu is not None else input_ids
target_dev = source_state.device
llm_grids: List[Optional[torch.Tensor]] = []
for b in range(ids_src.shape[0]):
grid_t, grid_h, grid_w = rows[b]
h_m, w_m = grid_h // m, grid_w // m
n_tokens = grid_t * h_m * w_m
img_pos = (ids_src[b] == self.image_token_id).nonzero(as_tuple=True)[0]
if img_pos.numel() == n_tokens:
if img_pos.device != target_dev:
img_pos = img_pos.to(device=target_dev, non_blocking=True)
vis_seq = source_state[b, img_pos, :] # [n_tokens, llm_dim]
vis_2d = (
vis_seq.view(grid_t, h_m, w_m, -1)
.mean(dim=0)
.permute(2, 0, 1)
.unsqueeze(0)
.contiguous()
)
llm_grids.append(vis_2d)
else:
llm_grids.append(None)
return llm_grids
def _extract_query_vectors(
self,
last_hidden_states: torch.Tensor,
input_ids: torch.Tensor,
layer_states: Optional[Sequence[torch.Tensor]] = None,
input_ids_cpu: Optional[torch.Tensor] = None,
) -> List[torch.Tensor]:
"""
Extracts the K `<|object_ref_end|>` decision vectors per batch item.
If multi-layer states are captured, returns [K, L, llm_dim] per batch item;
otherwise returns [K, llm_dim].
"""
states_to_index = list(layer_states) if layer_states else [last_hidden_states]
ids_src = input_ids_cpu if input_ids_cpu is not None else input_ids
target_dev = states_to_index[0].device
batch_queries: List[torch.Tensor] = []
for b in range(ids_src.shape[0]):
pos = (ids_src[b] == self.query_token_id).nonzero(as_tuple=True)[0]
if pos.numel() == 0:
pos = (ids_src[b] == self.im_end_token_id).nonzero(as_tuple=True)[0]
if pos.numel() > 0:
pos = pos[-1:]
else:
pos = torch.tensor([ids_src.shape[1] - 1], dtype=torch.long, device=ids_src.device)
if pos.device != target_dev:
pos = pos.to(device=target_dev, non_blocking=True)
if len(states_to_index) > 1:
per_layer = [
st[b, (pos.to(st.device) if pos.device != st.device else pos), :].to(target_dev)
for st in states_to_index
]
batch_queries.append(torch.stack(per_layer, dim=1)) # [K, L, llm_dim]
else:
batch_queries.append(states_to_index[0][b, pos, :]) # [K, llm_dim]
return batch_queries
def forward(
self,
input_ids: torch.LongTensor,
attention_mask: torch.Tensor,
pixel_values: torch.Tensor,
image_grid_thw: torch.LongTensor,
target_sizes: Optional[Sequence[Tuple[int, int]]] = None,
mm_token_type_ids: Optional[torch.Tensor] = None,
grid_thw_list: Optional[Sequence[Sequence[int]]] = None,
input_ids_cpu: Optional[torch.Tensor] = None,
) -> Dict[str, Any]:
"""
Single forward pass from tokenized JSON + image patches to pixel masks.
Args:
input_ids: [B, seq_len]
attention_mask: [B, seq_len]
pixel_values: [sum_patches, patch_dim]
image_grid_thw: [B, 3] (T, grid_h, grid_w)
target_sizes: optional list of (orig_H, orig_W) per image in batch.
Defaults to (grid_h * patch_size, grid_w * patch_size).
mm_token_type_ids: optional [B, seq_len] multimodal token type IDs (auto-computed if needed).
grid_thw_list: optional pre-extracted Python list of [T, grid_h, grid_w] per image to avoid CUDA syncs.
input_ids_cpu: optional CPU copy of `input_ids` to extract token positions without CUDA `.nonzero()` syncs.
Returns:
dict with:
- "mask_logits": List[Tensor] of shape [K_b, orig_H, orig_W] per image
- "mask_probs": List[Tensor] of shape [K_b, orig_H, orig_W] per image
- "pred_scores": List[Tensor] of shape [K_b] per image
"""
self._captured_visual.clear()
self._captured_llm_layers.clear()
resolved_grid_thw = list(grid_thw_list) if grid_thw_list is not None else image_grid_thw.tolist()
backbone_kwargs: Dict[str, Any] = {
"input_ids": input_ids,
"attention_mask": attention_mask,
"pixel_values": pixel_values,
"image_grid_thw": image_grid_thw,
"use_cache": False,
"output_hidden_states": False,
"return_dict": True,
}
if self._needs_mm_token_type_ids:
if mm_token_type_ids is None:
mm_token_type_ids = (input_ids == self.image_token_id).to(
dtype=torch.long, device=input_ids.device
)
backbone_kwargs["mm_token_type_ids"] = mm_token_type_ids
grad_ctx = torch.no_grad() if self.freeze_backbone else torch.enable_grad()
with grad_ctx:
outputs = self.backbone(**backbone_kwargs)
last_hidden_states = outputs.last_hidden_state
merged_vis = self._captured_visual.get("merged")
unmerged_vis = self._captured_visual.get("unmerged")
if merged_vis is None:
raise RuntimeError("Visual forward hook did not capture features; check pixel_values input.")
coarse_grids, fine_grids = self._reconstruct_2d_vision_features(
merged_tokens=merged_vis,
unmerged_tokens=unmerged_vis,
image_grid_thw=image_grid_thw,
grid_thw_list=resolved_grid_thw,
)
llm_vis_grids = self._extract_llm_visual_grids(
last_hidden_states=last_hidden_states,
input_ids=input_ids,
image_grid_thw=image_grid_thw,
layer_states=self._captured_llm_layers,
grid_thw_list=resolved_grid_thw,
input_ids_cpu=input_ids_cpu,
)
batch_queries = self._extract_query_vectors(
last_hidden_states=last_hidden_states,
input_ids=input_ids,
layer_states=self._captured_llm_layers,
input_ids_cpu=input_ids_cpu,
)
head_param = next(self.mask_head.parameters())
head_dtype = head_param.dtype
head_device = head_param.device
mask_logits_list: List[torch.Tensor] = []
mask_probs_list: List[torch.Tensor] = []
pred_scores_list: List[torch.Tensor] = []
for b in range(len(batch_queries)):
if target_sizes is not None and b < len(target_sizes):
tgt_size = (int(target_sizes[b][0]), int(target_sizes[b][1]))
else:
tgt_size = (
int(resolved_grid_thw[b][1] * self.patch_size),
int(resolved_grid_thw[b][2] * self.patch_size),
)
f_llm_b = (
llm_vis_grids[b].to(device=head_device, dtype=head_dtype)
if llm_vis_grids[b] is not None
else None
)
logits_b, scores_b = self.mask_head.forward_single_image(
queries=batch_queries[b].to(device=head_device, dtype=head_dtype),
f_coarse=coarse_grids[b].to(device=head_device, dtype=head_dtype),
f_fine=fine_grids[b].to(device=head_device, dtype=head_dtype),
target_size=tgt_size,
f_llm_coarse=f_llm_b,
)
mask_logits_list.append(logits_b)
mask_probs_list.append(torch.sigmoid(logits_b))
pred_scores_list.append(scores_b)
return {
"mask_logits": mask_logits_list,
"mask_probs": mask_probs_list,
"pred_scores": pred_scores_list,
}
@torch.no_grad()
def predict_json(
self,
image: Any,
query_json: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]],
threshold: float = 0.5,
) -> Dict[str, Any]:
"""
End-to-end System 1 inference:
Takes a PIL image + JSON object descriptions, runs 1 forward pass, and returns
a JSON dict with COCO polygon segmentations (`segmentation`), `bbox`, `area`, and `score`.
"""
if self.processor is None:
raise RuntimeError("`processor` is required for `predict_json`.")
self.eval()
orig_w, orig_h = image.size
messages, objects = build_segmentation_messages(image=image, query_input=query_json)
formatted_text = self.processor.apply_chat_template(
messages, tokenize=False, add_generation_prompt=False
)
inputs = self.processor(
text=[formatted_text],
images=[image],
padding=True,
return_tensors="pt",
)
input_ids_cpu = inputs["input_ids"]
grid_thw_list = inputs["image_grid_thw"].tolist()
device = next(self.mask_head.parameters()).device
inputs = {k: (v.to(device) if isinstance(v, torch.Tensor) else v) for k, v in inputs.items()}
outputs = self.forward(
input_ids=inputs["input_ids"],
attention_mask=inputs["attention_mask"],
pixel_values=inputs["pixel_values"],
image_grid_thw=inputs["image_grid_thw"],
target_sizes=[(orig_h, orig_w)],
grid_thw_list=grid_thw_list,
input_ids_cpu=input_ids_cpu,
)
probs = outputs["mask_probs"][0] # [K, orig_H, orig_W]
scores = outputs["pred_scores"][0] # [K]
results = []
for idx, obj in enumerate(objects):
if idx >= probs.shape[0]:
break
prob_map = probs[idx]
binary_mask = (prob_map >= threshold).to(torch.uint8)
polygons, bbox, area = mask_to_polygons(binary_mask)
# Combine IoU head score with mean foreground probability
fg_prob = float(prob_map[binary_mask > 0].mean().item()) if area > 0 else 0.0
score = round(0.5 * float(scores[idx].item()) + 0.5 * fg_prob, 4)
results.append(
{
"id": obj["id"],
"description": obj["description"],
"score": score,
"bbox": bbox,
"area": area,
"segmentation": polygons,
}
)
return {
"image_size": {"height": orig_h, "width": orig_w},
"objects": results,
}
class SegmentationLoss(nn.Module):
"""
Combined Focal + Soft Dice + BCE + IoU Score Calibration loss for training the mask head.
"""
def __init__(
self,
bce_weight: float = 1.0,
focal_weight: float = 2.0,
dice_weight: float = 2.0,
score_weight: float = 0.5,
focal_alpha: float = 0.25,
focal_gamma: float = 2.0,
):
super().__init__()
self.bce_weight = bce_weight
self.focal_weight = focal_weight
self.dice_weight = dice_weight
self.score_weight = score_weight
self.focal_alpha = focal_alpha
self.focal_gamma = focal_gamma
def _dice_loss(self, logits: torch.Tensor, targets: torch.Tensor, eps: float = 1.0) -> torch.Tensor:
probs = torch.sigmoid(logits).flatten(1)
tgt = targets.flatten(1)
intersection = (probs * tgt).sum(dim=1)
union = probs.sum(dim=1) + tgt.sum(dim=1)
dice = (2.0 * intersection + eps) / (union + eps)
return (1.0 - dice).mean()
def _sigmoid_focal_loss(self, logits: torch.Tensor, targets: torch.Tensor) -> torch.Tensor:
bce = F.binary_cross_entropy_with_logits(logits, targets, reduction="none")
probs = torch.sigmoid(logits)
p_t = probs * targets + (1.0 - probs) * (1.0 - targets)
alpha_t = self.focal_alpha * targets + (1.0 - self.focal_alpha) * (1.0 - targets)
focal = alpha_t * ((1.0 - p_t) ** self.focal_gamma) * bce
return focal.mean()
def forward(
self,
mask_logits_list: Sequence[torch.Tensor],
pred_scores_list: Sequence[torch.Tensor],
gt_masks_list: Sequence[torch.Tensor],
) -> Dict[str, torch.Tensor]:
total_bce = torch.tensor(0.0, device=mask_logits_list[0].device)
total_focal = torch.tensor(0.0, device=mask_logits_list[0].device)
total_dice = torch.tensor(0.0, device=mask_logits_list[0].device)
total_score = torch.tensor(0.0, device=mask_logits_list[0].device)
n_batches = max(len(mask_logits_list), 1)
for logits, scores, gt in zip(mask_logits_list, pred_scores_list, gt_masks_list):
gt = gt.to(device=logits.device, dtype=logits.dtype)
if gt.shape[-2:] != logits.shape[-2:]:
gt = F.interpolate(gt.unsqueeze(1), size=logits.shape[-2:], mode="nearest").squeeze(1)
total_bce = total_bce + F.binary_cross_entropy_with_logits(logits, gt)
total_focal = total_focal + self._sigmoid_focal_loss(logits, gt)
total_dice = total_dice + self._dice_loss(logits, gt)
with torch.no_grad():
pred_bin = (torch.sigmoid(logits) >= 0.5).to(gt.dtype).flatten(1)
gt_flat = gt.flatten(1)
inter = (pred_bin * gt_flat).sum(dim=1)
union = (pred_bin + gt_flat - pred_bin * gt_flat).sum(dim=1).clamp(min=1e-6)
true_iou = inter / union
total_score = total_score + F.mse_loss(scores, true_iou.to(scores.dtype))
total_bce = total_bce / n_batches
total_focal = total_focal / n_batches
total_dice = total_dice / n_batches
total_score = total_score / n_batches
loss = (
self.bce_weight * total_bce
+ self.focal_weight * total_focal
+ self.dice_weight * total_dice
+ self.score_weight * total_score
)
return {
"loss": loss,
"bce_loss": total_bce.detach(),
"focal_loss": total_focal.detach(),
"dice_loss": total_dice.detach(),
"score_loss": total_score.detach(),
}
import importlib.metadata as _im
import inspect
import json
from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
import cv2
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
# Guard against huggingface-hub>=2.0 version check in transformers while keeping full functionality
_orig_version = _im.version
def _safe_version(pkg: str) -> str:
if pkg == "huggingface-hub":
try:
ver = _orig_version(pkg)
if ver.startswith("2."):
return "1.99.0"
return ver
except Exception:
return "1.99.0"
return _orig_version(pkg)
_im.version = _safe_version
from transformers import AutoModel, AutoProcessor
OBJECT_START_TOKEN = "<|object_ref_start|>"
OBJECT_QUERY_TOKEN = "<|object_ref_end|>"
DEFAULT_SYSTEM_PROMPT = (
"You are a fast single-pass visual reasoning and segmentation model. "
"For each query or riddle in the input JSON, infer the intended target object in the image and ground it precisely."
)
def normalize_input_json(
query_input: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]],
) -> List[Dict[str, str]]:
"""
Normalizes user input (JSON string, dict with 'objects', or list of strings/dicts)
into a canonical list of [{'id': str, 'description': str}, ...].
"""
if isinstance(query_input, str):
parsed = json.loads(query_input)
else:
parsed = query_input
if isinstance(parsed, dict):
raw_objects = parsed.get("objects", parsed.get("queries", [parsed]))
elif isinstance(parsed, (list, tuple)):
raw_objects = list(parsed)
else:
raise ValueError(f"Unsupported query_input format: {type(query_input)}")
normalized: List[Dict[str, str]] = []
for idx, item in enumerate(raw_objects):
if isinstance(item, str):
normalized.append({"id": str(idx), "description": item.strip()})
elif isinstance(item, dict):
obj_id = str(item.get("id", idx))
desc = str(
item.get("description")
or item.get("prompt")
or item.get("text")
or item.get("sent")
or ""
).strip()
if not desc:
raise ValueError(f"Object at index {idx} has an empty description: {item}")
normalized.append({"id": obj_id, "description": desc})
else:
raise ValueError(f"Unsupported object entry at index {idx}: {item}")
if not normalized:
raise ValueError("Input JSON must contain at least one object description.")
return normalized
def build_segmentation_prompt(
query_input: Union[str, Dict[str, Any], Sequence[Union[str, Dict[str, Any]]]],
query_token: str = OBJECT_QUERY_TOKEN,
include_query_token: bool = False,
) -> Tuple[str, List[Dict[str, str]]]:
"""
Builds the user-side JSON prompt describing all objects/riddles to segment.
"""
objects = normalize_input_json(query_input)
entries = []
for obj in objects:
entry_json = json.dumps(
{"id": obj["id"], "description": obj["description"]},
ensure_ascii=False,
)
suffix = query_token if include_query_token else ""
entries.append(f" {entry_json}{suffix}")