from typing import List import os import sys import torch import torch.nn as nn import torch.nn.functional as F from torch.linalg import norm, eigh from transformers import BitsAndBytesConfig, CLIPVisionModel from utils.utils import (DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IMAGE_PATCH_TOKEN) from .llava.model.language_model.llava_llama import (LlavaLlamaForCausalLM, LlavaLlamaModel) from .segment_anything import build_sam_vit_h PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PROJECT_PARENT = os.path.dirname(PROJECT_ROOT) if PROJECT_PARENT not in sys.path: sys.path.insert(0, PROJECT_PARENT) from sam2.build_sam import build_sam2 from sam2.utils.transforms import SAM2Transforms from PIL import Image import numpy as np def read_images(image_paths, target_size=(1024, 1024)): images = [] for image_path in image_paths: # 用 Pillow 库打开图片 image = Image.open(image_path) # 将图片转换为 RGB 模式(如果图片不是 RGB) image = image.convert('RGB') # 统一图片大小到 target_size (1024x1024) image = image.resize(target_size) # 将图片转换为 NumPy 数组 image_array = np.array(image) images.append(image_array) return images def Gaussian_2d(mask): batch_size, h, w = mask.shape y_coords, x_coords = torch.meshgrid(torch.arange(h), torch.arange(w), indexing="ij") device = mask.device x_coords = x_coords.to(device) y_coords = y_coords.to(device) y_coords = y_coords.unsqueeze(0).expand(batch_size, -1, -1) x_coords = x_coords.unsqueeze(0).expand(batch_size, -1, -1) mass = mask.sum(dim=(1, 2), keepdim=True) + 1e-6 mu_x = (x_coords * mask).sum(dim=(1, 2)) / mass.squeeze(-1).squeeze(-1) mu_y = (y_coords * mask).sum(dim=(1, 2)) / mass.squeeze(-1).squeeze(-1) denom = mass.squeeze(-1).squeeze(-1) -1 + 1e-6 variance_x = ((((x_coords - mu_x))**2) * mask).sum(dim=(1, 2)) / denom variance_y = ((((y_coords - mu_y))**2) * mask).sum(dim=(1, 2)) / denom covariance_xy = (((x_coords - mu_x) * (y_coords - mu_y)) * mask).sum(dim=(1, 2)) / denom mean = torch.stack([mu_x, mu_y], dim=1) cov = torch.stack([ torch.stack([variance_x, covariance_xy], dim=-1), torch.stack([covariance_xy, variance_y], dim=-1) ], dim=-2) if not torch.isfinite(mean).all() or not torch.isfinite(cov).all(): raise ValueError("Gaussian_2d outputs contain NaN or Inf") return mean, cov def loss_sqrt_2(wass_dist): return 1 - 1 / (2 + torch.sqrt(wass_dist + 1e-4)) def torch_sqrtm(matrix, eps=1e-4): matrix = (matrix + matrix.T) / 2 trace = matrix[0][0] + matrix[1][1] determinant = matrix[0][0] * matrix[1][1] - matrix[0][1] * matrix[1][0] if determinant < -eps: raise ValueError("Negative determinant") s = torch.sqrt(determinant + eps) t = torch.sqrt(trace + 2 * s) if t < eps: raise ValueError("Zero t in sqrtm") sqrt_matrix = (matrix + s * torch.eye(2, device=matrix.device)) / t return sqrt_matrix def kl_divergence(Mu1, Cov1, Mu2, Cov2): kl_loss = 0 for i in range(Mu1.shape[0]): mu1 = Mu1[i] mu2 = Mu2[i] cov1 = Cov1[i] cov2 = Cov2[i] d = mu1.shape[-1] try: cov2_inv = torch.linalg.inv(cov2) trace_term = torch.trace(cov2_inv @ cov1) mean_diff = mu2 - mu1 mean_term = mean_diff @ cov2_inv @ mean_diff logdet_term = torch.logdet(cov2 + 1e-6 * torch.eye(d, device=cov2.device)) - \ torch.logdet(cov1 + 1e-6 * torch.eye(d, device=cov1.device)) kl = 0.5 * (logdet_term - d + trace_term + mean_term) except Exception: return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype) if not torch.isfinite(kl): return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype) kl_loss += loss_sqrt_2(kl) if not torch.isfinite(kl_loss): return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype) return kl_loss def bd_distance(Mu1, Cov1, Mu2, Cov2): bd_loss = 0 for i in range(Mu1.shape[0]): mu1 = Mu1[i] mu2 = Mu2[i] cov1 = Cov1[i] cov2 = Cov2[i] try: cov_avg = 0.5 * (cov1 + cov2) cov_avg_inv = torch.linalg.inv(cov_avg + 1e-6 * torch.eye(2, device=cov_avg.device)) mean_diff = (mu1 - mu2).unsqueeze(0) mean_term = 0.125 * (mean_diff @ cov_avg_inv @ mean_diff.T).squeeze() det_cov1 = torch.linalg.det(cov1 + 1e-6 * torch.eye(2, device=cov1.device)) det_cov2 = torch.linalg.det(cov2 + 1e-6 * torch.eye(2, device=cov2.device)) det_cov_avg = torch.linalg.det(cov_avg + 1e-6 * torch.eye(2, device=cov_avg.device)) det_term = 0.5 * torch.log(det_cov_avg / torch.sqrt(det_cov1 * det_cov2)) bd = mean_term + det_term except Exception: return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype) if not torch.isfinite(bd): return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype) bd_loss += loss_sqrt_2(bd) if not torch.isfinite(bd_loss): return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype) return bd_loss def wasserstein_distance(Mean1, Cov1, Mean2, Cov2): assert Mean1.shape[0] == Cov1.shape[0] wasserstein_loss = 0 for i in range(Mean1.shape[0]): mean1 = Mean1[i] mean2 = Mean2[i] cov1 = Cov1[i] cov2 = Cov2[i] if not all(torch.isfinite(x).all() for x in [mean1, mean2, cov1, cov2]): return torch.tensor(0.0, device=Mean1.device, dtype=Mean1.dtype) mean_diff = norm(mean1 - mean2)**2 try: sqrt_cov1 = torch_sqrtm(cov1) cov_prod = sqrt_cov1 @ cov2 @ sqrt_cov1 sqrt_cov_prod = torch_sqrtm(cov_prod) except Exception: return torch.tensor(0.0, device=Mean1.device, dtype=Mean1.dtype) if not all(torch.isfinite(x).all() for x in [sqrt_cov1, sqrt_cov_prod]): return torch.tensor(0.0, device=Mean1.device, dtype=Mean1.dtype) cov_term = torch.trace(cov1 + cov2 - 2 * sqrt_cov_prod) wasserstein_dist = torch.clamp(mean_diff + cov_term, min=0.0) if not torch.isfinite(wasserstein_dist): return torch.tensor(0.0, device=Mean1.device, dtype=Mean1.dtype) wasserstein_loss += loss_sqrt_2(wasserstein_dist) if not torch.isfinite(wasserstein_loss): return torch.tensor(0.0, device=Mean1.device, dtype=Mean1.dtype) return wasserstein_loss def contains_zero(tensor_list): return any((t == 0).any() for t in tensor_list) def rao_loss(pred_mask, gt_mask, num_masks): pred_mask = pred_mask.sigmoid() try: mean_pred, cov_pred = Gaussian_2d(pred_mask) mean_gt, cov_gt = Gaussian_2d(gt_mask) except Exception: return torch.tensor(0.0, device=pred_mask.device, dtype=pred_mask.dtype, requires_grad=True) if not all(torch.isfinite(x).all() for x in [mean_pred, cov_pred, mean_gt, cov_gt]): return torch.tensor(0.0, device=pred_mask.device, dtype=pred_mask.dtype, requires_grad=True) if contains_zero([mean_pred, cov_pred, mean_gt, cov_gt]): return torch.tensor(0.0, device=pred_mask.device, dtype=pred_mask.dtype, requires_grad=True) try: loss = wasserstein_distance(mean_pred, cov_pred, mean_gt, cov_gt) #loss = kl_divergence(mean_pred, cov_pred, mean_gt, cov_gt) #loss = bd_distance(mean_pred, cov_pred, mean_gt, cov_gt) except Exception: return torch.tensor(0.0, device=pred_mask.device, dtype=pred_mask.dtype, requires_grad=True) if not torch.isfinite(loss): return torch.tensor(0.0, device=pred_mask.device, dtype=pred_mask.dtype, requires_grad=True) return loss / (num_masks + 1e-8) def dice_loss( inputs: torch.Tensor, targets: torch.Tensor, num_masks: float, scale=1000, # 100000.0, eps=1e-6, ): """ Compute the DICE loss, similar to generalized IOU for masks Args: inputs: A float tensor of arbitrary shape. The predictions for each example. targets: A float tensor with the same shape as inputs. Stores the binary classification label for each element in inputs (0 for the negative class and 1 for the positive class). """ inputs = inputs.sigmoid() inputs = inputs.flatten(1, 2) targets = targets.flatten(1, 2) numerator = 2 * (inputs / scale * targets).sum(-1) denominator = (inputs / scale).sum(-1) + (targets / scale).sum(-1) loss = 1 - (numerator + eps) / (denominator + eps) loss = loss.sum() / (num_masks + 1e-8) return loss def sigmoid_ce_loss( inputs: torch.Tensor, targets: torch.Tensor, num_masks: float, ): """ Args: inputs: A float tensor of arbitrary shape. The predictions for each example. targets: A float tensor with the same shape as inputs. Stores the binary classification label for each element in inputs (0 for the negative class and 1 for the positive class). Returns: Loss tensor """ loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction="none") loss = loss.flatten(1, 2).mean(1).sum() / (num_masks + 1e-8) return loss class LisaMetaModel: def __init__( self, config, **kwargs, ): super(LisaMetaModel, self).__init__(config) self.config = config self.config.sam2_config = kwargs.get( "sam2_config", getattr(self.config, "sam2_config", "configs/sam2.1/sam2.1_hiera_l.yaml"), ) self.sam2_config = self.config.sam2_config if not hasattr(self.config, "train_mask_decoder"): self.config.train_mask_decoder = kwargs["train_mask_decoder"] self.config.out_dim = kwargs["out_dim"] self.vision_pretrained = kwargs.get("vision_pretrained", None) else: self.vision_pretrained = kwargs.get("vision_pretrained", None) self.initialize_lisa_modules(self.config) def initialize_lisa_modules(self, config): # SAM #self.visual_model = build_sam_vit_h(self.vision_pretrained) print("----------building sam2-------------") # ZeroGPU packs the fully loaded model after startup. Build the SAM2 # architecture on CPU first; the fixed model snapshot supplies all # trained SAM2 parameters before the complete model is moved to CUDA. self.visual_model = build_sam2( self.sam2_config, self.vision_pretrained, device="cpu", ) for param in self.visual_model.parameters(): param.requires_grad = False if config.train_mask_decoder: self.visual_model.sam_mask_decoder.train() for param in self.visual_model.sam_mask_decoder.parameters(): param.requires_grad = True self._transforms = SAM2Transforms( resolution=self.visual_model.image_size, mask_threshold=self.visual_model.mask_threshold, max_hole_area=self.visual_model.max_hole_area, max_sprinkle_area=self.visual_model.max_sprinkle_area, ) # Projection layer in_dim = config.hidden_size out_dim = config.out_dim text_fc = [ nn.Linear(in_dim, in_dim), nn.ReLU(inplace=True), nn.Linear(in_dim, out_dim), nn.Dropout(0.0), ] self.text_hidden_fcs = nn.ModuleList([nn.Sequential(*text_fc)]) self.text_hidden_fcs.train() for param in self.text_hidden_fcs.parameters(): param.requires_grad = True class LisaModel(LisaMetaModel, LlavaLlamaModel): def __init__( self, config, **kwargs, ): super(LisaModel, self).__init__(config, **kwargs) self.config.use_cache = False self.config.vision_tower = self.config.mm_vision_tower self.config.mm_vision_select_feature = "patch" self.config.image_aspect_ratio = "square" self.config.image_grid_pinpoints = None self.config.tune_mm_mlp_adapter = False self.config.freeze_mm_mlp_adapter = True self.config.pretrain_mm_mlp_adapter = None self.config.mm_use_im_patch_token = False class LISAForCausalLM(LlavaLlamaForCausalLM): def __init__( self, config, **kwargs, ): if not hasattr(config, "train_mask_decoder"): config.mm_use_im_start_end = kwargs.pop("use_mm_start_end", True) config.mm_vision_tower = kwargs.get( "vision_tower", "openai/clip-vit-large-patch14" ) self.ce_loss_weight = kwargs.pop("ce_loss_weight", None) self.dice_loss_weight = kwargs.pop("dice_loss_weight", None) self.rao_loss_weight = kwargs.pop("rao_loss_weight", None) self.bce_loss_weight = kwargs.pop("bce_loss_weight", None) else: config.mm_vision_tower = config.vision_tower self.seg_token_idx = kwargs.pop("seg_token_idx") super().__init__(config) self.model = LisaModel(config, **kwargs) self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) # Spatial dim for backbone feature maps self._bb_feat_sizes = [ (256, 256), (128, 128), (64, 64), ] # Initialize weights and apply final processing self.post_init() ''' self._transforms = SAM2Transforms( resolution=self.model.visual_model.image_size, mask_threshold=self.model.visual_model.mask_threshold, max_hole_area=self.model.visual_model.max_hole_area, max_sprinkle_area=self.model.visual_model.max_sprinkle_area, ) ''' def get_visual_embs(self, pixel_values: torch.FloatTensor): with torch.no_grad(): image_embeddings_list = [] for i in range(pixel_values.shape[0]): torch.cuda.empty_cache() image_embeddings = self.model.visual_model.image_encoder( pixel_values[i].unsqueeze(0) ) #print("-----------------------") #print(image_embeddings['vision_features'].shape) image_embeddings_list.append(image_embeddings['vision_features']) torch.cuda.empty_cache() image_embeddings = torch.cat(image_embeddings_list, 0) return image_embeddings def forward(self, **kwargs): if "past_key_values" in kwargs: return super().forward(**kwargs) return self.model_forward(**kwargs) def model_forward( self, #image_paths: list, images: torch.FloatTensor, images_clip: torch.FloatTensor, input_ids: torch.LongTensor, labels: torch.LongTensor, attention_masks: torch.LongTensor, offset: torch.LongTensor, masks_list: List[torch.FloatTensor], label_list: List[torch.Tensor], resize_list: List[tuple], inference: bool = False, **kwargs, ): image_embeddings = self.get_visual_embs(images) batch_size = image_embeddings.shape[0] #get high res feat img_batch = self.model._transforms.forward_batch(read_images(kwargs["image_paths"])) img_batch = img_batch.to(device=self.device, dtype=images.dtype) assert ( len(img_batch.shape) == 4 and img_batch.shape[1] == 3 ), f"img_batch must be of size Bx3xHxW, got {img_batch.shape}" backbone_out = self.model.visual_model.forward_image(img_batch) _, vision_feats, _, _ = self.model.visual_model._prepare_backbone_features(backbone_out) if self.model.visual_model.directly_add_no_mem_embed: vision_feats[-1] = vision_feats[-1] + self.model.visual_model.no_mem_embed feats = [ feat.permute(1, 2, 0).view(batch_size, -1, *feat_size) for feat, feat_size in zip(vision_feats[::-1], self._bb_feat_sizes[::-1]) ][::-1] self._features = {"image_embed": feats[-1], "high_res_feats": feats[:-1]} assert batch_size == len(offset) - 1 seg_token_mask = input_ids[:, 1:] == self.seg_token_idx seg_token_mask = torch.cat( [ seg_token_mask, torch.zeros((seg_token_mask.shape[0], 1)).bool().cuda(), ], dim=1, ) # hack for IMAGE_TOKEN_INDEX (we suppose that there is only one image, and it is in the front) seg_token_mask = torch.cat( [torch.zeros((seg_token_mask.shape[0], 255)).bool().cuda(), seg_token_mask], dim=1, ) if inference: n_batch = 1 length = input_ids.shape[0] assert images_clip.shape[0] == 1 images_clip_extend = images_clip.expand(length, -1, -1, -1).contiguous() output_hidden_states = [] for i in range(n_batch): start_i, end_i = i * length, min((i + 1) * length, input_ids.shape[0]) output_i = super().forward( images=images_clip_extend[: end_i - start_i], attention_mask=attention_masks[start_i:end_i], input_ids=input_ids[start_i:end_i], output_hidden_states=True, ) output_hidden_states.append(output_i.hidden_states) torch.cuda.empty_cache() output_hidden_states_list = [] output_hidden_states_level = torch.cat(output_hidden_states, dim=0) output_hidden_states_list.append(output_hidden_states_level) output_hidden_states = output_hidden_states_list output = None else: images_clip_list = [] for i in range(len(offset) - 1): start_i, end_i = offset[i], offset[i + 1] images_clip_i = ( images_clip[i] .unsqueeze(0) .expand(end_i - start_i, -1, -1, -1) .contiguous() ) images_clip_list.append(images_clip_i) images_clip = torch.cat(images_clip_list, dim=0) output = super().forward( images=images_clip, attention_mask=attention_masks, input_ids=input_ids, labels=labels, output_hidden_states=True, ) output_hidden_states = output.hidden_states hidden_states = [] assert len(self.model.text_hidden_fcs) == 1 hidden_states.append(self.model.text_hidden_fcs[0](output_hidden_states[-1])) last_hidden_state = torch.stack(hidden_states, dim=-1).sum(dim=-1) pred_embeddings = last_hidden_state[seg_token_mask] seg_token_counts = seg_token_mask.int().sum(-1) # [bs, ] seg_token_offset = seg_token_counts.cumsum(-1) seg_token_offset = torch.cat( [torch.zeros(1).long().cuda(), seg_token_offset], dim=0 ) seg_token_offset = seg_token_offset[offset] pred_embeddings_ = [] for i in range(len(seg_token_offset) - 1): start_i, end_i = seg_token_offset[i], seg_token_offset[i + 1] pred_embeddings_.append(pred_embeddings[start_i:end_i]) pred_embeddings = pred_embeddings_ multimask_output = False pred_masks = [] for i in range(len(pred_embeddings)): ( sparse_embeddings, dense_embeddings, ) = self.model.visual_model.sam_prompt_encoder( points=None, boxes=None, masks=None, text_embeds=pred_embeddings[i].unsqueeze(1), ) sparse_embeddings = sparse_embeddings.to(pred_embeddings[i].dtype) high_res_features = [ feat_level[i].unsqueeze(0) for feat_level in self._features["high_res_feats"] ] low_res_masks, iou_predictions, _, _ = self.model.visual_model.sam_mask_decoder( image_embeddings=image_embeddings[i].unsqueeze(0), image_pe=self.model.visual_model.sam_prompt_encoder.get_dense_pe(), sparse_prompt_embeddings=sparse_embeddings, dense_prompt_embeddings=dense_embeddings, multimask_output=multimask_output, high_res_features=high_res_features, repeat_image=True ) pred_mask = self.model.visual_model.postprocess_masks( low_res_masks, #input_size=resize_list[i], #original_size=label_list[i].shape, orig_hw = label_list[i].shape, #sam2 ) pred_masks.append(pred_mask[:, 0]) model_output = output gt_masks = masks_list if inference: return { "pred_masks": pred_masks, "gt_masks": gt_masks, } output = model_output.logits ce_loss = model_output.loss ce_loss = ce_loss * self.ce_loss_weight mask_bce_loss = 0 mask_rao_loss = 0 mask_dice_loss = 0 num_masks = 0 for batch_idx in range(len(pred_masks)): gt_mask = gt_masks[batch_idx] if gt_mask.sum() < 2: print(kwargs['image_paths'][batch_idx] + "'s gtmask is none") continue pred_mask = pred_masks[batch_idx] assert ( gt_mask.shape[0] == pred_mask.shape[0] ), "gt_mask.shape: {}, pred_mask.shape: {}".format( gt_mask.shape, pred_mask.shape ) mask_bce_loss += ( sigmoid_ce_loss(pred_mask, gt_mask, num_masks=gt_mask.shape[0]) * gt_mask.shape[0] ) t = rao_loss(pred_mask, gt_mask, num_masks=gt_mask.shape[0]) if t == 0: print(kwargs['image_paths'][batch_idx]) mask_rao_loss += ( t*gt_mask.shape[0] ) mask_dice_loss += ( dice_loss(pred_mask, gt_mask, num_masks=gt_mask.shape[0]) * gt_mask.shape[0] ) num_masks += gt_mask.shape[0] mask_bce_loss = self.bce_loss_weight * mask_bce_loss / (num_masks + 1e-8) mask_rao_loss = self.rao_loss_weight * mask_rao_loss / (num_masks + 1e-8) mask_dice_loss = self.dice_loss_weight * mask_dice_loss / (num_masks + 1e-8) mask_loss = mask_bce_loss + mask_dice_loss + mask_rao_loss loss = ce_loss + mask_loss return { "loss": loss, "ce_loss": ce_loss, "mask_bce_loss": mask_bce_loss, "mask_rao_loss": mask_rao_loss, "mask_dice_loss": mask_dice_loss, "mask_loss": mask_loss, } def evaluate( self, image_paths, images_clip, images, input_ids, resize_list, original_size_list, max_new_tokens=32, tokenizer=None, ): with torch.no_grad(): outputs = self.generate( images=images_clip, input_ids=input_ids, max_new_tokens=max_new_tokens, num_beams=1, output_hidden_states=True, return_dict_in_generate=True, ) generation_hidden_states = outputs.hidden_states if torch.is_tensor(generation_hidden_states): output_hidden_states = generation_hidden_states elif not generation_hidden_states: raise RuntimeError("Generation returned no hidden states") elif torch.is_tensor(generation_hidden_states[0]): output_hidden_states = torch.cat(generation_hidden_states, dim=1) else: output_hidden_states = torch.cat( [step_hidden_states[-1] for step_hidden_states in generation_hidden_states], dim=1, ) output_ids = outputs.sequences seg_token_mask = output_ids[:, 1:] == self.seg_token_idx # hack for IMAGE_TOKEN_INDEX (we suppose that there is only one image, and it is in the front) seg_token_mask = torch.cat( [ torch.zeros((seg_token_mask.shape[0], 255)).bool().cuda(), seg_token_mask, ], dim=1, ) if output_hidden_states.shape[:2] != seg_token_mask.shape: raise RuntimeError( "Generation hidden states and segmentation token mask have " f"incompatible shapes: {tuple(output_hidden_states.shape[:2])} " f"vs {tuple(seg_token_mask.shape)}" ) hidden_states = [] assert len(self.model.text_hidden_fcs) == 1 hidden_states.append(self.model.text_hidden_fcs[0](output_hidden_states)) last_hidden_state = torch.stack(hidden_states, dim=-1).sum(dim=-1) pred_embeddings = last_hidden_state[seg_token_mask] seg_token_counts = seg_token_mask.int().sum(-1) # [bs, ] seg_token_offset = seg_token_counts.cumsum(-1) seg_token_offset = torch.cat( [torch.zeros(1).long().cuda(), seg_token_offset], dim=0 ) pred_embeddings_ = [] for i in range(len(seg_token_offset) - 1): start_i, end_i = seg_token_offset[i], seg_token_offset[i + 1] pred_embeddings_.append(pred_embeddings[start_i:end_i]) pred_embeddings = pred_embeddings_ image_embeddings = self.get_visual_embs(images) #get high res feat img_batch = self.model._transforms.forward_batch(read_images(image_paths)) img_batch = img_batch.to(device=self.device, dtype=images.dtype) batch_size = image_embeddings.shape[0] assert ( len(img_batch.shape) == 4 and img_batch.shape[1] == 3 ), f"img_batch must be of size Bx3xHxW, got {img_batch.shape}" backbone_out = self.model.visual_model.forward_image(img_batch) _, vision_feats, _, _ = self.model.visual_model._prepare_backbone_features(backbone_out) if self.model.visual_model.directly_add_no_mem_embed: vision_feats[-1] = vision_feats[-1] + self.model.visual_model.no_mem_embed feats = [ feat.permute(1, 2, 0).view(batch_size, -1, *feat_size) for feat, feat_size in zip(vision_feats[::-1], self._bb_feat_sizes[::-1]) ][::-1] self._features = {"image_embed": feats[-1], "high_res_feats": feats[:-1]} multimask_output = False pred_masks = [] for i in range(len(pred_embeddings)): ( sparse_embeddings, dense_embeddings, ) = self.model.visual_model.sam_prompt_encoder( points=None, boxes=None, masks=None, text_embeds=pred_embeddings[i].unsqueeze(1), ) high_res_features = [ feat_level[i].unsqueeze(0) for feat_level in self._features["high_res_feats"] ] sparse_embeddings = sparse_embeddings.to(pred_embeddings[i].dtype) low_res_masks, iou_predictions, _, _ = self.model.visual_model.sam_mask_decoder( image_embeddings=image_embeddings[i].unsqueeze(0), image_pe=self.model.visual_model.sam_prompt_encoder.get_dense_pe(), sparse_prompt_embeddings=sparse_embeddings, dense_prompt_embeddings=dense_embeddings, high_res_features=high_res_features, multimask_output=multimask_output, repeat_image=True ) pred_mask = self.model.visual_model.postprocess_masks( low_res_masks, original_size_list[i], ) pred_masks.append(pred_mask[:, 0]) return output_ids, pred_masks