Spaces:
Sleeping
Sleeping
Download model/LISA_sam2.py from sosppxo/RSReasonNet: direct link, hf CLI and curl.
- Browser
- Download file 29.1 kB
-
https://huggingface.co/spaces/sosppxo/RSReasonNet/resolve/main/model/LISA_sam2.py
- Command line
-
hf download hf://spaces/sosppxo/RSReasonNet/model/LISA_sam2.py
-
curl -L -o LISA_sam2.py https://huggingface.co/spaces/sosppxo/RSReasonNet/resolve/main/model/LISA_sam2.py
29.1 kB
| from typing import List | |
| import os | |
| import sys | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from torch.linalg import norm, eigh | |
| from transformers import BitsAndBytesConfig, CLIPVisionModel | |
| from utils.utils import (DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN, | |
| DEFAULT_IMAGE_PATCH_TOKEN) | |
| from .llava.model.language_model.llava_llama import (LlavaLlamaForCausalLM, | |
| LlavaLlamaModel) | |
| from .segment_anything import build_sam_vit_h | |
| PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| PROJECT_PARENT = os.path.dirname(PROJECT_ROOT) | |
| if PROJECT_PARENT not in sys.path: | |
| sys.path.insert(0, PROJECT_PARENT) | |
| from sam2.build_sam import build_sam2 | |
| from sam2.utils.transforms import SAM2Transforms | |
| from PIL import Image | |
| import numpy as np | |
| def read_images(image_paths, target_size=(1024, 1024)): | |
| images = [] | |
| for image_path in image_paths: | |
| # 用 Pillow 库打开图片 | |
| image = Image.open(image_path) | |
| # 将图片转换为 RGB 模式(如果图片不是 RGB) | |
| image = image.convert('RGB') | |
| # 统一图片大小到 target_size (1024x1024) | |
| image = image.resize(target_size) | |
| # 将图片转换为 NumPy 数组 | |
| image_array = np.array(image) | |
| images.append(image_array) | |
| return images | |
| def Gaussian_2d(mask): | |
| batch_size, h, w = mask.shape | |
| y_coords, x_coords = torch.meshgrid(torch.arange(h), torch.arange(w), indexing="ij") | |
| device = mask.device | |
| x_coords = x_coords.to(device) | |
| y_coords = y_coords.to(device) | |
| y_coords = y_coords.unsqueeze(0).expand(batch_size, -1, -1) | |
| x_coords = x_coords.unsqueeze(0).expand(batch_size, -1, -1) | |
| mass = mask.sum(dim=(1, 2), keepdim=True) + 1e-6 | |
| mu_x = (x_coords * mask).sum(dim=(1, 2)) / mass.squeeze(-1).squeeze(-1) | |
| mu_y = (y_coords * mask).sum(dim=(1, 2)) / mass.squeeze(-1).squeeze(-1) | |
| denom = mass.squeeze(-1).squeeze(-1) -1 + 1e-6 | |
| variance_x = ((((x_coords - mu_x))**2) * mask).sum(dim=(1, 2)) / denom | |
| variance_y = ((((y_coords - mu_y))**2) * mask).sum(dim=(1, 2)) / denom | |
| covariance_xy = (((x_coords - mu_x) * (y_coords - mu_y)) * mask).sum(dim=(1, 2)) / denom | |
| mean = torch.stack([mu_x, mu_y], dim=1) | |
| cov = torch.stack([ | |
| torch.stack([variance_x, covariance_xy], dim=-1), | |
| torch.stack([covariance_xy, variance_y], dim=-1) | |
| ], dim=-2) | |
| if not torch.isfinite(mean).all() or not torch.isfinite(cov).all(): | |
| raise ValueError("Gaussian_2d outputs contain NaN or Inf") | |
| return mean, cov | |
| def loss_sqrt_2(wass_dist): | |
| return 1 - 1 / (2 + torch.sqrt(wass_dist + 1e-4)) | |
| def torch_sqrtm(matrix, eps=1e-4): | |
| matrix = (matrix + matrix.T) / 2 | |
| trace = matrix[0][0] + matrix[1][1] | |
| determinant = matrix[0][0] * matrix[1][1] - matrix[0][1] * matrix[1][0] | |
| if determinant < -eps: | |
| raise ValueError("Negative determinant") | |
| s = torch.sqrt(determinant + eps) | |
| t = torch.sqrt(trace + 2 * s) | |
| if t < eps: | |
| raise ValueError("Zero t in sqrtm") | |
| sqrt_matrix = (matrix + s * torch.eye(2, device=matrix.device)) / t | |
| return sqrt_matrix | |
| def kl_divergence(Mu1, Cov1, Mu2, Cov2): | |
| kl_loss = 0 | |
| for i in range(Mu1.shape[0]): | |
| mu1 = Mu1[i] | |
| mu2 = Mu2[i] | |
| cov1 = Cov1[i] | |
| cov2 = Cov2[i] | |
| d = mu1.shape[-1] | |
| try: | |
| cov2_inv = torch.linalg.inv(cov2) | |
| trace_term = torch.trace(cov2_inv @ cov1) | |
| mean_diff = mu2 - mu1 | |
| mean_term = mean_diff @ cov2_inv @ mean_diff | |
| logdet_term = torch.logdet(cov2 + 1e-6 * torch.eye(d, device=cov2.device)) - \ | |
| torch.logdet(cov1 + 1e-6 * torch.eye(d, device=cov1.device)) | |
| kl = 0.5 * (logdet_term - d + trace_term + mean_term) | |
| except Exception: | |
| return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype) | |
| if not torch.isfinite(kl): | |
| return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype) | |
| kl_loss += loss_sqrt_2(kl) | |
| if not torch.isfinite(kl_loss): | |
| return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype) | |
| return kl_loss | |
| def bd_distance(Mu1, Cov1, Mu2, Cov2): | |
| bd_loss = 0 | |
| for i in range(Mu1.shape[0]): | |
| mu1 = Mu1[i] | |
| mu2 = Mu2[i] | |
| cov1 = Cov1[i] | |
| cov2 = Cov2[i] | |
| try: | |
| cov_avg = 0.5 * (cov1 + cov2) | |
| cov_avg_inv = torch.linalg.inv(cov_avg + 1e-6 * torch.eye(2, device=cov_avg.device)) | |
| mean_diff = (mu1 - mu2).unsqueeze(0) | |
| mean_term = 0.125 * (mean_diff @ cov_avg_inv @ mean_diff.T).squeeze() | |
| det_cov1 = torch.linalg.det(cov1 + 1e-6 * torch.eye(2, device=cov1.device)) | |
| det_cov2 = torch.linalg.det(cov2 + 1e-6 * torch.eye(2, device=cov2.device)) | |
| det_cov_avg = torch.linalg.det(cov_avg + 1e-6 * torch.eye(2, device=cov_avg.device)) | |
| det_term = 0.5 * torch.log(det_cov_avg / torch.sqrt(det_cov1 * det_cov2)) | |
| bd = mean_term + det_term | |
| except Exception: | |
| return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype) | |
| if not torch.isfinite(bd): | |
| return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype) | |
| bd_loss += loss_sqrt_2(bd) | |
| if not torch.isfinite(bd_loss): | |
| return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype) | |
| return bd_loss | |
| def wasserstein_distance(Mean1, Cov1, Mean2, Cov2): | |
| assert Mean1.shape[0] == Cov1.shape[0] | |
| wasserstein_loss = 0 | |
| for i in range(Mean1.shape[0]): | |
| mean1 = Mean1[i] | |
| mean2 = Mean2[i] | |
| cov1 = Cov1[i] | |
| cov2 = Cov2[i] | |
| if not all(torch.isfinite(x).all() for x in [mean1, mean2, cov1, cov2]): | |
| return torch.tensor(0.0, device=Mean1.device, dtype=Mean1.dtype) | |
| mean_diff = norm(mean1 - mean2)**2 | |
| try: | |
| sqrt_cov1 = torch_sqrtm(cov1) | |
| cov_prod = sqrt_cov1 @ cov2 @ sqrt_cov1 | |
| sqrt_cov_prod = torch_sqrtm(cov_prod) | |
| except Exception: | |
| return torch.tensor(0.0, device=Mean1.device, dtype=Mean1.dtype) | |
| if not all(torch.isfinite(x).all() for x in [sqrt_cov1, sqrt_cov_prod]): | |
| return torch.tensor(0.0, device=Mean1.device, dtype=Mean1.dtype) | |
| cov_term = torch.trace(cov1 + cov2 - 2 * sqrt_cov_prod) | |
| wasserstein_dist = torch.clamp(mean_diff + cov_term, min=0.0) | |
| if not torch.isfinite(wasserstein_dist): | |
| return torch.tensor(0.0, device=Mean1.device, dtype=Mean1.dtype) | |
| wasserstein_loss += loss_sqrt_2(wasserstein_dist) | |
| if not torch.isfinite(wasserstein_loss): | |
| return torch.tensor(0.0, device=Mean1.device, dtype=Mean1.dtype) | |
| return wasserstein_loss | |
| def contains_zero(tensor_list): | |
| return any((t == 0).any() for t in tensor_list) | |
| def rao_loss(pred_mask, gt_mask, num_masks): | |
| pred_mask = pred_mask.sigmoid() | |
| try: | |
| mean_pred, cov_pred = Gaussian_2d(pred_mask) | |
| mean_gt, cov_gt = Gaussian_2d(gt_mask) | |
| except Exception: | |
| return torch.tensor(0.0, device=pred_mask.device, dtype=pred_mask.dtype, requires_grad=True) | |
| if not all(torch.isfinite(x).all() for x in [mean_pred, cov_pred, mean_gt, cov_gt]): | |
| return torch.tensor(0.0, device=pred_mask.device, dtype=pred_mask.dtype, requires_grad=True) | |
| if contains_zero([mean_pred, cov_pred, mean_gt, cov_gt]): | |
| return torch.tensor(0.0, device=pred_mask.device, dtype=pred_mask.dtype, requires_grad=True) | |
| try: | |
| loss = wasserstein_distance(mean_pred, cov_pred, mean_gt, cov_gt) | |
| #loss = kl_divergence(mean_pred, cov_pred, mean_gt, cov_gt) | |
| #loss = bd_distance(mean_pred, cov_pred, mean_gt, cov_gt) | |
| except Exception: | |
| return torch.tensor(0.0, device=pred_mask.device, dtype=pred_mask.dtype, requires_grad=True) | |
| if not torch.isfinite(loss): | |
| return torch.tensor(0.0, device=pred_mask.device, dtype=pred_mask.dtype, requires_grad=True) | |
| return loss / (num_masks + 1e-8) | |
| def dice_loss( | |
| inputs: torch.Tensor, | |
| targets: torch.Tensor, | |
| num_masks: float, | |
| scale=1000, # 100000.0, | |
| eps=1e-6, | |
| ): | |
| """ | |
| Compute the DICE loss, similar to generalized IOU for masks | |
| Args: | |
| inputs: A float tensor of arbitrary shape. | |
| The predictions for each example. | |
| targets: A float tensor with the same shape as inputs. Stores the binary | |
| classification label for each element in inputs | |
| (0 for the negative class and 1 for the positive class). | |
| """ | |
| inputs = inputs.sigmoid() | |
| inputs = inputs.flatten(1, 2) | |
| targets = targets.flatten(1, 2) | |
| numerator = 2 * (inputs / scale * targets).sum(-1) | |
| denominator = (inputs / scale).sum(-1) + (targets / scale).sum(-1) | |
| loss = 1 - (numerator + eps) / (denominator + eps) | |
| loss = loss.sum() / (num_masks + 1e-8) | |
| return loss | |
| def sigmoid_ce_loss( | |
| inputs: torch.Tensor, | |
| targets: torch.Tensor, | |
| num_masks: float, | |
| ): | |
| """ | |
| Args: | |
| inputs: A float tensor of arbitrary shape. | |
| The predictions for each example. | |
| targets: A float tensor with the same shape as inputs. Stores the binary | |
| classification label for each element in inputs | |
| (0 for the negative class and 1 for the positive class). | |
| Returns: | |
| Loss tensor | |
| """ | |
| loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction="none") | |
| loss = loss.flatten(1, 2).mean(1).sum() / (num_masks + 1e-8) | |
| return loss | |
| class LisaMetaModel: | |
| def __init__( | |
| self, | |
| config, | |
| **kwargs, | |
| ): | |
| super(LisaMetaModel, self).__init__(config) | |
| self.config = config | |
| self.config.sam2_config = kwargs.get( | |
| "sam2_config", | |
| getattr(self.config, "sam2_config", "configs/sam2.1/sam2.1_hiera_l.yaml"), | |
| ) | |
| self.sam2_config = self.config.sam2_config | |
| if not hasattr(self.config, "train_mask_decoder"): | |
| self.config.train_mask_decoder = kwargs["train_mask_decoder"] | |
| self.config.out_dim = kwargs["out_dim"] | |
| self.vision_pretrained = kwargs.get("vision_pretrained", None) | |
| else: | |
| self.vision_pretrained = kwargs.get("vision_pretrained", None) | |
| self.initialize_lisa_modules(self.config) | |
| def initialize_lisa_modules(self, config): | |
| # SAM | |
| #self.visual_model = build_sam_vit_h(self.vision_pretrained) | |
| print("----------building sam2-------------") | |
| # ZeroGPU packs the fully loaded model after startup. Build the SAM2 | |
| # architecture on CPU first; the fixed model snapshot supplies all | |
| # trained SAM2 parameters before the complete model is moved to CUDA. | |
| self.visual_model = build_sam2( | |
| self.sam2_config, | |
| self.vision_pretrained, | |
| device="cpu", | |
| ) | |
| for param in self.visual_model.parameters(): | |
| param.requires_grad = False | |
| if config.train_mask_decoder: | |
| self.visual_model.sam_mask_decoder.train() | |
| for param in self.visual_model.sam_mask_decoder.parameters(): | |
| param.requires_grad = True | |
| self._transforms = SAM2Transforms( | |
| resolution=self.visual_model.image_size, | |
| mask_threshold=self.visual_model.mask_threshold, | |
| max_hole_area=self.visual_model.max_hole_area, | |
| max_sprinkle_area=self.visual_model.max_sprinkle_area, | |
| ) | |
| # Projection layer | |
| in_dim = config.hidden_size | |
| out_dim = config.out_dim | |
| text_fc = [ | |
| nn.Linear(in_dim, in_dim), | |
| nn.ReLU(inplace=True), | |
| nn.Linear(in_dim, out_dim), | |
| nn.Dropout(0.0), | |
| ] | |
| self.text_hidden_fcs = nn.ModuleList([nn.Sequential(*text_fc)]) | |
| self.text_hidden_fcs.train() | |
| for param in self.text_hidden_fcs.parameters(): | |
| param.requires_grad = True | |
| class LisaModel(LisaMetaModel, LlavaLlamaModel): | |
| def __init__( | |
| self, | |
| config, | |
| **kwargs, | |
| ): | |
| super(LisaModel, self).__init__(config, **kwargs) | |
| self.config.use_cache = False | |
| self.config.vision_tower = self.config.mm_vision_tower | |
| self.config.mm_vision_select_feature = "patch" | |
| self.config.image_aspect_ratio = "square" | |
| self.config.image_grid_pinpoints = None | |
| self.config.tune_mm_mlp_adapter = False | |
| self.config.freeze_mm_mlp_adapter = True | |
| self.config.pretrain_mm_mlp_adapter = None | |
| self.config.mm_use_im_patch_token = False | |
| class LISAForCausalLM(LlavaLlamaForCausalLM): | |
| def __init__( | |
| self, | |
| config, | |
| **kwargs, | |
| ): | |
| if not hasattr(config, "train_mask_decoder"): | |
| config.mm_use_im_start_end = kwargs.pop("use_mm_start_end", True) | |
| config.mm_vision_tower = kwargs.get( | |
| "vision_tower", "openai/clip-vit-large-patch14" | |
| ) | |
| self.ce_loss_weight = kwargs.pop("ce_loss_weight", None) | |
| self.dice_loss_weight = kwargs.pop("dice_loss_weight", None) | |
| self.rao_loss_weight = kwargs.pop("rao_loss_weight", None) | |
| self.bce_loss_weight = kwargs.pop("bce_loss_weight", None) | |
| else: | |
| config.mm_vision_tower = config.vision_tower | |
| self.seg_token_idx = kwargs.pop("seg_token_idx") | |
| super().__init__(config) | |
| self.model = LisaModel(config, **kwargs) | |
| self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) | |
| # Spatial dim for backbone feature maps | |
| self._bb_feat_sizes = [ | |
| (256, 256), | |
| (128, 128), | |
| (64, 64), | |
| ] | |
| # Initialize weights and apply final processing | |
| self.post_init() | |
| ''' | |
| self._transforms = SAM2Transforms( | |
| resolution=self.model.visual_model.image_size, | |
| mask_threshold=self.model.visual_model.mask_threshold, | |
| max_hole_area=self.model.visual_model.max_hole_area, | |
| max_sprinkle_area=self.model.visual_model.max_sprinkle_area, | |
| ) | |
| ''' | |
| def get_visual_embs(self, pixel_values: torch.FloatTensor): | |
| with torch.no_grad(): | |
| image_embeddings_list = [] | |
| for i in range(pixel_values.shape[0]): | |
| torch.cuda.empty_cache() | |
| image_embeddings = self.model.visual_model.image_encoder( | |
| pixel_values[i].unsqueeze(0) | |
| ) | |
| #print("-----------------------") | |
| #print(image_embeddings['vision_features'].shape) | |
| image_embeddings_list.append(image_embeddings['vision_features']) | |
| torch.cuda.empty_cache() | |
| image_embeddings = torch.cat(image_embeddings_list, 0) | |
| return image_embeddings | |
| def forward(self, **kwargs): | |
| if "past_key_values" in kwargs: | |
| return super().forward(**kwargs) | |
| return self.model_forward(**kwargs) | |
| def model_forward( | |
| self, | |
| #image_paths: list, | |
| images: torch.FloatTensor, | |
| images_clip: torch.FloatTensor, | |
| input_ids: torch.LongTensor, | |
| labels: torch.LongTensor, | |
| attention_masks: torch.LongTensor, | |
| offset: torch.LongTensor, | |
| masks_list: List[torch.FloatTensor], | |
| label_list: List[torch.Tensor], | |
| resize_list: List[tuple], | |
| inference: bool = False, | |
| **kwargs, | |
| ): | |
| image_embeddings = self.get_visual_embs(images) | |
| batch_size = image_embeddings.shape[0] | |
| #get high res feat | |
| img_batch = self.model._transforms.forward_batch(read_images(kwargs["image_paths"])) | |
| img_batch = img_batch.to(device=self.device, dtype=images.dtype) | |
| assert ( | |
| len(img_batch.shape) == 4 and img_batch.shape[1] == 3 | |
| ), f"img_batch must be of size Bx3xHxW, got {img_batch.shape}" | |
| backbone_out = self.model.visual_model.forward_image(img_batch) | |
| _, vision_feats, _, _ = self.model.visual_model._prepare_backbone_features(backbone_out) | |
| if self.model.visual_model.directly_add_no_mem_embed: | |
| vision_feats[-1] = vision_feats[-1] + self.model.visual_model.no_mem_embed | |
| feats = [ | |
| feat.permute(1, 2, 0).view(batch_size, -1, *feat_size) | |
| for feat, feat_size in zip(vision_feats[::-1], self._bb_feat_sizes[::-1]) | |
| ][::-1] | |
| self._features = {"image_embed": feats[-1], "high_res_feats": feats[:-1]} | |
| assert batch_size == len(offset) - 1 | |
| seg_token_mask = input_ids[:, 1:] == self.seg_token_idx | |
| seg_token_mask = torch.cat( | |
| [ | |
| seg_token_mask, | |
| torch.zeros((seg_token_mask.shape[0], 1)).bool().cuda(), | |
| ], | |
| dim=1, | |
| ) | |
| # hack for IMAGE_TOKEN_INDEX (we suppose that there is only one image, and it is in the front) | |
| seg_token_mask = torch.cat( | |
| [torch.zeros((seg_token_mask.shape[0], 255)).bool().cuda(), seg_token_mask], | |
| dim=1, | |
| ) | |
| if inference: | |
| n_batch = 1 | |
| length = input_ids.shape[0] | |
| assert images_clip.shape[0] == 1 | |
| images_clip_extend = images_clip.expand(length, -1, -1, -1).contiguous() | |
| output_hidden_states = [] | |
| for i in range(n_batch): | |
| start_i, end_i = i * length, min((i + 1) * length, input_ids.shape[0]) | |
| output_i = super().forward( | |
| images=images_clip_extend[: end_i - start_i], | |
| attention_mask=attention_masks[start_i:end_i], | |
| input_ids=input_ids[start_i:end_i], | |
| output_hidden_states=True, | |
| ) | |
| output_hidden_states.append(output_i.hidden_states) | |
| torch.cuda.empty_cache() | |
| output_hidden_states_list = [] | |
| output_hidden_states_level = torch.cat(output_hidden_states, dim=0) | |
| output_hidden_states_list.append(output_hidden_states_level) | |
| output_hidden_states = output_hidden_states_list | |
| output = None | |
| else: | |
| images_clip_list = [] | |
| for i in range(len(offset) - 1): | |
| start_i, end_i = offset[i], offset[i + 1] | |
| images_clip_i = ( | |
| images_clip[i] | |
| .unsqueeze(0) | |
| .expand(end_i - start_i, -1, -1, -1) | |
| .contiguous() | |
| ) | |
| images_clip_list.append(images_clip_i) | |
| images_clip = torch.cat(images_clip_list, dim=0) | |
| output = super().forward( | |
| images=images_clip, | |
| attention_mask=attention_masks, | |
| input_ids=input_ids, | |
| labels=labels, | |
| output_hidden_states=True, | |
| ) | |
| output_hidden_states = output.hidden_states | |
| hidden_states = [] | |
| assert len(self.model.text_hidden_fcs) == 1 | |
| hidden_states.append(self.model.text_hidden_fcs[0](output_hidden_states[-1])) | |
| last_hidden_state = torch.stack(hidden_states, dim=-1).sum(dim=-1) | |
| pred_embeddings = last_hidden_state[seg_token_mask] | |
| seg_token_counts = seg_token_mask.int().sum(-1) # [bs, ] | |
| seg_token_offset = seg_token_counts.cumsum(-1) | |
| seg_token_offset = torch.cat( | |
| [torch.zeros(1).long().cuda(), seg_token_offset], dim=0 | |
| ) | |
| seg_token_offset = seg_token_offset[offset] | |
| pred_embeddings_ = [] | |
| for i in range(len(seg_token_offset) - 1): | |
| start_i, end_i = seg_token_offset[i], seg_token_offset[i + 1] | |
| pred_embeddings_.append(pred_embeddings[start_i:end_i]) | |
| pred_embeddings = pred_embeddings_ | |
| multimask_output = False | |
| pred_masks = [] | |
| for i in range(len(pred_embeddings)): | |
| ( | |
| sparse_embeddings, | |
| dense_embeddings, | |
| ) = self.model.visual_model.sam_prompt_encoder( | |
| points=None, | |
| boxes=None, | |
| masks=None, | |
| text_embeds=pred_embeddings[i].unsqueeze(1), | |
| ) | |
| sparse_embeddings = sparse_embeddings.to(pred_embeddings[i].dtype) | |
| high_res_features = [ | |
| feat_level[i].unsqueeze(0) | |
| for feat_level in self._features["high_res_feats"] | |
| ] | |
| low_res_masks, iou_predictions, _, _ = self.model.visual_model.sam_mask_decoder( | |
| image_embeddings=image_embeddings[i].unsqueeze(0), | |
| image_pe=self.model.visual_model.sam_prompt_encoder.get_dense_pe(), | |
| sparse_prompt_embeddings=sparse_embeddings, | |
| dense_prompt_embeddings=dense_embeddings, | |
| multimask_output=multimask_output, | |
| high_res_features=high_res_features, | |
| repeat_image=True | |
| ) | |
| pred_mask = self.model.visual_model.postprocess_masks( | |
| low_res_masks, | |
| #input_size=resize_list[i], | |
| #original_size=label_list[i].shape, | |
| orig_hw = label_list[i].shape, #sam2 | |
| ) | |
| pred_masks.append(pred_mask[:, 0]) | |
| model_output = output | |
| gt_masks = masks_list | |
| if inference: | |
| return { | |
| "pred_masks": pred_masks, | |
| "gt_masks": gt_masks, | |
| } | |
| output = model_output.logits | |
| ce_loss = model_output.loss | |
| ce_loss = ce_loss * self.ce_loss_weight | |
| mask_bce_loss = 0 | |
| mask_rao_loss = 0 | |
| mask_dice_loss = 0 | |
| num_masks = 0 | |
| for batch_idx in range(len(pred_masks)): | |
| gt_mask = gt_masks[batch_idx] | |
| if gt_mask.sum() < 2: | |
| print(kwargs['image_paths'][batch_idx] + "'s gtmask is none") | |
| continue | |
| pred_mask = pred_masks[batch_idx] | |
| assert ( | |
| gt_mask.shape[0] == pred_mask.shape[0] | |
| ), "gt_mask.shape: {}, pred_mask.shape: {}".format( | |
| gt_mask.shape, pred_mask.shape | |
| ) | |
| mask_bce_loss += ( | |
| sigmoid_ce_loss(pred_mask, gt_mask, num_masks=gt_mask.shape[0]) | |
| * gt_mask.shape[0] | |
| ) | |
| t = rao_loss(pred_mask, gt_mask, num_masks=gt_mask.shape[0]) | |
| if t == 0: | |
| print(kwargs['image_paths'][batch_idx]) | |
| mask_rao_loss += ( | |
| t*gt_mask.shape[0] | |
| ) | |
| mask_dice_loss += ( | |
| dice_loss(pred_mask, gt_mask, num_masks=gt_mask.shape[0]) | |
| * gt_mask.shape[0] | |
| ) | |
| num_masks += gt_mask.shape[0] | |
| mask_bce_loss = self.bce_loss_weight * mask_bce_loss / (num_masks + 1e-8) | |
| mask_rao_loss = self.rao_loss_weight * mask_rao_loss / (num_masks + 1e-8) | |
| mask_dice_loss = self.dice_loss_weight * mask_dice_loss / (num_masks + 1e-8) | |
| mask_loss = mask_bce_loss + mask_dice_loss + mask_rao_loss | |
| loss = ce_loss + mask_loss | |
| return { | |
| "loss": loss, | |
| "ce_loss": ce_loss, | |
| "mask_bce_loss": mask_bce_loss, | |
| "mask_rao_loss": mask_rao_loss, | |
| "mask_dice_loss": mask_dice_loss, | |
| "mask_loss": mask_loss, | |
| } | |
| def evaluate( | |
| self, | |
| image_paths, | |
| images_clip, | |
| images, | |
| input_ids, | |
| resize_list, | |
| original_size_list, | |
| max_new_tokens=32, | |
| tokenizer=None, | |
| ): | |
| with torch.no_grad(): | |
| outputs = self.generate( | |
| images=images_clip, | |
| input_ids=input_ids, | |
| max_new_tokens=max_new_tokens, | |
| num_beams=1, | |
| output_hidden_states=True, | |
| return_dict_in_generate=True, | |
| ) | |
| generation_hidden_states = outputs.hidden_states | |
| if torch.is_tensor(generation_hidden_states): | |
| output_hidden_states = generation_hidden_states | |
| elif not generation_hidden_states: | |
| raise RuntimeError("Generation returned no hidden states") | |
| elif torch.is_tensor(generation_hidden_states[0]): | |
| output_hidden_states = torch.cat(generation_hidden_states, dim=1) | |
| else: | |
| output_hidden_states = torch.cat( | |
| [step_hidden_states[-1] for step_hidden_states in generation_hidden_states], | |
| dim=1, | |
| ) | |
| output_ids = outputs.sequences | |
| seg_token_mask = output_ids[:, 1:] == self.seg_token_idx | |
| # hack for IMAGE_TOKEN_INDEX (we suppose that there is only one image, and it is in the front) | |
| seg_token_mask = torch.cat( | |
| [ | |
| torch.zeros((seg_token_mask.shape[0], 255)).bool().cuda(), | |
| seg_token_mask, | |
| ], | |
| dim=1, | |
| ) | |
| if output_hidden_states.shape[:2] != seg_token_mask.shape: | |
| raise RuntimeError( | |
| "Generation hidden states and segmentation token mask have " | |
| f"incompatible shapes: {tuple(output_hidden_states.shape[:2])} " | |
| f"vs {tuple(seg_token_mask.shape)}" | |
| ) | |
| hidden_states = [] | |
| assert len(self.model.text_hidden_fcs) == 1 | |
| hidden_states.append(self.model.text_hidden_fcs[0](output_hidden_states)) | |
| last_hidden_state = torch.stack(hidden_states, dim=-1).sum(dim=-1) | |
| pred_embeddings = last_hidden_state[seg_token_mask] | |
| seg_token_counts = seg_token_mask.int().sum(-1) # [bs, ] | |
| seg_token_offset = seg_token_counts.cumsum(-1) | |
| seg_token_offset = torch.cat( | |
| [torch.zeros(1).long().cuda(), seg_token_offset], dim=0 | |
| ) | |
| pred_embeddings_ = [] | |
| for i in range(len(seg_token_offset) - 1): | |
| start_i, end_i = seg_token_offset[i], seg_token_offset[i + 1] | |
| pred_embeddings_.append(pred_embeddings[start_i:end_i]) | |
| pred_embeddings = pred_embeddings_ | |
| image_embeddings = self.get_visual_embs(images) | |
| #get high res feat | |
| img_batch = self.model._transforms.forward_batch(read_images(image_paths)) | |
| img_batch = img_batch.to(device=self.device, dtype=images.dtype) | |
| batch_size = image_embeddings.shape[0] | |
| assert ( | |
| len(img_batch.shape) == 4 and img_batch.shape[1] == 3 | |
| ), f"img_batch must be of size Bx3xHxW, got {img_batch.shape}" | |
| backbone_out = self.model.visual_model.forward_image(img_batch) | |
| _, vision_feats, _, _ = self.model.visual_model._prepare_backbone_features(backbone_out) | |
| if self.model.visual_model.directly_add_no_mem_embed: | |
| vision_feats[-1] = vision_feats[-1] + self.model.visual_model.no_mem_embed | |
| feats = [ | |
| feat.permute(1, 2, 0).view(batch_size, -1, *feat_size) | |
| for feat, feat_size in zip(vision_feats[::-1], self._bb_feat_sizes[::-1]) | |
| ][::-1] | |
| self._features = {"image_embed": feats[-1], "high_res_feats": feats[:-1]} | |
| multimask_output = False | |
| pred_masks = [] | |
| for i in range(len(pred_embeddings)): | |
| ( | |
| sparse_embeddings, | |
| dense_embeddings, | |
| ) = self.model.visual_model.sam_prompt_encoder( | |
| points=None, | |
| boxes=None, | |
| masks=None, | |
| text_embeds=pred_embeddings[i].unsqueeze(1), | |
| ) | |
| high_res_features = [ | |
| feat_level[i].unsqueeze(0) | |
| for feat_level in self._features["high_res_feats"] | |
| ] | |
| sparse_embeddings = sparse_embeddings.to(pred_embeddings[i].dtype) | |
| low_res_masks, iou_predictions, _, _ = self.model.visual_model.sam_mask_decoder( | |
| image_embeddings=image_embeddings[i].unsqueeze(0), | |
| image_pe=self.model.visual_model.sam_prompt_encoder.get_dense_pe(), | |
| sparse_prompt_embeddings=sparse_embeddings, | |
| dense_prompt_embeddings=dense_embeddings, | |
| high_res_features=high_res_features, | |
| multimask_output=multimask_output, | |
| repeat_image=True | |
| ) | |
| pred_mask = self.model.visual_model.postprocess_masks( | |
| low_res_masks, | |
| original_size_list[i], | |
| ) | |
| pred_masks.append(pred_mask[:, 0]) | |
| return output_ids, pred_masks | |