RSReasonNet / model /LISA_sam2.py
sosppxo's picture
Use SAM2 mask postprocessing API
7872cb7 verified
Raw History Blame Contribute Delete
29.1 kB
from typing import List
import os
import sys
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.linalg import norm, eigh
from transformers import BitsAndBytesConfig, CLIPVisionModel
from utils.utils import (DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN,
DEFAULT_IMAGE_PATCH_TOKEN)
from .llava.model.language_model.llava_llama import (LlavaLlamaForCausalLM,
LlavaLlamaModel)
from .segment_anything import build_sam_vit_h
PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
PROJECT_PARENT = os.path.dirname(PROJECT_ROOT)
if PROJECT_PARENT not in sys.path:
sys.path.insert(0, PROJECT_PARENT)
from sam2.build_sam import build_sam2
from sam2.utils.transforms import SAM2Transforms
from PIL import Image
import numpy as np
def read_images(image_paths, target_size=(1024, 1024)):
images = []
for image_path in image_paths:
# 用 Pillow 库打开图片
image = Image.open(image_path)
# 将图片转换为 RGB 模式(如果图片不是 RGB)
image = image.convert('RGB')
# 统一图片大小到 target_size (1024x1024)
image = image.resize(target_size)
# 将图片转换为 NumPy 数组
image_array = np.array(image)
images.append(image_array)
return images
def Gaussian_2d(mask):
batch_size, h, w = mask.shape
y_coords, x_coords = torch.meshgrid(torch.arange(h), torch.arange(w), indexing="ij")
device = mask.device
x_coords = x_coords.to(device)
y_coords = y_coords.to(device)
y_coords = y_coords.unsqueeze(0).expand(batch_size, -1, -1)
x_coords = x_coords.unsqueeze(0).expand(batch_size, -1, -1)
mass = mask.sum(dim=(1, 2), keepdim=True) + 1e-6
mu_x = (x_coords * mask).sum(dim=(1, 2)) / mass.squeeze(-1).squeeze(-1)
mu_y = (y_coords * mask).sum(dim=(1, 2)) / mass.squeeze(-1).squeeze(-1)
denom = mass.squeeze(-1).squeeze(-1) -1 + 1e-6
variance_x = ((((x_coords - mu_x))**2) * mask).sum(dim=(1, 2)) / denom
variance_y = ((((y_coords - mu_y))**2) * mask).sum(dim=(1, 2)) / denom
covariance_xy = (((x_coords - mu_x) * (y_coords - mu_y)) * mask).sum(dim=(1, 2)) / denom
mean = torch.stack([mu_x, mu_y], dim=1)
cov = torch.stack([
torch.stack([variance_x, covariance_xy], dim=-1),
torch.stack([covariance_xy, variance_y], dim=-1)
], dim=-2)
if not torch.isfinite(mean).all() or not torch.isfinite(cov).all():
raise ValueError("Gaussian_2d outputs contain NaN or Inf")
return mean, cov
def loss_sqrt_2(wass_dist):
return 1 - 1 / (2 + torch.sqrt(wass_dist + 1e-4))
def torch_sqrtm(matrix, eps=1e-4):
matrix = (matrix + matrix.T) / 2
trace = matrix[0][0] + matrix[1][1]
determinant = matrix[0][0] * matrix[1][1] - matrix[0][1] * matrix[1][0]
if determinant < -eps:
raise ValueError("Negative determinant")
s = torch.sqrt(determinant + eps)
t = torch.sqrt(trace + 2 * s)
if t < eps:
raise ValueError("Zero t in sqrtm")
sqrt_matrix = (matrix + s * torch.eye(2, device=matrix.device)) / t
return sqrt_matrix
def kl_divergence(Mu1, Cov1, Mu2, Cov2):
kl_loss = 0
for i in range(Mu1.shape[0]):
mu1 = Mu1[i]
mu2 = Mu2[i]
cov1 = Cov1[i]
cov2 = Cov2[i]
d = mu1.shape[-1]
try:
cov2_inv = torch.linalg.inv(cov2)
trace_term = torch.trace(cov2_inv @ cov1)
mean_diff = mu2 - mu1
mean_term = mean_diff @ cov2_inv @ mean_diff
logdet_term = torch.logdet(cov2 + 1e-6 * torch.eye(d, device=cov2.device)) - \
torch.logdet(cov1 + 1e-6 * torch.eye(d, device=cov1.device))
kl = 0.5 * (logdet_term - d + trace_term + mean_term)
except Exception:
return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype)
if not torch.isfinite(kl):
return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype)
kl_loss += loss_sqrt_2(kl)
if not torch.isfinite(kl_loss):
return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype)
return kl_loss
def bd_distance(Mu1, Cov1, Mu2, Cov2):
bd_loss = 0
for i in range(Mu1.shape[0]):
mu1 = Mu1[i]
mu2 = Mu2[i]
cov1 = Cov1[i]
cov2 = Cov2[i]
try:
cov_avg = 0.5 * (cov1 + cov2)
cov_avg_inv = torch.linalg.inv(cov_avg + 1e-6 * torch.eye(2, device=cov_avg.device))
mean_diff = (mu1 - mu2).unsqueeze(0)
mean_term = 0.125 * (mean_diff @ cov_avg_inv @ mean_diff.T).squeeze()
det_cov1 = torch.linalg.det(cov1 + 1e-6 * torch.eye(2, device=cov1.device))
det_cov2 = torch.linalg.det(cov2 + 1e-6 * torch.eye(2, device=cov2.device))
det_cov_avg = torch.linalg.det(cov_avg + 1e-6 * torch.eye(2, device=cov_avg.device))
det_term = 0.5 * torch.log(det_cov_avg / torch.sqrt(det_cov1 * det_cov2))
bd = mean_term + det_term
except Exception:
return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype)
if not torch.isfinite(bd):
return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype)
bd_loss += loss_sqrt_2(bd)
if not torch.isfinite(bd_loss):
return torch.tensor(0.0, device=Mu1.device, dtype=Mu1.dtype)
return bd_loss
def wasserstein_distance(Mean1, Cov1, Mean2, Cov2):
assert Mean1.shape[0] == Cov1.shape[0]
wasserstein_loss = 0
for i in range(Mean1.shape[0]):
mean1 = Mean1[i]
mean2 = Mean2[i]
cov1 = Cov1[i]
cov2 = Cov2[i]
if not all(torch.isfinite(x).all() for x in [mean1, mean2, cov1, cov2]):
return torch.tensor(0.0, device=Mean1.device, dtype=Mean1.dtype)
mean_diff = norm(mean1 - mean2)**2
try:
sqrt_cov1 = torch_sqrtm(cov1)
cov_prod = sqrt_cov1 @ cov2 @ sqrt_cov1
sqrt_cov_prod = torch_sqrtm(cov_prod)
except Exception:
return torch.tensor(0.0, device=Mean1.device, dtype=Mean1.dtype)
if not all(torch.isfinite(x).all() for x in [sqrt_cov1, sqrt_cov_prod]):
return torch.tensor(0.0, device=Mean1.device, dtype=Mean1.dtype)
cov_term = torch.trace(cov1 + cov2 - 2 * sqrt_cov_prod)
wasserstein_dist = torch.clamp(mean_diff + cov_term, min=0.0)
if not torch.isfinite(wasserstein_dist):
return torch.tensor(0.0, device=Mean1.device, dtype=Mean1.dtype)
wasserstein_loss += loss_sqrt_2(wasserstein_dist)
if not torch.isfinite(wasserstein_loss):
return torch.tensor(0.0, device=Mean1.device, dtype=Mean1.dtype)
return wasserstein_loss
def contains_zero(tensor_list):
return any((t == 0).any() for t in tensor_list)
def rao_loss(pred_mask, gt_mask, num_masks):
pred_mask = pred_mask.sigmoid()
try:
mean_pred, cov_pred = Gaussian_2d(pred_mask)
mean_gt, cov_gt = Gaussian_2d(gt_mask)
except Exception:
return torch.tensor(0.0, device=pred_mask.device, dtype=pred_mask.dtype, requires_grad=True)
if not all(torch.isfinite(x).all() for x in [mean_pred, cov_pred, mean_gt, cov_gt]):
return torch.tensor(0.0, device=pred_mask.device, dtype=pred_mask.dtype, requires_grad=True)
if contains_zero([mean_pred, cov_pred, mean_gt, cov_gt]):
return torch.tensor(0.0, device=pred_mask.device, dtype=pred_mask.dtype, requires_grad=True)
try:
loss = wasserstein_distance(mean_pred, cov_pred, mean_gt, cov_gt)
#loss = kl_divergence(mean_pred, cov_pred, mean_gt, cov_gt)
#loss = bd_distance(mean_pred, cov_pred, mean_gt, cov_gt)
except Exception:
return torch.tensor(0.0, device=pred_mask.device, dtype=pred_mask.dtype, requires_grad=True)
if not torch.isfinite(loss):
return torch.tensor(0.0, device=pred_mask.device, dtype=pred_mask.dtype, requires_grad=True)
return loss / (num_masks + 1e-8)
def dice_loss(
inputs: torch.Tensor,
targets: torch.Tensor,
num_masks: float,
scale=1000, # 100000.0,
eps=1e-6,
):
"""
Compute the DICE loss, similar to generalized IOU for masks
Args:
inputs: A float tensor of arbitrary shape.
The predictions for each example.
targets: A float tensor with the same shape as inputs. Stores the binary
classification label for each element in inputs
(0 for the negative class and 1 for the positive class).
"""
inputs = inputs.sigmoid()
inputs = inputs.flatten(1, 2)
targets = targets.flatten(1, 2)
numerator = 2 * (inputs / scale * targets).sum(-1)
denominator = (inputs / scale).sum(-1) + (targets / scale).sum(-1)
loss = 1 - (numerator + eps) / (denominator + eps)
loss = loss.sum() / (num_masks + 1e-8)
return loss
def sigmoid_ce_loss(
inputs: torch.Tensor,
targets: torch.Tensor,
num_masks: float,
):
"""
Args:
inputs: A float tensor of arbitrary shape.
The predictions for each example.
targets: A float tensor with the same shape as inputs. Stores the binary
classification label for each element in inputs
(0 for the negative class and 1 for the positive class).
Returns:
Loss tensor
"""
loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction="none")
loss = loss.flatten(1, 2).mean(1).sum() / (num_masks + 1e-8)
return loss
class LisaMetaModel:
def __init__(
self,
config,
**kwargs,
):
super(LisaMetaModel, self).__init__(config)
self.config = config
self.config.sam2_config = kwargs.get(
"sam2_config",
getattr(self.config, "sam2_config", "configs/sam2.1/sam2.1_hiera_l.yaml"),
)
self.sam2_config = self.config.sam2_config
if not hasattr(self.config, "train_mask_decoder"):
self.config.train_mask_decoder = kwargs["train_mask_decoder"]
self.config.out_dim = kwargs["out_dim"]
self.vision_pretrained = kwargs.get("vision_pretrained", None)
else:
self.vision_pretrained = kwargs.get("vision_pretrained", None)
self.initialize_lisa_modules(self.config)
def initialize_lisa_modules(self, config):
# SAM
#self.visual_model = build_sam_vit_h(self.vision_pretrained)
print("----------building sam2-------------")
# ZeroGPU packs the fully loaded model after startup. Build the SAM2
# architecture on CPU first; the fixed model snapshot supplies all
# trained SAM2 parameters before the complete model is moved to CUDA.
self.visual_model = build_sam2(
self.sam2_config,
self.vision_pretrained,
device="cpu",
)
for param in self.visual_model.parameters():
param.requires_grad = False
if config.train_mask_decoder:
self.visual_model.sam_mask_decoder.train()
for param in self.visual_model.sam_mask_decoder.parameters():
param.requires_grad = True
self._transforms = SAM2Transforms(
resolution=self.visual_model.image_size,
mask_threshold=self.visual_model.mask_threshold,
max_hole_area=self.visual_model.max_hole_area,
max_sprinkle_area=self.visual_model.max_sprinkle_area,
)
# Projection layer
in_dim = config.hidden_size
out_dim = config.out_dim
text_fc = [
nn.Linear(in_dim, in_dim),
nn.ReLU(inplace=True),
nn.Linear(in_dim, out_dim),
nn.Dropout(0.0),
]
self.text_hidden_fcs = nn.ModuleList([nn.Sequential(*text_fc)])
self.text_hidden_fcs.train()
for param in self.text_hidden_fcs.parameters():
param.requires_grad = True
class LisaModel(LisaMetaModel, LlavaLlamaModel):
def __init__(
self,
config,
**kwargs,
):
super(LisaModel, self).__init__(config, **kwargs)
self.config.use_cache = False
self.config.vision_tower = self.config.mm_vision_tower
self.config.mm_vision_select_feature = "patch"
self.config.image_aspect_ratio = "square"
self.config.image_grid_pinpoints = None
self.config.tune_mm_mlp_adapter = False
self.config.freeze_mm_mlp_adapter = True
self.config.pretrain_mm_mlp_adapter = None
self.config.mm_use_im_patch_token = False
class LISAForCausalLM(LlavaLlamaForCausalLM):
def __init__(
self,
config,
**kwargs,
):
if not hasattr(config, "train_mask_decoder"):
config.mm_use_im_start_end = kwargs.pop("use_mm_start_end", True)
config.mm_vision_tower = kwargs.get(
"vision_tower", "openai/clip-vit-large-patch14"
)
self.ce_loss_weight = kwargs.pop("ce_loss_weight", None)
self.dice_loss_weight = kwargs.pop("dice_loss_weight", None)
self.rao_loss_weight = kwargs.pop("rao_loss_weight", None)
self.bce_loss_weight = kwargs.pop("bce_loss_weight", None)
else:
config.mm_vision_tower = config.vision_tower
self.seg_token_idx = kwargs.pop("seg_token_idx")
super().__init__(config)
self.model = LisaModel(config, **kwargs)
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
# Spatial dim for backbone feature maps
self._bb_feat_sizes = [
(256, 256),
(128, 128),
(64, 64),
]
# Initialize weights and apply final processing
self.post_init()
'''
self._transforms = SAM2Transforms(
resolution=self.model.visual_model.image_size,
mask_threshold=self.model.visual_model.mask_threshold,
max_hole_area=self.model.visual_model.max_hole_area,
max_sprinkle_area=self.model.visual_model.max_sprinkle_area,
)
'''
def get_visual_embs(self, pixel_values: torch.FloatTensor):
with torch.no_grad():
image_embeddings_list = []
for i in range(pixel_values.shape[0]):
torch.cuda.empty_cache()
image_embeddings = self.model.visual_model.image_encoder(
pixel_values[i].unsqueeze(0)
)
#print("-----------------------")
#print(image_embeddings['vision_features'].shape)
image_embeddings_list.append(image_embeddings['vision_features'])
torch.cuda.empty_cache()
image_embeddings = torch.cat(image_embeddings_list, 0)
return image_embeddings
def forward(self, **kwargs):
if "past_key_values" in kwargs:
return super().forward(**kwargs)
return self.model_forward(**kwargs)
def model_forward(
self,
#image_paths: list,
images: torch.FloatTensor,
images_clip: torch.FloatTensor,
input_ids: torch.LongTensor,
labels: torch.LongTensor,
attention_masks: torch.LongTensor,
offset: torch.LongTensor,
masks_list: List[torch.FloatTensor],
label_list: List[torch.Tensor],
resize_list: List[tuple],
inference: bool = False,
**kwargs,
):
image_embeddings = self.get_visual_embs(images)
batch_size = image_embeddings.shape[0]
#get high res feat
img_batch = self.model._transforms.forward_batch(read_images(kwargs["image_paths"]))
img_batch = img_batch.to(device=self.device, dtype=images.dtype)
assert (
len(img_batch.shape) == 4 and img_batch.shape[1] == 3
), f"img_batch must be of size Bx3xHxW, got {img_batch.shape}"
backbone_out = self.model.visual_model.forward_image(img_batch)
_, vision_feats, _, _ = self.model.visual_model._prepare_backbone_features(backbone_out)
if self.model.visual_model.directly_add_no_mem_embed:
vision_feats[-1] = vision_feats[-1] + self.model.visual_model.no_mem_embed
feats = [
feat.permute(1, 2, 0).view(batch_size, -1, *feat_size)
for feat, feat_size in zip(vision_feats[::-1], self._bb_feat_sizes[::-1])
][::-1]
self._features = {"image_embed": feats[-1], "high_res_feats": feats[:-1]}
assert batch_size == len(offset) - 1
seg_token_mask = input_ids[:, 1:] == self.seg_token_idx
seg_token_mask = torch.cat(
[
seg_token_mask,
torch.zeros((seg_token_mask.shape[0], 1)).bool().cuda(),
],
dim=1,
)
# hack for IMAGE_TOKEN_INDEX (we suppose that there is only one image, and it is in the front)
seg_token_mask = torch.cat(
[torch.zeros((seg_token_mask.shape[0], 255)).bool().cuda(), seg_token_mask],
dim=1,
)
if inference:
n_batch = 1
length = input_ids.shape[0]
assert images_clip.shape[0] == 1
images_clip_extend = images_clip.expand(length, -1, -1, -1).contiguous()
output_hidden_states = []
for i in range(n_batch):
start_i, end_i = i * length, min((i + 1) * length, input_ids.shape[0])
output_i = super().forward(
images=images_clip_extend[: end_i - start_i],
attention_mask=attention_masks[start_i:end_i],
input_ids=input_ids[start_i:end_i],
output_hidden_states=True,
)
output_hidden_states.append(output_i.hidden_states)
torch.cuda.empty_cache()
output_hidden_states_list = []
output_hidden_states_level = torch.cat(output_hidden_states, dim=0)
output_hidden_states_list.append(output_hidden_states_level)
output_hidden_states = output_hidden_states_list
output = None
else:
images_clip_list = []
for i in range(len(offset) - 1):
start_i, end_i = offset[i], offset[i + 1]
images_clip_i = (
images_clip[i]
.unsqueeze(0)
.expand(end_i - start_i, -1, -1, -1)
.contiguous()
)
images_clip_list.append(images_clip_i)
images_clip = torch.cat(images_clip_list, dim=0)
output = super().forward(
images=images_clip,
attention_mask=attention_masks,
input_ids=input_ids,
labels=labels,
output_hidden_states=True,
)
output_hidden_states = output.hidden_states
hidden_states = []
assert len(self.model.text_hidden_fcs) == 1
hidden_states.append(self.model.text_hidden_fcs[0](output_hidden_states[-1]))
last_hidden_state = torch.stack(hidden_states, dim=-1).sum(dim=-1)
pred_embeddings = last_hidden_state[seg_token_mask]
seg_token_counts = seg_token_mask.int().sum(-1) # [bs, ]
seg_token_offset = seg_token_counts.cumsum(-1)
seg_token_offset = torch.cat(
[torch.zeros(1).long().cuda(), seg_token_offset], dim=0
)
seg_token_offset = seg_token_offset[offset]
pred_embeddings_ = []
for i in range(len(seg_token_offset) - 1):
start_i, end_i = seg_token_offset[i], seg_token_offset[i + 1]
pred_embeddings_.append(pred_embeddings[start_i:end_i])
pred_embeddings = pred_embeddings_
multimask_output = False
pred_masks = []
for i in range(len(pred_embeddings)):
(
sparse_embeddings,
dense_embeddings,
) = self.model.visual_model.sam_prompt_encoder(
points=None,
boxes=None,
masks=None,
text_embeds=pred_embeddings[i].unsqueeze(1),
)
sparse_embeddings = sparse_embeddings.to(pred_embeddings[i].dtype)
high_res_features = [
feat_level[i].unsqueeze(0)
for feat_level in self._features["high_res_feats"]
]
low_res_masks, iou_predictions, _, _ = self.model.visual_model.sam_mask_decoder(
image_embeddings=image_embeddings[i].unsqueeze(0),
image_pe=self.model.visual_model.sam_prompt_encoder.get_dense_pe(),
sparse_prompt_embeddings=sparse_embeddings,
dense_prompt_embeddings=dense_embeddings,
multimask_output=multimask_output,
high_res_features=high_res_features,
repeat_image=True
)
pred_mask = self.model.visual_model.postprocess_masks(
low_res_masks,
#input_size=resize_list[i],
#original_size=label_list[i].shape,
orig_hw = label_list[i].shape, #sam2
)
pred_masks.append(pred_mask[:, 0])
model_output = output
gt_masks = masks_list
if inference:
return {
"pred_masks": pred_masks,
"gt_masks": gt_masks,
}
output = model_output.logits
ce_loss = model_output.loss
ce_loss = ce_loss * self.ce_loss_weight
mask_bce_loss = 0
mask_rao_loss = 0
mask_dice_loss = 0
num_masks = 0
for batch_idx in range(len(pred_masks)):
gt_mask = gt_masks[batch_idx]
if gt_mask.sum() < 2:
print(kwargs['image_paths'][batch_idx] + "'s gtmask is none")
continue
pred_mask = pred_masks[batch_idx]
assert (
gt_mask.shape[0] == pred_mask.shape[0]
), "gt_mask.shape: {}, pred_mask.shape: {}".format(
gt_mask.shape, pred_mask.shape
)
mask_bce_loss += (
sigmoid_ce_loss(pred_mask, gt_mask, num_masks=gt_mask.shape[0])
* gt_mask.shape[0]
)
t = rao_loss(pred_mask, gt_mask, num_masks=gt_mask.shape[0])
if t == 0:
print(kwargs['image_paths'][batch_idx])
mask_rao_loss += (
t*gt_mask.shape[0]
)
mask_dice_loss += (
dice_loss(pred_mask, gt_mask, num_masks=gt_mask.shape[0])
* gt_mask.shape[0]
)
num_masks += gt_mask.shape[0]
mask_bce_loss = self.bce_loss_weight * mask_bce_loss / (num_masks + 1e-8)
mask_rao_loss = self.rao_loss_weight * mask_rao_loss / (num_masks + 1e-8)
mask_dice_loss = self.dice_loss_weight * mask_dice_loss / (num_masks + 1e-8)
mask_loss = mask_bce_loss + mask_dice_loss + mask_rao_loss
loss = ce_loss + mask_loss
return {
"loss": loss,
"ce_loss": ce_loss,
"mask_bce_loss": mask_bce_loss,
"mask_rao_loss": mask_rao_loss,
"mask_dice_loss": mask_dice_loss,
"mask_loss": mask_loss,
}
def evaluate(
self,
image_paths,
images_clip,
images,
input_ids,
resize_list,
original_size_list,
max_new_tokens=32,
tokenizer=None,
):
with torch.no_grad():
outputs = self.generate(
images=images_clip,
input_ids=input_ids,
max_new_tokens=max_new_tokens,
num_beams=1,
output_hidden_states=True,
return_dict_in_generate=True,
)
generation_hidden_states = outputs.hidden_states
if torch.is_tensor(generation_hidden_states):
output_hidden_states = generation_hidden_states
elif not generation_hidden_states:
raise RuntimeError("Generation returned no hidden states")
elif torch.is_tensor(generation_hidden_states[0]):
output_hidden_states = torch.cat(generation_hidden_states, dim=1)
else:
output_hidden_states = torch.cat(
[step_hidden_states[-1] for step_hidden_states in generation_hidden_states],
dim=1,
)
output_ids = outputs.sequences
seg_token_mask = output_ids[:, 1:] == self.seg_token_idx
# hack for IMAGE_TOKEN_INDEX (we suppose that there is only one image, and it is in the front)
seg_token_mask = torch.cat(
[
torch.zeros((seg_token_mask.shape[0], 255)).bool().cuda(),
seg_token_mask,
],
dim=1,
)
if output_hidden_states.shape[:2] != seg_token_mask.shape:
raise RuntimeError(
"Generation hidden states and segmentation token mask have "
f"incompatible shapes: {tuple(output_hidden_states.shape[:2])} "
f"vs {tuple(seg_token_mask.shape)}"
)
hidden_states = []
assert len(self.model.text_hidden_fcs) == 1
hidden_states.append(self.model.text_hidden_fcs[0](output_hidden_states))
last_hidden_state = torch.stack(hidden_states, dim=-1).sum(dim=-1)
pred_embeddings = last_hidden_state[seg_token_mask]
seg_token_counts = seg_token_mask.int().sum(-1) # [bs, ]
seg_token_offset = seg_token_counts.cumsum(-1)
seg_token_offset = torch.cat(
[torch.zeros(1).long().cuda(), seg_token_offset], dim=0
)
pred_embeddings_ = []
for i in range(len(seg_token_offset) - 1):
start_i, end_i = seg_token_offset[i], seg_token_offset[i + 1]
pred_embeddings_.append(pred_embeddings[start_i:end_i])
pred_embeddings = pred_embeddings_
image_embeddings = self.get_visual_embs(images)
#get high res feat
img_batch = self.model._transforms.forward_batch(read_images(image_paths))
img_batch = img_batch.to(device=self.device, dtype=images.dtype)
batch_size = image_embeddings.shape[0]
assert (
len(img_batch.shape) == 4 and img_batch.shape[1] == 3
), f"img_batch must be of size Bx3xHxW, got {img_batch.shape}"
backbone_out = self.model.visual_model.forward_image(img_batch)
_, vision_feats, _, _ = self.model.visual_model._prepare_backbone_features(backbone_out)
if self.model.visual_model.directly_add_no_mem_embed:
vision_feats[-1] = vision_feats[-1] + self.model.visual_model.no_mem_embed
feats = [
feat.permute(1, 2, 0).view(batch_size, -1, *feat_size)
for feat, feat_size in zip(vision_feats[::-1], self._bb_feat_sizes[::-1])
][::-1]
self._features = {"image_embed": feats[-1], "high_res_feats": feats[:-1]}
multimask_output = False
pred_masks = []
for i in range(len(pred_embeddings)):
(
sparse_embeddings,
dense_embeddings,
) = self.model.visual_model.sam_prompt_encoder(
points=None,
boxes=None,
masks=None,
text_embeds=pred_embeddings[i].unsqueeze(1),
)
high_res_features = [
feat_level[i].unsqueeze(0)
for feat_level in self._features["high_res_feats"]
]
sparse_embeddings = sparse_embeddings.to(pred_embeddings[i].dtype)
low_res_masks, iou_predictions, _, _ = self.model.visual_model.sam_mask_decoder(
image_embeddings=image_embeddings[i].unsqueeze(0),
image_pe=self.model.visual_model.sam_prompt_encoder.get_dense_pe(),
sparse_prompt_embeddings=sparse_embeddings,
dense_prompt_embeddings=dense_embeddings,
high_res_features=high_res_features,
multimask_output=multimask_output,
repeat_image=True
)
pred_mask = self.model.visual_model.postprocess_masks(
low_res_masks,
original_size_list[i],
)
pred_masks.append(pred_mask[:, 0])
return output_ids, pred_masks