data_preprocess/infer_ground_objects.py

import cv2
import numpy as np
import supervision as sv
import json
import torch
import torchvision
import random
from tqdm import tqdm
from PIL import Image
from segment_anything import sam_model_registry, SamPredictor
import os
import os.path as osp
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from omnitry_bench.Grounded_Segment_Anything.GroundingDINO.groundingdino.util.inference import Model


DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# GroundingDINO config and checkpoint
GROUNDING_DINO_CONFIG_PATH = '../omnitry_bench/Grounded_Segment_Anything/GroundingDINO/groundingdino/config/GroundingDINO_SwinT_OGC.py'
GROUNDING_DINO_CHECKPOINT_PATH = '../checkpoints/groundingdino_swint_ogc.pth'

# Segment-Anything checkpoint
SAM_ENCODER_VERSION = "vit_h"
SAM_CHECKPOINT_PATH = "../checkpoints/sam_vit_h_4b8939.pth"

# Building GroundingDINO inference model
grounding_dino_model = Model(model_config_path=GROUNDING_DINO_CONFIG_PATH, model_checkpoint_path=GROUNDING_DINO_CHECKPOINT_PATH)

# Building SAM Model and SAM Predictor
sam = sam_model_registry[SAM_ENCODER_VERSION](checkpoint=SAM_CHECKPOINT_PATH)
sam.to(device=DEVICE)
sam_predictor = SamPredictor(sam)

# Predict classes and hyper-param for GroundingDINO
BOX_THRESHOLD = 0.25
TEXT_THRESHOLD = 0.25
NMS_THRESHOLD = 0.8


def generate_mask(path, prompt):
    CLASSES = [prompt]

    # load image
    image = Image.open(path).convert('RGB')
    image = np.array(image)[:, :, ::-1]

    # detect objects
    detections = grounding_dino_model.predict_with_classes(
        image=image,
        classes=CLASSES,
        box_threshold=BOX_THRESHOLD,
        text_threshold=TEXT_THRESHOLD
    )

    # NMS post process
    nms_idx = torchvision.ops.nms(
        torch.from_numpy(detections.xyxy), 
        torch.from_numpy(detections.confidence), 
        NMS_THRESHOLD
    ).numpy().tolist()

    detections.xyxy = detections.xyxy[nms_idx]
    detections.confidence = detections.confidence[nms_idx]
    detections.class_id = detections.class_id[nms_idx]

    if prompt.startswith('shoe') or prompt.startswith('earrings'):
        topk = 2
    else:
        topk = 1

    detections.xyxy = detections.xyxy[:topk]
    detections.confidence = detections.confidence[:topk]
    detections.class_id = detections.class_id[:topk]

    if detections.confidence[0] < 0.5:
        return None

    # Prompting SAM with detected boxes
    def segment(sam_predictor: SamPredictor, image: np.ndarray, xyxy: np.ndarray) -> np.ndarray:
        sam_predictor.set_image(image)
        result_masks = []
        for box in xyxy:
            masks, scores, logits = sam_predictor.predict(
                box=box,
                multimask_output=True
            )
            index = np.argmax(scores)
            result_masks.append(masks[index])
        return np.array(result_masks)


    # convert detections to masks
    detections.mask = segment(
        sam_predictor=sam_predictor,
        image=cv2.cvtColor(image, cv2.COLOR_BGR2RGB),
        xyxy=detections.xyxy
    )

    # annotate image with detections
    box_annotator = sv.BoxAnnotator()
    mask_annotator = sv.MaskAnnotator()
    labels = [
        f"{CLASSES[class_id]} {confidence:0.2f}" 
        for _, _, confidence, class_id, _, _ 
        in detections]
    annotated_image = mask_annotator.annotate(scene=image.copy(), detections=detections)
    annotated_image = box_annotator.annotate(scene=annotated_image, detections=detections, labels=labels)
    
    # output mask
    mask = torch.Tensor(np.any(detections.mask, axis=0, keepdims=True))
    return mask



if __name__ == '__main__':

    input_index_file = 'example_list_objects.json'
    output_index_file = 'example_ground_objects.json'
    
    data = json.load(open(input_index_file))
    outs = []
    for index in tqdm(data):
        
        new_objects = []
        for garment_description in index['objects']:
            tryon_path = index['image_path']
            mask = generate_mask(tryon_path, garment_description)
            if mask is None:
                continue
            mask_path = '.'.join(tryon_path.split('.')[:-1]) + '_{}_mask.jpg'.format('_'.join(garment_description.split(' ')))
            torchvision.utils.save_image(mask, mask_path)
            new_objects.append({
                'description': garment_description,
                'mask': mask_path
            })
        
        index['objects'] = new_objects
        outs.append(index)
    
    # save
    with open(output_index_file, 'w+') as f:
        f.write(json.dumps(outs, indent=4, ensure_ascii=False))