python_utils/get_model.py

import urllib.request
import tempfile


OPTIMAL_NMS_THRESHOLD = 0.7
_model_config_url = "https://huggingface.co/TZProject/final_tz_segmentor/resolve/main/final_model_config.yaml"
def get_set_up():
    import torch
    TORCH_VERSION = ".".join(torch.__version__.split(".")[:2])
    CUDA_VERSION = torch.__version__.split("+")[-1]
    print("torch: ", TORCH_VERSION, "; cuda: ", CUDA_VERSION)
    print(f'GPU available: {torch.cuda.is_available()}')
    print(torch.cuda.get_device_capability())

    # print("detectron2:", detectron2.__version__)

def load_model():
    # return None
    import torch
    from detectron2.engine import DefaultPredictor
    from detectron2.config import get_cfg

    ## define relevant parameters
    cfg = get_cfg()

    with tempfile.NamedTemporaryFile(suffix=".yaml") as tmp:
        print(tmp.name)
        urllib.request.urlretrieve(_model_config_url, filename=tmp.name)
        cfg.merge_from_file(tmp.name)
    if not torch.cuda.is_available():
        cfg.MODEL.DEVICE = "cpu"
    else:
        cfg.MODEL.DEVICE = 'cuda'

    ## when rerouting to use the final model (final_tz_segmentor) USE_FED_LOSS has to be set to False
    ## this setting requires the training data to calculate class imbalance that the app will not have access to and cause a runtime error
    ## some messages will appear when using the model that certain weights are not being used 
    ## but these are used during training and not inference and shouldn't affect the model performance
    ## code below
    cfg.MODEL.ROI_BOX_HEAD.USE_FED_LOSS = False
    
    predictor = DefaultPredictor(cfg)

    return predictor

def mask_nms(masks, scores, nms_threshold=OPTIMAL_NMS_THRESHOLD):
    """
    Runs class agnostic NMS on masks/segmentations instead of the bounding boxes.
    :param      masks: (list float) List of coordinates that make up the mask output from the model.
    :param      scores: (list float) List of corresponding confidence scores given to each mask.
    :param      nms_threshold: (float) Threshold to apply mask-based class agnostic NMS.
    :return     masks_kept (list float): List of masks kept after applying NMS.
    """
    import supervision as sv
    from shapely.geometry.polygon import Polygon
    
    polygons = []
    for mask in masks:
        contour = sv.mask_to_polygons(mask)
        if len(contour) > 0:
            polygons.append(Polygon(contour[0]))
        else:
            polygons.append(Polygon([]))
    order = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True)
    masks_kept = []
    while order:
        i = order.pop(0)
        masks_kept.append(i)
        for j in order:
            # Calculate the IoU between the two polygons
            intersection = polygons[i].intersection(polygons[j]).area
            union = polygons[i].union(polygons[j]).area
            iou = intersection / union

            # Remove masks with IoU greater than the threshold
            if iou > nms_threshold:
                order.remove(j)
    return masks_kept

def apply_nms(prediction, mask=False, cls_agnostic_nms=OPTIMAL_NMS_THRESHOLD):
    from torchvision.ops import nms
    from detectron2.structures import Instances
    
    if mask:
        nms_indices = mask_nms(prediction["instances"].pred_masks.numpy(),
                               prediction["instances"]._fields["scores"], cls_agnostic_nms)
    else:
        nms_indices = nms(prediction["instances"].pred_boxes.tensor,
                          prediction["instances"].scores, cls_agnostic_nms)

    pred = {"instances": Instances(image_size=prediction["instances"].image_size,
                                   pred_boxes=prediction["instances"].pred_boxes[nms_indices],
                                   scores=prediction["instances"].scores[nms_indices],
                                   pred_classes=prediction["instances"].pred_classes[nms_indices],
                                   pred_masks=prediction["instances"].pred_masks[nms_indices])}

    return pred

if __name__ == '__main__':
    # get_set_up()
    load_model()