import urllib.request import tempfile ## Urls and model variables that might change. ## If changing any of these, think about other places in repos where they might need changing (e.g. weights url inside config file). OPTIMAL_NMS_THRESHOLD = 0.7 model_page = "https://huggingface.co/TZTestAnalysis/final_tz_segmentor" _model_config_url = model_page + "/resolve/main/final_model_config.yaml" MODEL_VERSION = "v1.0" discussion_url = 'https://huggingface.co/spaces/TZTestAnalysis/OrchAId/discussions' github_repo_url = 'https://github.com/JATamura/TZSegmenting' def get_set_up(): import torch TORCH_VERSION = ".".join(torch.__version__.split(".")[:2]) CUDA_VERSION = torch.__version__.split("+")[-1] print("torch: ", TORCH_VERSION, "; cuda: ", CUDA_VERSION) print(f'GPU available: {torch.cuda.is_available()}') print(torch.cuda.get_device_capability()) # print("detectron2:", detectron2.__version__) def load_model(using_final_model: bool = True): """ Load and configure a Detectron2 model predictor. The method creates a configuration object, merges it with a specified configuration file fetched from a remote URL, and initializes the model using the `DefaultPredictor` from Detectron2. The model will be set up to run on either a GPU or CPU depending on the system's capabilities. :param using_final_model: A flag to indicate whether the final model should be used with specific configurations. When this is set to True, adjustments are made to suppress settings that are relevant during training but cause runtime errors during inference. This includes disabling certain loss calculations that depend on training data. :return: A Detectron2 predictor object configured and ready for inference. """ # return None import torch from detectron2.engine import DefaultPredictor from detectron2.config import get_cfg ## define relevant parameters cfg = get_cfg() with tempfile.NamedTemporaryFile(suffix=".yaml") as tmp: print(tmp.name) urllib.request.urlretrieve(_model_config_url, filename=tmp.name) cfg.merge_from_file(tmp.name) if not torch.cuda.is_available(): cfg.MODEL.DEVICE = "cpu" print('No GPU available, using CPU') else: cfg.MODEL.DEVICE = 'cuda' print('Using GPU') if using_final_model: ## when rerouting to use the final model (final_tz_segmentor) USE_FED_LOSS has to be set to False ## this setting requires the training data to calculate class imbalance that the app will not have access to and cause a runtime error ## some messages will appear when using the model that certain weights are not being used ## but these are used during training and not inference and shouldn't affect the model performance ## code below cfg.MODEL.ROI_BOX_HEAD.USE_FED_LOSS = False predictor = DefaultPredictor(cfg) return predictor def mask_nms(masks, scores, nms_threshold=OPTIMAL_NMS_THRESHOLD): """ Runs class agnostic NMS on masks/segmentations instead of the bounding boxes. :param masks: (list float) List of coordinates that make up the mask output from the model. :param scores: (list float) List of corresponding confidence scores given to each mask. :param nms_threshold: (float) Threshold to apply mask-based class agnostic NMS. :return masks_kept (list float): List of masks kept after applying NMS. """ import supervision as sv from shapely.geometry.polygon import Polygon polygons = [] for mask in masks: contour = sv.mask_to_polygons(mask) if len(contour) > 0: polygons.append(Polygon(contour[0])) else: polygons.append(Polygon([])) order = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True) masks_kept = [] while order: i = order.pop(0) masks_kept.append(i) for j in order: # Calculate the IoU between the two polygons intersection = polygons[i].intersection(polygons[j]).area union = polygons[i].union(polygons[j]).area iou = intersection / union # Remove masks with IoU greater than the threshold if iou > nms_threshold: order.remove(j) return masks_kept def apply_nms(prediction, mask=False, cls_agnostic_nms=OPTIMAL_NMS_THRESHOLD): """ Applies Non-Maximum Suppression (NMS) to filter redundant bounding boxes from the prediction output produced by an object detection model. The method is compatible with instances containing bounding boxes, scores, classes, and optionally mask predictions. The NMS operation is performed based on a specific IoU threshold, configurable via the `cls_agnostic_nms` parameter. If mask mode is enabled, a mask-based NMS step will be applied to filter out redundant masks among the predictions. :param prediction: The dictionary containing detection results, where the "instances" key points to an `Instances` object. This object should contain attributes including bounding boxes, scores, predicted classes, and optionally predicted masks. :type prediction: Dict[str, Instances] :param mask: A boolean flag indicating whether NMS should additionally be applied to instance masks. Defaults to ``False``. :type mask: bool :param cls_agnostic_nms: An IoU threshold for NMS, reflecting the level of overlap above which boxes are considered for suppression. Defaults to ``""" from torchvision.ops import nms from detectron2.structures import Instances print(f'applying nms with threshold {cls_agnostic_nms} and mask {mask}... \n') if mask: # print(prediction["instances"].pred_masks) # print(prediction["instances"].pred_masks.cpu()) nms_indices = mask_nms(prediction["instances"].pred_masks.cpu().numpy(), prediction["instances"]._fields["scores"], cls_agnostic_nms) else: nms_indices = nms(prediction["instances"].pred_boxes.tensor, prediction["instances"].scores, cls_agnostic_nms) pred = {"instances": Instances(image_size=prediction["instances"].image_size, pred_boxes=prediction["instances"].pred_boxes[nms_indices], scores=prediction["instances"].scores[nms_indices], pred_classes=prediction["instances"].pred_classes[nms_indices], pred_masks=prediction["instances"].pred_masks[nms_indices])} return pred if __name__ == '__main__': # get_set_up() load_model()