Update app.py

app.py

@@ -6,18 +6,14 @@ import torch
 from torchvision import models, transforms
 from PIL import Image
 
-# -- get torch and cuda version
-#TORCH_VERSION = ".".join(torch.__version__.split(".")[:2])
-#CUDA_VERSION = torch.__version__.split("+")[-1]
 
-# -- install pre-build detectron2
+# -- install detectron2 from source ------------------------------------------------------------------------------
 os.system('pip install git+https://github.com/facebookresearch/detectron2.git')
 os.system('pip install pyyaml==5.1')
 
 import detectron2
 
-from detectron2.utils.logger import setup_logger # ????
-# from google.colab.patches import cv2_imshow
+from detectron2.utils.logger import setup_logger
 
 from detectron2 import model_zoo
 from detectron2.engine import DefaultPredictor
@@ -26,10 +22,10 @@ from detectron2.utils.visualizer import Visualizer
 from detectron2.data import MetadataCatalog, DatasetCatalog
 import cv2
 
-# ????
+
 setup_logger()
 
-# -- load rcnn model
+# -- load rcnn model ---------------------------------------------------------------------------------------------
 cfg = get_cfg()
 # add project-specific config (e.g., TensorMask) here if you're not running a model in detectron2's core library
 cfg.merge_from_file(model_zoo.get_config_file("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml"))
@@ -38,84 +34,80 @@ cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5  # set threshold for this model
 cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml")
 cfg.MODEL.DEVICE= 'cpu'
 predictor = DefaultPredictor(cfg)
-'''
-os.system(wget http://images.cocodataset.org/val2017/000000439715.jpg -q -O input.jpg)
-im = cv2.imread("./input.jpg")
-cv2_imshow(im)
-
-outputs = predictor(im)
 
-print(outputs["instances"].pred_classes)
-print(outputs["instances"].pred_boxes)
-'''
-# -- load design modernity model for classification
+# -- load design modernity model for classification --------------------------------------------------------------
 DesignModernityModel = torch.load("DesignModernityModel.pt")
 
-#INPUT_FEATURES = DesignModernityModel.fc.in_features
-#linear = nn.linear(INPUT_FEATURES, 5)
-
 DesignModernityModel.eval() # set state of the model to inference
 
+# Set class labels
 LABELS = ['2000-2003', '2006-2008', '2009-2011', '2012-2014', '2015-2018']
 n_labels = len(LABELS)
 
+# define maéan and std dev for normalization
 MEAN = [0.485, 0.456, 0.406]
 STD = [0.229, 0.224, 0.225]
 
+# define image transformation steps
 carTransforms = transforms.Compose([transforms.Resize(224),
                                     transforms.ToTensor(),
                                     transforms.Normalize(mean=MEAN, std=STD)])
 
-                                                                        
+     
+# -- define a function for extraction of the detected car ---------------------------------------------------------                                                                                                                                          
 def cropImage(outputs, im, boxes, car_class_true):
     # Get the masks
-    #car_class_true = outputs["instances"].pred_classes == 2
-    #boxes = list(outputs["instances"].pred_boxes[car_class_true])
     masks = list(np.array(outputs["instances"].pred_masks[car_class_true]))
     max_idx = torch.tensor([(x[2] - x[0])*(x[3] - x[1]) for x in boxes]).argmax().item()
 
     # Pick an item to mask
     item_mask = masks[max_idx]
+    
     # Get the true bounding box of the mask
     segmentation = np.where(item_mask == True) # return a list of different position in the bow, which are the actual detected object
     x_min = int(np.min(segmentation[1])) # minimum x position
     x_max = int(np.max(segmentation[1]))
     y_min = int(np.min(segmentation[0]))
     y_max = int(np.max(segmentation[0]))
+    
     # Create cropped image from the just portion of the image we want
     cropped = Image.fromarray(im[y_min:y_max, x_min:x_max, :], mode = 'RGB')
     # Create a PIL Image out of the mask
     mask = Image.fromarray((item_mask * 255).astype('uint8')) ###### change 255 
     # Crop the mask to match the cropped image
     cropped_mask = mask.crop((x_min, y_min, x_max, y_max))
+    
     # Load in a background image and choose a paste position
     height = y_max-y_min
     width = x_max-x_min
     background = Image.new(mode='RGB', size=(width, height), color=(255, 255, 255, 0))
+    
     # Create a new foreground image as large as the composite and paste the cropped image on top
     new_fg_image = Image.new('RGB', background.size)
     new_fg_image.paste(cropped)
+    
     # Create a new alpha mask as large as the composite and paste the cropped mask
     new_alpha_mask = Image.new('L', background.size, color=0)
     new_alpha_mask.paste(cropped_mask)
+    
     #composite the foreground and background using the alpha mask
     composite = Image.composite(new_fg_image, background, new_alpha_mask)
+    
     return composite
     
-
+# -- define function for image segmentation and classification --------------------------------------------------------
 def classifyCar(im):
-  try:
+    # read image
     im = cv2.imread(im)
-  except:
-    label = "fail"
-  try:
+    # perform segmentation
     outputs = predictor(im)
     v = Visualizer(im[:, :, ::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1)
     out = v.draw_instance_predictions(outputs["instances"])
-    
+    # check if a car was detected in the image
     car_class_true = outputs["instances"].pred_classes == 2
     boxes = list(outputs["instances"].pred_boxes[car_class_true])
     
+    # if a car was detected, extract the car and perform modernity score classification
     if len(boxes) != 0:
       #max_idx = torch.tensor([(x[2] - x[0])*(x[3] - x[1]) for x in boxes]).argmax().item()
       
@@ -128,21 +120,14 @@ def classifyCar(im):
         scores = torch.nn.functional.softmax(DesignModernityModel(carTransforms(im2).unsqueeze(0))[0])
       label = {LABELS[i]: float(scores[i]) for i in range(n_labels)}
       
+    # if no car was detected, show original image and print "No car detected"  
     else:
         im2 = Image.fromarray(np.uint8(im)).convert('RGB')
         label = "No car detected"
-  except:
-    label = "fail2"
-    #im2 = carTransforms(im).unsqueeze(0)  # transform and add batch dimension
-    #with torch.no_grad():
-    #  scores = torch.nn.functional.softmax(DesignModernityModel(im2)[0])
-      #{LABELS[i]: float(scores[i]) for i in range(n_labels)}
-      #Image.fromarray(np.uint8(out.get_image())).convert('RGB')
-  return im2, label
 
-#examples = [[example_img.jpg], [example_img2.jpg]]  # must be uploaded in repo
+  return im2, label
 
-# create interface for model
+# -- create interface for model ----------------------------------------------------------------------------------------
 interface = gr.Interface(classifyCar, inputs='image', outputs=['image','label'], cache_examples=False, title='Modernity car classification')
 interface.launch()