Improve UI explanations, fix NMS slider default, and add model version to results

app.py

@@ -21,15 +21,13 @@ import matplotlib.pyplot as plt
 from PIL import Image
 from shiny import App, ui, render, reactive, Session, module
 from detectron2.utils.visualizer import Visualizer, ColorMode
-from detectron2.data import Metadata
 
-from python_utils import load_model, apply_nms
+from python_utils import load_model, apply_nms, OPTIMAL_NMS_THRESHOLD, MODEL_VERSION, discussion_url, model_page, github_repo_url
 
 # Load data and compute static values
 app_dir = Path(__file__).parent
 
 protocol_url = 'https://pgomba.github.io/orchid_protocol/'
-discussion_url = 'https://huggingface.co/spaces/TZProject/TZSeedApp/discussions'
 acknowledgement_text = "The OrchAId TZ viability dataset used to develop the model was created by the Royal Botanic Gardens, Kew, Silo National des Graines Forestieres, Madagascar, the Ministry of Agriculture, Lands, Housing and Environment, Monsterrat, Instituto de Investigação Agrária de Moçambique, Mozambique, Departmento de Recursos Naturales y Ambientales, Puerto Rico & the National Parks Trust of the Virgin Islands."
 
 # Load the prediction model
@@ -43,8 +41,8 @@ main_app = ui.page_fluid(
                               multiple=True,
                               accept=[".png", ".jpg", ".jpeg"]),
 
-                ui.input_slider("threshold", "Threshold for Discarding Overlapping Segmentations"
-                                             " (ADD a more descriptive label, and explain in text above).", 0, 1.0, 0.7),
+                ui.input_slider("threshold", f"Threshold for Discarding Overlapping Segmentations (Default: {OPTIMAL_NMS_THRESHOLD})", 0, 1.0,
+                                OPTIMAL_NMS_THRESHOLD),
 
                 ui.tags.style(""" 
                     .irs.irs--shiny .irs-single { /* square with number */
@@ -119,23 +117,35 @@ app_ui = ui.page_fluid(
     ),
 
     ui.div(
-        ui.h4("Using this App"),
+        ui.h4("Model Overview"),
         ui.p(
-            "This app is built to use a computer vision model to analyse images of orchid TZ tests and count the number of viable, non-viable and empty seeds."
-            " To use this app, upload images and click 'Analyse'."
-            " Segmented images will be displayed in the right-hand panel and results can be downloaded."),
+            "This app uses a computer vision model trained to analyse images of orchid tetrazolium chloride tests to count the number of viable, non-viable and empty seeds."
+            " Full details of the model, training process and evaluation can be found on the project ",
+            ui.a("GitHub repository", href=github_repo_url, target="_blank"),
+            "."),
+
+        ui.h5('Performance'),
+        ui.p('To briefly summarise model performance on our test dataset, '),
+        ui.p('Disclaimer: the evaluation of the model applies to our dataset and there are many factors that may influence performance of the model on new images.'
+            ' We recommend visually inspecting at least a few images to ensure the model is performing as expected on your batch of images.'),
+        ui.h4("Using this App"),
         ui.p(
-            "This app is built for use with specific types of images -- the protocol for taking images compatible with this model is available on ",
+            "This app is built for use with ", ui.HTML("<b>specific types of images</b>"),
+            " -- the protocol for taking images compatible with this model is available on ",
             ui.a("GitHub", href=protocol_url, target="_blank"),
             ". The protocol is available in English, Indonesian, Thai, French, Spanish, Portuguese, Arabic, Mandarin, Malagasy and Japanese."),
         ui.p(
-            "NMS set to 0.7 as this was found to be optimal for our data, but you can adjust this value in the slider and lower values may be useful for images"
-            "with more overlapping seeds."),
-        ui.p(" If you have any feedback on the app, please start a discussion on ",
-             ui.a("the HuggingFace space", href=discussion_url, target="_blank")
+            "To use this app, upload images and click 'Analyse'."
+            " Segmented images will be displayed in the right-hand panel, showing viable seeds in red, non-viable in yellow and empty in black. An opacity slider can be used to adjust the transparency of the segmentation masks."
+            " The counts will also be displayed as text and results can be downloaded using the 'Download Results' button."),
+
+        ui.p(
+            f"Before analysing images it is possible to change the threshold used to discard overlapping segmentations produced by the model. The default threshold is {OPTIMAL_NMS_THRESHOLD} as this was found to be optimal for our data, but you can adjust this value in the slider."
+            f" We recommend leaving this as the default, and only decreasing the value if you find that your images have many overlapping seeds and some of them are not being included in the output. Similarly, you can increase this value if your images have very few overlapping seeds and the output includes multiple segmentations of the same seed."),
+        ui.p(" If you have any feedback on the app, please start a discussion on the project ",
+             ui.a("HuggingFace space", href=discussion_url, target="_blank"), '.'
              ),
-        ui.h4("Model Details and Accuracy"),
-        ui.p(),
+
         class_="body-bar"
     ),
     main_app,
@@ -260,8 +270,11 @@ def server(input, output, session: Session):
 
                 # Run prediction with original BGR image
                 prediction = predictor(im)
-
+                print(f"Analyzing image {idx + 1} of {len(files)}")
+                print(f"NMS threshold: {input.threshold()}")
+                print(f'Number of instances: {len(prediction["instances"])}')
                 prediction = apply_nms(prediction, mask=True, cls_agnostic_nms=input.threshold())
+                print(f'Number of instances after NMS: {len(prediction["instances"])}')
 
                 classes = prediction["instances"].pred_classes.tolist()
 
@@ -332,6 +345,7 @@ def server(input, output, session: Session):
             "Empty": r.get("empty", ""),
             "Total": r.get("total", ""),
             'NMS Threshold': r.get('NMS threshold', ''),
+            'Model Version': MODEL_VERSION
         } for r in results])
 
         # Create in-memory CSV file

python_utils/get_model.py

@@ -4,7 +4,7 @@ import tempfile
 ## Urls and model variables that might change.
 OPTIMAL_NMS_THRESHOLD = 0.7
 model_page = "https://huggingface.co/TZProject/final_tz_segmentor"
-_model_config_url = model_page+"/resolve/main/final_model_config.yaml"
+_model_config_url = model_page + "/resolve/main/final_model_config.yaml"
 MODEL_VERSION = "v1.0"
 discussion_url = 'https://huggingface.co/spaces/TZProject/TZSeedApp/discussions'
 github_repo_url = 'https://github.com/JATamura/TZSegmenting'
@@ -19,7 +19,8 @@ def get_set_up():
 
     # print("detectron2:", detectron2.__version__)
 
-def load_model(using_final_model:bool=True):
+
+def load_model(using_final_model: bool = True):
     """
     Load and configure a Detectron2 model predictor. The method creates a configuration
     object, merges it with a specified configuration file fetched from a remote URL,
@@ -57,11 +58,12 @@ def load_model(using_final_model:bool=True):
         ## but these are used during training and not inference and shouldn't affect the model performance
         ## code below
         cfg.MODEL.ROI_BOX_HEAD.USE_FED_LOSS = False
-    
+
     predictor = DefaultPredictor(cfg)
 
     return predictor
 
+
 def mask_nms(masks, scores, nms_threshold=OPTIMAL_NMS_THRESHOLD):
     """
     Runs class agnostic NMS on masks/segmentations instead of the bounding boxes.
@@ -72,7 +74,7 @@ def mask_nms(masks, scores, nms_threshold=OPTIMAL_NMS_THRESHOLD):
     """
     import supervision as sv
     from shapely.geometry.polygon import Polygon
-    
+
     polygons = []
     for mask in masks:
         contour = sv.mask_to_polygons(mask)
@@ -96,6 +98,7 @@ def mask_nms(masks, scores, nms_threshold=OPTIMAL_NMS_THRESHOLD):
                 order.remove(j)
     return masks_kept
 
+
 def apply_nms(prediction, mask=False, cls_agnostic_nms=OPTIMAL_NMS_THRESHOLD):
     """
     Applies Non-Maximum Suppression (NMS) to filter redundant bounding boxes
@@ -118,7 +121,7 @@ def apply_nms(prediction, mask=False, cls_agnostic_nms=OPTIMAL_NMS_THRESHOLD):
         Defaults to ``"""
     from torchvision.ops import nms
     from detectron2.structures import Instances
-    
+    print(f'applying nms with threshold {cls_agnostic_nms} and mask {mask}... \n')
     if mask:
         nms_indices = mask_nms(prediction["instances"].pred_masks.numpy(),
                                prediction["instances"]._fields["scores"], cls_agnostic_nms)
@@ -134,6 +137,7 @@ def apply_nms(prediction, mask=False, cls_agnostic_nms=OPTIMAL_NMS_THRESHOLD):
 
     return pred
 
+
 if __name__ == '__main__':
     # get_set_up()
-    load_model()
+    load_model()