remove tests

tests.py

@@ -1,37 +0,0 @@
-import numpy as np
-test_cases = [
-    {
-        "predictions": [np.array(a) for a in [
-                            [1,1,10,20,30,40,0.85],
-                            [1,2,50,60,70,80,0.92],
-                            [1,3,80,90,100,110,0.75],
-                            [2,1,15,25,35,45,0.78],
-                            [2,2,55,65,75,85,0.95],
-                            [3,1,20,30,40,50,0.88],
-                            [3,2,60,70,80,90,0.82],
-                            [4,1,25,35,45,55,0.91],
-                            [4,2,65,75,85,95,0.89]
-                        ]],
-        "references": [np.array(a) for a in [
-                            [1, 1, 10, 20, 30, 40],
-                            [1, 2, 50, 60, 70, 80],
-                            [1, 3, 85, 95, 105, 115],
-                            [2, 1, 15, 25, 35, 45],
-                            [2, 2, 55, 65, 75, 85],
-                            [3, 1, 20, 30, 40, 50],
-                            [3, 2, 60, 70, 80, 90],
-                            [4, 1, 25, 35, 45, 55],
-                            [5, 1, 30, 40, 50, 60],
-                            [5, 2, 70, 80, 90, 100]
-                        ]],
-        "result": {'idf1': 0.8421052631578947, 'idp': 0.8888888888888888,
-                    'idr': 0.8, 'recall': 0.8, 'precision': 0.8888888888888888,
-                    'num_unique_objects': 3,'mostly_tracked': 2,
-                    'partially_tracked': 1, 'mostly_lost': 0,
-                    'num_false_positives': 1, 'num_misses': 2,
-                    'num_switches': 0, 'num_fragmentations': 0,
-                    'mota': 0.7, 'motp': 0.02981870229007634,
-                    'num_transfer': 0, 'num_ascend': 0,
-                    'num_migrate': 0}
-    },   
-]

user-friendly-metrics.py

@@ -169,11 +169,25 @@ def calculate(predictions,
     
     return summary
 
+def build_metrics_template(models, filters):
+    metrics_dict = {}
+    for model in models:
+        metrics_dict[model] = {}
+        metrics_dict[model]["all"] = {}
+        for filter, filter_ranges in filters.items():
+            metrics_dict[model][filter] = {}
+            for filter_range in filter_ranges:
+                filter_range_name = filter_range[0]
+                metrics_dict[model][filter][filter_range_name] = {}
+    return metrics_dict
+
+    
 def calculate_from_payload(payload: dict, 
                            max_iou: float = 0.5,
                            filters = {},
                            recognition_thresholds = [0.3, 0.5, 0.8], 
                            debug: bool = False):
+    
     if not isinstance(payload, dict):
         try:
             payload = payload.to_dict()
@@ -191,15 +205,7 @@ def calculate_from_payload(payload: dict,
         print("sequence_list: ", sequence_list)
 
     metrics_per_sequence = {}
-    metrics_global = {}
-    for model in models:
-        metrics_global[model] = {}
-        metrics_global[model]["all"] = {}
-        for filter, filter_ranges in filters.items():
-            metrics_global[model][filter] = {}
-            for filter_range in filter_ranges:
-                filter_range_name = filter_range[0]
-                metrics_global[model][filter][filter_range_name] = {}
+    metrics_global = build_metrics_template(models, filters)
 
     for sequence in sequence_list:
         metrics_per_sequence[sequence] = {}
@@ -217,14 +223,16 @@ def calculate_from_payload(payload: dict,
                 index = detection['index']
                 x, y, w, h = detection['bounding_box']
                 all_formated_references["all"].append([frame_id+1, index, x, y, w, h])  
+                
                 for filter, filter_ranges in filters.items():
                     filter_value = detection[filter]
                     for filter_range in filter_ranges:
-                        filter_range_name = filter_range[0]
-                        filter_range_limits = filter_range[1]
+                        filter_range_name, filter_range_limits = filter_range[0], filter_range[1]
                         if filter_value >= filter_range_limits[0] and filter_value <= filter_range_limits[1]:
                             all_formated_references[filter][filter_range_name].append([frame_id+1, index, x, y, w, h])
 
+        metrics_per_sequence[sequence] = build_metrics_template(models, filters)
+
         for model in models:
             frames = payload['sequences'][sequence][model]
             formated_predictions = []
@@ -233,19 +241,20 @@ def calculate_from_payload(payload: dict,
                 for detection in frame:
                     index = detection['index']
                     x, y, w, h = detection['bounding_box']
-                    confidence = detection['confidence']
-                    confidence = 1  #TODO: remove this line
+                    confidence = 1
                     formated_predictions.append([frame_id+1, index, x, y, w, h, confidence])
+            
             if debug:
                 print("sequence/model: ", sequence, model)
                 print("formated_predictions: ", formated_predictions)
                 print("formated_references: ", all_formated_references)
+            
             if len(formated_predictions) == 0:
                 metrics_per_sequence[sequence][model] = "Model had no predictions."
             elif len(all_formated_references["all"]) == 0:
                 metrics_per_sequence[sequence][model] = "No ground truth."
+            
             else:
-                metrics_per_sequence[sequence][model] = {}
 
                 sequence_metrics = calculate(formated_predictions, all_formated_references["all"], max_iou=max_iou, recognition_thresholds = recognition_thresholds)
                 sequence_metrics = realize_metrics(sequence_metrics, recognition_thresholds)
@@ -255,7 +264,7 @@ def calculate_from_payload(payload: dict,
                 metrics_global[model]["all"] = realize_metrics(metrics_global[model]["all"], recognition_thresholds)
                                 
                 for filter, filter_ranges in filters.items():
-                    metrics_per_sequence[sequence][model][filter] = {}
+
                     for filter_range in filter_ranges:
                         
                         filter_range_name = filter_range[0]
@@ -291,6 +300,9 @@ def sum_dicts(dict1, dict2):
 
 def realize_metrics(metrics_dict, 
                    recognition_thresholds):
+    """
+    calculates metrics based on raw metrics
+    """
     
     metrics_dict["precision"] = metrics_dict["tp"]/(metrics_dict["tp"]+metrics_dict["fp"])
     metrics_dict["recall"] = metrics_dict["tp"]/(metrics_dict["tp"]+metrics_dict["fn"])
@@ -301,13 +313,3 @@ def realize_metrics(metrics_dict,
         
     return metrics_dict
     
-def per_sequence_to_global(metrics_dict):
-    global_metrics = {}
-    for sequence in metrics_dict:
-        for model in metrics_dict[sequence]:
-            if model not in global_metrics:
-                global_metrics[model] = metrics_dict[sequence][model]
-            else:
-                global_metrics[model] = sum_dicts(global_metrics[model], metrics_dict[sequence][model])
-
-    return global_metrics