evaluator.py

import shutil
from typing import Callable

from PIL.Image import Image
from coco_eval import CocoEvaluator
from tqdm import tqdm

from yolo_dataset import YoloDataset, MaterialYoloDataset
from yolo_model import YoloModel

image_loader = Callable[[str], Image]


def coco_evaluate(model: YoloModel, dataset: YoloDataset, confidence_threshold=0.6):
    coco_gt = dataset.to_coco()
    # initialize evaluator with ground truth (gt)
    evaluator = CocoEvaluator(coco_gt=coco_gt, iou_types=["bbox"])

    print("Running evaluation...")
    coco_anns = []

    for image_id, annotations in tqdm(coco_gt.imgToAnns.items()):
        # get the inputs
        image = coco_gt.imgs[image_id]
        results = model.model(source=dataset.load_image(image["file_name"]))
        for result in results:
            coco_anns.extend(yolo_boxes_to_coco_annotations(image_id, result.boxes,
                                                            confidence_threshold=confidence_threshold))

    evaluator.update(coco_anns)
    evaluator.synchronize_between_processes()
    evaluator.accumulate()
    evaluator.summarize()


def yolo_evaluator(model: YoloModel, dataset: YoloDataset):
    return Metrics(model=model, material=dataset.to_material())


class Metrics:
    def __init__(self, model: YoloModel, material: MaterialYoloDataset):
        self.model = model
        self.material = material
        self.val = None

    def __enter__(self):
        if self.val is None:
            self.material.__enter__()
            self.val = self.model.model.val(data=self.material.yaml)
        return self.val

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.material.__exit__(exc_type, exc_val, exc_tb)
        shutil.rmtree(self.val.save_dir)


def yolo_boxes_to_coco_annotations(image_id: int, yolo_boxes, confidence_threshold=0.6):
    return [
        {
            "image_id": int(image_id),
            "category_id": int(box.cls.tolist()[0]),
            "bbox": box.xywh.tolist()[0],
            "score": box.conf.tolist()[0],
        }
        for box in yolo_boxes if box.conf.tolist()[0] > confidence_threshold
    ]


def main():
    yolo_dataset = YoloDataset.from_path('tests/coco8.zip')
    coco_gt = yolo_dataset.to_coco()
    model = YoloModel("ultralyticsplus/yolov8s", "yolov8s.pt")
    # model = YoloModel("SHOU-ISD/fire-and-smoke", "yolov8n.pt")
    # evaluate(model=model, coco_gt=coco_gt, loader=yolo_dataset.load_image, confidence_threshold=0.1)
    # Validate the model
    with yolo_dataset.to_material() as material:
        metrics = model.model.val(data=material.yaml)
        print(metrics.box.map)  # map50-95
        print(metrics.box.map50)  # map50
        print(metrics.box.map75)  # map75
        print(metrics.box.maps)  # a list contains map50-95 of each category


if __name__ == '__main__':
    main()