Add files using upload-large-folder tool

approach/ovod/d-cube/d_cube/__init__.py

@@ -0,0 +1 @@
+from .d3 import D3

approach/ovod/d-cube/d_cube/d3.py

@@ -0,0 +1,775 @@
+# -*- coding: utf-8 -*-
+__author__ = "Chi Xie and Zhao Zhang"
+__maintainer__ = "Chi Xie"
+# this is the core of the d-cube toolkit
+import os
+import os.path as osp
+import json
+from collections import defaultdict
+
+import numpy as np
+from pycocotools import mask
+import cv2
+import matplotlib.pyplot as plt
+
+
+from .data_util import *
+
+
+class D3:
+    def __init__(self, img_root, anno_root):
+        self.image_dir = img_root
+        self.anno_dir = anno_root
+        self.load_data()
+
+    def load_data(self):
+        file_names = ["sentences.pkl", "annotations.pkl", "images.pkl", "groups.pkl"]
+        self.data = {
+            name.split(".")[0]: load_pkl(osp.join(self.anno_dir, name))
+            for name in file_names
+        }
+
+    def get_sent_ids(self, anno_ids=[], img_ids=[], group_ids=[], sent_ids=[]):
+        """get sentence ids for D-cube.
+
+        Args:
+            anno_ids (list, optional): annotation ids to get sentence ids. Defaults to [].
+            img_ids (list, optional): image ids to get sentence ids. Defaults to [].
+            group_ids (list, optional): group ids to get sentence ids. Defaults to [].
+            sent_ids (list, optional): additional sentence ids you want to include. Defaults to [].
+
+        Raises:
+            Exception: anno_ids, img_ids and group_ids cannot be used together.
+
+        Returns:
+            list: sentence ids.
+        """
+        img_ids = img_ids if isinstance(img_ids, list) else [img_ids]
+        anno_ids = anno_ids if isinstance(anno_ids, list) else [anno_ids]
+        group_ids = group_ids if isinstance(group_ids, list) else [group_ids]
+        sent_ids = sent_ids if isinstance(sent_ids, list) else [sent_ids]
+
+        if not any([img_ids, anno_ids, group_ids, sent_ids]):
+            return list(self.data["sentences"].keys())
+
+        if (
+            (anno_ids and img_ids)
+            or (anno_ids and group_ids)
+            or (img_ids and group_ids)
+        ):
+            raise Exception("anno_ids, img_ids, group_ids can only be used alone")
+
+        out_ids_set = set()
+        if img_ids:
+            for img_id in img_ids:
+                imganno_ids = self.data["images"][img_id]["anno_id"]
+                for ianno_id in imganno_ids:
+                    out_ids_set |= set(self.data["annotations"][ianno_id]["sent_id"])
+
+        if group_ids:
+            for group_id in group_ids:
+                out_ids_set |= set(self.data["groups"][group_id]["inner_sent_id"])
+
+        if anno_ids:
+            for ianno_id in anno_ids:
+                out_ids_set |= set(self.data["annotations"][ianno_id]["sent_id"])
+
+        if sent_ids:
+            out_ids_set &= set(sent_ids)
+
+        return list(out_ids_set)
+
+    def get_anno_ids(self, anno_ids=[], img_ids=[], group_ids=[], sent_ids=[]):
+        """get annotation ids for D-cube.
+
+        Args:
+            anno_ids (list, optional): additional annotation ids you want to include. Defaults to [].
+            img_ids (list, optional): image ids to get annotation ids. Defaults to [].
+            group_ids (list, optional): group ids to get annotation ids. Defaults to [].
+            sent_ids (list, optional): sentence ids to get annotation ids. Defaults to [].
+
+        Raises:
+            Exception: img_ids and group_ids cannot be used together.
+
+        Returns:
+            list: annotation ids.
+        """
+        img_ids = img_ids if isinstance(img_ids, list) else [img_ids]
+        anno_ids = anno_ids if isinstance(anno_ids, list) else [anno_ids]
+        group_ids = group_ids if isinstance(group_ids, list) else [group_ids]
+        sent_ids = sent_ids if isinstance(sent_ids, list) else [sent_ids]
+
+        if not any([img_ids, anno_ids, group_ids, sent_ids]):
+            return list(self.data["annotations"].keys())
+
+        if img_ids and group_ids:
+            raise Exception("img_ids, group_ids can only be used alone")
+
+        out_ids_set = set()
+        if img_ids:
+            for img_id in img_ids:
+                out_ids_set |= set(self.data["images"][img_id]["anno_id"])
+
+        if group_ids:
+            for group_id in group_ids:
+                for groupimg_id in self.data["groups"][group_id]["img_id"]:
+                    out_ids_set |= set(self.data["images"][groupimg_id]["anno_id"])
+
+        if sent_ids and img_ids:
+            for sent_id in sent_ids:
+                out_ids_set &= set(self.data["sentences"][sent_id]["anno_id"])
+        else:
+            for sent_id in sent_ids:
+                out_ids_set |= set(self.data["sentences"][sent_id]["anno_id"])
+
+        if anno_ids:
+            out_ids_set &= set(anno_ids)
+
+        return list(out_ids_set)
+
+    def get_img_ids(self, anno_ids=[], img_ids=[], group_ids=[], sent_ids=[]):
+        """get image ids for D-cube.
+
+        Args:
+            anno_ids (list, optional): annotation ids to get image ids. Defaults to [].
+            img_ids (list, optional): additional image ids you want to include. Defaults to [].
+            group_ids (list, optional): group ids to get image ids. Defaults to [].
+            sent_ids (list, optional): sentence ids to get image ids. Defaults to [].
+
+        Raises:
+            Exception: anno_ids and img_ids cannot be used together.
+            Exception: anno_ids and group_ids cannot be used together.
+
+        Returns:
+            list: image ids.
+        """
+        img_ids = img_ids if isinstance(img_ids, list) else [img_ids]
+        anno_ids = anno_ids if isinstance(anno_ids, list) else [anno_ids]
+        group_ids = group_ids if isinstance(group_ids, list) else [group_ids]
+        sent_ids = sent_ids if isinstance(sent_ids, list) else [sent_ids]
+
+        if not any([img_ids, anno_ids, group_ids, sent_ids]):
+            return list(self.data["images"].keys())
+
+        if anno_ids and img_ids:
+            raise Exception("anno_ids and img_ids can only be used alone")
+        if anno_ids and group_ids:
+            raise Exception("anno_ids and group_ids can only be used alone")
+
+        out_ids_set = set()
+        if anno_ids:
+            for ianno_id in anno_ids:
+                out_ids_set.add(self.data["annotations"][ianno_id]["img_id"])
+
+        if group_ids:
+            for group_id in group_ids:
+                out_ids_set |= set(self.data["groups"][group_id]["img_id"])
+
+        if sent_ids:
+            for sent_id in sent_ids:
+                for sentanno_id in self.data["sentences"][sent_id]["anno_id"]:
+                    out_ids_set.add(self.data["annotations"][sentanno_id]["image_id"])
+
+        if img_ids:
+            out_ids_set &= set(img_ids)
+
+        return list(out_ids_set)
+
+    def get_group_ids(self, anno_ids=[], img_ids=[], group_ids=[], sent_ids=[]):
+        """get group ids for D-cube.
+
+        Args:
+            anno_ids (list, optional): annotation ids to get group ids. Defaults to [].
+            img_ids (list, optional): image ids to get group ids. Defaults to [].
+            group_ids (list, optional): additional group_ids you want to include. Defaults to [].
+            sent_ids (list, optional): sentence ids to get group ids. Defaults to [].
+
+        Raises:
+            Exception: anno_ids, img_ids and sent_ids cannot be used together.
+
+        Returns:
+            list: group ids.
+        """
+        img_ids = img_ids if isinstance(img_ids, list) else [img_ids]
+        anno_ids = anno_ids if isinstance(anno_ids, list) else [anno_ids]
+        group_ids = group_ids if isinstance(group_ids, list) else [group_ids]
+        sent_ids = sent_ids if isinstance(sent_ids, list) else [sent_ids]
+
+        if not any([img_ids, anno_ids, group_ids, sent_ids]):
+            return list(self.data["groups"].keys())
+
+        if anno_ids and img_ids:
+            raise Exception("anno_ids and img_ids can only be used alone")
+        if anno_ids and sent_ids:
+            raise Exception("anno_ids and sent_ids can only be used alone")
+        if img_ids and sent_ids:
+            raise Exception("img_ids and sent_ids can only be used alone")
+
+        out_ids_set = set()
+        if img_ids:
+            for img_id in img_ids:
+                out_ids_set.add(self.data["images"][img_id]["group_id"])
+
+        if anno_ids:
+            for anno_id in anno_ids:
+                out_ids_set.add(self.data["annotations"][anno_id]["group_id"])
+
+        if sent_ids:
+            for sent_id in sent_ids:
+                out_ids_set |= set(self.data["sentences"][sent_id]["group_id"])
+
+        if group_ids:
+            out_ids_set &= set(group_ids)
+
+        return list(out_ids_set)
+
+    def load_sents(self, sent_ids=None):
+        """load sentence info.
+
+        Args:
+            sent_ids (list, int, optional): sentence ids. Defaults to None.
+
+        Returns:
+            list: a list of sentence info.
+        """
+        if sent_ids is not None and not isinstance(sent_ids, list):
+            sent_ids = [sent_ids]
+        if isinstance(sent_ids, list):
+            return [self.data["sentences"][sent_id] for sent_id in sent_ids]
+        else:
+            return list(self.data["sentences"].values())
+
+    def load_annos(self, anno_ids=None):
+        """load annotation info.
+
+        Args:
+            anno_ids (list, int, optional): annotation ids. Defaults to None.
+
+        Returns:
+            list: a list of annotation info.
+        """
+        if anno_ids is not None and not isinstance(anno_ids, list):
+            anno_ids = [anno_ids]
+        if isinstance(anno_ids, list):
+            return [self.data["annotations"][anno_id] for anno_id in anno_ids]
+        else:
+            return list(self.data["annotations"].values())
+
+    def load_imgs(self, img_ids=None):
+        """load image info.
+
+        Args:
+            img_ids (list, int, optional): image ids. Defaults to None.
+
+        Returns:
+            list: a list of image info.
+        """
+        if img_ids is not None and not isinstance(img_ids, list):
+            img_ids = [img_ids]
+        if isinstance(img_ids, list):
+            return [self.data["images"][img_ids] for img_ids in img_ids]
+        else:
+            return list(self.data["images"].values())
+
+    def load_groups(self, group_ids=None):
+        """load group info.
+
+        Args:
+            group_ids (list, int, optional): group ids. Defaults to None.
+
+        Returns:
+            list: a list of group info.
+        """
+        if group_ids is not None and not isinstance(group_ids, list):
+            group_ids = [group_ids]
+        if isinstance(group_ids, list):
+            return [self.data["groups"][group_ids] for group_ids in group_ids]
+        else:
+            return list(self.data["groups"].values())
+
+    def get_mask(self, anno):
+        rle = anno[0]["segmentation"]
+        m = mask.decode(rle)
+        m = np.sum(
+            m, axis=2
+        )  # sometimes there are multiple binary map (corresponding to multiple segs)
+        m = m.astype(np.uint8)  # convert to np.uint8
+        # compute area
+        area = sum(mask.area(rle))  # should be close to ann['area']
+        return {"mask": m, "area": area}
+
+    def show_mask(self, anno):
+        M = self.get_mask(anno)
+        msk = M["mask"]
+        ax = plt.gca()
+        ax.imshow(msk)
+
+    def show_image_seg(
+        self,
+        img_ids=[],
+        save_dir=None,
+        show_sent=False,
+        on_image=False,
+        checkerboard_bg=False,
+        is_instance=True,
+    ):
+        if is_instance and checkerboard_bg:
+            raise ValueError(
+                "Cannot apply both is_instance and checkboard_bg at the same time."
+            )
+        img_infos = self.load_imgs(img_ids=img_ids)
+        for img_idx, img_info in enumerate(img_infos):
+            img = cv2.imread(osp.join(self.image_dir, img_info["file_name"]))
+            anno_infos = self.load_annos(img_info["anno_id"])
+
+            bm_canvas = defaultdict(list)
+            merge_canvas = defaultdict(list)
+            for anno_info in anno_infos:
+                for sent_id in anno_info["sent_id"]:
+                    bm_canvas[sent_id].append(anno_info["segmentation"])
+
+            for sent_id, bm_list in bm_canvas.items():
+                merge_canvas[sent_id] = merge_rle(
+                    bm_list, is_instance=is_instance, on_image=on_image
+                )
+
+            cv2.imwrite(osp.join(save_dir, f"{img_info['id']}.png"), img)
+            for sent_id, merge_mask in merge_canvas.items():
+                if checkerboard_bg:
+                    merge_mask = add_checkerboard_bg(img, merge_mask)
+                elif on_image:
+                    merge_mask = visualize_mask_on_image(img, merge_mask, add_edge=True)
+                if show_sent:
+                    sent_en = self.load_sents(sent_ids=sent_id)[0]["raw_sent"]
+                    merge_mask = paste_text(merge_mask, sent_en)
+                cv2.imwrite(
+                    osp.join(save_dir, f"{img_info['id']}_{sent_id}.png"), merge_mask
+                )
+
+        return merge_canvas
+
+    def show_group_seg(
+        self,
+        group_ids,
+        save_root,
+        show_sent=True,
+        is_instance=True,
+        on_image=False,
+        checkerboard_bg=False,
+    ):
+        group_infos = self.load_groups(group_ids=group_ids)
+        for group_info in group_infos:
+            save_dir = osp.join(save_root, group_info["group_name"])
+            os.makedirs(save_dir, exist_ok=True)
+            self.show_image_seg(
+                img_ids=group_info["img_id"],
+                save_dir=save_dir,
+                show_sent=show_sent,
+                is_instance=is_instance,
+                on_image=on_image,
+                checkerboard_bg=checkerboard_bg,
+            )
+
+    def show_image_seg_bbox(
+        self,
+        img_ids=[],
+        save_dir=None,
+        show_sent=False,
+        on_image=False,
+        checkerboard_bg=False,
+        is_instance=True,
+    ):
+        if is_instance and checkerboard_bg:
+            raise ValueError(
+                "Cannot apply both is_instance and checkboard_bg at the same time."
+            )
+        img_infos = self.load_imgs(img_ids=img_ids)
+        for img_idx, img_info in enumerate(img_infos):
+            img = cv2.imread(osp.join(self.image_dir, img_info["file_name"]))
+            anno_infos = self.load_annos(img_info["anno_id"])
+
+            bm_canvas = defaultdict(list)
+            merge_canvas = defaultdict(list)
+            sent_boxes = defaultdict(list)
+            for anno_info in anno_infos:
+                for sent_id in anno_info["sent_id"]:
+                    bm_canvas[sent_id].append(anno_info["segmentation"])
+                    sent_boxes[sent_id].append(anno_info["bbox"][0].tolist())
+
+            for sent_id, bm_list in bm_canvas.items():
+                merge_canvas[sent_id] = merge_rle(
+                    bm_list, is_instance=is_instance, on_image=on_image
+                )
+
+            cv2.imwrite(osp.join(save_dir, f"{img_info['id']}.png"), img)
+            for sent_id, merge_mask in merge_canvas.items():
+                # vis mask
+                if checkerboard_bg:
+                    merge_mask = add_checkerboard_bg(img, merge_mask)
+                elif on_image:
+                    merge_mask = visualize_mask_on_image(img, merge_mask, add_edge=True)
+                # vis box
+                bboxes = sent_boxes[sent_id]
+                merge_mask = visualize_bbox_on_image(merge_mask, bboxes)
+                # vis sent
+                if show_sent:
+                    sent_en = self.load_sents(sent_ids=sent_id)[0]["raw_sent"]
+                    merge_mask = paste_text(merge_mask, sent_en)
+                cv2.imwrite(
+                    osp.join(save_dir, f"{img_info['id']}_{sent_id}.png"), merge_mask
+                )
+
+        return merge_canvas
+
+    def show_group_seg_bbox(
+        self,
+        group_ids,
+        save_root,
+        show_sent=True,
+        is_instance=True,
+        on_image=False,
+        checkerboard_bg=False,
+    ):
+        group_infos = self.load_groups(group_ids=group_ids)
+        for group_info in group_infos:
+            save_dir = osp.join(save_root, group_info["group_name"])
+            os.makedirs(save_dir, exist_ok=True)
+            self.show_image_seg_bbox(
+                img_ids=group_info["img_id"],
+                save_dir=save_dir,
+                show_sent=show_sent,
+                is_instance=is_instance,
+                on_image=on_image,
+                checkerboard_bg=checkerboard_bg,
+            )
+
+    def show_image_bbox(self, img_ids=[], save_dir=None, show_sent=False):
+        img_infos = self.load_imgs(img_ids=img_ids)
+        for img_idx, img_info in enumerate(img_infos):
+            img = cv2.imread(osp.join(self.image_dir, img_info["file_name"]))
+            anno_infos = self.load_annos(img_info["anno_id"])
+
+            sent_boxes = defaultdict(list)
+            for anno_info in anno_infos:
+                for sent_id in anno_info["sent_id"]:
+                    sent_boxes[sent_id].append(anno_info["bbox"][0].tolist())
+
+            cv2.imwrite(osp.join(save_dir, f"{img_info['id']}.png"), img)
+            for sent_id, bboxes in sent_boxes.items():
+                merge_img = visualize_bbox_on_image(img, bboxes)
+                if show_sent:
+                    sent_en = self.load_sents(sent_ids=sent_id)[0]["raw_sent"]
+                    merge_img = paste_text(merge_img, sent_en)
+                cv2.imwrite(
+                    osp.join(save_dir, f"{img_info['id']}_{sent_id}.png"), merge_img
+                )
+
+    def show_group_bbox(self, group_ids, save_root, show_sent=True):
+        group_infos = self.load_groups(group_ids=group_ids)
+        for group_info in group_infos:
+            save_dir = osp.join(save_root, group_info["group_name"])
+            os.makedirs(save_dir, exist_ok=True)
+            self.show_image_bbox(
+                img_ids=group_info["img_id"], save_dir=save_dir, show_sent=show_sent
+            )
+
+    def stat_description(self, with_rev=False, inter_group=False):
+        """calculate and print dataset statistics.
+
+        Args:
+            with_rev (bool, optional): consider absence descriptions or not. Defaults to False.
+            inter_group (bool, optional): calculate under intra- or inter-group settings. Defaults to False.
+        """
+        stat_dict = {}
+        # Number of sents
+        sent_ids = list(self.data["sentences"].keys())
+        if not with_rev:
+            sent_ids = [sent_id for sent_id in sent_ids if not self.is_revsent(sent_id)]
+        stat_dict["nsent"] = len(sent_ids)
+        # Number of annos / instance  # TODO: rm rev
+        stat_dict["nanno"] = len(self.data["annotations"].keys())
+        # Number of images
+        stat_dict["nimg"] = len(self.data["images"].keys())
+        # Number of groups
+        stat_dict["ngroup"] = len(self.data["groups"].keys())
+
+        # Number of img-sent pair
+        num_img_sent = 0
+        for img_id in self.data["images"].keys():
+            anno_ids = self.get_anno_ids(img_ids=img_id)
+            anno_infos = self.load_annos(anno_ids=anno_ids)
+            cur_sent_set = set()
+            group_sent_ids = set(
+                self.load_groups(self.get_group_ids(img_ids=img_id))[0]["inner_sent_id"]
+            )
+            for anno_info in anno_infos:
+                cur_sent_set |= set(
+                    [i for i in anno_info["sent_id"] if i in group_sent_ids]
+                )
+            if not with_rev:
+                cur_sent_set = [
+                    sent_id for sent_id in cur_sent_set if not self.is_revsent(sent_id)
+                ]
+            num_img_sent += len(cur_sent_set)
+        stat_dict["num_img_sent"] = num_img_sent
+
+        # Number of absence img-sent pair
+        num_anti_img_sent = 0
+        for img_id in self.data["images"].keys():
+            anno_ids = self.get_anno_ids(img_ids=img_id)
+            anno_infos = self.load_annos(anno_ids=anno_ids)
+            cur_sent_set = set()
+            group_sent_ids = set(
+                self.load_groups(self.get_group_ids(img_ids=img_id))[0]["inner_sent_id"]
+            )
+            for anno_info in anno_infos:
+                cur_sent_set |= set(
+                    [i for i in anno_info["sent_id"] if i in group_sent_ids]
+                )
+            assert group_sent_ids.issuperset(
+                cur_sent_set
+            ), f"{group_sent_ids}, {cur_sent_set}"
+            cur_anti_sent_set = group_sent_ids - cur_sent_set
+            if not with_rev:
+                cur_anti_sent_set = [
+                    sent_id
+                    for sent_id in cur_anti_sent_set
+                    if not self.is_revsent(sent_id)
+                ]
+            num_anti_img_sent += len(cur_anti_sent_set)
+        stat_dict["num_anti_img_sent"] = num_anti_img_sent
+
+        # Number of anno-sent pair
+        num_anno_sent = 0
+        anno_infos = self.load_annos()
+        for anno_info in anno_infos:
+            if inter_group:
+                anno_sent_ids = [i for i in anno_info["sent_id"]]
+            else:
+                group_sent_ids = set(
+                    self.load_groups(anno_info["group_id"])[0]["inner_sent_id"]
+                )
+                anno_sent_ids = [i for i in anno_info["sent_id"] if i in group_sent_ids]
+            if not with_rev:
+                anno_sent_ids = [
+                    sent_id for sent_id in anno_sent_ids if not self.is_revsent(sent_id)
+                ]
+            num_anno_sent += len(anno_sent_ids)
+
+        stat_dict["num_anno_sent"] = num_anno_sent
+
+        # Number of anti anno-sent pair
+        num_anti_anno_sent = 0
+        anno_infos = self.load_annos()
+        for anno_info in anno_infos:
+            if inter_group:
+                all_sent_ids = set(self.get_sent_ids())
+                anno_sent_ids = anno_info["sent_id"]
+
+                anti_sent_ids = [
+                    sent_id for sent_id in all_sent_ids if sent_id not in anno_sent_ids
+                ]
+            else:
+                group_sent_ids = set(
+                    self.load_groups(anno_info["group_id"])[0]["inner_sent_id"]
+                )
+                anno_sent_ids = [i for i in anno_info["sent_id"] if i in group_sent_ids]
+
+                anti_sent_ids = [
+                    sent_id
+                    for sent_id in group_sent_ids
+                    if sent_id not in anno_sent_ids
+                ]
+
+            if not with_rev:
+                anti_sent_ids = [
+                    sent_id for sent_id in anti_sent_ids if not self.is_revsent(sent_id)
+                ]
+            num_anti_anno_sent += len(anti_sent_ids)
+
+        stat_dict["num_anti_anno_sent"] = num_anti_anno_sent
+
+        # Len of sentence
+        totle_len = 0
+        for sent_info in self.load_sents(sent_ids):
+            totle_len += len(sent_info["raw_sent"].split())
+
+        stat_dict["avg_sent_len"] = totle_len / stat_dict["nsent"]
+
+        print(stat_dict)
+
+    def is_revsent(self, sent_id):
+        sent_info = self.load_sents(sent_ids=sent_id)
+        return sent_info[0]["is_negative"]
+
+    def data2coca(self, out_root, with_rev=False):
+        group_infos = self.load_groups()
+        for group_info in group_infos:
+            sent_ids = group_info["inner_sent_id"]
+            if not with_rev:
+                sent_ids = [
+                    sent_id for sent_id in sent_ids if not self.is_revsent(sent_id)
+                ]
+            sent_infos = self.load_sents(sent_ids)
+            for sent_info in sent_infos:
+                sent = sent_info["raw_sent"]
+                img_infos = self.load_imgs(group_info["img_id"])
+                for img_info in img_infos:
+                    src_img_path = osp.join(self.image_dir, img_info["file_name"])
+                    raw_name = img_info["file_name"].split("/")[-1]
+                    out_img_dir = osp.join(out_root, "images", sent)
+                    os.makedirs(out_img_dir, exist_ok=True)
+                    out_img_path = osp.join(out_img_dir, raw_name)
+                    copy_file(src_img_path, out_img_path)
+
+                    out_mask_dir = osp.join(out_root, "masks", sent)
+                    os.makedirs(out_mask_dir, exist_ok=True)
+                    out_mask_path = osp.join(
+                        out_mask_dir, raw_name.replace(".jpg", ".png")
+                    )
+
+                    cur_anno_ids = self.get_anno_ids(
+                        img_ids=img_info["id"], sent_ids=sent_info["id"]
+                    )
+                    anno_infos = self.load_annos(cur_anno_ids)
+                    rle_list = [anno_info["segmentation"] for anno_info in anno_infos]
+                    bmask = merge2bin(rle_list, img_info["height"], img_info["width"])
+                    cv2.imwrite(out_mask_path, bmask)
+
+    def convert2coco(self, out_root, anti_mode=False, is_group_separated=True):
+        """
+        Convert the annotation format of D^3 dataset to COCO.
+        1. The sent_id can be viewed as category_id in COCO.
+        2. If `is_group_separated` is True, `outer_sent_id` does not need to be considered.
+        3. if `with_rev` is False,  sents that meet `is_revsent` will be ignore.
+        """
+        os.makedirs(out_root, exist_ok=True)
+        coco_dict = {
+            "images": [],
+            "categories": [],
+            "annotations": [],
+        }
+
+        sent_ids = self.get_sent_ids()
+        if anti_mode == 1:
+            sent_ids = [sent_id for sent_id in sent_ids if not self.is_revsent(sent_id)]
+        elif anti_mode == 2:
+            sent_ids = [sent_id for sent_id in sent_ids if self.is_revsent(sent_id)]
+        elif anti_mode == 0:
+            pass
+        else:
+            raise Exception("Unimplemented anti_mode.")
+
+        sent_infos = self.load_sents(sent_ids)
+        for isent_info in sent_infos:
+            coco_dict["categories"].append(
+                {
+                    "id": isent_info["id"],
+                    "name": isent_info["raw_sent"],
+                }
+            )
+
+        item_id = 0
+        img_infos = self.load_imgs()
+        for iimg_info in img_infos:
+            coco_dict["images"].append(
+                {
+                    "id": iimg_info["id"],
+                    "file_name": iimg_info["file_name"],
+                    "height": iimg_info["height"],
+                    "width": iimg_info["width"],
+                }
+            )
+
+            anno_ids = self.get_anno_ids(img_ids=iimg_info["id"])
+            anno_infos = self.load_annos(anno_ids)
+
+            for ianno_info in anno_infos:
+                if is_group_separated:
+                    inner_group_sent_ids = [
+                        isent_id
+                        for isent_id in ianno_info["sent_id"]
+                        if isent_id
+                        in self.load_groups(ianno_info["group_id"])[0]["inner_sent_id"]
+                    ]
+                    cur_sent_ids = inner_group_sent_ids
+                else:
+                    cur_sent_ids = ianno_info["sent_id"]
+
+                for isent_id in cur_sent_ids:
+                    if isent_id not in sent_ids:
+                        continue
+
+                    seg = ianno_info["segmentation"][0].copy()
+                    if isinstance(seg, dict):  # RLE
+                        counts = seg["counts"]
+                        if not isinstance(counts, str):
+                            # make it json-serializable
+                            seg["counts"] = counts.decode("ascii")
+
+                    coco_dict["annotations"].append(
+                        {
+                            "id": item_id,
+                            "image_id": iimg_info["id"],
+                            "category_id": isent_id,
+                            "segmentation": seg,
+                            "area": int(ianno_info["area"][0]),
+                            "bbox": [
+                                int(cord) for cord in ianno_info["bbox"][0].tolist()
+                            ],
+                            "iscrowd": 0,  # TODO: ianno_info["iscrowd"]
+                        }
+                    )
+                    item_id += 1
+
+        with open(osp.join(out_root, "coco_annotations.json"), "w") as f:
+            json.dump(coco_dict, f, indent=4)
+
+    def sent_analyse(self, save_dir, with_rev=False):
+        """analyze word info in D-cube and generate word length histograms, word clouds, etc.
+
+        Args:
+            save_dir (str): path to save the visualized results.
+            with_rev (bool, optional): consider absence descriptions or not. Defaults to False.
+        """
+        sent_ids = self.get_sent_ids()
+        if not with_rev:
+            sent_ids = [sent_id for sent_id in sent_ids if not self.is_revsent(sent_id)]
+
+        sent_lens, sent_raws = [], []
+        sent_infos = self.load_sents(sent_ids)
+        for isent_info in sent_infos:
+            sent_raws.append(isent_info["raw_sent"])
+            sent_lens.append(len(isent_info["raw_sent"].split()))
+
+        os.makedirs(save_dir, exist_ok=True)
+        # plot_hist(
+        #     sent_lens,
+        #     bins=max(sent_lens) - min(sent_lens) + 1,
+        #     save_path=osp.join(save_dir, "words_hist.pdf"),
+        #     x="Lengths of descriptions",
+        # )
+        # generate_wordclouds(sent_raws, osp.join(save_dir, "word_clouds"))
+
+    def group_analysis(self, save_dir, with_rev=False):
+        group_infos = self.load_groups()
+        scene_tree = defaultdict(dict)
+
+        for group_info in group_infos:
+            scene_tree[group_info["scene"]][group_info["group_name"]] = {"nimg": 0.1}
+
+        # vis_group_tree(scene_tree, osp.join(save_dir, 'scene_tree.png'))  # the visualized result is ugly
+
+    def bbox_num_analyze(self):
+        n_cat = len(self.data["sentences"].keys())
+        all_img_ids = self.data["images"].keys()
+        n_img = len(all_img_ids)
+        cat_obj_count = np.zeros((n_cat, n_img), dtype=int)
+        for img_id in all_img_ids:
+            # img_cat_ids = self.get_sent_ids(img_ids=img_id)
+            anno_ids = self.get_anno_ids(img_ids=img_id)
+            anno_infos = self.load_annos(anno_ids=anno_ids)
+            for anno in anno_infos:
+                for sid in anno["sent_id"]:
+                    cat_obj_count[sid - 1, img_id] += 1
+        return cat_obj_count

approach/ovod/d-cube/d_cube/data_util.py

@@ -0,0 +1,269 @@
+# -*- coding: utf-8 -*-
+__author__ = "Chi Xie and Zhao Zhang"
+__maintainer__ = "Chi Xie"
+# data utility functions are defined in the script
+import json
+import pickle
+import shutil
+
+# from io import StringIO
+# import string
+
+import numpy as np
+import cv2
+from pycocotools import mask as cocomask
+
+VOC_COLORMAP = [
+    [128, 0, 0],
+    [0, 128, 0],
+    [128, 128, 0],
+    [0, 0, 128],
+    [128, 0, 128],
+    [0, 128, 128],
+    [128, 128, 128],
+    [64, 0, 0],
+    [192, 0, 0],
+    [64, 128, 0],
+    [192, 128, 0],
+    [64, 0, 128],
+    [192, 0, 128],
+    [64, 128, 128],
+    [192, 128, 128],
+    [0, 64, 0],
+    [128, 64, 0],
+    [0, 192, 0],
+    [128, 192, 0],
+    [0, 64, 128],
+]
+
+
+def visualize_bbox_on_image(img, bbox_list, save_path=None, thickness=3):
+    img_copy = img.copy()
+    for i, bbox in enumerate(bbox_list):
+        color = tuple(VOC_COLORMAP[i % len(VOC_COLORMAP)])
+        x, y, w, h = bbox
+        img_copy = cv2.rectangle(
+            img_copy, (int(x), int(y)), (int((x + w)), int(y + h)), color, thickness
+        )
+    if save_path:
+        cv2.imwrite(save_path, img_copy)
+    return img_copy
+
+
+def rle2bmask(rle):
+    bm = cocomask.decode(rle)
+    if len(bm.shape) == 3:
+        bm = np.sum(
+            bm, axis=2
+        )  # sometimes there are multiple binary map (corresponding to multiple segs)
+    bm = bm.astype(np.uint8)  # convert to np.uint8
+    return bm
+
+
+def merge_rle(rle_list, is_instance=True, on_image=False):
+    if is_instance:
+        cm_list = []
+        for rle_idx, rle in enumerate(rle_list):
+            color = VOC_COLORMAP[rle_idx]
+            bm = rle2bmask(rle)
+            cm = cv2.cvtColor(bm, cv2.COLOR_GRAY2BGR)
+            cm_list.append(cm * color)
+        merge_map = np.sum(cm_list, axis=0, dtype=np.uint8)
+    else:
+        bm_list = [rle2bmask(rle) for rle in rle_list]
+        merge_map = np.sum(bm_list, axis=0, dtype=np.uint8)
+    merge_map[merge_map >= 1] = 1
+    if not on_image:
+        color = VOC_COLORMAP[0]
+        merge_map = cv2.cvtColor(merge_map, cv2.COLOR_GRAY2BGR)
+        merge_map *= np.array(color, dtype=np.uint8)
+
+    merge_map[merge_map > 255] = 255
+
+    if not on_image:
+        tmp_sum_map = np.sum(merge_map, axis=-1)
+        merge_map[tmp_sum_map == 0] = 220
+    return merge_map
+
+
+def merge2bin(rle_list, img_h, img_w):
+    if rle_list:
+        bm_list = [rle2bmask(rle) for rle in rle_list]
+        merge_map = np.sum(bm_list, axis=0, dtype=np.uint8)
+        merge_map[merge_map >= 1] = 255
+        merge_map = np.expand_dims(merge_map, axis=-1)
+        return merge_map
+    else:
+        return np.zeros([img_h, img_w, 1], dtype=np.uint8)
+
+
+def paste_text(img, text):
+    fontFace = cv2.FONT_HERSHEY_COMPLEX_SMALL
+    overlay = img.copy()
+    # fontFace = cv2.FONT_HERSHEY_TRIPLEX
+    fontScale = 1
+    thickness = 1
+    backgroud_alpha = 0.8
+
+    retval, baseLine = cv2.getTextSize(
+        text, fontFace=fontFace, fontScale=fontScale, thickness=thickness
+    )
+    topleft = (0, 0)
+    # bottomright = (topleft[0] + retval[0], topleft[1] + retval[1]+10)
+    bottomright = (img.shape[1], topleft[1] + retval[1] + 10)
+
+    cv2.rectangle(overlay, topleft, bottomright, thickness=-1, color=(250, 250, 250))
+    img = cv2.addWeighted(overlay, backgroud_alpha, img, 1 - backgroud_alpha, 0)
+
+    cv2.putText(
+        img,
+        text,
+        (0, baseLine + 10),
+        fontScale=fontScale,
+        fontFace=fontFace,
+        thickness=thickness,
+        color=(10, 10, 10),
+    )
+    return img
+
+
+def load_json(json_path, to_int=False):
+    clean_res_dic = {}
+    with open(json_path, "r", encoding="utf-8") as f_in:
+        res_dic = json.load(f_in)
+
+    for ikey, iv in res_dic.items():
+        ikey = int(ikey.strip()) if to_int else ikey.strip()
+        clean_res_dic[ikey] = iv
+
+    return clean_res_dic
+
+
+def path_map(src_path, obj_path):
+    def inner_map(full_path):
+        return full_path.replace(src_path, obj_path)
+
+
+def save_pkl(src, obj_path):
+    with open(obj_path, "wb") as f_out:
+        pickle.dump(src, f_out)
+
+
+def load_pkl(src_path):
+    with open(src_path, "rb") as f_in:
+        in_pkl = pickle.load(f_in)
+    return in_pkl
+
+
+def copy_file(src_path, obj_path):
+    shutil.copy(src_path, obj_path)
+
+
+def sentence_analysis():
+    return 0
+
+
+def add_checkerboard_bg(image, mask, save_path=None):
+    # Create a new image with the same size as the original image
+    new_image = np.zeros_like(image)
+
+    # Define the size of the checkerboard pattern
+    checkerboard_size = 24
+
+    # Loop over each pixel in the mask
+    for x in range(mask.shape[1]):
+        for y in range(mask.shape[0]):
+            # If the pixel is transparent, draw a checkerboard pattern
+            if mask[y, x] == 0:
+                if (x // checkerboard_size) % 2 == (y // checkerboard_size) % 2:
+                    new_image[y, x] = (255, 255, 255)
+                else:
+                    new_image[y, x] = (128, 128, 128)
+            # Otherwise, copy the corresponding pixel from the original image
+            else:
+                new_image[y, x] = image[y, x]
+
+    # Save the new image with the checkerboard background
+    if save_path:
+        cv2.imwrite(save_path, new_image)
+    return new_image
+
+
+def visualize_mask_on_image(
+    img, mask, save_path=None, add_edge=False, dark_background=False
+):
+    # Convert the mask to a binary mask if it's not already
+    if mask.max() > 1:
+        mask = mask.astype(np.uint8) // 255
+
+    # Convert the mask to a 3-channel mask if it's not already
+    if len(mask.shape) == 2:
+        mask = np.expand_dims(mask, axis=2)
+        mask = np.tile(mask, (1, 1, 3))
+
+    # Create a color map for the mask
+    cmap = np.array([255, 117, 44], dtype=np.uint8)
+    mask_colors = mask * cmap
+
+    # Add an opaque white edge to the mask if desired
+    if add_edge:
+        if len(mask.shape) == 2:
+            mask = np.expand_dims(mask, axis=2)
+            mask = np.tile(mask, (1, 1, 3))
+
+        kernel = np.ones((5, 5), dtype=np.uint8)
+        mask_edge = cv2.erode(mask, kernel, iterations=1)
+        mask_edge = mask - mask_edge
+
+        # mask_edge = np.tile(mask_edge[:, :, np.newaxis], [1, 1, 3])
+        mask_colors[mask_edge > 0] = 255
+
+    # Overlay the mask on the masked image
+    if dark_background:
+        masked_img = cv2.addWeighted(img, 0.4, mask_colors, 0.6, 0)
+    else:
+        masked_img = img.copy()
+        masked_img[mask > 0] = cv2.addWeighted(img, 0.4, mask_colors, 0.6, 0)[mask > 0]
+
+    # Save the result to the specified path if provided
+    if save_path is not None:
+        cv2.imwrite(save_path, masked_img)
+
+    return masked_img
+
+
+# def visualize_mask_on_image(img, mask, save_path=None, add_edge=False):
+#     # Convert the mask to a binary mask if it's not already
+#     if mask.max() > 1:
+#         mask = mask.astype(np.uint8) // 255
+
+#     # Convert the mask to a 3-channel mask if it's not already
+#     if len(mask.shape) == 2:
+#         mask = np.expand_dims(mask, axis=2)
+#         mask = np.tile(mask, (1, 1, 3))
+
+#     # Create a color map for the mask
+#     cmap = np.array([255, 117, 44], dtype=np.uint8)
+#     mask_colors = mask * cmap
+
+#     # Add an opaque white edge to the mask if desired
+#     if add_edge:
+#         if len(mask.shape) == 2:
+#             mask = np.expand_dims(mask, axis=2)
+#             mask = np.tile(mask, (1, 1, 3))
+
+#         kernel = np.ones((5, 5), dtype=np.uint8)
+#         mask_edge = cv2.erode(mask, kernel, iterations=1)
+#         mask_edge = mask - mask_edge
+
+#         # mask_edge = np.tile(mask_edge[:, :, np.newaxis], [1, 1, 3])
+#         mask_colors[mask_edge > 0] = 255
+
+#     # Overlay the mask on the masked image
+#     masked_img = cv2.addWeighted(img, 0.5, mask_colors, 0.5, 0)
+
+#     # Save the result to the specified path if provided
+#     if save_path is not None:
+#         cv2.imwrite(save_path, masked_img)
+
+#     return masked_img

approach/ovod/d-cube/d_cube/vis_util.py

@@ -0,0 +1,199 @@
+# -*- coding: utf-8 -*-
+__author__ = "Chi Xie and Zhao Zhang"
+__maintainer__ = "Chi Xie"
+import os
+from collections import Counter
+
+import spacy
+import matplotlib.pyplot as plt
+import seaborn as sns
+from wordcloud import WordCloud
+
+# from pycirclize import Circos
+# from Bio.Phylo.BaseTree import Tree
+# from Bio import Phylo
+# from newick import Node
+
+
+def plot_hist(data, bins=10, is_norm=False, save_path=None, x=None):
+    sns.set_theme(style="whitegrid", font_scale=2.0)
+    ax = sns.histplot(data, bins=bins, common_norm=is_norm, kde=False)
+    ax.set_xlabel(x)
+    plt.tight_layout()
+    plt.savefig(save_path)
+    plt.close()
+
+
+def plot_bars(names, nums, is_sort, save_path=None):
+    sns.set(style="whitegrid")
+
+    if is_sort:
+        zipped = zip(nums, names)
+        sort_zipped = sorted(zipped, key=lambda x: (x[0], x[1]))
+        result = zip(*sort_zipped)
+        nums, names = [list(x) for x in result]
+
+    fontx = {"family": "Times New Roman", "size": 10}
+    fig, ax = plt.subplots()
+    fig = plt.figure(figsize=(16, 4))
+    # sns.set_palette("PuBuGn_d")
+    sns.barplot(names, nums, palette=sns.cubehelix_palette(80, start=0.5, rot=-0.75))
+    fig.autofmt_xdate(rotation=90)
+    plt.tick_params(axis="x", labelsize=10)
+    labels = ax.get_xticklabels() + ax.get_yticklabels()
+    [label.set_fontname("Times New Roman") for label in labels]
+    plt.tight_layout()
+    plt.savefig(save_path)
+
+
+def generate_wordclouds(sentences, save_dir):
+    """Generates word clouds for different parts of speech in a list of sentences.
+
+    Args:
+        sentences: A list of sentences.
+        save_dir: The directory to save the word cloud images.
+    """
+
+    os.makedirs(save_dir, exist_ok=True)
+    # Load the spacy model
+    nlp = spacy.load("en_core_web_sm")
+
+    # Define the parts of speech to include in the word clouds
+    pos_to_include = ["NOUN", "VERB", "ADJ", "ADV"]
+
+    # Process each sentence and collect the relevant words for each part of speech
+    words_by_pos = {pos: [] for pos in pos_to_include}
+    for sent in sentences:
+        doc = nlp(sent)
+        for token in doc:
+            if token.pos_ in pos_to_include:
+                words_by_pos[token.pos_].append(token.lemma_.lower())
+
+    # Generate a word cloud for each part of speech
+    for pos, words in words_by_pos.items():
+        if len(words) == 0:
+            continue  # skip parts of speech with no words
+
+        # Count the frequency of each word
+        word_counts = Counter(words)
+
+        # Generate the word cloud
+        wordcloud = WordCloud(
+            width=800,
+            height=800,
+            background_color="white",
+            max_words=200,
+            colormap="Set2",
+            max_font_size=150,
+        ).generate_from_frequencies(word_counts)
+
+        # Save the word cloud image
+        filename = f"{pos.lower()}_wordcloud.png"
+        filepath = os.path.join(save_dir, filename)
+        wordcloud.to_file(filepath)
+
+
+# def vis_group_tree(data_dict, save_path):
+
+#     # Create 3 randomized trees
+#     tree_size_list = [60, 40, 50]
+#     trees = [Tree.randomized(string.ascii_uppercase, branch_stdev=0.5) for size in tree_size_list]
+
+#     # Initialize circos sector with 3 randomized tree size
+#     sectors = {name: size for name, size in zip(list("ABC"), tree_size_list)}
+#     circos = Circos(sectors, space=5)
+
+#     colors = ["tomato", "skyblue", "limegreen"]
+#     cmaps = ["bwr", "viridis", "Spectral"]
+#     for idx, sector in enumerate(circos.sectors):
+#         sector.text(sector.name, r=120, size=12)
+#         # Plot randomized tree
+#         tree = trees[idx]
+#         tree_track = sector.add_track((30, 70))
+#         tree_track.axis(fc=colors[idx], alpha=0.2)
+#         tree_track.tree(tree, leaf_label_size=3, leaf_label_margin=21)
+#         # Plot randomized bar
+#         bar_track = sector.add_track((70, 90))
+#         x = np.arange(0, int(sector.size)) + 0.5
+#         height = np.random.randint(1, 10, int(sector.size))
+#         bar_track.bar(x, height, facecolor=colors[idx], ec="grey", lw=0.5, hatch="//")
+
+#     circos.savefig(save_path, dpi=600)
+
+# def clean_newick_key(in_str):
+#     bad_chars = [':', ';', ',', '(', ')']
+#     for bad_char in bad_chars:
+#         in_str = in_str.replace(bad_char, ' ')
+#     return in_str
+
+# def build_tree_from_dict(data):
+#     root = Node()  # create the root node
+#     for key, value in data.items():
+#         node = Node(name=clean_newick_key(key)) # name doesn't need to be cleaned
+#         if value is not None:
+#             child_node = build_tree_from_dict(value)
+#             node.add_descendant(child_node)
+#         root.add_descendant(node)
+
+#     return root
+
+
+def replace_chars_in_dict_keys(d):
+    """
+    Replaces the characters ':', ';', ',', '(', and ')' in the keys of a nested dictionary with '_'.
+    """
+    new_dict = {}
+    for k, v in d.items():
+        if isinstance(v, dict):
+            v = replace_chars_in_dict_keys(v)
+        new_key = k.translate(str.maketrans(":;,()", "_____"))
+        new_dict[new_key] = v
+    return new_dict
+
+
+def build_newick_tree(tree_dict):
+    newick_tree = ""
+    if isinstance(tree_dict, dict):
+        for key, value in tree_dict.items():
+            if isinstance(value, dict):
+                subtree = build_newick_tree(value)
+                if subtree:
+                    newick_tree += "(" + subtree + ")" + key + ","
+                else:
+                    newick_tree += key + ","
+            else:
+                newick_tree += key + ":" + str(value) + ","
+        newick_tree = newick_tree.rstrip(",") + ")"
+        return newick_tree
+    else:
+        return None
+
+
+# def vis_group_tree(data_dict, save_path):
+#     data_dic = replace_chars_in_dict_keys(data_dict)
+#     super_group_names = data_dict.keys()
+
+#     # Create 3 randomized trees
+#     tree_size_list = [60, 40, 50]
+#     trees = [Phylo.read(StringIO(build_newick_tree(data_dict[super_group_name])), "newick") for super_group_name in super_group_names]
+
+#     # Initialize circos sector with 3 randomized tree size
+#     sectors = {name: size for name, size in zip(list("ABC"), tree_size_list)}
+#     circos = Circos(sectors, space=5)
+
+#     colors = ["tomato", "skyblue", "limegreen"]
+#     cmaps = ["bwr", "viridis", "Spectral"]
+#     for idx, sector in enumerate(circos.sectors):
+#         sector.text(sector.name, r=120, size=12)
+#         # Plot randomized tree
+#         tree = trees[idx]
+#         tree_track = sector.add_track((30, 70))
+#         tree_track.axis(fc=colors[idx], alpha=0.2)
+#         tree_track.tree(tree, leaf_label_size=3, leaf_label_margin=21)
+#         # Plot randomized bar
+#         bar_track = sector.add_track((70, 90))
+#         x = np.arange(0, int(sector.size)) + 0.5
+#         height = np.random.randint(1, 10, int(sector.size))
+#         bar_track.bar(x, height, facecolor=colors[idx], ec="grey", lw=0.5, hatch="//")
+
+#     circos.savefig(save_path, dpi=600)

approach/ovod/d-cube/eval_sota/README.md

@@ -0,0 +1,27 @@
+# Evaluting SOTA Methods on $D^3$
+
+## Leaderboard
+
+In this directory, we keep the scripts or github links (official or custom) to evaluate SOTA methods (REC/OVD/DOD/MLLM) on $D^3$:
+
+| Name | Paper | Original Tasks | Training Data | Evaluation Code | Intra-FULL/PRES/ABS/Inter-FULL/PRES/ABS | Source | Note |
+|:-----|:-----:|:----:|:-----:|:-----:|:-----:|:-----:|:-----:|
+| OFA-large | [OFA: Unifying Architectures, Tasks, and Modalities Through a Simple Sequence-to-Sequence Learning Framework (ICML 2022)](https://arxiv.org/abs/2202.03052) | REC | - | - | 4.2/4.1/4.6/0.1/0.1/0.1 | [DOD paper](https://arxiv.org/abs/2307.12813) | - |
+| CORA-R50 | [CORA: Adapting CLIP for Open-Vocabulary Detection with Region Prompting and Anchor Pre-Matching (CVPR 2023)](https://openaccess.thecvf.com/content/CVPR2023/papers/Wu_CORA_Adapting_CLIP_for_Open-Vocabulary_Detection_With_Region_Prompting_and_CVPR_2023_paper.pdf) | OVD | - | - | 6.2/6.7/5.0/2.0/2.2/1.3 | [DOD paper](https://arxiv.org/abs/2307.12813) | - |
+| OWL-ViT-large | [Simple Open-Vocabulary Object Detection with Vision Transformers (ECCV 2022)](https://www.ecva.net/papers/eccv_2022/papers_ECCV/papers/136700714.pdf) | OVD | - | [DOD official](./owl_vit.py) | 9.6/10.7/6.4/2.5/2.9/2.1 | [DOD paper](https://arxiv.org/abs/2307.12813) | Post-processing hyper-parameters may affect the performance and the result may not exactly match the paper |
+| SPHINX-7B | [SPHINX: The Joint Mixing of Weights, Tasks, and Visual Embeddings for Multi-modal Large Language Models (arxiv 2023)](https://arxiv.org/abs/2311.07575) | **MLLM** capable of REC | - | [DOD official](./sphinx.py) | 10.6/11.4/7.9/-/-/- | DOD authors | A lot of contribution from [Jie Li](https://github.com/theFool32) |
+| GLIP-T | [Grounded Language-Image Pre-training (CVPR 2022)](https://arxiv.org/abs/2112.03857)  | OVD & PG | - | - | 19.1/18.3/21.5/-/-/- | GEN paper | - |
+| UNINEXT-huge | [Universal Instance Perception as Object Discovery and Retrieval (CVPR 2023)](https://arxiv.org/abs/2303.06674v2) | OVD & REC | - | [DOD official](https://github.com/Charles-Xie/UNINEXT_D3) | 20.0/20.6/18.1/3.3/3.9/1.6 | [DOD paper](https://arxiv.org/abs/2307.12813) | - |
+| Grounding-DINO-base | [Grounding DINO: Marrying DINO with Grounded Pre-Training for Open-Set Object Detection (arxiv 2023)](https://arxiv.org/abs/2303.05499) | OVD & REC | - | [DOD official](./groundingdino.py) | 20.7/20.1/22.5/2.7/2.4/3.5 | [DOD paper](https://arxiv.org/abs/2307.12813) | Post-processing hyper-parameters may affect the performance and the result may not exactly match the paper |
+| OFA-DOD-base | [Described Object Detection: Liberating Object Detection with Flexible Expressions (NeurIPS 2023)](https://arxiv.org/abs/2307.12813) | DOD | - | - | 21.6/23.7/15.4/5.7/6.9/2.3 | [DOD paper](https://arxiv.org/abs/2307.12813) | - |
+| FIBER-B | [Coarse-to-Fine Vision-Language Pre-training with Fusion in the Backbone (NeurIPS 2022)](https://arxiv.org/abs/2206.07643) | OVD & REC | - | - | 22.7/21.5/26.0/-/-/- | GEN paper | - |
+| MM-Grounding-DINO | [An Open and Comprehensive Pipeline for Unified Object Grounding and Detection (arxiv 2024)](https://arxiv.org/abs/2401.02361) | DOD & OVD & REC | O365, GoldG, GRIT, V3Det | [MM-GDINO official](https://github.com/open-mmlab/mmdetection/tree/main/configs/mm_grounding_dino#zero-shot-description-detection-datasetdod) | 22.9/21.9/26.0/-/-/- | MM-GDINO paper | - |
+| GEN (FIBER-B) | [Generating Enhanced Negatives for Training Language-Based Object Detectors (arxiv 2024](https://arxiv.org/abs/2401.00094) | DOD | - | - | 26.0/25.2/28.1/-/-/- | GEN paper | Enhancement based on FIBER-B |
+| APE-large (D) | [Aligning and Prompting Everything All at Once for Universal Visual Perception (arxiv 2023)](https://arxiv.org/abs/2312.02153) | DOD & OVD & REC | COCO, LVIS, O365, OpenImages, Visual Genome, RefCOCO/+/g, SA-1B, GQA, PhraseCut, Flickr30k | [APE official](https://github.com/shenyunhang/APE) | 37.5/38.8/33.9/21.0/22.0/17.9 | APE paper | Extra training data helps for this amazing performance |
+
+
+Some extra notes:
+- Each method is currently recorded by *the variant with the highest performance* in this table, if there are multiple variants available, so it's only a leaderboard, not meant for fair comparison.
+- Methods like GLIP, FIBER, etc. are actually not evaluated on OVD benchmarks. For zero-shot eval on DOD, We currently do not distinguish between methods for OVD benchmarks and methods for ZS-OD, as long as it is verified with open-set detection capability.
+
+For other variants (e.g. for a fair comparison regarding data, backbone, etc.), please refer to the papers.

approach/ovod/d-cube/eval_sota/groundingdino.py

@@ -0,0 +1,304 @@
+# -*- coding: utf-8 -*-
+__author__ = "Chi Xie"
+__maintainer__ = "Chi Xie"
+
+# An example for how to run this script:
+# CUDA_VISIBLE_DEVICES=0
+# python groundingdino.py \
+#     -c ./groundingdino/config/GroundingDINO_SwinB.cfg.py \
+#     -p ./ckpt/groundingdino_swinb_cogcoor.pth \
+#     -o "outputs/gdino_d3" \
+#     --box_threshold 0.05 \
+#     --text_threshold 0.05 \
+#     --img-top1
+
+import argparse
+import json
+import os
+
+import numpy as np
+import torch
+from PIL import Image, ImageDraw, ImageFont
+from pycocotools.coco import COCO
+from pycocotools.cocoeval import COCOeval
+from tqdm import tqdm
+
+import groundingdino.datasets.transforms as T
+from groundingdino.models import build_model
+from groundingdino.util.slconfig import SLConfig
+from groundingdino.util.utils import clean_state_dict, get_phrases_from_posmap
+from d_cube import D3
+
+
+def plot_boxes_to_image(image_pil, tgt):
+    H, W = tgt["size"]
+    boxes = tgt["boxes"]
+    labels = tgt["labels"]
+    assert len(boxes) == len(labels), "boxes and labels must have same length"
+
+    draw = ImageDraw.Draw(image_pil)
+    mask = Image.new("L", image_pil.size, 0)
+    mask_draw = ImageDraw.Draw(mask)
+
+    # draw boxes and masks
+    for box, label in zip(boxes, labels):
+        # from 0..1 to 0..W, 0..H
+        box = box * torch.Tensor([W, H, W, H])
+        # from xywh to xyxy
+        box[:2] -= box[2:] / 2
+        box[2:] += box[:2]
+        # random color
+        color = tuple(np.random.randint(0, 255, size=3).tolist())
+        # draw
+        x0, y0, x1, y1 = box
+        x0, y0, x1, y1 = int(x0), int(y0), int(x1), int(y1)
+
+        draw.rectangle([x0, y0, x1, y1], outline=color, width=6)
+        # draw.text((x0, y0), str(label), fill=color)
+
+        font = ImageFont.load_default()
+        if hasattr(font, "getbbox"):
+            bbox = draw.textbbox((x0, y0), str(label), font)
+        else:
+            w, h = draw.textsize(str(label), font)
+            bbox = (x0, y0, w + x0, y0 + h)
+        # bbox = draw.textbbox((x0, y0), str(label))
+        draw.rectangle(bbox, fill=color)
+        draw.text((x0, y0), str(label), fill="white")
+
+        mask_draw.rectangle([x0, y0, x1, y1], fill=255, width=6)
+    return image_pil, mask
+
+
+def load_image(image_path):
+    # load image
+    image_pil = Image.open(image_path).convert("RGB")  # load image
+
+    transform = T.Compose(
+        [
+            T.RandomResize([800], max_size=1333),
+            T.ToTensor(),
+            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
+        ]
+    )
+    image, _ = transform(image_pil, None)  # 3, h, w
+    return image_pil, image
+
+
+def load_model(model_config_path, model_checkpoint_path, cpu_only=False):
+    args = SLConfig.fromfile(model_config_path)
+    args.device = "cuda" if not cpu_only else "cpu"
+    model = build_model(args)
+    checkpoint = torch.load(model_checkpoint_path, map_location="cpu")
+    load_res = model.load_state_dict(clean_state_dict(checkpoint["model"]), strict=False)
+    print(load_res)
+    _ = model.eval()
+    return model
+
+
+def get_grounding_output(model, image, caption, box_threshold, text_threshold, with_logits=True, cpu_only=False):
+    caption = caption.lower()
+    caption = caption.strip()
+    if not caption.endswith("."):
+        caption = caption + "."
+    device = "cuda" if not cpu_only else "cpu"
+    model = model.to(device)
+    image = image.to(device)
+    with torch.no_grad():
+        outputs = model(image[None], captions=[caption])
+    logits = outputs["pred_logits"].cpu().sigmoid()[0]  # (nq, 256)
+    boxes = outputs["pred_boxes"].cpu()[0]  # (nq, 4)
+    logits.shape[0]
+
+    # filter output
+    logits_filt = logits.clone()
+    boxes_filt = boxes.clone()
+    filt_mask = logits_filt.max(dim=1)[0] > box_threshold
+    logits_filt = logits_filt[filt_mask]  # num_filt, 256
+    boxes_filt = boxes_filt[filt_mask]  # num_filt, 4
+    logits_filt.shape[0]
+
+    # get phrase
+    tokenlizer = model.tokenizer
+    tokenized = tokenlizer(caption)
+    # build pred
+    pred_phrases = []
+    logits_list = []
+    for logit, box in zip(logits_filt, boxes_filt):
+        pred_phrase = get_phrases_from_posmap(logit > text_threshold, tokenized, tokenlizer)
+        logits_list.append(logit.max().item())
+        if with_logits:
+            pred_phrases.append(pred_phrase + f"({str(logit.max().item())[:4]})")
+        else:
+            pred_phrases.append(pred_phrase)
+
+    return boxes_filt, pred_phrases, logits_list
+
+
+def get_dataset_iter(coco):
+    img_ids = coco.get_img_ids()
+    for img_id in img_ids:
+        img_info = coco.load_imgs(img_id)[0]
+        file_name = img_info["file_name"]
+        img_path = os.path.join(IMG_ROOT, file_name)
+        yield img_id, img_path
+
+
+def eval_on_d3(pred_path, mode="pn"):
+    assert mode in ("pn", "p", "n")
+    if mode == "pn":
+        gt_path = os.path.join(JSON_ANNO_PATH, "d3_full_annotations.json")
+    elif mode == "p":
+        gt_path = os.path.join(JSON_ANNO_PATH, "d3_pres_annotations.json")
+    else:
+        gt_path = os.path.join(JSON_ANNO_PATH, "d3_abs_annotations.json")
+    coco = COCO(gt_path)
+    d3_res = coco.loadRes(pred_path)
+    cocoEval = COCOeval(coco, d3_res, "bbox")
+    cocoEval.evaluate()
+    cocoEval.accumulate()
+    cocoEval.summarize()
+
+    # comment the following if u only need intra/inter map for full/pres/abs
+    # ===================== uncomment this if u need detailed analysis =====================
+    # aps = cocoEval.eval["precision"][:, :, :, 0, -1]
+    # category_ids = coco.getCatIds()
+    # category_names = [cat["name"] for cat in coco.loadCats(category_ids)]
+
+    # aps_lens = defaultdict(list)
+    # counter_lens = defaultdict(int)
+    # for i in range(len(category_names)):
+    #     ap = aps[:, :, i]
+    #     ap_value = ap[ap > -1].mean()
+    #     if not np.isnan(ap_value):
+    #         len_ref = len(category_names[i].split(" "))
+    #         aps_lens[len_ref].append(ap_value)
+    #         counter_lens[len_ref] += 1
+
+    # ap_sum_short = sum([sum(aps_lens[i]) for i in range(0, 4)])
+    # ap_sum_mid = sum([sum(aps_lens[i]) for i in range(4, 7)])
+    # ap_sum_long = sum([sum(aps_lens[i]) for i in range(7, 10)])
+    # ap_sum_very_long = sum(
+    #     [sum(aps_lens[i]) for i in range(10, max(counter_lens.keys()) + 1)]
+    # )
+    # c_sum_short = sum([counter_lens[i] for i in range(1, 4)])
+    # c_sum_mid = sum([counter_lens[i] for i in range(4, 7)])
+    # c_sum_long = sum([counter_lens[i] for i in range(7, 10)])
+    # c_sum_very_long = sum(
+    #     [counter_lens[i] for i in range(10, max(counter_lens.keys()) + 1)]
+    # )
+    # map_short = ap_sum_short / c_sum_short
+    # map_mid = ap_sum_mid / c_sum_mid
+    # map_long = ap_sum_long / c_sum_long
+    # map_very_long = ap_sum_very_long / c_sum_very_long
+    # print(
+    #     f"mAP over reference length: short - {map_short:.4f}, mid - {map_mid:.4f}, long - {map_long:.4f}, very long - {map_very_long:.4f}"
+    # )
+    # ===================== uncomment this if u need detailed analysis =====================
+
+
+def inference_on_d3(data_iter, model, args, box_threshold, text_threshold):
+    pred = []
+    for idx, (img_id, image_path) in enumerate(tqdm(data_iter)):
+        # load image
+        image_pil, image = load_image(image_path)
+        size = image_pil.size
+        W, H = size
+
+        group_ids = d3.get_group_ids(img_ids=[img_id])
+        sent_ids = d3.get_sent_ids(group_ids=group_ids)
+        sent_list = d3.load_sents(sent_ids=sent_ids)
+        text_list = [sent['raw_sent'] for sent in sent_list]
+
+        for sent_id, text_prompt in zip(sent_ids, text_list):
+            # run model
+            boxes_filt, pred_phrases, logit_list = get_grounding_output(
+                model, image, text_prompt, box_threshold, text_threshold, cpu_only=args.cpu_only, with_logits=False,
+            )
+            if args.vis:
+                pred_dict = {
+                    "boxes": boxes_filt,  # [x_center, y_center, w, h]
+                    "size": [size[1], size[0]],
+                    "labels": [f"{phrase}({str(logit)[:4]})" for phrase, logit in zip(pred_phrases, logit_list)],
+                }
+                image_with_box = plot_boxes_to_image(image_pil.copy(), pred_dict)[0]
+                image_with_box.save(os.path.join(output_dir, f"{img_id}_{text_prompt}.jpg"))
+            if not logit_list:
+                continue
+            if args.img_top1:
+                max_score_idx = logit_list.index(max(logit_list))
+                bboxes, phrases, logits = [boxes_filt[max_score_idx]], [pred_phrases[max_score_idx]], [logit_list[max_score_idx]]
+            else:
+                bboxes, phrases, logits = boxes_filt, pred_phrases, logit_list
+            for box, phrase, logit in zip(bboxes, phrases, logits):
+                if len(phrase) > args.overlap_percent * len(text_prompt) or phrase == text_prompt:
+                    x1, y1, w, h = box.tolist()
+                    x0, y0 = x1 - w / 2, y1 - h / 2
+                    pred_item = {
+                        "image_id": img_id,
+                        "category_id": sent_id,
+                        "bbox": [x0 * W, y0 * H, w * W, h * H],
+                        "score": float(logit),
+                    }
+                    pred.append(pred_item)
+
+    return pred
+
+
+if __name__ == "__main__":
+    IMG_ROOT = None  # set here
+    JSON_ANNO_PATH = None  # set here
+    PKL_ANNO_PATH = None  # set here
+    assert IMG_ROOT is not None, "Please set IMG_ROOT in the script first"
+    assert JSON_ANNO_PATH is not None, "Please set JSON_ANNO_PATH in the script first"
+    assert PKL_ANNO_PATH is not None, "Please set PKL_ANNO_PATH in the script first"
+
+    d3 = D3(IMG_ROOT, PKL_ANNO_PATH)
+
+    parser = argparse.ArgumentParser("Grounding DINO evaluation on D-cube (https://arxiv.org/abs/2307.12813)", add_help=True)
+    parser.add_argument("--config_file", "-c", type=str, required=True, help="path to config file")
+    parser.add_argument(
+        "--checkpoint_path", "-p", type=str, required=True, help="path to checkpoint file"
+    )
+    # parser.add_argument("--image_path", "-i", type=str, required=True, help="path to image file")
+    # parser.add_argument("--text_prompt", "-t", type=str, required=True, help="text prompt")
+    parser.add_argument(
+        "--output_dir", "-o", type=str, default="outputs", required=True, help="output directory"
+    )
+    parser.add_argument("--vis", action="store_true", help="visualization on D3")
+
+    parser.add_argument("--box_threshold", type=float, default=0.3, help="box threshold")
+    parser.add_argument("--text_threshold", type=float, default=0.25, help="text threshold")
+
+    parser.add_argument("--cpu-only", action="store_true", help="running on cpu only!, default=False")
+    parser.add_argument("--img-top1", action="store_true", help="select only the box with top max score")
+    # parser.add_argument("--overlap-percent", type=float, default=1.0, help="overlapping percentage between input prompt and output label")
+    # this overlapping percentage denotes an additional post-processing technique we designed. if you turn this on, you may get higher performance by tuning this parameter.
+    args = parser.parse_args()
+    args.overlap_percent = 1  # by default, we do not use this technique.
+    print(args)
+
+    # cfg
+    config_file = args.config_file  # change the path of the model config file
+    checkpoint_path = args.checkpoint_path  # change the path of the model
+    # image_path = args.image_path
+    # text_prompt = args.text_prompt
+    output_dir = args.output_dir
+    box_threshold = args.box_threshold
+    text_threshold = args.text_threshold
+
+    # make dir
+    os.makedirs(output_dir, exist_ok=True)
+    # load model
+    model = load_model(config_file, checkpoint_path, cpu_only=args.cpu_only)
+
+    data_iter = get_dataset_iter(d3)
+
+    pred = inference_on_d3(data_iter, model, args, box_threshold=box_threshold, text_threshold=text_threshold)
+
+    pred_path = os.path.join(output_dir, f"prediction.json")
+    with open(pred_path, "w") as f_:
+        json.dump(pred, f_)
+    eval_on_d3(pred_path, mode='pn')
+    eval_on_d3(pred_path, mode='p')
+    eval_on_d3(pred_path, mode='n')

approach/ovod/d-cube/eval_sota/owl_vit.py

@@ -0,0 +1,192 @@
+import json
+import os
+from collections import defaultdict
+
+from tqdm import tqdm
+from PIL import Image
+import numpy as np
+from pycocotools.coco import COCO
+from pycocotools.cocoeval import COCOeval
+import torch
+from transformers import OwlViTProcessor, OwlViTForObjectDetection
+
+from d_cube import D3
+
+
+def write_json(json_path, json_data):
+    with open(json_path, "w") as f_:
+        json.dump(json_data, f_)
+
+
+def read_json(json_path):
+    with open(json_path, "r") as f_:
+        json_data = json.load(f_)
+    return json_data
+
+
+def load_image_general(image_path):
+    image_pil = Image.open(image_path)
+    return image_pil
+
+
+def get_prediction(model, image, captions, cpu_only=False):
+    for i in range(len(captions)):
+        captions[i] = captions[i].lower()
+        captions[i] = captions[i].strip()
+        if not captions[i].endswith("."):
+            captions[i] = captions[i] + "."
+    device = "cuda" if not cpu_only else "cpu"
+    model = model.to(device)
+    with torch.no_grad():
+        inputs = processor(text=[captions], images=image, return_tensors="pt").to(
+            device
+        )
+        outputs = model(**inputs)
+    target_size = torch.Tensor([image.size[::-1]]).to(device)
+    results = processor.post_process_object_detection(
+        outputs=outputs, target_sizes=target_size, threshold=0.1
+        # the post precessing threshold will affect the performance obviously
+        # you may tune it to get better performance, e.g., 0.05
+    )
+    boxes, scores, labels = (
+        results[0]["boxes"],
+        results[0]["scores"],
+        results[0]["labels"],
+    )
+    return boxes, scores, labels
+
+
+def get_dataset_iter(coco):
+    img_ids = coco.get_img_ids()
+    for img_id in img_ids:
+        img_info = coco.load_imgs(img_id)[0]
+        file_name = img_info["file_name"]
+        img_path = os.path.join(IMG_ROOT, file_name)
+        yield img_id, img_path
+
+
+def eval_on_d3(pred_path, mode="pn"):
+    assert mode in ("pn", "p", "n")
+    if mode == "pn":
+        gt_path = os.path.join(JSON_ANNO_PATH, "d3_full_annotations.json")
+    elif mode == "p":
+        gt_path = os.path.join(JSON_ANNO_PATH, "d3_pres_annotations.json")
+    else:
+        gt_path = os.path.join(JSON_ANNO_PATH, "d3_abs_annotations.json")
+    coco = COCO(gt_path)
+    d3_res = coco.loadRes(pred_path)
+    cocoEval = COCOeval(coco, d3_res, "bbox")
+    cocoEval.evaluate()
+    cocoEval.accumulate()
+    cocoEval.summarize()
+
+    # comment the following if u only need intra/inter map for full/pres/abs
+    # ===================== uncomment this if u need detailed analysis =====================
+    # aps = cocoEval.eval["precision"][:, :, :, 0, -1]
+    # category_ids = coco.getCatIds()
+    # category_names = [cat["name"] for cat in coco.loadCats(category_ids)]
+
+    # aps_lens = defaultdict(list)
+    # counter_lens = defaultdict(int)
+    # for i in range(len(category_names)):
+    #     ap = aps[:, :, i]
+    #     ap_value = ap[ap > -1].mean()
+    #     if not np.isnan(ap_value):
+    #         len_ref = len(category_names[i].split(" "))
+    #         aps_lens[len_ref].append(ap_value)
+    #         counter_lens[len_ref] += 1
+
+    # ap_sum_short = sum([sum(aps_lens[i]) for i in range(0, 4)])
+    # ap_sum_mid = sum([sum(aps_lens[i]) for i in range(4, 7)])
+    # ap_sum_long = sum([sum(aps_lens[i]) for i in range(7, 10)])
+    # ap_sum_very_long = sum(
+    #     [sum(aps_lens[i]) for i in range(10, max(counter_lens.keys()) + 1)]
+    # )
+    # c_sum_short = sum([counter_lens[i] for i in range(1, 4)])
+    # c_sum_mid = sum([counter_lens[i] for i in range(4, 7)])
+    # c_sum_long = sum([counter_lens[i] for i in range(7, 10)])
+    # c_sum_very_long = sum(
+    #     [counter_lens[i] for i in range(10, max(counter_lens.keys()) + 1)]
+    # )
+    # map_short = ap_sum_short / c_sum_short
+    # map_mid = ap_sum_mid / c_sum_mid
+    # map_long = ap_sum_long / c_sum_long
+    # map_very_long = ap_sum_very_long / c_sum_very_long
+    # print(
+    #     f"mAP over reference length: short - {map_short:.4f}, mid - {map_mid:.4f}, long - {map_long:.4f}, very long - {map_very_long:.4f}"
+    # )
+    # ===================== uncomment this if u need detailed analysis =====================
+
+
+def inference_on_d3(data_iter, model):
+    pred = []
+    error = []
+    for img_id, image_path in tqdm(data_iter):
+        image = load_image_general(image_path)
+
+        # ==================================== intra-group setting ==================================== 
+        # each image is evaluated with the categories in its group (usually 4)
+        group_ids = d3.get_group_ids(img_ids=[img_id])
+        sent_ids = d3.get_sent_ids(group_ids=group_ids)
+        # ==================================== intra-group setting ====================================
+        # ==================================== inter-group setting ====================================
+        # each image is evaluated with all categories in the dataset (422 for the first version of the dataset)
+        # sent_ids = d3.get_sent_ids()
+        # ==================================== inter-group setting ====================================
+        sent_list = d3.load_sents(sent_ids=sent_ids)
+        text_list = [sent["raw_sent"] for sent in sent_list]
+
+        try:
+            boxes, scores, labels = get_prediction(model, image, text_list, cpu_only=False)
+            for box, score, label in zip(boxes, scores, labels):
+                pred_item = {
+                    "image_id": img_id,
+                    "category_id": sent_ids[label],
+                    "bbox": convert_to_xywh(box.tolist()),  # use xywh
+                    "score": float(score),
+                }
+                pred.append(pred_item)  # the output to be saved to JSON.
+        except:
+            print("error!!!")
+    return pred, error
+
+
+def convert_to_xywh(bbox_xyxy):
+    """
+    Convert top-left and bottom-right corner coordinates to [x, y, width, height] format.
+    """
+    x1, y1, x2, y2 = bbox_xyxy
+    width = x2 - x1
+    height = y2 - y1
+    return [x1, y1, width, height]
+
+
+if __name__ == "__main__":
+    IMG_ROOT = None  # set here
+    JSON_ANNO_PATH = None  # set here
+    PKL_ANNO_PATH = None  # set here
+    assert IMG_ROOT is not None, "Please set IMG_ROOT in the script first"
+    assert JSON_ANNO_PATH is not None, "Please set JSON_ANNO_PATH in the script first"
+    assert PKL_ANNO_PATH is not None, "Please set PKL_ANNO_PATH in the script first"
+
+    d3 = D3(IMG_ROOT, PKL_ANNO_PATH)
+
+    output_dir = "ovd/owlvit/"
+    os.makedirs(output_dir, exist_ok=True)
+
+    # model prediction
+    processor = OwlViTProcessor.from_pretrained("owl-vit")
+    model = OwlViTForObjectDetection.from_pretrained("owl-vit")
+    data_iter = get_dataset_iter(d3)
+    pred, error = inference_on_d3(data_iter, model)
+
+    pred_path = os.path.join(output_dir, f"prediction.json")
+    pred_path_error = os.path.join(output_dir, "error.json")
+    write_json(pred_path, pred)
+    write_json(pred_path_error, error)
+    # see https://github.com/shikras/d-cube/blob/main/doc.md#output-format for the output format
+    # the output format is identical to COCO.
+
+    eval_on_d3(pred_path, mode="pn")  # the FULL setting
+    eval_on_d3(pred_path, mode="p")  # the PRES setting
+    eval_on_d3(pred_path, mode="n")  # the ABS setting

approach/ovod/d-cube/eval_sota/sphinx.py

@@ -0,0 +1,184 @@
+# -*- coding: utf-8 -*-
+__author__ = "Chi Xie and Jie Li"
+__maintainer__ = "Chi Xie"
+
+import json
+import os
+from collections import defaultdict
+import re
+
+from PIL import Image
+from pycocotools.coco import COCO
+from pycocotools.cocoeval import COCOeval
+
+from d_cube import D3
+
+
+def write_json(json_path, json_data):
+    with open(json_path, "w") as f_:
+        json.dump(json_data, f_)
+
+
+def read_json(json_path):
+    with open(json_path, "r") as f_:
+        json_data = json.load(f_)
+    return json_data
+
+
+def load_image_general(image_path):
+    image_pil = Image.open(image_path)
+    return image_pil
+
+
+def extract_boxes(input_string):
+    # if input_string.startswith("None"):
+    #     return []
+    # Define the pattern using regular expression
+    pattern = r'\[([\d.,; ]+)\]'
+    
+    # Search for the pattern in the input string
+    match = re.search(pattern, input_string)
+    
+    # If a match is found, extract and return the boxes as a list
+    if match:
+        boxes_str = match.group(1)
+        boxes_list = [list(map(float, box.split(','))) for box in boxes_str.split(';')]
+        return boxes_list
+    else:
+        return []
+
+
+def get_prediction(mllm_res, image, captions, cpu_only=False):
+    boxes, scores, labels = [], [], []
+    width, height = image.size
+    for idx, res_item in enumerate(mllm_res):
+        boxes_list = extract_boxes(res_item["answer"])
+        for bbox in boxes_list:
+            bbox_rescaled = get_true_bbox(image.size, bbox)
+            boxes.append(bbox_rescaled)
+            scores.append(1.0)
+            labels.append(idx)
+    return boxes, scores, labels
+
+
+def get_dataset_iter(coco):
+    img_ids = coco.get_img_ids()
+    for img_id in img_ids:
+        img_info = coco.load_imgs(img_id)[0]
+        file_name = img_info["file_name"]
+        img_path = os.path.join(IMG_ROOT, file_name)
+        yield img_id, file_name, img_path
+
+
+def eval_on_d3(pred_path, mode="pn"):
+    assert mode in ("pn", "p", "n")
+    if mode == "pn":
+        gt_path = os.path.join(JSON_ANNO_PATH, "d3_full_annotations.json")
+    elif mode == "p":
+        gt_path = os.path.join(JSON_ANNO_PATH, "d3_pres_annotations.json")
+    else:
+        gt_path = os.path.join(JSON_ANNO_PATH, "d3_abs_annotations.json")
+    coco = COCO(gt_path)
+    d3_res = coco.loadRes(pred_path)
+    cocoEval = COCOeval(coco, d3_res, "bbox")
+    cocoEval.evaluate()
+    cocoEval.accumulate()
+    cocoEval.summarize()
+
+
+def group_sphinx_res_by_img(inference_res):
+    inference_res_by_img = defaultdict(list)
+    for res_item in inference_res:
+        img_path = "/".join(res_item["image_path"].split("/")[-2:])
+        inference_res_by_img[img_path].append(res_item)
+    inference_res_by_img = dict(inference_res_by_img)
+    return inference_res_by_img
+
+
+def get_true_bbox(img_size, bbox):
+    width, height = img_size
+    max_edge = max(height, width)
+    bbox = [v * max_edge for v in bbox]
+    diff = abs(width - height) // 2
+    if height < width:
+        bbox[1] -= diff
+        bbox[3] -= diff
+    else:
+        bbox[0] -= diff
+        bbox[2] -= diff
+    return bbox
+
+
+def inference_on_d3(data_iter, inference_res):
+    pred = []
+    inf_res_by_img = group_sphinx_res_by_img(inference_res)
+    for idx, (img_id, img_name, img_path) in enumerate(data_iter):
+        image = load_image_general(img_path)
+
+        # ==================================== intra-group setting ==================================== 
+        # each image is evaluated with the categories in its group (usually 4)
+        group_ids = d3.get_group_ids(img_ids=[img_id])
+        sent_ids = d3.get_sent_ids(group_ids=group_ids)
+        # ==================================== intra-group setting ====================================
+        # ==================================== inter-group setting ====================================
+        # each image is evaluated with all categories in the dataset (422 for the first version of the dataset)
+        # sent_ids = d3.get_sent_ids()
+        # ==================================== inter-group setting ====================================
+        sent_list = d3.load_sents(sent_ids=sent_ids)
+        text_list = [sent["raw_sent"] for sent in sent_list]
+
+        boxes, scores, labels = get_prediction(inf_res_by_img[img_name], image, text_list, cpu_only=False)
+        for box, score, label in zip(boxes, scores, labels):
+            pred_item = {
+                "image_id": img_id,
+                "category_id": sent_ids[label],
+                "bbox": convert_to_xywh(box),  # use xywh
+                "score": float(score),
+            }
+            pred.append(pred_item)  # the output to be saved to JSON.
+    return pred
+
+
+def convert_to_xywh(bbox_xyxy):
+    """
+    Convert top-left and bottom-right corner coordinates to [x, y, width, height] format.
+    """
+    x1, y1, x2, y2 = bbox_xyxy
+    width = x2 - x1
+    height = y2 - y1
+    return [x1, y1, width, height]
+
+
+if __name__ == "__main__":
+    IMG_ROOT = None  # set here
+    JSON_ANNO_PATH = None  # set here
+    PKL_ANNO_PATH = None  # set here
+    # ============================== SPHINX inference result file ===============
+    SPHINX_INFERENCE_RES_PATH = None
+    # You can download the SPHINX d3 inference result example from:
+    # https://github.com/shikras/d-cube/files/14276682/sphinx_d3_result.json
+    # For the inference process, please refer to SPHINX official repo (https://github.com/Alpha-VLLM/LLaMA2-Accessory)
+    # the prompts we used are available in this JSON file
+    # Thanks for the contribution from Jie Li (https://github.com/theFool32)
+    # ============================== SPHINX inference result file ===============
+    assert IMG_ROOT is not None, "Please set IMG_ROOT in the script first"
+    assert JSON_ANNO_PATH is not None, "Please set JSON_ANNO_PATH in the script first"
+    assert PKL_ANNO_PATH is not None, "Please set PKL_ANNO_PATH in the script first"
+
+    d3 = D3(IMG_ROOT, PKL_ANNO_PATH)
+
+    output_dir = "mllm/sphinx/"  # or whatever you prefer
+    inference_res = read_json(SPHINX_INFERENCE_RES_PATH)
+
+    # model prediction
+    data_iter = get_dataset_iter(d3)
+    pred = inference_on_d3(data_iter, inference_res)
+
+    pred_path = os.path.join(output_dir, f"prediction.json")
+    write_json(pred_path, pred)
+    # see https://github.com/shikras/d-cube/blob/main/doc.md#output-format for the output format
+    # the output format is identical to COCO.
+
+    eval_on_d3(pred_path, mode="pn")  # the FULL setting
+    eval_on_d3(pred_path, mode="p")  # the PRES setting
+    eval_on_d3(pred_path, mode="n")  # the ABS setting

approach/ovod/d-cube/scripts/eval_and_analysis_json.py

@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+__author__ = "Chi Xie and Zhao Zhang"
+__maintainer__ = "Chi Xie"
+# this script takes the result json in, and print evaluation and analysis result on D-cube (FULL/PRES/ABS, etc.)
+import os
+import json
+import argparse
+from collections import defaultdict
+
+import numpy as np
+from pycocotools.coco import COCO
+from pycocotools.cocoeval import COCOeval
+
+from d_cube import D3
+
+def eval_on_d3(pred_path, mode="pn", nbox_partition=None, lref_partition=False):
+    assert mode in ("pn", "p", "n")
+    if mode == "pn":
+        gt_path = os.path.join(JSON_ANNO_PATH, "d3_full_annotations.json")
+    elif mode == "p":
+        gt_path = os.path.join(JSON_ANNO_PATH, "d3_pres_annotations.json")
+    else:
+        gt_path = os.path.join(JSON_ANNO_PATH, "d3_abs_annotations.json")
+
+    if nbox_partition:
+        gt_path, pred_path = nbox_partition_json(gt_path, pred_path, nbox_partition)
+
+    # Eval results
+    coco = COCO(gt_path)
+    d3_res = coco.loadRes(pred_path)
+    cocoEval = COCOeval(coco, d3_res, "bbox")
+    cocoEval.evaluate()
+    cocoEval.accumulate()
+    cocoEval.summarize()
+
+    aps = cocoEval.eval["precision"][:, :, :, 0, -1]
+    category_ids = coco.getCatIds()
+    category_names = [cat["name"] for cat in coco.loadCats(category_ids)]
+
+    if lref_partition:
+        aps_lens = defaultdict(list)
+        counter_lens = defaultdict(int)
+        for i in range(len(category_names)):
+            ap = aps[:, :, i]
+            ap_value = ap[ap > -1].mean()
+            if not np.isnan(ap_value):
+                len_ref = len(category_names[i].split(" "))
+                aps_lens[len_ref].append(ap_value)
+                counter_lens[len_ref] += 1
+
+        ap_sum_short = sum([sum(aps_lens[i]) for i in range(0, 4)])
+        ap_sum_mid = sum([sum(aps_lens[i]) for i in range(4, 7)])
+        ap_sum_long = sum([sum(aps_lens[i]) for i in range(7, 10)])
+        ap_sum_very_long = sum(
+            [sum(aps_lens[i]) for i in range(10, max(counter_lens.keys()) + 1)]
+        )
+        c_sum_short = sum([counter_lens[i] for i in range(1, 4)])
+        c_sum_mid = sum([counter_lens[i] for i in range(4, 7)])
+        c_sum_long = sum([counter_lens[i] for i in range(7, 10)])
+        c_sum_very_long = sum(
+            [counter_lens[i] for i in range(10, max(counter_lens.keys()) + 1)]
+        )
+        map_short = ap_sum_short / c_sum_short
+        map_mid = ap_sum_mid / c_sum_mid
+        map_long = ap_sum_long / c_sum_long
+        map_very_long = ap_sum_very_long / c_sum_very_long
+        print(
+            f"mAP over reference length: short - {map_short:.4f}, mid - {map_mid:.4f}, long - {map_long:.4f}, very long - {map_very_long:.4f}"
+        )
+
+
+def nbox_partition_json(gt_path, pred_path, nbox_partition):
+    with open(gt_path, "r") as f_gt:
+        gts = json.load(f_gt)
+    with open(pred_path, "r") as f_pred:
+        preds = json.load(f_pred)
+
+    cat_obj_count = d3.bbox_num_analyze()
+    annos = gts["annotations"]
+    new_annos = []
+    for ann in annos:
+        img_id = ann["image_id"]
+        category_id = ann["category_id"]
+        if nbox_partition == "one" and cat_obj_count[category_id - 1, img_id] == 1:
+            new_annos.append(ann)
+        if nbox_partition == "multi" and cat_obj_count[category_id - 1, img_id] > 1:
+            new_annos.append(ann)
+        if nbox_partition == "two" and cat_obj_count[category_id - 1, img_id] == 2:
+            new_annos.append(ann)
+        if nbox_partition == "three" and cat_obj_count[category_id - 1, img_id] == 3:
+            new_annos.append(ann)
+        if nbox_partition == "four" and cat_obj_count[category_id - 1, img_id] == 4:
+            new_annos.append(ann)
+        if nbox_partition == "four_more" and cat_obj_count[category_id - 1, img_id] > 4:
+            new_annos.append(ann)
+    gts["annotations"] = new_annos
+    new_gts = gts
+    new_preds = []
+    for prd in preds:
+        img_id = prd["image_id"]
+        category_id = prd["category_id"]
+        if nbox_partition == "no" and cat_obj_count[category_id - 1, img_id] == 0:
+            new_preds.append(prd)
+        if nbox_partition == "one" and cat_obj_count[category_id - 1, img_id] == 1:
+            new_preds.append(prd)
+        if nbox_partition == "multi" and cat_obj_count[category_id - 1, img_id] > 1:
+            new_preds.append(prd)
+        if nbox_partition == "two" and cat_obj_count[category_id - 1, img_id] == 2:
+            new_preds.append(prd)
+        if nbox_partition == "three" and cat_obj_count[category_id - 1, img_id] == 3:
+            new_preds.append(prd)
+        if nbox_partition == "four" and cat_obj_count[category_id - 1, img_id] == 4:
+            new_preds.append(prd)
+        if nbox_partition == "four_more" and cat_obj_count[category_id - 1, img_id] > 4:
+            new_preds.append(prd)
+
+    new_gt_path = gt_path.replace(".json", f".{nbox_partition}-instance.json")
+    new_pred_path = pred_path.replace(".json", f".{nbox_partition}-instance.json")
+    with open(new_gt_path, "w") as fo_gt:
+        json.dump(new_gts, fo_gt)
+    with open(new_pred_path, "w") as fo_pred:
+        json.dump(new_preds, fo_pred)
+    return new_gt_path, new_pred_path
+
+
+def convert_to_xywh(x1, y1, x2, y2):
+    """
+    Convert top-left and bottom-right corner coordinates to [x,y,width,height] format.
+    """
+    width = x2 - x1
+    height = y2 - y1
+    return x1, y1, width, height
+
+
+def transform_json_boxes(pred_path):
+    with open(pred_path, "r") as f_:
+        res = json.load(f_)
+    for item in res:
+        item["bbox"] = convert_to_xywh(*item["bbox"])
+    res_path = pred_path.replace(".json", ".xywh.json")
+    with open(res_path, "w") as f_w:
+        json.dump(res, f_w)
+    return res_path
+
+
+if __name__ == "__main__":
+    D3_DATASET_ROOT = os.environ.get("D3_DATASET_ROOT")
+    if not D3_DATASET_ROOT:
+        raise RuntimeError("Set D3_DATASET_ROOT to the extracted D3 dataset directory.")
+    IMG_ROOT = D3_DATASET_ROOT
+    JSON_ANNO_PATH = os.path.join(D3_DATASET_ROOT, "d3_json")
+    PKL_ANNO_PATH = os.path.join(D3_DATASET_ROOT, "d3_pkl")
+    d3 = D3(IMG_ROOT, PKL_ANNO_PATH)
+
+    parser = argparse.ArgumentParser(
+        "An example script for D-cube evaluation with prediction file (JSON)",
+        add_help=True,
+    )
+    parser.add_argument("pred_path", type=str, help="path to the prediction JSON file")
+    parser.add_argument(
+        "--partition-by-nbox",
+        action="store_true",
+        help="divide the images by num of boxes for each ref",
+    )
+    parser.add_argument(
+        "--partition-by-lens",
+        action="store_true",
+        help="divide the references by their lengths",
+    )
+    parser.add_argument(
+        "--xyxy2xywh",
+        action="store_true",
+        help="transform box coords from xyxy to xywh",
+    )
+    args = parser.parse_args()
+    if args.xyxy2xywh:
+        pred_path = transform_json_boxes(args.pred_path)
+    else:
+        pred_path = args.pred_path
+    pred_path = args.pred_path
+    if args.partition_by_nbox:
+        # partiton: no-instance, one-instance, multi-instance
+        for mode in ("pn", "p", "n"):
+            # for ptt in ('no', 'one', 'multi'):
+            for ptt in ("no", "one", "two", "three", "four", "four_more"):
+                eval_on_d3(pred_path, mode=mode, nbox_partition=ptt)
+    else:
+        eval_on_d3(pred_path, mode="pn", lref_partition=args.partition_by_lens)
+        eval_on_d3(pred_path, mode="p", lref_partition=args.partition_by_lens)
+        eval_on_d3(pred_path, mode="n", lref_partition=args.partition_by_lens)

approach/ovod/d-cube/scripts/eval_json_example.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+__author__ = "Chi Xie and Zhao Zhang"
+__maintainer__ = "Chi Xie"
+# this script takes the result json in, and print evaluation and analysis result on D-cube (FULL/PRES/ABS, etc.)
+from pycocotools.coco import COCO
+from pycocotools.cocoeval import COCOeval
+
+# Eval results with COCOAPI
+gt_path = "./d3_full_annotations.json"  # FULL, PRES or ABS
+pred_path = None  # set your prediction JSON path
+coco = COCO(gt_path)
+d3_res = coco.loadRes(pred_path)
+cocoEval = COCOeval(coco, d3_res, "bbox")
+cocoEval.evaluate()
+cocoEval.accumulate()
+cocoEval.summarize()

approach/ovod/d-cube/scripts/get_d3_stat.py

@@ -0,0 +1,98 @@
+import numpy as np
+
+from d_cube.vis_util import plot_hist
+from d_cube import D3
+
+
+def vis_num_instance(cat_obj_count):
+    # Assuming `cat_obj_count` is your numpy array of shape [n_cat, n_img]
+
+    # Calculate the total number of instances in each image
+    total_instances_per_image = np.sum(cat_obj_count, axis=0)
+
+    # # Plot the histogram
+    # plt.hist(total_instances_per_image, bins=20)
+    # plt.xlabel('Number of Instances')
+    # plt.ylabel('Frequency')
+    # plt.title('Distribution of Number of Instances on a Image')
+
+    # # Save the figure
+    # plt.savefig('vis_fig/instance_distribution.png', bbox_inches='tight')
+    # plt.close()
+    plot_hist(
+        total_instances_per_image,
+        bins=max(total_instances_per_image) - min(total_instances_per_image) + 1,
+        save_path="vis_fig/instance_dist_hist.pdf",
+    )
+
+
+def vis_num_category(cat_obj_count):
+    # Assuming `cat_obj_count` is your numpy array of shape [n_cat, n_img]
+
+    # Calculate the number of categories in each image
+    num_categories_per_image = np.sum(cat_obj_count > 0, axis=0)
+
+    # # Plot the histogram
+    # plt.hist(num_categories_per_image, bins=20)
+    # plt.xlabel('Number of Categories')
+    # plt.ylabel('Frequency')
+    # plt.title('Distribution of Number of Categories on a Image')
+
+    # # Save the figure
+    # plt.savefig('vis_fig/category_distribution.png', bbox_inches='tight')
+    # plt.close()
+    plot_hist(
+        num_categories_per_image,
+        bins=max(num_categories_per_image) - min(num_categories_per_image) + 1,
+        save_path="vis_fig/category_dist_hist.pdf",
+    )
+
+
+def vis_num_img_per_cat(cat_obj_count):
+    num_img_per_cat = np.sum(cat_obj_count > 0, axis=1)
+    plot_hist(
+        num_img_per_cat,
+        bins=20,
+        save_path="vis_fig/nimg_pcat_hist.pdf",
+        x="Num. of images",
+    )
+
+
+def vis_num_box_per_cat(cat_obj_count):
+    num_box_per_cat = np.sum(cat_obj_count, axis=1)
+    plot_hist(
+        num_box_per_cat,
+        bins=20,
+        save_path="vis_fig/nbox_pcat_hist.pdf",
+        x="Num. of instances",
+    )
+
+
+def vis_num_box_per_cat_per_img(cat_obj_count):
+    img_obj_count = cat_obj_count.reshape(-1)
+    plot_hist(
+        img_obj_count[img_obj_count > 0],
+        bins=max(img_obj_count) - min(img_obj_count) + 1,
+        save_path="vis_fig/nbox_pcat_pimg_hist.pdf",
+        x="Num. of instances on a image",
+    )
+
+
+if __name__ == "__main__":
+    IMG_ROOT = None  # set here
+    PKL_ANNO_PATH = None  # set here
+    assert IMG_ROOT is not None, "Please set IMG_ROOT in the script first"
+    assert PKL_ANNO_PATH is not None, "Please set PKL_ANNO_PATH in the script first"
+    d3 = D3(IMG_ROOT, PKL_ANNO_PATH)
+
+    cat_obj_count = d3.bbox_num_analyze()
+    vis_num_instance(cat_obj_count)
+    vis_num_category(cat_obj_count)
+    vis_num_img_per_cat(cat_obj_count)
+    vis_num_box_per_cat(cat_obj_count)
+    vis_num_box_per_cat_per_img(cat_obj_count)
+
+    d3.stat_description(with_rev=False)
+    d3.stat_description(with_rev=True)
+    d3.stat_description(with_rev=False, inter_group=True)
+    d3.stat_description(with_rev=True, inter_group=True)

approach/ovod/detectron2/.circleci/config.yml

@@ -0,0 +1,270 @@
+version: 2.1
+
+# -------------------------------------------------------------------------------------
+# Environments to run the jobs in
+# -------------------------------------------------------------------------------------
+cpu: &cpu
+  machine:
+    image: ubuntu-2004:202107-02
+  resource_class: medium
+
+gpu: &gpu
+  machine:
+    # NOTE: use a cuda version that's supported by all our pytorch versions
+    image: ubuntu-1604-cuda-11.1:202012-01
+  resource_class: gpu.nvidia.small
+
+windows-cpu: &windows_cpu
+  machine:
+    resource_class: windows.medium
+    image: windows-server-2019-vs2019:stable
+    shell: powershell.exe
+
+# windows-gpu: &windows_gpu
+#     machine:
+#       resource_class: windows.gpu.nvidia.medium
+#       image: windows-server-2019-nvidia:stable
+
+version_parameters: &version_parameters
+  parameters:
+    pytorch_version:
+      type: string
+    torchvision_version:
+      type: string
+    pytorch_index:
+      type: string
+      # use test wheels index to have access to RC wheels
+      # https://download.pytorch.org/whl/test/torch_test.html
+      default: "https://download.pytorch.org/whl/torch_stable.html"
+    python_version:  # NOTE: only affect linux
+      type: string
+      default: '3.7.9'
+
+  environment:
+    PYTORCH_VERSION: << parameters.pytorch_version >>
+    TORCHVISION_VERSION: << parameters.torchvision_version >>
+    PYTORCH_INDEX: << parameters.pytorch_index >>
+    PYTHON_VERSION: << parameters.python_version>>
+    # point datasets to ~/.torch so it's cached in CI
+    DETECTRON2_DATASETS: ~/.torch/datasets
+
+# -------------------------------------------------------------------------------------
+# Re-usable commands
+# -------------------------------------------------------------------------------------
+# install_nvidia_driver: &install_nvidia_driver
+#   - run:
+#       name: Install nvidia driver
+#       working_directory: ~/
+#       command: |
+#         wget -q 'https://s3.amazonaws.com/ossci-linux/nvidia_driver/NVIDIA-Linux-x86_64-430.40.run'
+#         sudo /bin/bash ./NVIDIA-Linux-x86_64-430.40.run -s --no-drm
+#         nvidia-smi
+
+add_ssh_keys: &add_ssh_keys
+  # https://circleci.com/docs/2.0/add-ssh-key/
+  - add_ssh_keys:
+      fingerprints:
+        - "e4:13:f2:22:d4:49:e8:e4:57:5a:ac:20:2f:3f:1f:ca"
+
+install_python: &install_python
+  - run:
+      name: Install Python
+      working_directory: ~/
+      command: |
+        # upgrade pyenv
+        cd /opt/circleci/.pyenv/plugins/python-build/../.. && git pull && cd -
+        pyenv install -s $PYTHON_VERSION
+        pyenv global $PYTHON_VERSION
+        python --version
+        which python
+        pip install --upgrade pip
+
+setup_venv: &setup_venv
+  - run:
+      name: Setup Virtual Env
+      working_directory: ~/
+      command: |
+        python -m venv ~/venv
+        echo ". ~/venv/bin/activate" >> $BASH_ENV
+        . ~/venv/bin/activate
+        python --version
+        which python
+        which pip
+        pip install --upgrade pip
+
+setup_venv_win: &setup_venv_win
+  - run:
+      name: Setup Virtual Env for Windows
+      command: |
+        pip install virtualenv
+        python -m virtualenv env
+        .\env\Scripts\activate
+        python --version
+        which python
+        which pip
+
+install_linux_dep: &install_linux_dep
+  - run:
+      name: Install Dependencies
+      command: |
+        # disable crash coredump, so unittests fail fast
+        sudo systemctl stop apport.service
+        # install from github to get latest; install iopath first since fvcore depends on it
+        pip install --progress-bar off -U 'git+https://github.com/facebookresearch/iopath'
+        pip install --progress-bar off -U 'git+https://github.com/facebookresearch/fvcore'
+        # Don't use pytest-xdist: cuda tests are unstable under multi-process workers.
+        pip install --progress-bar off ninja opencv-python-headless pytest tensorboard pycocotools onnx
+        pip install --progress-bar off torch==$PYTORCH_VERSION -f $PYTORCH_INDEX
+        if [[ "$TORCHVISION_VERSION" == "master" ]]; then
+          pip install git+https://github.com/pytorch/vision.git
+        else
+          pip install --progress-bar off torchvision==$TORCHVISION_VERSION -f $PYTORCH_INDEX
+        fi
+
+        python -c 'import torch; print("CUDA:", torch.cuda.is_available())'
+        gcc --version
+
+install_detectron2: &install_detectron2
+  - run:
+      name: Install Detectron2
+      command: |
+        # Remove first, in case it's in the CI cache
+        pip uninstall -y detectron2
+
+        pip install --progress-bar off -e .[all]
+        python -m detectron2.utils.collect_env
+        ./datasets/prepare_for_tests.sh
+
+run_unittests: &run_unittests
+  - run:
+      name: Run Unit Tests
+      command: |
+        pytest -sv --durations=15 tests  # parallel causes some random failures
+
+uninstall_tests: &uninstall_tests
+  - run:
+      name: Run Tests After Uninstalling
+      command: |
+        pip uninstall -y detectron2
+        # Remove built binaries
+        rm -rf build/ detectron2/*.so
+        # Tests that code is importable without installation
+        PYTHONPATH=. ./.circleci/import-tests.sh
+
+
+# -------------------------------------------------------------------------------------
+# Jobs to run
+# -------------------------------------------------------------------------------------
+jobs:
+  linux_cpu_tests:
+    <<: *cpu
+    <<: *version_parameters
+
+    working_directory: ~/detectron2
+
+    steps:
+      - checkout
+
+      # Cache the venv directory that contains python, dependencies, and checkpoints
+      # Refresh the key when dependencies should be updated (e.g. when pytorch releases)
+      - restore_cache:
+          keys:
+            - cache-{{ arch }}-<< parameters.pytorch_version >>-{{ .Branch }}-20210827
+
+      - <<: *install_python
+      - <<: *install_linux_dep
+      - <<: *install_detectron2
+      - <<: *run_unittests
+      - <<: *uninstall_tests
+
+      - save_cache:
+          paths:
+            - /opt/circleci/.pyenv
+            - ~/.torch
+          key: cache-{{ arch }}-<< parameters.pytorch_version >>-{{ .Branch }}-20210827
+
+
+  linux_gpu_tests:
+    <<: *gpu
+    <<: *version_parameters
+
+    working_directory: ~/detectron2
+
+    steps:
+      - checkout
+
+      - restore_cache:
+          keys:
+            - cache-{{ arch }}-<< parameters.pytorch_version >>-{{ .Branch }}-20210827
+
+      - <<: *install_python
+      - <<: *install_linux_dep
+      - <<: *install_detectron2
+      - <<: *run_unittests
+      - <<: *uninstall_tests
+
+      - save_cache:
+          paths:
+            - /opt/circleci/.pyenv
+            - ~/.torch
+          key: cache-{{ arch }}-<< parameters.pytorch_version >>-{{ .Branch }}-20210827
+
+  windows_cpu_build:
+    <<: *windows_cpu
+    <<: *version_parameters
+    steps:
+      - <<: *add_ssh_keys
+      - checkout
+      - <<: *setup_venv_win
+
+      # Cache the env directory that contains dependencies
+      - restore_cache:
+          keys:
+            - cache-{{ arch }}-<< parameters.pytorch_version >>-{{ .Branch }}-20210404
+
+      - run:
+          name: Install Dependencies
+          command: |
+            pip install certifi --ignore-installed  # required on windows to workaround some cert issue
+            pip install numpy cython  # required on windows before pycocotools
+            pip install opencv-python-headless pytest-xdist pycocotools tensorboard onnx
+            pip install -U git+https://github.com/facebookresearch/iopath
+            pip install -U git+https://github.com/facebookresearch/fvcore
+            pip install torch==$env:PYTORCH_VERSION torchvision==$env:TORCHVISION_VERSION -f $env:PYTORCH_INDEX
+
+      - save_cache:
+          paths:
+            - env
+          key: cache-{{ arch }}-<< parameters.pytorch_version >>-{{ .Branch }}-20210404
+
+      - <<: *install_detectron2
+      # TODO: unittest fails for now
+
+workflows:
+  version: 2
+  regular_test:
+    jobs:
+      - linux_cpu_tests:
+          name: linux_cpu_tests_pytorch1.10
+          pytorch_version: '1.10.0+cpu'
+          torchvision_version: '0.11.1+cpu'
+      - linux_gpu_tests:
+          name: linux_gpu_tests_pytorch1.8
+          pytorch_version: '1.8.1+cu111'
+          torchvision_version: '0.9.1+cu111'
+      - linux_gpu_tests:
+          name: linux_gpu_tests_pytorch1.9
+          pytorch_version: '1.9+cu111'
+          torchvision_version: '0.10+cu111'
+      - linux_gpu_tests:
+          name: linux_gpu_tests_pytorch1.10
+          pytorch_version: '1.10+cu111'
+          torchvision_version: '0.11.1+cu111'
+      - linux_gpu_tests:
+          name: linux_gpu_tests_pytorch1.10_python39
+          pytorch_version: '1.10+cu111'
+          torchvision_version: '0.11.1+cu111'
+          python_version: '3.9.6'
+      - windows_cpu_build:
+          pytorch_version: '1.10+cpu'
+          torchvision_version: '0.11.1+cpu'

approach/ovod/detectron2/.circleci/import-tests.sh

@@ -0,0 +1,16 @@
+#!/bin/bash -e
+# Copyright (c) Facebook, Inc. and its affiliates.
+
+# Test that import works without building detectron2.
+
+# Check that _C is not importable
+python -c "from detectron2 import _C" > /dev/null 2>&1 && {
+  echo "This test should be run without building detectron2."
+  exit 1
+}
+
+# Check that other modules are still importable, even when _C is not importable
+python -c "from detectron2 import modeling"
+python -c "from detectron2 import modeling, data"
+python -c "from detectron2 import evaluation, export, checkpoint"
+python -c "from detectron2 import utils, engine"

approach/ovod/detectron2/.github/CODE_OF_CONDUCT.md

@@ -0,0 +1,5 @@
+# Code of Conduct
+
+Facebook has adopted a Code of Conduct that we expect project participants to adhere to.
+Please read the [full text](https://code.fb.com/codeofconduct/)
+so that you can understand what actions will and will not be tolerated.

approach/ovod/detectron2/.github/CONTRIBUTING.md

@@ -0,0 +1,68 @@
+# Contributing to detectron2
+
+## Issues
+We use GitHub issues to track public bugs and questions.
+Please make sure to follow one of the
+[issue templates](https://github.com/facebookresearch/detectron2/issues/new/choose)
+when reporting any issues.
+
+Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe
+disclosure of security bugs. In those cases, please go through the process
+outlined on that page and do not file a public issue.
+
+## Pull Requests
+We actively welcome pull requests.
+
+However, if you're adding any significant features (e.g. > 50 lines), please
+make sure to discuss with maintainers about your motivation and proposals in an issue
+before sending a PR. This is to save your time so you don't spend time on a PR that we'll not accept.
+
+We do not always accept new features, and we take the following
+factors into consideration:
+
+1. Whether the same feature can be achieved without modifying detectron2.
+   Detectron2 is designed so that you can implement many extensions from the outside, e.g.
+   those in [projects](https://github.com/facebookresearch/detectron2/tree/master/projects).
+   * If some part of detectron2 is not extensible enough, you can also bring up a more general issue to
+     improve it. Such feature request may be useful to more users.
+2. Whether the feature is potentially useful to a large audience (e.g. an impactful detection paper, a popular dataset,
+   a significant speedup, a widely useful utility),
+   or only to a small portion of users (e.g., a less-known paper, an improvement not in the object
+   detection field, a trick that's not very popular in the community, code to handle a non-standard type of data)
+   * Adoption of additional models, datasets, new task are by default not added to detectron2 before they
+     receive significant popularity in the community.
+     We sometimes accept such features in `projects/`, or as a link in `projects/README.md`.
+3. Whether the proposed solution has a good design / interface. This can be discussed in the issue prior to PRs, or
+   in the form of a draft PR.
+4. Whether the proposed solution adds extra mental/practical overhead to users who don't
+   need such feature.
+5. Whether the proposed solution breaks existing APIs.
+
+To add a feature to an existing function/class `Func`, there are always two approaches:
+(1) add new arguments to `Func`; (2) write a new `Func_with_new_feature`.
+To meet the above criteria, we often prefer approach (2), because:
+
+1. It does not involve modifying or potentially breaking existing code.
+2. It does not add overhead to users who do not need the new feature.
+3. Adding new arguments to a function/class is not scalable w.r.t. all the possible new research ideas in the future.
+
+When sending a PR, please do:
+
+1. If a PR contains multiple orthogonal changes, split it to several PRs.
+2. If you've added code that should be tested, add tests.
+3. For PRs that need experiments (e.g. adding a new model or new methods),
+   you don't need to update model zoo, but do provide experiment results in the description of the PR.
+4. If APIs are changed, update the documentation.
+5. We use the [Google style docstrings](https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html) in python.
+6. Make sure your code lints with `./dev/linter.sh`.
+
+
+## Contributor License Agreement ("CLA")
+In order to accept your pull request, we need you to submit a CLA. You only need
+to do this once to work on any of Facebook's open source projects.
+
+Complete your CLA here: <https://code.facebook.com/cla>
+
+## License
+By contributing to detectron2, you agree that your contributions will be licensed
+under the LICENSE file in the root directory of this source tree.

approach/ovod/detectron2/.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,5 @@
+
+Please select an issue template from
+https://github.com/facebookresearch/detectron2/issues/new/choose .
+
+Otherwise your issue will be closed.

approach/ovod/detectron2/.github/pull_request_template.md

@@ -0,0 +1,10 @@
+Thanks for your contribution!
+
+If you're sending a large PR (e.g., >100 lines),
+please open an issue first about the feature / bug, and indicate how you want to contribute.
+
+We do not always accept features.
+See https://detectron2.readthedocs.io/notes/contributing.html#pull-requests about how we handle PRs.
+
+Before submitting a PR, please run `dev/linter.sh` to lint the code.
+

approach/ovod/detectron2/configs/Base-RCNN-C4.yaml

@@ -0,0 +1,18 @@
+MODEL:
+  META_ARCHITECTURE: "GeneralizedRCNN"
+  RPN:
+    PRE_NMS_TOPK_TEST: 6000
+    POST_NMS_TOPK_TEST: 1000
+  ROI_HEADS:
+    NAME: "Res5ROIHeads"
+DATASETS:
+  TRAIN: ("coco_2017_train",)
+  TEST: ("coco_2017_val",)
+SOLVER:
+  IMS_PER_BATCH: 16
+  BASE_LR: 0.02
+  STEPS: (60000, 80000)
+  MAX_ITER: 90000
+INPUT:
+  MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800)
+VERSION: 2

approach/ovod/detectron2/configs/Base-RCNN-DilatedC5.yaml

@@ -0,0 +1,31 @@
+MODEL:
+  META_ARCHITECTURE: "GeneralizedRCNN"
+  RESNETS:
+    OUT_FEATURES: ["res5"]
+    RES5_DILATION: 2
+  RPN:
+    IN_FEATURES: ["res5"]
+    PRE_NMS_TOPK_TEST: 6000
+    POST_NMS_TOPK_TEST: 1000
+  ROI_HEADS:
+    NAME: "StandardROIHeads"
+    IN_FEATURES: ["res5"]
+  ROI_BOX_HEAD:
+    NAME: "FastRCNNConvFCHead"
+    NUM_FC: 2
+    POOLER_RESOLUTION: 7
+  ROI_MASK_HEAD:
+    NAME: "MaskRCNNConvUpsampleHead"
+    NUM_CONV: 4
+    POOLER_RESOLUTION: 14
+DATASETS:
+  TRAIN: ("coco_2017_train",)
+  TEST: ("coco_2017_val",)
+SOLVER:
+  IMS_PER_BATCH: 16
+  BASE_LR: 0.02
+  STEPS: (60000, 80000)
+  MAX_ITER: 90000
+INPUT:
+  MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800)
+VERSION: 2

approach/ovod/detectron2/configs/Base-RCNN-FPN.yaml

@@ -0,0 +1,42 @@
+MODEL:
+  META_ARCHITECTURE: "GeneralizedRCNN"
+  BACKBONE:
+    NAME: "build_resnet_fpn_backbone"
+  RESNETS:
+    OUT_FEATURES: ["res2", "res3", "res4", "res5"]
+  FPN:
+    IN_FEATURES: ["res2", "res3", "res4", "res5"]
+  ANCHOR_GENERATOR:
+    SIZES: [[32], [64], [128], [256], [512]]  # One size for each in feature map
+    ASPECT_RATIOS: [[0.5, 1.0, 2.0]]  # Three aspect ratios (same for all in feature maps)
+  RPN:
+    IN_FEATURES: ["p2", "p3", "p4", "p5", "p6"]
+    PRE_NMS_TOPK_TRAIN: 2000  # Per FPN level
+    PRE_NMS_TOPK_TEST: 1000  # Per FPN level
+    # Detectron1 uses 2000 proposals per-batch,
+    # (See "modeling/rpn/rpn_outputs.py" for details of this legacy issue)
+    # which is approximately 1000 proposals per-image since the default batch size for FPN is 2.
+    POST_NMS_TOPK_TRAIN: 1000
+    POST_NMS_TOPK_TEST: 1000
+  ROI_HEADS:
+    NAME: "StandardROIHeads"
+    IN_FEATURES: ["p2", "p3", "p4", "p5"]
+  ROI_BOX_HEAD:
+    NAME: "FastRCNNConvFCHead"
+    NUM_FC: 2
+    POOLER_RESOLUTION: 7
+  ROI_MASK_HEAD:
+    NAME: "MaskRCNNConvUpsampleHead"
+    NUM_CONV: 4
+    POOLER_RESOLUTION: 14
+DATASETS:
+  TRAIN: ("coco_2017_train",)
+  TEST: ("coco_2017_val",)
+SOLVER:
+  IMS_PER_BATCH: 16
+  BASE_LR: 0.02
+  STEPS: (60000, 80000)
+  MAX_ITER: 90000
+INPUT:
+  MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800)
+VERSION: 2

approach/ovod/detectron2/configs/Base-RetinaNet.yaml

@@ -0,0 +1,25 @@
+MODEL:
+  META_ARCHITECTURE: "RetinaNet"
+  BACKBONE:
+    NAME: "build_retinanet_resnet_fpn_backbone"
+  RESNETS:
+    OUT_FEATURES: ["res3", "res4", "res5"]
+  ANCHOR_GENERATOR:
+    SIZES: !!python/object/apply:eval ["[[x, x * 2**(1.0/3), x * 2**(2.0/3) ] for x in [32, 64, 128, 256, 512 ]]"]
+  FPN:
+    IN_FEATURES: ["res3", "res4", "res5"]
+  RETINANET:
+    IOU_THRESHOLDS: [0.4, 0.5]
+    IOU_LABELS: [0, -1, 1]
+    SMOOTH_L1_LOSS_BETA: 0.0
+DATASETS:
+  TRAIN: ("coco_2017_train",)
+  TEST: ("coco_2017_val",)
+SOLVER:
+  IMS_PER_BATCH: 16
+  BASE_LR: 0.01  # Note that RetinaNet uses a different default learning rate
+  STEPS: (60000, 80000)
+  MAX_ITER: 90000
+INPUT:
+  MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800)
+VERSION: 2

approach/ovod/detectron2/demo/README.md

@@ -0,0 +1,8 @@
+
+## Detectron2 Demo
+
+We provide a command line tool to run a simple demo of builtin configs.
+The usage is explained in [GETTING_STARTED.md](../GETTING_STARTED.md).
+
+See our [blog post](https://ai.facebook.com/blog/-detectron2-a-pytorch-based-modular-object-detection-library-)
+for a high-quality demo generated with this tool.

approach/ovod/detectron2/demo/demo.py

@@ -0,0 +1,188 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+import argparse
+import glob
+import multiprocessing as mp
+import numpy as np
+import os
+import tempfile
+import time
+import warnings
+import cv2
+import tqdm
+
+from detectron2.config import get_cfg
+from detectron2.data.detection_utils import read_image
+from detectron2.utils.logger import setup_logger
+
+from predictor import VisualizationDemo
+
+# constants
+WINDOW_NAME = "COCO detections"
+
+
+def setup_cfg(args):
+    # load config from file and command-line arguments
+    cfg = get_cfg()
+    # To use demo for Panoptic-DeepLab, please uncomment the following two lines.
+    # from detectron2.projects.panoptic_deeplab import add_panoptic_deeplab_config  # noqa
+    # add_panoptic_deeplab_config(cfg)
+    cfg.merge_from_file(args.config_file)
+    cfg.merge_from_list(args.opts)
+    # Set score_threshold for builtin models
+    cfg.MODEL.RETINANET.SCORE_THRESH_TEST = args.confidence_threshold
+    cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = args.confidence_threshold
+    cfg.MODEL.PANOPTIC_FPN.COMBINE.INSTANCES_CONFIDENCE_THRESH = args.confidence_threshold
+    cfg.freeze()
+    return cfg
+
+
+def get_parser():
+    parser = argparse.ArgumentParser(description="Detectron2 demo for builtin configs")
+    parser.add_argument(
+        "--config-file",
+        default="configs/quick_schedules/mask_rcnn_R_50_FPN_inference_acc_test.yaml",
+        metavar="FILE",
+        help="path to config file",
+    )
+    parser.add_argument("--webcam", action="store_true", help="Take inputs from webcam.")
+    parser.add_argument("--video-input", help="Path to video file.")
+    parser.add_argument(
+        "--input",
+        nargs="+",
+        help="A list of space separated input images; "
+        "or a single glob pattern such as 'directory/*.jpg'",
+    )
+    parser.add_argument(
+        "--output",
+        help="A file or directory to save output visualizations. "
+        "If not given, will show output in an OpenCV window.",
+    )
+
+    parser.add_argument(
+        "--confidence-threshold",
+        type=float,
+        default=0.5,
+        help="Minimum score for instance predictions to be shown",
+    )
+    parser.add_argument(
+        "--opts",
+        help="Modify config options using the command-line 'KEY VALUE' pairs",
+        default=[],
+        nargs=argparse.REMAINDER,
+    )
+    return parser
+
+
+def test_opencv_video_format(codec, file_ext):
+    with tempfile.TemporaryDirectory(prefix="video_format_test") as dir:
+        filename = os.path.join(dir, "test_file" + file_ext)
+        writer = cv2.VideoWriter(
+            filename=filename,
+            fourcc=cv2.VideoWriter_fourcc(*codec),
+            fps=float(30),
+            frameSize=(10, 10),
+            isColor=True,
+        )
+        [writer.write(np.zeros((10, 10, 3), np.uint8)) for _ in range(30)]
+        writer.release()
+        if os.path.isfile(filename):
+            return True
+        return False
+
+
+if __name__ == "__main__":
+    mp.set_start_method("spawn", force=True)
+    args = get_parser().parse_args()
+    setup_logger(name="fvcore")
+    logger = setup_logger()
+    logger.info("Arguments: " + str(args))
+
+    cfg = setup_cfg(args)
+
+    demo = VisualizationDemo(cfg)
+
+    if args.input:
+        if len(args.input) == 1:
+            args.input = glob.glob(os.path.expanduser(args.input[0]))
+            assert args.input, "The input path(s) was not found"
+        for path in tqdm.tqdm(args.input, disable=not args.output):
+            # use PIL, to be consistent with evaluation
+            img = read_image(path, format="BGR")
+            start_time = time.time()
+            predictions, visualized_output = demo.run_on_image(img)
+            logger.info(
+                "{}: {} in {:.2f}s".format(
+                    path,
+                    "detected {} instances".format(len(predictions["instances"]))
+                    if "instances" in predictions
+                    else "finished",
+                    time.time() - start_time,
+                )
+            )
+
+            if args.output:
+                if os.path.isdir(args.output):
+                    assert os.path.isdir(args.output), args.output
+                    out_filename = os.path.join(args.output, os.path.basename(path))
+                else:
+                    assert len(args.input) == 1, "Please specify a directory with args.output"
+                    out_filename = args.output
+                visualized_output.save(out_filename)
+            else:
+                cv2.namedWindow(WINDOW_NAME, cv2.WINDOW_NORMAL)
+                cv2.imshow(WINDOW_NAME, visualized_output.get_image()[:, :, ::-1])
+                if cv2.waitKey(0) == 27:
+                    break  # esc to quit
+    elif args.webcam:
+        assert args.input is None, "Cannot have both --input and --webcam!"
+        assert args.output is None, "output not yet supported with --webcam!"
+        cam = cv2.VideoCapture(0)
+        for vis in tqdm.tqdm(demo.run_on_video(cam)):
+            cv2.namedWindow(WINDOW_NAME, cv2.WINDOW_NORMAL)
+            cv2.imshow(WINDOW_NAME, vis)
+            if cv2.waitKey(1) == 27:
+                break  # esc to quit
+        cam.release()
+        cv2.destroyAllWindows()
+    elif args.video_input:
+        video = cv2.VideoCapture(args.video_input)
+        width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH))
+        height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        frames_per_second = video.get(cv2.CAP_PROP_FPS)
+        num_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
+        basename = os.path.basename(args.video_input)
+        codec, file_ext = (
+            ("x264", ".mkv") if test_opencv_video_format("x264", ".mkv") else ("mp4v", ".mp4")
+        )
+        if codec == ".mp4v":
+            warnings.warn("x264 codec not available, switching to mp4v")
+        if args.output:
+            if os.path.isdir(args.output):
+                output_fname = os.path.join(args.output, basename)
+                output_fname = os.path.splitext(output_fname)[0] + file_ext
+            else:
+                output_fname = args.output
+            assert not os.path.isfile(output_fname), output_fname
+            output_file = cv2.VideoWriter(
+                filename=output_fname,
+                # some installation of opencv may not support x264 (due to its license),
+                # you can try other format (e.g. MPEG)
+                fourcc=cv2.VideoWriter_fourcc(*codec),
+                fps=float(frames_per_second),
+                frameSize=(width, height),
+                isColor=True,
+            )
+        assert os.path.isfile(args.video_input)
+        for vis_frame in tqdm.tqdm(demo.run_on_video(video), total=num_frames):
+            if args.output:
+                output_file.write(vis_frame)
+            else:
+                cv2.namedWindow(basename, cv2.WINDOW_NORMAL)
+                cv2.imshow(basename, vis_frame)
+                if cv2.waitKey(1) == 27:
+                    break  # esc to quit
+        video.release()
+        if args.output:
+            output_file.release()
+        else:
+            cv2.destroyAllWindows()

approach/ovod/detectron2/demo/predictor.py

@@ -0,0 +1,220 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+import atexit
+import bisect
+import multiprocessing as mp
+from collections import deque
+import cv2
+import torch
+
+from detectron2.data import MetadataCatalog
+from detectron2.engine.defaults import DefaultPredictor
+from detectron2.utils.video_visualizer import VideoVisualizer
+from detectron2.utils.visualizer import ColorMode, Visualizer
+
+
+class VisualizationDemo(object):
+    def __init__(self, cfg, instance_mode=ColorMode.IMAGE, parallel=False):
+        """
+        Args:
+            cfg (CfgNode):
+            instance_mode (ColorMode):
+            parallel (bool): whether to run the model in different processes from visualization.
+                Useful since the visualization logic can be slow.
+        """
+        self.metadata = MetadataCatalog.get(
+            cfg.DATASETS.TEST[0] if len(cfg.DATASETS.TEST) else "__unused"
+        )
+        self.cpu_device = torch.device("cpu")
+        self.instance_mode = instance_mode
+
+        self.parallel = parallel
+        if parallel:
+            num_gpu = torch.cuda.device_count()
+            self.predictor = AsyncPredictor(cfg, num_gpus=num_gpu)
+        else:
+            self.predictor = DefaultPredictor(cfg)
+
+    def run_on_image(self, image):
+        """
+        Args:
+            image (np.ndarray): an image of shape (H, W, C) (in BGR order).
+                This is the format used by OpenCV.
+
+        Returns:
+            predictions (dict): the output of the model.
+            vis_output (VisImage): the visualized image output.
+        """
+        vis_output = None
+        predictions = self.predictor(image)
+        # Convert image from OpenCV BGR format to Matplotlib RGB format.
+        image = image[:, :, ::-1]
+        visualizer = Visualizer(image, self.metadata, instance_mode=self.instance_mode)
+        if "panoptic_seg" in predictions:
+            panoptic_seg, segments_info = predictions["panoptic_seg"]
+            vis_output = visualizer.draw_panoptic_seg_predictions(
+                panoptic_seg.to(self.cpu_device), segments_info
+            )
+        else:
+            if "sem_seg" in predictions:
+                vis_output = visualizer.draw_sem_seg(
+                    predictions["sem_seg"].argmax(dim=0).to(self.cpu_device)
+                )
+            if "instances" in predictions:
+                instances = predictions["instances"].to(self.cpu_device)
+                vis_output = visualizer.draw_instance_predictions(predictions=instances)
+
+        return predictions, vis_output
+
+    def _frame_from_video(self, video):
+        while video.isOpened():
+            success, frame = video.read()
+            if success:
+                yield frame
+            else:
+                break
+
+    def run_on_video(self, video):
+        """
+        Visualizes predictions on frames of the input video.
+
+        Args:
+            video (cv2.VideoCapture): a :class:`VideoCapture` object, whose source can be
+                either a webcam or a video file.
+
+        Yields:
+            ndarray: BGR visualizations of each video frame.
+        """
+        video_visualizer = VideoVisualizer(self.metadata, self.instance_mode)
+
+        def process_predictions(frame, predictions):
+            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            if "panoptic_seg" in predictions:
+                panoptic_seg, segments_info = predictions["panoptic_seg"]
+                vis_frame = video_visualizer.draw_panoptic_seg_predictions(
+                    frame, panoptic_seg.to(self.cpu_device), segments_info
+                )
+            elif "instances" in predictions:
+                predictions = predictions["instances"].to(self.cpu_device)
+                vis_frame = video_visualizer.draw_instance_predictions(frame, predictions)
+            elif "sem_seg" in predictions:
+                vis_frame = video_visualizer.draw_sem_seg(
+                    frame, predictions["sem_seg"].argmax(dim=0).to(self.cpu_device)
+                )
+
+            # Converts Matplotlib RGB format to OpenCV BGR format
+            vis_frame = cv2.cvtColor(vis_frame.get_image(), cv2.COLOR_RGB2BGR)
+            return vis_frame
+
+        frame_gen = self._frame_from_video(video)
+        if self.parallel:
+            buffer_size = self.predictor.default_buffer_size
+
+            frame_data = deque()
+
+            for cnt, frame in enumerate(frame_gen):
+                frame_data.append(frame)
+                self.predictor.put(frame)
+
+                if cnt >= buffer_size:
+                    frame = frame_data.popleft()
+                    predictions = self.predictor.get()
+                    yield process_predictions(frame, predictions)
+
+            while len(frame_data):
+                frame = frame_data.popleft()
+                predictions = self.predictor.get()
+                yield process_predictions(frame, predictions)
+        else:
+            for frame in frame_gen:
+                yield process_predictions(frame, self.predictor(frame))
+
+
+class AsyncPredictor:
+    """
+    A predictor that runs the model asynchronously, possibly on >1 GPUs.
+    Because rendering the visualization takes considerably amount of time,
+    this helps improve throughput a little bit when rendering videos.
+    """
+
+    class _StopToken:
+        pass
+
+    class _PredictWorker(mp.Process):
+        def __init__(self, cfg, task_queue, result_queue):
+            self.cfg = cfg
+            self.task_queue = task_queue
+            self.result_queue = result_queue
+            super().__init__()
+
+        def run(self):
+            predictor = DefaultPredictor(self.cfg)
+
+            while True:
+                task = self.task_queue.get()
+                if isinstance(task, AsyncPredictor._StopToken):
+                    break
+                idx, data = task
+                result = predictor(data)
+                self.result_queue.put((idx, result))
+
+    def __init__(self, cfg, num_gpus: int = 1):
+        """
+        Args:
+            cfg (CfgNode):
+            num_gpus (int): if 0, will run on CPU
+        """
+        num_workers = max(num_gpus, 1)
+        self.task_queue = mp.Queue(maxsize=num_workers * 3)
+        self.result_queue = mp.Queue(maxsize=num_workers * 3)
+        self.procs = []
+        for gpuid in range(max(num_gpus, 1)):
+            cfg = cfg.clone()
+            cfg.defrost()
+            cfg.MODEL.DEVICE = "cuda:{}".format(gpuid) if num_gpus > 0 else "cpu"
+            self.procs.append(
+                AsyncPredictor._PredictWorker(cfg, self.task_queue, self.result_queue)
+            )
+
+        self.put_idx = 0
+        self.get_idx = 0
+        self.result_rank = []
+        self.result_data = []
+
+        for p in self.procs:
+            p.start()
+        atexit.register(self.shutdown)
+
+    def put(self, image):
+        self.put_idx += 1
+        self.task_queue.put((self.put_idx, image))
+
+    def get(self):
+        self.get_idx += 1  # the index needed for this request
+        if len(self.result_rank) and self.result_rank[0] == self.get_idx:
+            res = self.result_data[0]
+            del self.result_data[0], self.result_rank[0]
+            return res
+
+        while True:
+            # make sure the results are returned in the correct order
+            idx, res = self.result_queue.get()
+            if idx == self.get_idx:
+                return res
+            insert = bisect.bisect(self.result_rank, idx)
+            self.result_rank.insert(insert, idx)
+            self.result_data.insert(insert, res)
+
+    def __len__(self):
+        return self.put_idx - self.get_idx
+
+    def __call__(self, image):
+        self.put(image)
+        return self.get()
+
+    def shutdown(self):
+        for _ in self.procs:
+            self.task_queue.put(AsyncPredictor._StopToken())
+
+    @property
+    def default_buffer_size(self):
+        return len(self.procs) * 5

approach/ovod/detectron2/detectron2/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+
+from .utils.env import setup_environment
+
+setup_environment()
+
+
+# This line will be programatically read/write by setup.py.
+# Leave them at the bottom of this file and don't touch them.
+__version__ = "0.6"

approach/ovod/detectron2/dev/README.md

@@ -0,0 +1,7 @@
+
+## Some scripts for developers to use, include:
+
+- `linter.sh`: lint the codebase before commit.
+- `run_{inference,instant}_tests.sh`: run inference/training for a few iterations.
+   Note that these tests require 2 GPUs.
+- `parse_results.sh`: parse results from a log file.

approach/ovod/detectron2/dev/linter.sh

@@ -0,0 +1,42 @@
+#!/bin/bash -e
+# Copyright (c) Facebook, Inc. and its affiliates.
+
+# cd to detectron2 project root
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+
+{
+  black --version | grep -E "22\." > /dev/null
+} || {
+  echo "Linter requires 'black==22.*' !"
+  exit 1
+}
+
+ISORT_VERSION=$(isort --version-number)
+if [[ "$ISORT_VERSION" != 4.3* ]]; then
+  echo "Linter requires isort==4.3.21 !"
+  exit 1
+fi
+
+set -v
+
+echo "Running isort ..."
+isort -y -sp . --atomic
+
+echo "Running black ..."
+black -l 100 .
+
+echo "Running flake8 ..."
+if [ -x "$(command -v flake8)" ]; then
+  flake8 .
+else
+  python3 -m flake8 .
+fi
+
+# echo "Running mypy ..."
+# Pytorch does not have enough type annotations
+# mypy detectron2/solver detectron2/structures detectron2/config
+
+echo "Running clang-format ..."
+find . -regex ".*\.\(cpp\|c\|cc\|cu\|cxx\|h\|hh\|hpp\|hxx\|tcc\|mm\|m\)" -print0 | xargs -0 clang-format -i
+
+command -v arc > /dev/null && arc lint

approach/ovod/detectron2/dev/parse_results.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Copyright (c) Facebook, Inc. and its affiliates.
+
+# A shell script that parses metrics from the log file.
+# Make it easier for developers to track performance of models.
+
+LOG="$1"
+
+if [[ -z "$LOG" ]]; then
+	echo "Usage: $0 /path/to/log/file"
+	exit 1
+fi
+
+# [12/15 11:47:32] trainer INFO: Total training time: 12:15:04.446477 (0.4900 s / it)
+# [12/15 11:49:03] inference INFO: Total inference time: 0:01:25.326167 (0.13652186737060548 s / img per device, on 8 devices)
+# [12/15 11:49:03] inference INFO: Total inference pure compute time: .....
+
+# training time
+trainspeed=$(grep -o 'Overall training.*' "$LOG" | grep -Eo '\(.*\)' | grep -o '[0-9\.]*')
+echo "Training speed: $trainspeed s/it"
+
+# inference time: there could be multiple inference during training
+inferencespeed=$(grep -o 'Total inference pure.*' "$LOG" | tail -n1 | grep -Eo '\(.*\)' | grep -o '[0-9\.]*' | head -n1)
+echo "Inference speed: $inferencespeed s/it"
+
+# [12/15 11:47:18] trainer INFO: eta: 0:00:00  iter: 90000  loss: 0.5407 (0.7256)  loss_classifier: 0.1744 (0.2446)  loss_box_reg: 0.0838 (0.1160)  loss_mask: 0.2159 (0.2722)  loss_objectness: 0.0244 (0.0429)  loss_rpn_box_reg: 0.0279 (0.0500)  time: 0.4487 (0.4899)  data: 0.0076 (0.0975) lr: 0.000200  max mem: 4161
+memory=$(grep -o 'max[_ ]mem: [0-9]*' "$LOG" | tail -n1 | grep -o '[0-9]*')
+echo "Training memory: $memory MB"
+
+echo "Easy to copypaste:"
+echo "$trainspeed","$inferencespeed","$memory"
+
+echo "------------------------------"
+
+# [12/26 17:26:32] engine.coco_evaluation: copypaste: Task: bbox
+# [12/26 17:26:32] engine.coco_evaluation: copypaste: AP,AP50,AP75,APs,APm,APl
+# [12/26 17:26:32] engine.coco_evaluation: copypaste: 0.0017,0.0024,0.0017,0.0005,0.0019,0.0011
+# [12/26 17:26:32] engine.coco_evaluation: copypaste: Task: segm
+# [12/26 17:26:32] engine.coco_evaluation: copypaste: AP,AP50,AP75,APs,APm,APl
+# [12/26 17:26:32] engine.coco_evaluation: copypaste: 0.0014,0.0021,0.0016,0.0005,0.0016,0.0011
+
+echo "COCO Results:"
+num_tasks=$(grep -o 'copypaste:.*Task.*' "$LOG" | sort -u | wc -l)
+# each task has 3 lines
+grep -o 'copypaste:.*' "$LOG" | cut -d ' ' -f 2- | tail -n $((num_tasks * 3))

approach/ovod/detectron2/dev/run_inference_tests.sh

@@ -0,0 +1,44 @@
+#!/bin/bash -e
+# Copyright (c) Facebook, Inc. and its affiliates.
+
+BIN="python tools/train_net.py"
+OUTPUT="inference_test_output"
+NUM_GPUS=2
+
+CFG_LIST=( "${@:1}" )
+
+if [ ${#CFG_LIST[@]} -eq 0 ]; then
+  CFG_LIST=( ./configs/quick_schedules/*inference_acc_test.yaml )
+fi
+
+echo "========================================================================"
+echo "Configs to run:"
+echo "${CFG_LIST[@]}"
+echo "========================================================================"
+
+
+for cfg in "${CFG_LIST[@]}"; do
+    echo "========================================================================"
+    echo "Running $cfg ..."
+    echo "========================================================================"
+    $BIN \
+      --eval-only \
+      --num-gpus $NUM_GPUS \
+      --config-file "$cfg" \
+      OUTPUT_DIR $OUTPUT
+      rm -rf $OUTPUT
+done
+
+
+echo "========================================================================"
+echo "Running demo.py ..."
+echo "========================================================================"
+DEMO_BIN="python demo/demo.py"
+COCO_DIR=datasets/coco/val2014
+mkdir -pv $OUTPUT
+
+set -v
+
+$DEMO_BIN --config-file ./configs/quick_schedules/panoptic_fpn_R_50_inference_acc_test.yaml \
+  --input $COCO_DIR/COCO_val2014_0000001933* --output $OUTPUT
+rm -rf $OUTPUT

approach/ovod/detectron2/dev/run_instant_tests.sh

@@ -0,0 +1,27 @@
+#!/bin/bash -e
+# Copyright (c) Facebook, Inc. and its affiliates.
+
+BIN="python tools/train_net.py"
+OUTPUT="instant_test_output"
+NUM_GPUS=2
+
+CFG_LIST=( "${@:1}" )
+if [ ${#CFG_LIST[@]} -eq 0 ]; then
+  CFG_LIST=( ./configs/quick_schedules/*instant_test.yaml )
+fi
+
+echo "========================================================================"
+echo "Configs to run:"
+echo "${CFG_LIST[@]}"
+echo "========================================================================"
+
+for cfg in "${CFG_LIST[@]}"; do
+    echo "========================================================================"
+    echo "Running $cfg ..."
+    echo "========================================================================"
+    $BIN --num-gpus $NUM_GPUS --config-file "$cfg" \
+      SOLVER.IMS_PER_BATCH $(($NUM_GPUS * 2)) \
+      OUTPUT_DIR "$OUTPUT"
+    rm -rf "$OUTPUT"
+done
+

approach/ovod/detectron2/docs/.gitignore

@@ -0,0 +1 @@
+_build

approach/ovod/detectron2/docs/Makefile

@@ -0,0 +1,19 @@
+# Minimal makefile for Sphinx documentation
+# Copyright (c) Facebook, Inc. and its affiliates.
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

approach/ovod/detectron2/docs/README.md

@@ -0,0 +1,15 @@
+# Read the docs:
+
+The latest documentation built from this directory is available at [detectron2.readthedocs.io](https://detectron2.readthedocs.io/).
+Documents in this directory are not meant to be read on github.
+
+# Build the docs:
+
+1. Install detectron2 according to [INSTALL.md](../INSTALL.md).
+2. Install additional libraries required to build docs:
+  - docutils==0.16
+  - Sphinx==3.2.0
+  - recommonmark==0.6.0
+  - sphinx_rtd_theme
+
+3. Run `make html` from this directory.

approach/ovod/detectron2/docs/conf.py

@@ -0,0 +1,391 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) Facebook, Inc. and its affiliates.
+
+# flake8: noqa
+
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+from unittest import mock
+from sphinx.domains import Domain
+from typing import Dict, List, Tuple
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+import sphinx_rtd_theme
+
+
+class GithubURLDomain(Domain):
+    """
+    Resolve certain links in markdown files to github source.
+    """
+
+    name = "githuburl"
+    ROOT = "https://github.com/facebookresearch/detectron2/blob/main/"
+    LINKED_DOC = ["tutorials/install", "tutorials/getting_started"]
+
+    def resolve_any_xref(self, env, fromdocname, builder, target, node, contnode):
+        github_url = None
+        if not target.endswith("html") and target.startswith("../../"):
+            url = target.replace("../", "")
+            github_url = url
+        if fromdocname in self.LINKED_DOC:
+            # unresolved links in these docs are all github links
+            github_url = target
+
+        if github_url is not None:
+            if github_url.endswith("MODEL_ZOO") or github_url.endswith("README"):
+                # bug of recommonmark.
+                # https://github.com/readthedocs/recommonmark/blob/ddd56e7717e9745f11300059e4268e204138a6b1/recommonmark/parser.py#L152-L155
+                github_url += ".md"
+            print("Ref {} resolved to github:{}".format(target, github_url))
+            contnode["refuri"] = self.ROOT + github_url
+            return [("githuburl:any", contnode)]
+        else:
+            return []
+
+
+# to support markdown
+from recommonmark.parser import CommonMarkParser
+
+sys.path.insert(0, os.path.abspath("../"))
+os.environ["_DOC_BUILDING"] = "True"
+DEPLOY = os.environ.get("READTHEDOCS") == "True"
+
+
+# -- Project information -----------------------------------------------------
+
+# fmt: off
+try:
+    import torch  # noqa
+except ImportError:
+    for m in [
+        "torch", "torchvision", "torch.nn", "torch.nn.parallel", "torch.distributed", "torch.multiprocessing", "torch.autograd",
+        "torch.autograd.function", "torch.nn.modules", "torch.nn.modules.utils", "torch.utils", "torch.utils.data", "torch.onnx",
+        "torchvision", "torchvision.ops",
+    ]:
+        sys.modules[m] = mock.Mock(name=m)
+    sys.modules['torch'].__version__ = "1.7"  # fake version
+    HAS_TORCH = False
+else:
+    try:
+        torch.ops.detectron2 = mock.Mock(name="torch.ops.detectron2")
+    except:
+        pass
+    HAS_TORCH = True
+
+for m in [
+    "cv2", "scipy", "portalocker", "detectron2._C",
+    "pycocotools", "pycocotools.mask", "pycocotools.coco", "pycocotools.cocoeval",
+    "google", "google.protobuf", "google.protobuf.internal", "onnx",
+    "caffe2", "caffe2.proto", "caffe2.python", "caffe2.python.utils", "caffe2.python.onnx", "caffe2.python.onnx.backend",
+]:
+    sys.modules[m] = mock.Mock(name=m)
+# fmt: on
+sys.modules["cv2"].__version__ = "3.4"
+
+import detectron2  # isort: skip
+
+if HAS_TORCH:
+    from detectron2.utils.env import fixup_module_metadata
+
+    fixup_module_metadata("torch.nn", torch.nn.__dict__)
+    fixup_module_metadata("torch.utils.data", torch.utils.data.__dict__)
+
+
+project = "detectron2"
+copyright = "2019-2020, detectron2 contributors"
+author = "detectron2 contributors"
+
+# The short X.Y version
+version = detectron2.__version__
+# The full version, including alpha/beta/rc tags
+release = version
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+needs_sphinx = "3.0"
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "recommonmark",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.todo",
+    "sphinx.ext.coverage",
+    "sphinx.ext.mathjax",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.githubpages",
+]
+
+# -- Configurations for plugins ------------
+napoleon_google_docstring = True
+napoleon_include_init_with_doc = True
+napoleon_include_special_with_doc = True
+napoleon_numpy_docstring = False
+napoleon_use_rtype = False
+autodoc_inherit_docstrings = False
+autodoc_member_order = "bysource"
+
+if DEPLOY:
+    intersphinx_timeout = 10
+else:
+    # skip this when building locally
+    intersphinx_timeout = 0.5
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3.7", None),
+    "numpy": ("https://docs.scipy.org/doc/numpy/", None),
+    "torch": ("https://pytorch.org/docs/master/", None),
+}
+# -------------------------
+
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+source_suffix = [".rst", ".md"]
+
+# The master toctree document.
+master_doc = "index"
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "build", "README.md", "tutorials/README.md"]
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = "sphinx"
+
+
+# -- Options for HTML output -------------------------------------------------
+
+html_theme = "sphinx_rtd_theme"
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+html_css_files = ["css/custom.css"]
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = "detectron2doc"
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, "detectron2.tex", "detectron2 Documentation", "detectron2 contributors", "manual")
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [(master_doc, "detectron2", "detectron2 Documentation", [author], 1)]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (
+        master_doc,
+        "detectron2",
+        "detectron2 Documentation",
+        author,
+        "detectron2",
+        "One line description of project.",
+        "Miscellaneous",
+    )
+]
+
+
+# -- Options for todo extension ----------------------------------------------
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+def autodoc_skip_member(app, what, name, obj, skip, options):
+    # we hide something deliberately
+    if getattr(obj, "__HIDE_SPHINX_DOC__", False):
+        return True
+
+    # Hide some that are deprecated or not intended to be used
+    HIDDEN = {
+        "ResNetBlockBase",
+        "GroupedBatchSampler",
+        "build_transform_gen",
+        "apply_transform_gens",
+        "TransformGen",
+        "apply_augmentations",
+        "StandardAugInput",
+        "build_batch_data_loader",
+        "draw_panoptic_seg_predictions",
+        "WarmupCosineLR",
+        "WarmupMultiStepLR",
+        "downgrade_config",
+        "upgrade_config",
+        "add_export_config",
+    }
+    try:
+        if name in HIDDEN or (
+            hasattr(obj, "__doc__") and obj.__doc__.lower().strip().startswith("deprecated")
+        ):
+            print("Skipping deprecated object: {}".format(name))
+            return True
+    except:
+        pass
+    return skip
+
+
+_PAPER_DATA = {
+    "resnet": ("1512.03385", "Deep Residual Learning for Image Recognition"),
+    "fpn": ("1612.03144", "Feature Pyramid Networks for Object Detection"),
+    "mask r-cnn": ("1703.06870", "Mask R-CNN"),
+    "faster r-cnn": (
+        "1506.01497",
+        "Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks",
+    ),
+    "deformconv": ("1703.06211", "Deformable Convolutional Networks"),
+    "deformconv2": ("1811.11168", "Deformable ConvNets v2: More Deformable, Better Results"),
+    "panopticfpn": ("1901.02446", "Panoptic Feature Pyramid Networks"),
+    "retinanet": ("1708.02002", "Focal Loss for Dense Object Detection"),
+    "cascade r-cnn": ("1712.00726", "Cascade R-CNN: Delving into High Quality Object Detection"),
+    "lvis": ("1908.03195", "LVIS: A Dataset for Large Vocabulary Instance Segmentation"),
+    "rrpn": ("1703.01086", "Arbitrary-Oriented Scene Text Detection via Rotation Proposals"),
+    "imagenet in 1h": ("1706.02677", "Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour"),
+    "xception": ("1610.02357", "Xception: Deep Learning with Depthwise Separable Convolutions"),
+    "mobilenet": (
+        "1704.04861",
+        "MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications",
+    ),
+    "deeplabv3+": (
+        "1802.02611",
+        "Encoder-Decoder with Atrous Separable Convolution for Semantic Image Segmentation",
+    ),
+    "dds": ("2003.13678", "Designing Network Design Spaces"),
+    "scaling": ("2103.06877", "Fast and Accurate Model Scaling"),
+    "fcos": ("2006.09214", "FCOS: A Simple and Strong Anchor-free Object Detector"),
+    "rethinking-batchnorm": ("2105.07576", 'Rethinking "Batch" in BatchNorm'),
+    "vitdet": ("2203.16527", "Exploring Plain Vision Transformer Backbones for Object Detection"),
+    "mvitv2": (
+        "2112.01526",
+        "MViTv2: Improved Multiscale Vision Transformers for Classification and Detection",
+    ),
+    "swin": (
+        "2103.14030",
+        "Swin Transformer: Hierarchical Vision Transformer using Shifted Windows",
+    ),
+}
+
+
+def paper_ref_role(
+    typ: str,
+    rawtext: str,
+    text: str,
+    lineno: int,
+    inliner,
+    options: Dict = {},
+    content: List[str] = [],
+):
+    """
+    Parse :paper:`xxx`. Similar to the "extlinks" sphinx extension.
+    """
+    from docutils import nodes, utils
+    from sphinx.util.nodes import split_explicit_title
+
+    text = utils.unescape(text)
+    has_explicit_title, title, link = split_explicit_title(text)
+    link = link.lower()
+    if link not in _PAPER_DATA:
+        inliner.reporter.warning("Cannot find paper " + link)
+        paper_url, paper_title = "#", link
+    else:
+        paper_url, paper_title = _PAPER_DATA[link]
+        if "/" not in paper_url:
+            paper_url = "https://arxiv.org/abs/" + paper_url
+    if not has_explicit_title:
+        title = paper_title
+    pnode = nodes.reference(title, title, internal=False, refuri=paper_url)
+    return [pnode], []
+
+
+def setup(app):
+    from recommonmark.transform import AutoStructify
+
+    app.add_domain(GithubURLDomain)
+    app.connect("autodoc-skip-member", autodoc_skip_member)
+    app.add_role("paper", paper_ref_role)
+    app.add_config_value(
+        "recommonmark_config",
+        {"enable_math": True, "enable_inline_math": True, "enable_eval_rst": True},
+        True,
+    )
+    app.add_transform(AutoStructify)

approach/ovod/detectron2/docs/index.rst

@@ -0,0 +1,14 @@
+.. detectron2 documentation master file, created by
+   sphinx-quickstart on Sat Sep 21 13:46:45 2019.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to detectron2's documentation!
+======================================
+
+.. toctree::
+   :maxdepth: 2
+
+   tutorials/index
+   notes/index
+   modules/index

approach/ovod/detectron2/docs/notes/benchmarks.md

@@ -0,0 +1,196 @@
+
+# Benchmarks
+
+Here we benchmark the training speed of a Mask R-CNN in detectron2,
+with some other popular open source Mask R-CNN implementations.
+
+
+### Settings
+
+* Hardware: 8 NVIDIA V100s with NVLink.
+* Software: Python 3.7, CUDA 10.1, cuDNN 7.6.5, PyTorch 1.5,
+  TensorFlow 1.15.0rc2, Keras 2.2.5, MxNet 1.6.0b20190820.
+* Model: an end-to-end R-50-FPN Mask-RCNN model, using the same hyperparameter as the
+  [Detectron baseline config](https://github.com/facebookresearch/Detectron/blob/master/configs/12_2017_baselines/e2e_mask_rcnn_R-50-FPN_1x.yaml)
+  (it does not have scale augmentation).
+* Metrics: We use the average throughput in iterations 100-500 to skip GPU warmup time.
+  Note that for R-CNN-style models, the throughput of a model typically changes during training, because
+  it depends on the predictions of the model. Therefore this metric is not directly comparable with
+  "train speed" in model zoo, which is the average speed of the entire training run.
+
+
+### Main Results
+
+```eval_rst
++-------------------------------+--------------------+
+| Implementation                | Throughput (img/s) |
++===============================+====================+
+| |D2| |PT|                     | 62                 |
++-------------------------------+--------------------+
+| mmdetection_  |PT|            | 53                 |
++-------------------------------+--------------------+
+| maskrcnn-benchmark_  |PT|     | 53                 |
++-------------------------------+--------------------+
+| tensorpack_ |TF|              | 50                 |
++-------------------------------+--------------------+
+| simpledet_ |mxnet|            | 39                 |
++-------------------------------+--------------------+
+| Detectron_  |C2|              | 19                 |
++-------------------------------+--------------------+
+| `matterport/Mask_RCNN`__ |TF| | 14                 |
++-------------------------------+--------------------+
+
+.. _maskrcnn-benchmark: https://github.com/facebookresearch/maskrcnn-benchmark/
+.. _tensorpack: https://github.com/tensorpack/tensorpack/tree/master/examples/FasterRCNN
+.. _mmdetection: https://github.com/open-mmlab/mmdetection/
+.. _simpledet: https://github.com/TuSimple/simpledet/
+.. _Detectron: https://github.com/facebookresearch/Detectron
+__ https://github.com/matterport/Mask_RCNN/
+
+.. |D2| image:: https://github.com/facebookresearch/detectron2/raw/main/.github/Detectron2-Logo-Horz.svg?sanitize=true
+   :height: 15pt
+   :target: https://github.com/facebookresearch/detectron2/
+.. |PT| image:: https://pytorch.org/assets/images/logo-icon.svg
+   :width: 15pt
+   :height: 15pt
+   :target: https://pytorch.org
+.. |TF| image:: https://static.nvidiagrid.net/ngc/containers/tensorflow.png
+   :width: 15pt
+   :height: 15pt
+   :target: https://tensorflow.org
+.. |mxnet| image:: https://github.com/dmlc/web-data/raw/master/mxnet/image/mxnet_favicon.png
+   :width: 15pt
+   :height: 15pt
+   :target: https://mxnet.apache.org/
+.. |C2| image:: https://caffe2.ai/static/logo.svg
+   :width: 15pt
+   :height: 15pt
+   :target: https://caffe2.ai
+```
+
+
+Details for each implementation:
+
+* __Detectron2__: with release v0.1.2, run:
+  ```
+  python tools/train_net.py  --config-file configs/Detectron1-Comparisons/mask_rcnn_R_50_FPN_noaug_1x.yaml --num-gpus 8
+  ```
+
+* __mmdetection__: at commit `b0d845f`, run
+  ```
+  ./tools/dist_train.sh configs/mask_rcnn/mask_rcnn_r50_caffe_fpn_1x_coco.py 8
+  ```
+
+* __maskrcnn-benchmark__: use commit `0ce8f6f` with `sed -i 's/torch.uint8/torch.bool/g' **/*.py; sed -i 's/AT_CHECK/TORCH_CHECK/g' **/*.cu`
+  to make it compatible with PyTorch 1.5. Then, run training with
+  ```
+  python -m torch.distributed.launch --nproc_per_node=8 tools/train_net.py --config-file configs/e2e_mask_rcnn_R_50_FPN_1x.yaml
+  ```
+  The speed we observed is faster than its model zoo, likely due to different software versions.
+
+* __tensorpack__: at commit `caafda`, `export TF_CUDNN_USE_AUTOTUNE=0`, then run
+  ```
+  mpirun -np 8 ./train.py --config DATA.BASEDIR=/data/coco TRAINER=horovod BACKBONE.STRIDE_1X1=True TRAIN.STEPS_PER_EPOCH=50 --load ImageNet-R50-AlignPadding.npz
+  ```
+
+* __SimpleDet__: at commit `9187a1`, run
+  ```
+  python detection_train.py --config config/mask_r50v1_fpn_1x.py
+  ```
+
+* __Detectron__: run
+  ```
+  python tools/train_net.py --cfg configs/12_2017_baselines/e2e_mask_rcnn_R-50-FPN_1x.yaml
+  ```
+  Note that many of its ops run on CPUs, therefore the performance is limited.
+
+* __matterport/Mask_RCNN__: at commit `3deaec`, apply the following diff, `export TF_CUDNN_USE_AUTOTUNE=0`, then run
+  ```
+  python coco.py train --dataset=/data/coco/ --model=imagenet
+  ```
+  Note that many small details in this implementation might be different
+  from Detectron's standards.
+
+  <details>
+  <summary>
+  (diff to make it use the same hyperparameters - click to expand)
+  </summary>
+
+  ```diff
+  diff --git i/mrcnn/model.py w/mrcnn/model.py
+  index 62cb2b0..61d7779 100644
+  --- i/mrcnn/model.py
+  +++ w/mrcnn/model.py
+  @@ -2367,8 +2367,8 @@ class MaskRCNN():
+        epochs=epochs,
+        steps_per_epoch=self.config.STEPS_PER_EPOCH,
+        callbacks=callbacks,
+  -            validation_data=val_generator,
+  -            validation_steps=self.config.VALIDATION_STEPS,
+  +            #validation_data=val_generator,
+  +            #validation_steps=self.config.VALIDATION_STEPS,
+        max_queue_size=100,
+        workers=workers,
+        use_multiprocessing=True,
+  diff --git i/mrcnn/parallel_model.py w/mrcnn/parallel_model.py
+  index d2bf53b..060172a 100644
+  --- i/mrcnn/parallel_model.py
+  +++ w/mrcnn/parallel_model.py
+  @@ -32,6 +32,7 @@ class ParallelModel(KM.Model):
+      keras_model: The Keras model to parallelize
+      gpu_count: Number of GPUs. Must be > 1
+      """
+  +        super().__init__()
+      self.inner_model = keras_model
+      self.gpu_count = gpu_count
+      merged_outputs = self.make_parallel()
+  diff --git i/samples/coco/coco.py w/samples/coco/coco.py
+  index 5d172b5..239ed75 100644
+  --- i/samples/coco/coco.py
+  +++ w/samples/coco/coco.py
+  @@ -81,7 +81,10 @@ class CocoConfig(Config):
+    IMAGES_PER_GPU = 2
+
+    # Uncomment to train on 8 GPUs (default is 1)
+  -    # GPU_COUNT = 8
+  +    GPU_COUNT = 8
+  +    BACKBONE = "resnet50"
+  +    STEPS_PER_EPOCH = 50
+  +    TRAIN_ROIS_PER_IMAGE = 512
+
+    # Number of classes (including background)
+    NUM_CLASSES = 1 + 80  # COCO has 80 classes
+  @@ -496,29 +499,10 @@ if __name__ == '__main__':
+      # *** This training schedule is an example. Update to your needs ***
+
+      # Training - Stage 1
+  -        print("Training network heads")
+      model.train(dataset_train, dataset_val,
+            learning_rate=config.LEARNING_RATE,
+            epochs=40,
+  -                    layers='heads',
+  -                    augmentation=augmentation)
+  -
+  -        # Training - Stage 2
+  -        # Finetune layers from ResNet stage 4 and up
+  -        print("Fine tune Resnet stage 4 and up")
+  -        model.train(dataset_train, dataset_val,
+  -                    learning_rate=config.LEARNING_RATE,
+  -                    epochs=120,
+  -                    layers='4+',
+  -                    augmentation=augmentation)
+  -
+  -        # Training - Stage 3
+  -        # Fine tune all layers
+  -        print("Fine tune all layers")
+  -        model.train(dataset_train, dataset_val,
+  -                    learning_rate=config.LEARNING_RATE / 10,
+  -                    epochs=160,
+  -                    layers='all',
+  +                    layers='3+',
+            augmentation=augmentation)
+
+    elif args.command == "evaluate":
+  ```
+
+  </details>

approach/ovod/detectron2/docs/notes/changelog.md

@@ -0,0 +1,48 @@
+# Change Log and Backward Compatibility
+
+### Releases
+See release logs at
+[https://github.com/facebookresearch/detectron2/releases](https://github.com/facebookresearch/detectron2/releases)
+for new updates.
+
+### Backward Compatibility
+
+Due to the research nature of what the library does, there might be backward incompatible changes.
+But we try to reduce users' disruption by the following ways:
+* APIs listed in [API documentation](https://detectron2.readthedocs.io/modules/index.html), including
+  function/class names, their arguments, and documented class attributes, are considered *stable* unless
+  otherwise noted in the documentation.
+  They are less likely to be broken, but if needed, will trigger a deprecation warning for a reasonable period
+  before getting broken, and will be documented in release logs.
+* Others functions/classses/attributes are considered internal, and are more likely to change.
+  However, we're aware that some of them may be already used by other projects, and in particular we may
+  use them for convenience among projects under `detectron2/projects`.
+  For such APIs, we may treat them as stable APIs and also apply the above strategies.
+  They may be promoted to stable when we're ready.
+* Projects under "detectron2/projects" or imported with "detectron2.projects" are research projects
+  and are all considered experimental.
+* Classes/functions that contain the word "default" or are explicitly documented to produce
+  "default behavior" may change their behaviors when new features are added.
+
+Despite of the possible breakage, if a third-party project would like to keep up with the latest updates
+in detectron2, using it as a library will still be less disruptive than forking, because
+the frequency and scope of API changes will be much smaller than code changes.
+
+To see such changes, search for "incompatible changes" in [release logs](https://github.com/facebookresearch/detectron2/releases).
+
+### Config Version Change Log
+
+Detectron2's config version has not been changed since open source.
+There is no need for an open source user to worry about this.
+
+* v1: Rename `RPN_HEAD.NAME` to `RPN.HEAD_NAME`.
+* v2: A batch of rename of many configurations before release.
+
+### Silent Regressions in Historical Versions:
+
+We list a few silent regressions, since they may silently produce incorrect results and will be hard to debug.
+
+* 04/01/2020 - 05/11/2020: Bad accuracy if `TRAIN_ON_PRED_BOXES` is set to True.
+* 03/30/2020 - 04/01/2020: ResNets are not correctly built.
+* 12/19/2019 - 12/26/2019: Using aspect ratio grouping causes a drop in accuracy.
+* - 11/9/2019: Test time augmentation does not predict the last category.

approach/ovod/detectron2/docs/notes/compatibility.md

@@ -0,0 +1,84 @@
+# Compatibility with Other Libraries
+
+## Compatibility with Detectron (and maskrcnn-benchmark)
+
+Detectron2 addresses some legacy issues left in Detectron. As a result, their models
+are not compatible:
+running inference with the same model weights will produce different results in the two code bases.
+
+The major differences regarding inference are:
+
+- The height and width of a box with corners (x1, y1) and (x2, y2) is now computed more naturally as
+  width = x2 - x1 and height = y2 - y1;
+  In Detectron, a "+ 1" was added both height and width.
+
+  Note that the relevant ops in Caffe2 have [adopted this change of convention](https://github.com/pytorch/pytorch/pull/20550)
+  with an extra option.
+  So it is still possible to run inference with a Detectron2-trained model in Caffe2.
+
+  The change in height/width calculations most notably changes:
+  - encoding/decoding in bounding box regression.
+  - non-maximum suppression. The effect here is very negligible, though.
+
+- RPN now uses simpler anchors with fewer quantization artifacts.
+
+  In Detectron, the anchors were quantized and
+  [do not have accurate areas](https://github.com/facebookresearch/Detectron/issues/227).
+  In Detectron2, the anchors are center-aligned to feature grid points and not quantized.
+
+- Classification layers have a different ordering of class labels.
+
+  This involves any trainable parameter with shape (..., num_categories + 1, ...).
+  In Detectron2, integer labels [0, K-1] correspond to the K = num_categories object categories
+  and the label "K" corresponds to the special "background" category.
+  In Detectron, label "0" means background, and labels [1, K] correspond to the K categories.
+
+- ROIAlign is implemented differently. The new implementation is [available in Caffe2](https://github.com/pytorch/pytorch/pull/23706).
+
+  1. All the ROIs are shifted by half a pixel compared to Detectron in order to create better image-feature-map alignment.
+     See `layers/roi_align.py` for details.
+     To enable the old behavior, use `ROIAlign(aligned=False)`, or `POOLER_TYPE=ROIAlign` instead of
+     `ROIAlignV2` (the default).
+
+  1. The ROIs are not required to have a minimum size of 1.
+     This will lead to tiny differences in the output, but should be negligible.
+
+- Mask inference function is different.
+
+  In Detectron2, the "paste_mask" function is different and should be more accurate than in Detectron. This change
+  can improve mask AP on COCO by ~0.5% absolute.
+
+There are some other differences in training as well, but they won't affect
+model-level compatibility. The major ones are:
+
+- We fixed a [bug](https://github.com/facebookresearch/Detectron/issues/459) in
+  Detectron, by making `RPN.POST_NMS_TOPK_TRAIN` per-image, rather than per-batch.
+  The fix may lead to a small accuracy drop for a few models (e.g. keypoint
+  detection) and will require some parameter tuning to match the Detectron results.
+- For simplicity, we change the default loss in bounding box regression to L1 loss, instead of smooth L1 loss.
+  We have observed that this tends to slightly decrease box AP50 while improving box AP for higher
+  overlap thresholds (and leading to a slight overall improvement in box AP).
+- We interpret the coordinates in COCO bounding box and segmentation annotations
+  as coordinates in range `[0, width]` or `[0, height]`. The coordinates in
+  COCO keypoint annotations are interpreted as pixel indices in range `[0, width - 1]` or `[0, height - 1]`.
+  Note that this affects how flip augmentation is implemented.
+
+
+[This article](https://ppwwyyxx.com/blog/2021/Where-are-Pixels/)
+explains more details on the above mentioned issues
+about pixels, coordinates, and "+1"s.
+
+
+## Compatibility with Caffe2
+
+As mentioned above, despite the incompatibilities with Detectron, the relevant
+ops have been implemented in Caffe2.
+Therefore, models trained with detectron2 can be converted in Caffe2.
+See [Deployment](../tutorials/deployment.md) for the tutorial.
+
+## Compatibility with TensorFlow
+
+Most ops are available in TensorFlow, although some tiny differences in
+the implementation of resize / ROIAlign / padding need to be addressed.
+A working conversion script is provided by [tensorpack Faster R-CNN](https://github.com/tensorpack/tensorpack/tree/master/examples/FasterRCNN/convert_d2)
+to run a standard detectron2 model in TensorFlow.

approach/ovod/detectron2/docs/notes/contributing.md

@@ -0,0 +1,68 @@
+# Contributing to detectron2
+
+## Issues
+We use GitHub issues to track public bugs and questions.
+Please make sure to follow one of the
+[issue templates](https://github.com/facebookresearch/detectron2/issues/new/choose)
+when reporting any issues.
+
+Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe
+disclosure of security bugs. In those cases, please go through the process
+outlined on that page and do not file a public issue.
+
+## Pull Requests
+We actively welcome pull requests.
+
+However, if you're adding any significant features (e.g. > 50 lines), please
+make sure to discuss with maintainers about your motivation and proposals in an issue
+before sending a PR. This is to save your time so you don't spend time on a PR that we'll not accept.
+
+We do not always accept new features, and we take the following
+factors into consideration:
+
+1. Whether the same feature can be achieved without modifying detectron2.
+   Detectron2 is designed so that you can implement many extensions from the outside, e.g.
+   those in [projects](https://github.com/facebookresearch/detectron2/tree/master/projects).
+   * If some part of detectron2 is not extensible enough, you can also bring up a more general issue to
+     improve it. Such feature request may be useful to more users.
+2. Whether the feature is potentially useful to a large audience (e.g. an impactful detection paper, a popular dataset,
+   a significant speedup, a widely useful utility),
+   or only to a small portion of users (e.g., a less-known paper, an improvement not in the object
+   detection field, a trick that's not very popular in the community, code to handle a non-standard type of data)
+   * Adoption of additional models, datasets, new task are by default not added to detectron2 before they
+     receive significant popularity in the community.
+     We sometimes accept such features in `projects/`, or as a link in `projects/README.md`.
+3. Whether the proposed solution has a good design / interface. This can be discussed in the issue prior to PRs, or
+   in the form of a draft PR.
+4. Whether the proposed solution adds extra mental/practical overhead to users who don't
+   need such feature.
+5. Whether the proposed solution breaks existing APIs.
+
+To add a feature to an existing function/class `Func`, there are always two approaches:
+(1) add new arguments to `Func`; (2) write a new `Func_with_new_feature`.
+To meet the above criteria, we often prefer approach (2), because:
+
+1. It does not involve modifying or potentially breaking existing code.
+2. It does not add overhead to users who do not need the new feature.
+3. Adding new arguments to a function/class is not scalable w.r.t. all the possible new research ideas in the future.
+
+When sending a PR, please do:
+
+1. If a PR contains multiple orthogonal changes, split it to several PRs.
+2. If you've added code that should be tested, add tests.
+3. For PRs that need experiments (e.g. adding a new model or new methods),
+   you don't need to update model zoo, but do provide experiment results in the description of the PR.
+4. If APIs are changed, update the documentation.
+5. We use the [Google style docstrings](https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html) in python.
+6. Make sure your code lints with `./dev/linter.sh`.
+
+
+## Contributor License Agreement ("CLA")
+In order to accept your pull request, we need you to submit a CLA. You only need
+to do this once to work on any of Facebook's open source projects.
+
+Complete your CLA here: <https://code.facebook.com/cla>
+
+## License
+By contributing to detectron2, you agree that your contributions will be licensed
+under the LICENSE file in the root directory of this source tree.

approach/ovod/detectron2/docs/notes/index.rst

@@ -0,0 +1,10 @@
+Notes
+======================================
+
+.. toctree::
+   :maxdepth: 2
+
+   benchmarks
+   compatibility
+   contributing
+   changelog

approach/ovod/detectron2/docs/requirements.txt

@@ -0,0 +1,24 @@
+docutils==0.16
+# https://github.com/sphinx-doc/sphinx/commit/7acd3ada3f38076af7b2b5c9f3b60bb9c2587a3d
+sphinx==3.2.0
+recommonmark==0.6.0
+sphinx_rtd_theme
+# Dependencies here are only those required by import
+termcolor
+numpy
+tqdm
+matplotlib
+termcolor
+yacs
+tabulate
+cloudpickle
+Pillow
+future
+git+https://github.com/facebookresearch/fvcore.git
+https://download.pytorch.org/whl/cpu/torch-1.8.1%2Bcpu-cp37-cp37m-linux_x86_64.whl
+https://download.pytorch.org/whl/cpu/torchvision-0.9.1%2Bcpu-cp37-cp37m-linux_x86_64.whl
+omegaconf>=2.1.0.dev24
+hydra-core>=1.1.0.dev5
+scipy
+timm
+fairscale

approach/ovod/detectron2/docs/tutorials/README.md

@@ -0,0 +1,4 @@
+# Read the docs:
+
+The latest documentation built from this directory is available at [detectron2.readthedocs.io](https://detectron2.readthedocs.io/).
+Documents in this directory are not meant to be read on github.

approach/ovod/detectron2/docs/tutorials/builtin_datasets.md

@@ -0,0 +1,140 @@
+# Use Builtin Datasets
+
+A dataset can be used by accessing [DatasetCatalog](https://detectron2.readthedocs.io/modules/data.html#detectron2.data.DatasetCatalog)
+for its data, or [MetadataCatalog](https://detectron2.readthedocs.io/modules/data.html#detectron2.data.MetadataCatalog) for its metadata (class names, etc).
+This document explains how to setup the builtin datasets so they can be used by the above APIs.
+[Use Custom Datasets](https://detectron2.readthedocs.io/tutorials/datasets.html) gives a deeper dive on how to use `DatasetCatalog` and `MetadataCatalog`,
+and how to add new datasets to them.
+
+Detectron2 has builtin support for a few datasets.
+The datasets are assumed to exist in a directory specified by the environment variable
+`DETECTRON2_DATASETS`.
+Under this directory, detectron2 will look for datasets in the structure described below, if needed.
+```
+$DETECTRON2_DATASETS/
+  coco/
+  lvis/
+  cityscapes/
+  VOC20{07,12}/
+```
+
+You can set the location for builtin datasets by `export DETECTRON2_DATASETS=/path/to/datasets`.
+If left unset, the default is `./datasets` relative to your current working directory.
+
+The [model zoo](https://github.com/facebookresearch/detectron2/blob/master/MODEL_ZOO.md)
+contains configs and models that use these builtin datasets.
+
+## Expected dataset structure for [COCO instance/keypoint detection](https://cocodataset.org/#download):
+
+```
+coco/
+  annotations/
+    instances_{train,val}2017.json
+    person_keypoints_{train,val}2017.json
+  {train,val}2017/
+    # image files that are mentioned in the corresponding json
+```
+
+You can use the 2014 version of the dataset as well.
+
+Some of the builtin tests (`dev/run_*_tests.sh`) uses a tiny version of the COCO dataset,
+which you can download with `./datasets/prepare_for_tests.sh`.
+
+## Expected dataset structure for PanopticFPN:
+
+Extract panoptic annotations from [COCO website](https://cocodataset.org/#download)
+into the following structure:
+```
+coco/
+  annotations/
+    panoptic_{train,val}2017.json
+  panoptic_{train,val}2017/  # png annotations
+  panoptic_stuff_{train,val}2017/  # generated by the script mentioned below
+```
+
+Install panopticapi by:
+```
+pip install git+https://github.com/cocodataset/panopticapi.git
+```
+Then, run `python datasets/prepare_panoptic_fpn.py`, to extract semantic annotations from panoptic annotations.
+
+## Expected dataset structure for [LVIS instance segmentation](https://www.lvisdataset.org/dataset):
+```
+coco/
+  {train,val,test}2017/
+lvis/
+  lvis_v0.5_{train,val}.json
+  lvis_v0.5_image_info_test.json
+  lvis_v1_{train,val}.json
+  lvis_v1_image_info_test{,_challenge}.json
+```
+
+Install lvis-api by:
+```
+pip install git+https://github.com/lvis-dataset/lvis-api.git
+```
+
+To evaluate models trained on the COCO dataset using LVIS annotations,
+run `python datasets/prepare_cocofied_lvis.py` to prepare "cocofied" LVIS annotations.
+
+## Expected dataset structure for [cityscapes](https://www.cityscapes-dataset.com/downloads/):
+```
+cityscapes/
+  gtFine/
+    train/
+      aachen/
+        color.png, instanceIds.png, labelIds.png, polygons.json,
+        labelTrainIds.png
+      ...
+    val/
+    test/
+    # below are generated Cityscapes panoptic annotation
+    cityscapes_panoptic_train.json
+    cityscapes_panoptic_train/
+    cityscapes_panoptic_val.json
+    cityscapes_panoptic_val/
+    cityscapes_panoptic_test.json
+    cityscapes_panoptic_test/
+  leftImg8bit/
+    train/
+    val/
+    test/
+```
+Install cityscapes scripts by:
+```
+pip install git+https://github.com/mcordts/cityscapesScripts.git
+```
+
+Note: to create labelTrainIds.png, first prepare the above structure, then run cityscapesescript with:
+```
+CITYSCAPES_DATASET=/path/to/abovementioned/cityscapes python cityscapesscripts/preparation/createTrainIdLabelImgs.py
+```
+These files are not needed for instance segmentation.
+
+Note: to generate Cityscapes panoptic dataset, run cityscapesescript with:
+```
+CITYSCAPES_DATASET=/path/to/abovementioned/cityscapes python cityscapesscripts/preparation/createPanopticImgs.py
+```
+These files are not needed for semantic and instance segmentation.
+
+## Expected dataset structure for [Pascal VOC](http://host.robots.ox.ac.uk/pascal/VOC/index.html):
+```
+VOC20{07,12}/
+  Annotations/
+  ImageSets/
+    Main/
+      trainval.txt
+      test.txt
+      # train.txt or val.txt, if you use these splits
+  JPEGImages/
+```
+
+## Expected dataset structure for [ADE20k Scene Parsing](http://sceneparsing.csail.mit.edu/):
+```
+ADEChallengeData2016/
+  annotations/
+  annotations_detectron2/
+  images/
+  objectInfo150.txt
+```
+The directory `annotations_detectron2` is generated by running `python datasets/prepare_ade20k_sem_seg.py`.

approach/ovod/detectron2/docs/tutorials/configs.md

@@ -0,0 +1,62 @@
+# Yacs Configs
+
+Detectron2 provides a key-value based config system that can be
+used to obtain standard, common behaviors.
+
+This system uses YAML and [yacs](https://github.com/rbgirshick/yacs).
+Yaml is a very limited language,
+so we do not expect all features in detectron2 to be available through configs.
+If you need something that's not available in the config space,
+please write code using detectron2's API.
+
+With the introduction of a more powerful [LazyConfig system](lazyconfigs.md),
+we no longer add functionality / new keys to the Yacs/Yaml-based config system.
+
+### Basic Usage
+
+Some basic usage of the `CfgNode` object is shown here. See more in [documentation](../modules/config.html#detectron2.config.CfgNode).
+```python
+from detectron2.config import get_cfg
+cfg = get_cfg()    # obtain detectron2's default config
+cfg.xxx = yyy      # add new configs for your own custom components
+cfg.merge_from_file("my_cfg.yaml")   # load values from a file
+
+cfg.merge_from_list(["MODEL.WEIGHTS", "weights.pth"])   # can also load values from a list of str
+print(cfg.dump())  # print formatted configs
+with open("output.yaml", "w") as f:
+  f.write(cfg.dump())   # save config to file
+```
+
+In addition to the basic Yaml syntax, the config file can
+define a `_BASE_: base.yaml` field, which will load a base config file first.
+Values in the base config will be overwritten in sub-configs, if there are any conflicts.
+We provided several base configs for standard model architectures.
+
+Many builtin tools in detectron2 accept command line config overwrite:
+Key-value pairs provided in the command line will overwrite the existing values in the config file.
+For example, [demo.py](../../demo/demo.py) can be used with
+```sh
+./demo.py --config-file config.yaml [--other-options] \
+  --opts MODEL.WEIGHTS /path/to/weights INPUT.MIN_SIZE_TEST 1000
+```
+
+To see a list of available configs in detectron2 and what they mean,
+check [Config References](../modules/config.html#config-references)
+
+### Configs in Projects
+
+A project that lives outside the detectron2 library may define its own configs, which will need to be added
+for the project to be functional, e.g.:
+```python
+from detectron2.projects.point_rend import add_pointrend_config
+cfg = get_cfg()    # obtain detectron2's default config
+add_pointrend_config(cfg)  # add pointrend's default config
+# ... ...
+```
+
+### Best Practice with Configs
+
+1. Treat the configs you write as "code": avoid copying them or duplicating them; use `_BASE_`
+   to share common parts between configs.
+
+2. Keep the configs you write simple: don't include keys that do not affect the experimental setting.

approach/ovod/detectron2/docs/tutorials/data_loading.md

@@ -0,0 +1,95 @@
+
+# Dataloader
+
+Dataloader is the component that provides data to models.
+A dataloader usually (but not necessarily) takes raw information from [datasets](./datasets.md),
+and process them into a format needed by the model.
+
+## How the Existing Dataloader Works
+
+Detectron2 contains a builtin data loading pipeline.
+It's good to understand how it works, in case you need to write a custom one.
+
+Detectron2 provides two functions
+[build_detection_{train,test}_loader](../modules/data.html#detectron2.data.build_detection_train_loader)
+that create a default data loader from a given config.
+Here is how `build_detection_{train,test}_loader` work:
+
+1. It takes the name of a registered dataset (e.g., "coco_2017_train") and loads a `list[dict]` representing the dataset items
+   in a lightweight format. These dataset items are not yet ready to be used by the model (e.g., images are
+   not loaded into memory, random augmentations have not been applied, etc.).
+   Details about the dataset format and dataset registration can be found in
+   [datasets](./datasets.md).
+2. Each dict in this list is mapped by a function ("mapper"):
+   * Users can customize this mapping function by specifying the "mapper" argument in
+        `build_detection_{train,test}_loader`. The default mapper is [DatasetMapper](../modules/data.html#detectron2.data.DatasetMapper).
+   * The output format of the mapper can be arbitrary, as long as it is accepted by the consumer of this data loader (usually the model).
+     The outputs of the default mapper, after batching, follow the default model input format documented in
+     [Use Models](./models.html#model-input-format).
+   * The role of the mapper is to transform the lightweight representation of a dataset item into a format
+     that is ready for the model to consume (including, e.g., read images, perform random data augmentation and convert to torch Tensors).
+     If you would like to perform custom transformations to data, you often want a custom mapper.
+3. The outputs of the mapper are batched (simply into a list).
+4. This batched data is the output of the data loader. Typically, it's also the input of
+   `model.forward()`.
+
+
+## Write a Custom Dataloader
+
+Using a different "mapper" with `build_detection_{train,test}_loader(mapper=)` works for most use cases
+of custom data loading.
+For example, if you want to resize all images to a fixed size for training, use:
+
+```python
+import detectron2.data.transforms as T
+from detectron2.data import DatasetMapper   # the default mapper
+dataloader = build_detection_train_loader(cfg,
+   mapper=DatasetMapper(cfg, is_train=True, augmentations=[
+      T.Resize((800, 800))
+   ]))
+# use this dataloader instead of the default
+```
+If the arguments of the default [DatasetMapper](../modules/data.html#detectron2.data.DatasetMapper)
+does not provide what you need, you may write a custom mapper function and use it instead, e.g.:
+
+```python
+from detectron2.data import detection_utils as utils
+ # Show how to implement a minimal mapper, similar to the default DatasetMapper
+def mapper(dataset_dict):
+    dataset_dict = copy.deepcopy(dataset_dict)  # it will be modified by code below
+    # can use other ways to read image
+    image = utils.read_image(dataset_dict["file_name"], format="BGR")
+    # See "Data Augmentation" tutorial for details usage
+    auginput = T.AugInput(image)
+    transform = T.Resize((800, 800))(auginput)
+    image = torch.from_numpy(auginput.image.transpose(2, 0, 1))
+    annos = [
+        utils.transform_instance_annotations(annotation, [transform], image.shape[1:])
+        for annotation in dataset_dict.pop("annotations")
+    ]
+    return {
+       # create the format that the model expects
+       "image": image,
+       "instances": utils.annotations_to_instances(annos, image.shape[1:])
+    }
+dataloader = build_detection_train_loader(cfg, mapper=mapper)
+```
+
+If you want to change not only the mapper (e.g., in order to implement different sampling or batching logic),
+`build_detection_train_loader` won't work and you will need to write a different data loader.
+The data loader is simply a
+python iterator that produces [the format](./models.md) that the model accepts.
+You can implement it using any tools you like.
+
+No matter what to implement, it's recommended to
+check out [API documentation of detectron2.data](../modules/data) to learn more about the APIs of
+these functions.
+
+## Use a Custom Dataloader
+
+If you use [DefaultTrainer](../modules/engine.html#detectron2.engine.defaults.DefaultTrainer),
+you can overwrite its `build_{train,test}_loader` method to use your own dataloader.
+See the [deeplab dataloader](../../projects/DeepLab/train_net.py)
+for an example.
+
+If you write your own training loop, you can plug in your data loader easily.

approach/ovod/detectron2/docs/tutorials/datasets.md

@@ -0,0 +1,290 @@
+# Use Custom Datasets
+
+This document explains how the dataset APIs
+([DatasetCatalog](../modules/data.html#detectron2.data.DatasetCatalog), [MetadataCatalog](../modules/data.html#detectron2.data.MetadataCatalog))
+work, and how to use them to add custom datasets.
+
+Datasets that have builtin support in detectron2 are listed in [builtin datasets](builtin_datasets.md).
+If you want to use a custom dataset while also reusing detectron2's data loaders,
+you will need to:
+
+1. __Register__ your dataset (i.e., tell detectron2 how to obtain your dataset).
+2. Optionally, __register metadata__ for your dataset.
+
+Next, we explain the above two concepts in detail.
+
+The [Colab tutorial](https://colab.research.google.com/drive/16jcaJoc6bCFAQ96jDe2HwtXj7BMD_-m5)
+has a live example of how to register and train on a dataset of custom formats.
+
+### Register a Dataset
+
+To let detectron2 know how to obtain a dataset named "my_dataset", users need to implement
+a function that returns the items in your dataset and then tell detectron2 about this
+function:
+```python
+def my_dataset_function():
+  ...
+  return list[dict] in the following format
+
+from detectron2.data import DatasetCatalog
+DatasetCatalog.register("my_dataset", my_dataset_function)
+# later, to access the data:
+data: List[Dict] = DatasetCatalog.get("my_dataset")
+```
+
+Here, the snippet associates a dataset named "my_dataset" with a function that returns the data.
+The function must return the same data (with same order) if called multiple times.
+The registration stays effective until the process exits.
+
+The function can do arbitrary things and should return the data in `list[dict]`, each dict in either
+of the following formats:
+1. Detectron2's standard dataset dict, described below. This will make it work with many other builtin
+   features in detectron2, so it's recommended to use it when it's sufficient.
+2. Any custom format. You can also return arbitrary dicts in your own format,
+   such as adding extra keys for new tasks.
+   Then you will need to handle them properly downstream as well.
+   See below for more details.
+
+#### Standard Dataset Dicts
+
+For standard tasks
+(instance detection, instance/semantic/panoptic segmentation, keypoint detection),
+we load the original dataset into `list[dict]` with a specification similar to COCO's annotations.
+This is our standard representation for a dataset.
+
+Each dict contains information about one image.
+The dict may have the following fields,
+and the required fields vary based on what the dataloader or the task needs (see more below).
+
+```eval_rst
+.. list-table::
+  :header-rows: 1
+
+  * - Task
+    - Fields
+  * - Common
+    - file_name, height, width, image_id
+
+  * - Instance detection/segmentation
+    - annotations
+
+  * - Semantic segmentation
+    - sem_seg_file_name
+
+  * - Panoptic segmentation
+    - pan_seg_file_name, segments_info
+```
+
++ `file_name`: the full path to the image file.
++ `height`, `width`: integer. The shape of the image.
++ `image_id` (str or int): a unique id that identifies this image. Required by many
+  evaluators to identify the images, but a dataset may use it for different purposes.
++ `annotations` (list[dict]): Required by __instance detection/segmentation or keypoint detection__ tasks.
+  Each dict corresponds to annotations of one instance in this image, and
+  may contain the following keys:
+  + `bbox` (list[float], required): list of 4 numbers representing the bounding box of the instance.
+  + `bbox_mode` (int, required): the format of bbox.  It must be a member of
+    [structures.BoxMode](../modules/structures.html#detectron2.structures.BoxMode).
+    Currently supports: `BoxMode.XYXY_ABS`, `BoxMode.XYWH_ABS`.
+  + `category_id` (int, required): an integer in the range [0, num_categories-1] representing the category label.
+    The value num_categories is reserved to represent the "background" category, if applicable.
+  + `segmentation` (list[list[float]] or dict): the segmentation mask of the instance.
+    + If `list[list[float]]`, it represents a list of polygons, one for each connected component
+      of the object. Each `list[float]` is one simple polygon in the format of `[x1, y1, ..., xn, yn]` (n≥3).
+      The Xs and Ys are absolute coordinates in unit of pixels.
+    + If `dict`, it represents the per-pixel segmentation mask in COCO's compressed RLE format.
+      The dict should have keys "size" and "counts". You can convert a uint8 segmentation mask of 0s and
+      1s into such dict by `pycocotools.mask.encode(np.asarray(mask, order="F"))`.
+      `cfg.INPUT.MASK_FORMAT` must be set to `bitmask` if using the default data loader with such format.
+  + `keypoints` (list[float]): in the format of [x1, y1, v1,..., xn, yn, vn].
+    v[i] means the [visibility](http://cocodataset.org/#format-data) of this keypoint.
+    `n` must be equal to the number of keypoint categories.
+    The Xs and Ys are absolute real-value coordinates in range [0, W or H].
+
+    (Note that the keypoint coordinates in COCO format are integers in range [0, W-1 or H-1], which is different
+    from our standard format. Detectron2 adds 0.5 to COCO keypoint coordinates to convert them from discrete
+    pixel indices to floating point coordinates.)
+  + `iscrowd`: 0 (default) or 1. Whether this instance is labeled as COCO's "crowd
+    region". Don't include this field if you don't know what it means.
+
+  If `annotations` is an empty list, it means the image is labeled to have no objects.
+  Such images will by default be removed from training,
+  but can be included using `DATALOADER.FILTER_EMPTY_ANNOTATIONS`.
+
++ `sem_seg_file_name` (str):
+  The full path to the semantic segmentation ground truth file.
+  It should be a grayscale image whose pixel values are integer labels.
++ `pan_seg_file_name` (str):
+  The full path to panoptic segmentation ground truth file.
+  It should be an RGB image whose pixel values are integer ids encoded using the
+  [panopticapi.utils.id2rgb](https://github.com/cocodataset/panopticapi/) function.
+  The ids are defined by `segments_info`.
+  If an id does not appear in `segments_info`, the pixel is considered unlabeled
+  and is usually ignored in training & evaluation.
++ `segments_info` (list[dict]): defines the meaning of each id in panoptic segmentation ground truth.
+  Each dict has the following keys:
+  + `id` (int): integer that appears in the ground truth image.
+  + `category_id` (int): an integer in the range [0, num_categories-1] representing the category label.
+  + `iscrowd`: 0 (default) or 1. Whether this instance is labeled as COCO's "crowd region".
+
+
+```eval_rst
+
+.. note::
+
+   The PanopticFPN model does not use the panoptic segmentation
+   format defined here, but a combination of both instance segmentation and semantic segmentation data
+   format. See :doc:`builtin_datasets` for instructions on COCO.
+
+```
+
+Fast R-CNN (with pre-computed proposals) models are rarely used today.
+To train a Fast R-CNN, the following extra keys are needed:
+
++ `proposal_boxes` (array): 2D numpy array with shape (K, 4) representing K precomputed proposal boxes for this image.
++ `proposal_objectness_logits` (array): numpy array with shape (K, ), which corresponds to the objectness
+ logits of proposals in 'proposal_boxes'.
++ `proposal_bbox_mode` (int): the format of the precomputed proposal bbox.
+ It must be a member of
+ [structures.BoxMode](../modules/structures.html#detectron2.structures.BoxMode).
+ Default is `BoxMode.XYXY_ABS`.
+
+
+
+#### Custom Dataset Dicts for New Tasks
+
+In the `list[dict]` that your dataset function returns, the dictionary can also have __arbitrary custom data__.
+This will be useful for a new task that needs extra information not covered
+by the standard dataset dicts. In this case, you need to make sure the downstream code can handle your data
+correctly. Usually this requires writing a new `mapper` for the dataloader (see [Use Custom Dataloaders](./data_loading.md)).
+
+When designing a custom format, note that all dicts are stored in memory
+(sometimes serialized and with multiple copies).
+To save memory, each dict is meant to contain __small__ but sufficient information
+about each sample, such as file names and annotations.
+Loading full samples typically happens in the data loader.
+
+For attributes shared among the entire dataset, use `Metadata` (see below).
+To avoid extra memory, do not save such information inside each sample.
+
+### "Metadata" for Datasets
+
+Each dataset is associated with some metadata, accessible through
+`MetadataCatalog.get(dataset_name).some_metadata`.
+Metadata is a key-value mapping that contains information that's shared among
+the entire dataset, and usually is used to interpret what's in the dataset, e.g.,
+names of classes, colors of classes, root of files, etc.
+This information will be useful for augmentation, evaluation, visualization, logging, etc.
+The structure of metadata depends on what is needed from the corresponding downstream code.
+
+If you register a new dataset through `DatasetCatalog.register`,
+you may also want to add its corresponding metadata through
+`MetadataCatalog.get(dataset_name).some_key = some_value`, to enable any features that need the metadata.
+You can do it like this (using the metadata key "thing_classes" as an example):
+
+```python
+from detectron2.data import MetadataCatalog
+MetadataCatalog.get("my_dataset").thing_classes = ["person", "dog"]
+```
+
+Here is a list of metadata keys that are used by builtin features in detectron2.
+If you add your own dataset without these metadata, some features may be
+unavailable to you:
+
+* `thing_classes` (list[str]): Used by all instance detection/segmentation tasks.
+  A list of names for each instance/thing category.
+  If you load a COCO format dataset, it will be automatically set by the function `load_coco_json`.
+
+* `thing_colors` (list[tuple(r, g, b)]): Pre-defined color (in [0, 255]) for each thing category.
+  Used for visualization. If not given, random colors will be used.
+
+* `stuff_classes` (list[str]): Used by semantic and panoptic segmentation tasks.
+  A list of names for each stuff category.
+
+* `stuff_colors` (list[tuple(r, g, b)]): Pre-defined color (in [0, 255]) for each stuff category.
+  Used for visualization. If not given, random colors are used.
+
+* `ignore_label` (int): Used by semantic and panoptic segmentation tasks. Pixels in ground-truth
+  annotations with this category label should be ignored in evaluation. Typically these are "unlabeled"
+  pixels.
+
+* `keypoint_names` (list[str]): Used by keypoint detection. A list of names for each keypoint.
+
+* `keypoint_flip_map` (list[tuple[str]]): Used by keypoint detection. A list of pairs of names,
+  where each pair are the two keypoints that should be flipped if the image is
+  flipped horizontally during augmentation.
+* `keypoint_connection_rules`: list[tuple(str, str, (r, g, b))]. Each tuple specifies a pair of keypoints
+  that are connected and the color (in [0, 255]) to use for the line between them when visualized.
+
+Some additional metadata that are specific to the evaluation of certain datasets (e.g. COCO):
+
+* `thing_dataset_id_to_contiguous_id` (dict[int->int]): Used by all instance detection/segmentation tasks in the COCO format.
+  A mapping from instance class ids in the dataset to contiguous ids in range [0, #class).
+  Will be automatically set by the function `load_coco_json`.
+
+* `stuff_dataset_id_to_contiguous_id` (dict[int->int]): Used when generating prediction json files for
+  semantic/panoptic segmentation.
+  A mapping from semantic segmentation class ids in the dataset
+  to contiguous ids in [0, num_categories). It is useful for evaluation only.
+
+* `json_file`: The COCO annotation json file. Used by COCO evaluation for COCO-format datasets.
+* `panoptic_root`, `panoptic_json`: Used by COCO-format panoptic evaluation.
+* `evaluator_type`: Used by the builtin main training script to select
+   evaluator. Don't use it in a new training script.
+   You can just provide the [DatasetEvaluator](../modules/evaluation.html#detectron2.evaluation.DatasetEvaluator)
+   for your dataset directly in your main script.
+
+```eval_rst
+.. note::
+
+   In recognition, sometimes we use the term "thing" for instance-level tasks,
+   and "stuff" for semantic segmentation tasks.
+   Both are used in panoptic segmentation tasks.
+   For background on the concept of "thing" and "stuff", see
+   `On Seeing Stuff: The Perception of Materials by Humans and Machines
+   <http://persci.mit.edu/pub_pdfs/adelson_spie_01.pdf>`_.
+```
+
+### Register a COCO Format Dataset
+
+If your instance-level (detection, segmentation, keypoint) dataset is already a json file in the COCO format,
+the dataset and its associated metadata can be registered easily with:
+```python
+from detectron2.data.datasets import register_coco_instances
+register_coco_instances("my_dataset", {}, "json_annotation.json", "path/to/image/dir")
+```
+
+If your dataset is in COCO format but need to be further processed, or has extra custom per-instance annotations,
+the [load_coco_json](../modules/data.html#detectron2.data.datasets.load_coco_json)
+function might be useful.
+
+### Update the Config for New Datasets
+
+Once you've registered the dataset, you can use the name of the dataset (e.g., "my_dataset" in
+example above) in `cfg.DATASETS.{TRAIN,TEST}`.
+There are other configs you might want to change to train or evaluate on new datasets:
+
+* `MODEL.ROI_HEADS.NUM_CLASSES` and `MODEL.RETINANET.NUM_CLASSES` are the number of thing classes
+  for R-CNN and RetinaNet models, respectively.
+* `MODEL.ROI_KEYPOINT_HEAD.NUM_KEYPOINTS` sets the number of keypoints for Keypoint R-CNN.
+  You'll also need to set [Keypoint OKS](http://cocodataset.org/#keypoints-eval)
+  with `TEST.KEYPOINT_OKS_SIGMAS` for evaluation.
+* `MODEL.SEM_SEG_HEAD.NUM_CLASSES` sets the number of stuff classes for Semantic FPN & Panoptic FPN.
+* `TEST.DETECTIONS_PER_IMAGE` controls the maximum number of objects to be detected.
+  Set it to a larger number if test images may contain >100 objects.
+* If you're training Fast R-CNN (with precomputed proposals), `DATASETS.PROPOSAL_FILES_{TRAIN,TEST}`
+  need to match the datasets. The format of proposal files are documented
+  [here](../modules/data.html#detectron2.data.load_proposals_into_dataset).
+
+New models
+(e.g. [TensorMask](../../projects/TensorMask),
+[PointRend](../../projects/PointRend))
+often have similar configs of their own that need to be changed as well.
+
+```eval_rst
+.. tip::
+
+   After changing the number of classes, certain layers in a pre-trained model will become incompatible
+   and therefore cannot be loaded to the new model.
+   This is expected, and loading such pre-trained models will produce warnings about such layers.
+```

approach/ovod/detectron2/docs/tutorials/deployment.md

@@ -0,0 +1,137 @@
+# Deployment
+
+Models written in Python need to go through an export process to become a deployable artifact.
+A few basic concepts about this process:
+
+__"Export method"__ is how a Python model is fully serialized to a deployable format.
+We support the following export methods:
+
+* `tracing`: see [pytorch documentation](https://pytorch.org/tutorials/beginner/Intro_to_TorchScript_tutorial.html) to learn about it
+* `scripting`: see [pytorch documentation](https://pytorch.org/tutorials/beginner/Intro_to_TorchScript_tutorial.html) to learn about it
+* `caffe2_tracing`: replace parts of the model by caffe2 operators, then use tracing.
+
+__"Format"__ is how a serialized model is described in a file, e.g.
+TorchScript, Caffe2 protobuf, ONNX format.
+__"Runtime"__ is an engine that loads a serialized model and executes it,
+e.g., PyTorch, Caffe2, TensorFlow, onnxruntime, TensorRT, etc.
+A runtime is often tied to a specific format
+(e.g. PyTorch needs TorchScript format, Caffe2 needs protobuf format).
+We currently support the following combination and each has some limitations:
+
+```eval_rst
++----------------------------+-------------+-------------+-----------------------------+
+|       Export Method        |   tracing   |  scripting  |       caffe2_tracing        |
++============================+=============+=============+=============================+
+| **Formats**                | TorchScript | TorchScript | Caffe2, TorchScript, ONNX   |
++----------------------------+-------------+-------------+-----------------------------+
+| **Runtime**                | PyTorch     | PyTorch     | Caffe2, PyTorch             |
++----------------------------+-------------+-------------+-----------------------------+
+| C++/Python inference       | ✅          | ✅          | ✅                          |
++----------------------------+-------------+-------------+-----------------------------+
+| Dynamic resolution         | ✅          | ✅          | ✅                          |
++----------------------------+-------------+-------------+-----------------------------+
+| Batch size requirement     | Constant    | Dynamic     | Batch inference unsupported |
++----------------------------+-------------+-------------+-----------------------------+
+| Extra runtime deps         | torchvision | torchvision | Caffe2 ops (usually already |
+|                            |             |             |                             |
+|                            |             |             | included in PyTorch)        |
++----------------------------+-------------+-------------+-----------------------------+
+| Faster/Mask/Keypoint R-CNN | ✅          | ✅          | ✅                          |
++----------------------------+-------------+-------------+-----------------------------+
+| RetinaNet                  | ✅          | ✅          | ✅                          |
++----------------------------+-------------+-------------+-----------------------------+
+| PointRend R-CNN            | ✅          | ❌          | ❌                          |
++----------------------------+-------------+-------------+-----------------------------+
+| Cascade R-CNN              | ✅          | ❌          | ❌                          |
++----------------------------+-------------+-------------+-----------------------------+
+
+```
+
+`caffe2_tracing` is going to be deprecated.
+We don't plan to work on additional support for other formats/runtime, but contributions are welcome.
+
+
+## Deployment with Tracing or Scripting
+
+Models can be exported to TorchScript format, by either
+[tracing or scripting](https://pytorch.org/tutorials/beginner/Intro_to_TorchScript_tutorial.html).
+The output model file can be loaded without detectron2 dependency in either Python or C++.
+The exported model often requires torchvision (or its C++ library) dependency for some custom ops.
+
+This feature requires PyTorch ≥ 1.8.
+
+### Coverage
+Most official models under the meta architectures `GeneralizedRCNN` and `RetinaNet`
+are supported in both tracing and scripting mode.
+Cascade R-CNN and PointRend are currently supported in tracing.
+Users' custom extensions are supported if they are also scriptable or traceable.
+
+For models exported with tracing, dynamic input resolution is allowed, but batch size
+(number of input images) must be fixed.
+Scripting can support dynamic batch size.
+
+### Usage
+
+The main export APIs for tracing and scripting are [TracingAdapter](../modules/export.html#detectron2.export.TracingAdapter)
+and [scripting_with_instances](../modules/export.html#detectron2.export.scripting_with_instances).
+Their usage is currently demonstrated in [test_export_torchscript.py](../../tests/test_export_torchscript.py)
+(see `TestScripting` and `TestTracing`)
+as well as the [deployment example](../../tools/deploy).
+Please check that these examples can run, and then modify for your use cases.
+The usage now requires some user effort and necessary knowledge for each model to workaround the limitation of scripting and tracing.
+In the future we plan to wrap these under simpler APIs to lower the bar to use them.
+
+## Deployment with Caffe2-tracing
+We provide [Caffe2Tracer](../modules/export.html#detectron2.export.Caffe2Tracer)
+that performs the export logic.
+It replaces parts of the model with Caffe2 operators,
+and then export the model into Caffe2, TorchScript or ONNX format.
+
+The converted model is able to run in either Python or C++ without detectron2/torchvision dependency, on CPU or GPUs.
+It has a runtime optimized for CPU & mobile inference, but not optimized for GPU inference.
+
+This feature requires ONNX ≥ 1.6.
+
+### Coverage
+
+Most official models under these 3 common meta architectures: `GeneralizedRCNN`, `RetinaNet`, `PanopticFPN`
+are supported. Cascade R-CNN is not supported. Batch inference is not supported.
+
+Users' custom extensions under these architectures (added through registration) are supported
+as long as they do not contain control flow or operators not available in Caffe2 (e.g. deformable convolution).
+For example, custom backbones and heads are often supported out of the box.
+
+### Usage
+
+The APIs are listed at [the API documentation](../modules/export).
+We provide [export_model.py](../../tools/deploy/) as an example that uses
+these APIs to convert a standard model. For custom models/datasets, you can add them to this script.
+
+### Use the model in C++/Python
+
+The model can be loaded in C++ and deployed with
+either Caffe2 or Pytorch runtime.. [C++ examples](../../tools/deploy/) for Mask R-CNN
+are given as a reference. Note that:
+
+* Models exported with `caffe2_tracing` method take a special input format
+  described in [documentation](../modules/export.html#detectron2.export.Caffe2Tracer).
+  This was taken care of in the C++ example.
+
+* The converted models do not contain post-processing operations that
+  transform raw layer outputs into formatted predictions.
+  For example, the C++ examples only produce raw outputs (28x28 masks) from the final
+  layers that are not post-processed, because in actual deployment, an application often needs
+  its custom lightweight post-processing, so this step is left for users.
+
+To help use the Caffe2-format model in python,
+we provide a python wrapper around the converted model, in the
+[Caffe2Model.\_\_call\_\_](../modules/export.html#detectron2.export.Caffe2Model.__call__) method.
+This method has an interface that's identical to the [pytorch versions of models](./models.md),
+and it internally applies pre/post-processing code to match the formats.
+This wrapper can serve as a reference for how to use Caffe2's python API,
+or for how to implement pre/post-processing in actual deployment.
+
+## Conversion to TensorFlow
+[tensorpack Faster R-CNN](https://github.com/tensorpack/tensorpack/tree/master/examples/FasterRCNN/convert_d2)
+provides scripts to convert a few standard detectron2 R-CNN models to TensorFlow's pb format.
+It works by translating configs and weights, therefore only support a few models.

approach/ovod/detectron2/docs/tutorials/extend.md

@@ -0,0 +1,141 @@
+# Extend Detectron2's Defaults
+
+__Research is about doing things in new ways__.
+This brings a tension in how to create abstractions in code,
+which is a challenge for any research engineering project of a significant size:
+
+1. On one hand, it needs to have very thin abstractions to allow for the possibility of doing
+   everything in new ways. It should be reasonably easy to break existing
+   abstractions and replace them with new ones.
+
+2. On the other hand, such a project also needs reasonably high-level
+   abstractions, so that users can easily do things in standard ways,
+   without worrying too much about the details that only certain researchers care about.
+
+In detectron2, there are two types of interfaces that address this tension together:
+
+1. Functions and classes that take a config (`cfg`) argument
+   created from a yaml file
+   (sometimes with few extra arguments).
+
+   Such functions and classes implement
+   the "standard default" behavior: it will read what it needs from a given
+   config and do the "standard" thing.
+   Users only need to load an expert-made config and pass it around, without having to worry about
+   which arguments are used and what they all mean.
+
+   See [Yacs Configs](configs.md) for a detailed tutorial.
+
+2. Functions and classes that have well-defined explicit arguments.
+
+   Each of these is a small building block of the entire system.
+   They require users' expertise to understand what each argument should be,
+   and require more effort to stitch together to a larger system.
+   But they can be stitched together in more flexible ways.
+
+   When you need to implement something not supported by the "standard defaults"
+   included in detectron2, these well-defined components can be reused.
+
+   The [LazyConfig system](lazyconfigs.md) relies on such functions and classes.
+
+3. A few functions and classes are implemented with the
+   [@configurable](../modules/config.html#detectron2.config.configurable)
+   decorator - they can be called with either a config, or with explicit arguments, or a mixture of both.
+   Their explicit argument interfaces are currently experimental.
+
+   As an example, a Mask R-CNN model can be built in the following ways:
+
+   1. Config-only:
+      ```python
+      # load proper yaml config file, then
+      model = build_model(cfg)
+      ```
+
+   2. Mixture of config and additional argument overrides:
+      ```python
+      model = GeneralizedRCNN(
+        cfg,
+        roi_heads=StandardROIHeads(cfg, batch_size_per_image=666),
+        pixel_std=[57.0, 57.0, 57.0])
+      ```
+
+   3. Full explicit arguments:
+   <details>
+   <summary>
+   (click to expand)
+   </summary>
+
+   ```python
+   model = GeneralizedRCNN(
+       backbone=FPN(
+           ResNet(
+               BasicStem(3, 64, norm="FrozenBN"),
+               ResNet.make_default_stages(50, stride_in_1x1=True, norm="FrozenBN"),
+               out_features=["res2", "res3", "res4", "res5"],
+           ).freeze(2),
+           ["res2", "res3", "res4", "res5"],
+           256,
+           top_block=LastLevelMaxPool(),
+       ),
+       proposal_generator=RPN(
+           in_features=["p2", "p3", "p4", "p5", "p6"],
+           head=StandardRPNHead(in_channels=256, num_anchors=3),
+           anchor_generator=DefaultAnchorGenerator(
+               sizes=[[32], [64], [128], [256], [512]],
+               aspect_ratios=[0.5, 1.0, 2.0],
+               strides=[4, 8, 16, 32, 64],
+               offset=0.0,
+           ),
+           anchor_matcher=Matcher([0.3, 0.7], [0, -1, 1], allow_low_quality_matches=True),
+           box2box_transform=Box2BoxTransform([1.0, 1.0, 1.0, 1.0]),
+           batch_size_per_image=256,
+           positive_fraction=0.5,
+           pre_nms_topk=(2000, 1000),
+           post_nms_topk=(1000, 1000),
+           nms_thresh=0.7,
+       ),
+       roi_heads=StandardROIHeads(
+           num_classes=80,
+           batch_size_per_image=512,
+           positive_fraction=0.25,
+           proposal_matcher=Matcher([0.5], [0, 1], allow_low_quality_matches=False),
+           box_in_features=["p2", "p3", "p4", "p5"],
+           box_pooler=ROIPooler(7, (1.0 / 4, 1.0 / 8, 1.0 / 16, 1.0 / 32), 0, "ROIAlignV2"),
+           box_head=FastRCNNConvFCHead(
+               ShapeSpec(channels=256, height=7, width=7), conv_dims=[], fc_dims=[1024, 1024]
+           ),
+           box_predictor=FastRCNNOutputLayers(
+               ShapeSpec(channels=1024),
+               test_score_thresh=0.05,
+               box2box_transform=Box2BoxTransform((10, 10, 5, 5)),
+               num_classes=80,
+           ),
+           mask_in_features=["p2", "p3", "p4", "p5"],
+           mask_pooler=ROIPooler(14, (1.0 / 4, 1.0 / 8, 1.0 / 16, 1.0 / 32), 0, "ROIAlignV2"),
+           mask_head=MaskRCNNConvUpsampleHead(
+               ShapeSpec(channels=256, width=14, height=14),
+               num_classes=80,
+               conv_dims=[256, 256, 256, 256, 256],
+           ),
+       ),
+       pixel_mean=[103.530, 116.280, 123.675],
+       pixel_std=[1.0, 1.0, 1.0],
+       input_format="BGR",
+   )
+   ```
+
+   </details>
+
+
+If you only need the standard behavior, the [Beginner's Tutorial](./getting_started.md)
+should suffice. If you need to extend detectron2 to your own needs,
+see the following tutorials for more details:
+
+* Detectron2 includes a few standard datasets. To use custom ones, see
+  [Use Custom Datasets](./datasets.md).
+* Detectron2 contains the standard logic that creates a data loader for training/testing from a
+  dataset, but you can write your own as well. See [Use Custom Data Loaders](./data_loading.md).
+* Detectron2 implements many standard detection models, and provide ways for you
+  to overwrite their behaviors. See [Use Models](./models.md) and [Write Models](./write-models.md).
+* Detectron2 provides a default training loop that is good for common training tasks.
+  You can customize it with hooks, or write your own loop instead. See [training](./training.md).