diff --git a/approach/ovod/d-cube/.assets/d-cube_logo.png b/approach/ovod/d-cube/.assets/d-cube_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..b3491cc9f00b6d7f5847a9e55790166175d4cfd2 Binary files /dev/null and b/approach/ovod/d-cube/.assets/d-cube_logo.png differ diff --git a/approach/ovod/d-cube/d_cube/__init__.py b/approach/ovod/d-cube/d_cube/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e5207774e9ab152a282379e7666c5bd7c1d81d02 --- /dev/null +++ b/approach/ovod/d-cube/d_cube/__init__.py @@ -0,0 +1 @@ +from .d3 import D3 diff --git a/approach/ovod/d-cube/d_cube/d3.py b/approach/ovod/d-cube/d_cube/d3.py new file mode 100644 index 0000000000000000000000000000000000000000..480c3d0171e4c6a39f233ed17469fd5ab11f2779 --- /dev/null +++ b/approach/ovod/d-cube/d_cube/d3.py @@ -0,0 +1,775 @@ +# -*- coding: utf-8 -*- +__author__ = "Chi Xie and Zhao Zhang" +__maintainer__ = "Chi Xie" +# this is the core of the d-cube toolkit +import os +import os.path as osp +import json +from collections import defaultdict + +import numpy as np +from pycocotools import mask +import cv2 +import matplotlib.pyplot as plt + + +from .data_util import * + + +class D3: + def __init__(self, img_root, anno_root): + self.image_dir = img_root + self.anno_dir = anno_root + self.load_data() + + def load_data(self): + file_names = ["sentences.pkl", "annotations.pkl", "images.pkl", "groups.pkl"] + self.data = { + name.split(".")[0]: load_pkl(osp.join(self.anno_dir, name)) + for name in file_names + } + + def get_sent_ids(self, anno_ids=[], img_ids=[], group_ids=[], sent_ids=[]): + """get sentence ids for D-cube. + + Args: + anno_ids (list, optional): annotation ids to get sentence ids. Defaults to []. + img_ids (list, optional): image ids to get sentence ids. Defaults to []. + group_ids (list, optional): group ids to get sentence ids. Defaults to []. + sent_ids (list, optional): additional sentence ids you want to include. Defaults to []. + + Raises: + Exception: anno_ids, img_ids and group_ids cannot be used together. + + Returns: + list: sentence ids. + """ + img_ids = img_ids if isinstance(img_ids, list) else [img_ids] + anno_ids = anno_ids if isinstance(anno_ids, list) else [anno_ids] + group_ids = group_ids if isinstance(group_ids, list) else [group_ids] + sent_ids = sent_ids if isinstance(sent_ids, list) else [sent_ids] + + if not any([img_ids, anno_ids, group_ids, sent_ids]): + return list(self.data["sentences"].keys()) + + if ( + (anno_ids and img_ids) + or (anno_ids and group_ids) + or (img_ids and group_ids) + ): + raise Exception("anno_ids, img_ids, group_ids can only be used alone") + + out_ids_set = set() + if img_ids: + for img_id in img_ids: + imganno_ids = self.data["images"][img_id]["anno_id"] + for ianno_id in imganno_ids: + out_ids_set |= set(self.data["annotations"][ianno_id]["sent_id"]) + + if group_ids: + for group_id in group_ids: + out_ids_set |= set(self.data["groups"][group_id]["inner_sent_id"]) + + if anno_ids: + for ianno_id in anno_ids: + out_ids_set |= set(self.data["annotations"][ianno_id]["sent_id"]) + + if sent_ids: + out_ids_set &= set(sent_ids) + + return list(out_ids_set) + + def get_anno_ids(self, anno_ids=[], img_ids=[], group_ids=[], sent_ids=[]): + """get annotation ids for D-cube. + + Args: + anno_ids (list, optional): additional annotation ids you want to include. Defaults to []. + img_ids (list, optional): image ids to get annotation ids. Defaults to []. + group_ids (list, optional): group ids to get annotation ids. Defaults to []. + sent_ids (list, optional): sentence ids to get annotation ids. Defaults to []. + + Raises: + Exception: img_ids and group_ids cannot be used together. + + Returns: + list: annotation ids. + """ + img_ids = img_ids if isinstance(img_ids, list) else [img_ids] + anno_ids = anno_ids if isinstance(anno_ids, list) else [anno_ids] + group_ids = group_ids if isinstance(group_ids, list) else [group_ids] + sent_ids = sent_ids if isinstance(sent_ids, list) else [sent_ids] + + if not any([img_ids, anno_ids, group_ids, sent_ids]): + return list(self.data["annotations"].keys()) + + if img_ids and group_ids: + raise Exception("img_ids, group_ids can only be used alone") + + out_ids_set = set() + if img_ids: + for img_id in img_ids: + out_ids_set |= set(self.data["images"][img_id]["anno_id"]) + + if group_ids: + for group_id in group_ids: + for groupimg_id in self.data["groups"][group_id]["img_id"]: + out_ids_set |= set(self.data["images"][groupimg_id]["anno_id"]) + + if sent_ids and img_ids: + for sent_id in sent_ids: + out_ids_set &= set(self.data["sentences"][sent_id]["anno_id"]) + else: + for sent_id in sent_ids: + out_ids_set |= set(self.data["sentences"][sent_id]["anno_id"]) + + if anno_ids: + out_ids_set &= set(anno_ids) + + return list(out_ids_set) + + def get_img_ids(self, anno_ids=[], img_ids=[], group_ids=[], sent_ids=[]): + """get image ids for D-cube. + + Args: + anno_ids (list, optional): annotation ids to get image ids. Defaults to []. + img_ids (list, optional): additional image ids you want to include. Defaults to []. + group_ids (list, optional): group ids to get image ids. Defaults to []. + sent_ids (list, optional): sentence ids to get image ids. Defaults to []. + + Raises: + Exception: anno_ids and img_ids cannot be used together. + Exception: anno_ids and group_ids cannot be used together. + + Returns: + list: image ids. + """ + img_ids = img_ids if isinstance(img_ids, list) else [img_ids] + anno_ids = anno_ids if isinstance(anno_ids, list) else [anno_ids] + group_ids = group_ids if isinstance(group_ids, list) else [group_ids] + sent_ids = sent_ids if isinstance(sent_ids, list) else [sent_ids] + + if not any([img_ids, anno_ids, group_ids, sent_ids]): + return list(self.data["images"].keys()) + + if anno_ids and img_ids: + raise Exception("anno_ids and img_ids can only be used alone") + if anno_ids and group_ids: + raise Exception("anno_ids and group_ids can only be used alone") + + out_ids_set = set() + if anno_ids: + for ianno_id in anno_ids: + out_ids_set.add(self.data["annotations"][ianno_id]["img_id"]) + + if group_ids: + for group_id in group_ids: + out_ids_set |= set(self.data["groups"][group_id]["img_id"]) + + if sent_ids: + for sent_id in sent_ids: + for sentanno_id in self.data["sentences"][sent_id]["anno_id"]: + out_ids_set.add(self.data["annotations"][sentanno_id]["image_id"]) + + if img_ids: + out_ids_set &= set(img_ids) + + return list(out_ids_set) + + def get_group_ids(self, anno_ids=[], img_ids=[], group_ids=[], sent_ids=[]): + """get group ids for D-cube. + + Args: + anno_ids (list, optional): annotation ids to get group ids. Defaults to []. + img_ids (list, optional): image ids to get group ids. Defaults to []. + group_ids (list, optional): additional group_ids you want to include. Defaults to []. + sent_ids (list, optional): sentence ids to get group ids. Defaults to []. + + Raises: + Exception: anno_ids, img_ids and sent_ids cannot be used together. + + Returns: + list: group ids. + """ + img_ids = img_ids if isinstance(img_ids, list) else [img_ids] + anno_ids = anno_ids if isinstance(anno_ids, list) else [anno_ids] + group_ids = group_ids if isinstance(group_ids, list) else [group_ids] + sent_ids = sent_ids if isinstance(sent_ids, list) else [sent_ids] + + if not any([img_ids, anno_ids, group_ids, sent_ids]): + return list(self.data["groups"].keys()) + + if anno_ids and img_ids: + raise Exception("anno_ids and img_ids can only be used alone") + if anno_ids and sent_ids: + raise Exception("anno_ids and sent_ids can only be used alone") + if img_ids and sent_ids: + raise Exception("img_ids and sent_ids can only be used alone") + + out_ids_set = set() + if img_ids: + for img_id in img_ids: + out_ids_set.add(self.data["images"][img_id]["group_id"]) + + if anno_ids: + for anno_id in anno_ids: + out_ids_set.add(self.data["annotations"][anno_id]["group_id"]) + + if sent_ids: + for sent_id in sent_ids: + out_ids_set |= set(self.data["sentences"][sent_id]["group_id"]) + + if group_ids: + out_ids_set &= set(group_ids) + + return list(out_ids_set) + + def load_sents(self, sent_ids=None): + """load sentence info. + + Args: + sent_ids (list, int, optional): sentence ids. Defaults to None. + + Returns: + list: a list of sentence info. + """ + if sent_ids is not None and not isinstance(sent_ids, list): + sent_ids = [sent_ids] + if isinstance(sent_ids, list): + return [self.data["sentences"][sent_id] for sent_id in sent_ids] + else: + return list(self.data["sentences"].values()) + + def load_annos(self, anno_ids=None): + """load annotation info. + + Args: + anno_ids (list, int, optional): annotation ids. Defaults to None. + + Returns: + list: a list of annotation info. + """ + if anno_ids is not None and not isinstance(anno_ids, list): + anno_ids = [anno_ids] + if isinstance(anno_ids, list): + return [self.data["annotations"][anno_id] for anno_id in anno_ids] + else: + return list(self.data["annotations"].values()) + + def load_imgs(self, img_ids=None): + """load image info. + + Args: + img_ids (list, int, optional): image ids. Defaults to None. + + Returns: + list: a list of image info. + """ + if img_ids is not None and not isinstance(img_ids, list): + img_ids = [img_ids] + if isinstance(img_ids, list): + return [self.data["images"][img_ids] for img_ids in img_ids] + else: + return list(self.data["images"].values()) + + def load_groups(self, group_ids=None): + """load group info. + + Args: + group_ids (list, int, optional): group ids. Defaults to None. + + Returns: + list: a list of group info. + """ + if group_ids is not None and not isinstance(group_ids, list): + group_ids = [group_ids] + if isinstance(group_ids, list): + return [self.data["groups"][group_ids] for group_ids in group_ids] + else: + return list(self.data["groups"].values()) + + def get_mask(self, anno): + rle = anno[0]["segmentation"] + m = mask.decode(rle) + m = np.sum( + m, axis=2 + ) # sometimes there are multiple binary map (corresponding to multiple segs) + m = m.astype(np.uint8) # convert to np.uint8 + # compute area + area = sum(mask.area(rle)) # should be close to ann['area'] + return {"mask": m, "area": area} + + def show_mask(self, anno): + M = self.get_mask(anno) + msk = M["mask"] + ax = plt.gca() + ax.imshow(msk) + + def show_image_seg( + self, + img_ids=[], + save_dir=None, + show_sent=False, + on_image=False, + checkerboard_bg=False, + is_instance=True, + ): + if is_instance and checkerboard_bg: + raise ValueError( + "Cannot apply both is_instance and checkboard_bg at the same time." + ) + img_infos = self.load_imgs(img_ids=img_ids) + for img_idx, img_info in enumerate(img_infos): + img = cv2.imread(osp.join(self.image_dir, img_info["file_name"])) + anno_infos = self.load_annos(img_info["anno_id"]) + + bm_canvas = defaultdict(list) + merge_canvas = defaultdict(list) + for anno_info in anno_infos: + for sent_id in anno_info["sent_id"]: + bm_canvas[sent_id].append(anno_info["segmentation"]) + + for sent_id, bm_list in bm_canvas.items(): + merge_canvas[sent_id] = merge_rle( + bm_list, is_instance=is_instance, on_image=on_image + ) + + cv2.imwrite(osp.join(save_dir, f"{img_info['id']}.png"), img) + for sent_id, merge_mask in merge_canvas.items(): + if checkerboard_bg: + merge_mask = add_checkerboard_bg(img, merge_mask) + elif on_image: + merge_mask = visualize_mask_on_image(img, merge_mask, add_edge=True) + if show_sent: + sent_en = self.load_sents(sent_ids=sent_id)[0]["raw_sent"] + merge_mask = paste_text(merge_mask, sent_en) + cv2.imwrite( + osp.join(save_dir, f"{img_info['id']}_{sent_id}.png"), merge_mask + ) + + return merge_canvas + + def show_group_seg( + self, + group_ids, + save_root, + show_sent=True, + is_instance=True, + on_image=False, + checkerboard_bg=False, + ): + group_infos = self.load_groups(group_ids=group_ids) + for group_info in group_infos: + save_dir = osp.join(save_root, group_info["group_name"]) + os.makedirs(save_dir, exist_ok=True) + self.show_image_seg( + img_ids=group_info["img_id"], + save_dir=save_dir, + show_sent=show_sent, + is_instance=is_instance, + on_image=on_image, + checkerboard_bg=checkerboard_bg, + ) + + def show_image_seg_bbox( + self, + img_ids=[], + save_dir=None, + show_sent=False, + on_image=False, + checkerboard_bg=False, + is_instance=True, + ): + if is_instance and checkerboard_bg: + raise ValueError( + "Cannot apply both is_instance and checkboard_bg at the same time." + ) + img_infos = self.load_imgs(img_ids=img_ids) + for img_idx, img_info in enumerate(img_infos): + img = cv2.imread(osp.join(self.image_dir, img_info["file_name"])) + anno_infos = self.load_annos(img_info["anno_id"]) + + bm_canvas = defaultdict(list) + merge_canvas = defaultdict(list) + sent_boxes = defaultdict(list) + for anno_info in anno_infos: + for sent_id in anno_info["sent_id"]: + bm_canvas[sent_id].append(anno_info["segmentation"]) + sent_boxes[sent_id].append(anno_info["bbox"][0].tolist()) + + for sent_id, bm_list in bm_canvas.items(): + merge_canvas[sent_id] = merge_rle( + bm_list, is_instance=is_instance, on_image=on_image + ) + + cv2.imwrite(osp.join(save_dir, f"{img_info['id']}.png"), img) + for sent_id, merge_mask in merge_canvas.items(): + # vis mask + if checkerboard_bg: + merge_mask = add_checkerboard_bg(img, merge_mask) + elif on_image: + merge_mask = visualize_mask_on_image(img, merge_mask, add_edge=True) + # vis box + bboxes = sent_boxes[sent_id] + merge_mask = visualize_bbox_on_image(merge_mask, bboxes) + # vis sent + if show_sent: + sent_en = self.load_sents(sent_ids=sent_id)[0]["raw_sent"] + merge_mask = paste_text(merge_mask, sent_en) + cv2.imwrite( + osp.join(save_dir, f"{img_info['id']}_{sent_id}.png"), merge_mask + ) + + return merge_canvas + + def show_group_seg_bbox( + self, + group_ids, + save_root, + show_sent=True, + is_instance=True, + on_image=False, + checkerboard_bg=False, + ): + group_infos = self.load_groups(group_ids=group_ids) + for group_info in group_infos: + save_dir = osp.join(save_root, group_info["group_name"]) + os.makedirs(save_dir, exist_ok=True) + self.show_image_seg_bbox( + img_ids=group_info["img_id"], + save_dir=save_dir, + show_sent=show_sent, + is_instance=is_instance, + on_image=on_image, + checkerboard_bg=checkerboard_bg, + ) + + def show_image_bbox(self, img_ids=[], save_dir=None, show_sent=False): + img_infos = self.load_imgs(img_ids=img_ids) + for img_idx, img_info in enumerate(img_infos): + img = cv2.imread(osp.join(self.image_dir, img_info["file_name"])) + anno_infos = self.load_annos(img_info["anno_id"]) + + sent_boxes = defaultdict(list) + for anno_info in anno_infos: + for sent_id in anno_info["sent_id"]: + sent_boxes[sent_id].append(anno_info["bbox"][0].tolist()) + + cv2.imwrite(osp.join(save_dir, f"{img_info['id']}.png"), img) + for sent_id, bboxes in sent_boxes.items(): + merge_img = visualize_bbox_on_image(img, bboxes) + if show_sent: + sent_en = self.load_sents(sent_ids=sent_id)[0]["raw_sent"] + merge_img = paste_text(merge_img, sent_en) + cv2.imwrite( + osp.join(save_dir, f"{img_info['id']}_{sent_id}.png"), merge_img + ) + + def show_group_bbox(self, group_ids, save_root, show_sent=True): + group_infos = self.load_groups(group_ids=group_ids) + for group_info in group_infos: + save_dir = osp.join(save_root, group_info["group_name"]) + os.makedirs(save_dir, exist_ok=True) + self.show_image_bbox( + img_ids=group_info["img_id"], save_dir=save_dir, show_sent=show_sent + ) + + def stat_description(self, with_rev=False, inter_group=False): + """calculate and print dataset statistics. + + Args: + with_rev (bool, optional): consider absence descriptions or not. Defaults to False. + inter_group (bool, optional): calculate under intra- or inter-group settings. Defaults to False. + """ + stat_dict = {} + # Number of sents + sent_ids = list(self.data["sentences"].keys()) + if not with_rev: + sent_ids = [sent_id for sent_id in sent_ids if not self.is_revsent(sent_id)] + stat_dict["nsent"] = len(sent_ids) + # Number of annos / instance # TODO: rm rev + stat_dict["nanno"] = len(self.data["annotations"].keys()) + # Number of images + stat_dict["nimg"] = len(self.data["images"].keys()) + # Number of groups + stat_dict["ngroup"] = len(self.data["groups"].keys()) + + # Number of img-sent pair + num_img_sent = 0 + for img_id in self.data["images"].keys(): + anno_ids = self.get_anno_ids(img_ids=img_id) + anno_infos = self.load_annos(anno_ids=anno_ids) + cur_sent_set = set() + group_sent_ids = set( + self.load_groups(self.get_group_ids(img_ids=img_id))[0]["inner_sent_id"] + ) + for anno_info in anno_infos: + cur_sent_set |= set( + [i for i in anno_info["sent_id"] if i in group_sent_ids] + ) + if not with_rev: + cur_sent_set = [ + sent_id for sent_id in cur_sent_set if not self.is_revsent(sent_id) + ] + num_img_sent += len(cur_sent_set) + stat_dict["num_img_sent"] = num_img_sent + + # Number of absence img-sent pair + num_anti_img_sent = 0 + for img_id in self.data["images"].keys(): + anno_ids = self.get_anno_ids(img_ids=img_id) + anno_infos = self.load_annos(anno_ids=anno_ids) + cur_sent_set = set() + group_sent_ids = set( + self.load_groups(self.get_group_ids(img_ids=img_id))[0]["inner_sent_id"] + ) + for anno_info in anno_infos: + cur_sent_set |= set( + [i for i in anno_info["sent_id"] if i in group_sent_ids] + ) + assert group_sent_ids.issuperset( + cur_sent_set + ), f"{group_sent_ids}, {cur_sent_set}" + cur_anti_sent_set = group_sent_ids - cur_sent_set + if not with_rev: + cur_anti_sent_set = [ + sent_id + for sent_id in cur_anti_sent_set + if not self.is_revsent(sent_id) + ] + num_anti_img_sent += len(cur_anti_sent_set) + stat_dict["num_anti_img_sent"] = num_anti_img_sent + + # Number of anno-sent pair + num_anno_sent = 0 + anno_infos = self.load_annos() + for anno_info in anno_infos: + if inter_group: + anno_sent_ids = [i for i in anno_info["sent_id"]] + else: + group_sent_ids = set( + self.load_groups(anno_info["group_id"])[0]["inner_sent_id"] + ) + anno_sent_ids = [i for i in anno_info["sent_id"] if i in group_sent_ids] + if not with_rev: + anno_sent_ids = [ + sent_id for sent_id in anno_sent_ids if not self.is_revsent(sent_id) + ] + num_anno_sent += len(anno_sent_ids) + + stat_dict["num_anno_sent"] = num_anno_sent + + # Number of anti anno-sent pair + num_anti_anno_sent = 0 + anno_infos = self.load_annos() + for anno_info in anno_infos: + if inter_group: + all_sent_ids = set(self.get_sent_ids()) + anno_sent_ids = anno_info["sent_id"] + + anti_sent_ids = [ + sent_id for sent_id in all_sent_ids if sent_id not in anno_sent_ids + ] + else: + group_sent_ids = set( + self.load_groups(anno_info["group_id"])[0]["inner_sent_id"] + ) + anno_sent_ids = [i for i in anno_info["sent_id"] if i in group_sent_ids] + + anti_sent_ids = [ + sent_id + for sent_id in group_sent_ids + if sent_id not in anno_sent_ids + ] + + if not with_rev: + anti_sent_ids = [ + sent_id for sent_id in anti_sent_ids if not self.is_revsent(sent_id) + ] + num_anti_anno_sent += len(anti_sent_ids) + + stat_dict["num_anti_anno_sent"] = num_anti_anno_sent + + # Len of sentence + totle_len = 0 + for sent_info in self.load_sents(sent_ids): + totle_len += len(sent_info["raw_sent"].split()) + + stat_dict["avg_sent_len"] = totle_len / stat_dict["nsent"] + + print(stat_dict) + + def is_revsent(self, sent_id): + sent_info = self.load_sents(sent_ids=sent_id) + return sent_info[0]["is_negative"] + + def data2coca(self, out_root, with_rev=False): + group_infos = self.load_groups() + for group_info in group_infos: + sent_ids = group_info["inner_sent_id"] + if not with_rev: + sent_ids = [ + sent_id for sent_id in sent_ids if not self.is_revsent(sent_id) + ] + sent_infos = self.load_sents(sent_ids) + for sent_info in sent_infos: + sent = sent_info["raw_sent"] + img_infos = self.load_imgs(group_info["img_id"]) + for img_info in img_infos: + src_img_path = osp.join(self.image_dir, img_info["file_name"]) + raw_name = img_info["file_name"].split("/")[-1] + out_img_dir = osp.join(out_root, "images", sent) + os.makedirs(out_img_dir, exist_ok=True) + out_img_path = osp.join(out_img_dir, raw_name) + copy_file(src_img_path, out_img_path) + + out_mask_dir = osp.join(out_root, "masks", sent) + os.makedirs(out_mask_dir, exist_ok=True) + out_mask_path = osp.join( + out_mask_dir, raw_name.replace(".jpg", ".png") + ) + + cur_anno_ids = self.get_anno_ids( + img_ids=img_info["id"], sent_ids=sent_info["id"] + ) + anno_infos = self.load_annos(cur_anno_ids) + rle_list = [anno_info["segmentation"] for anno_info in anno_infos] + bmask = merge2bin(rle_list, img_info["height"], img_info["width"]) + cv2.imwrite(out_mask_path, bmask) + + def convert2coco(self, out_root, anti_mode=False, is_group_separated=True): + """ + Convert the annotation format of D^3 dataset to COCO. + 1. The sent_id can be viewed as category_id in COCO. + 2. If `is_group_separated` is True, `outer_sent_id` does not need to be considered. + 3. if `with_rev` is False, sents that meet `is_revsent` will be ignore. + """ + os.makedirs(out_root, exist_ok=True) + coco_dict = { + "images": [], + "categories": [], + "annotations": [], + } + + sent_ids = self.get_sent_ids() + if anti_mode == 1: + sent_ids = [sent_id for sent_id in sent_ids if not self.is_revsent(sent_id)] + elif anti_mode == 2: + sent_ids = [sent_id for sent_id in sent_ids if self.is_revsent(sent_id)] + elif anti_mode == 0: + pass + else: + raise Exception("Unimplemented anti_mode.") + + sent_infos = self.load_sents(sent_ids) + for isent_info in sent_infos: + coco_dict["categories"].append( + { + "id": isent_info["id"], + "name": isent_info["raw_sent"], + } + ) + + item_id = 0 + img_infos = self.load_imgs() + for iimg_info in img_infos: + coco_dict["images"].append( + { + "id": iimg_info["id"], + "file_name": iimg_info["file_name"], + "height": iimg_info["height"], + "width": iimg_info["width"], + } + ) + + anno_ids = self.get_anno_ids(img_ids=iimg_info["id"]) + anno_infos = self.load_annos(anno_ids) + + for ianno_info in anno_infos: + if is_group_separated: + inner_group_sent_ids = [ + isent_id + for isent_id in ianno_info["sent_id"] + if isent_id + in self.load_groups(ianno_info["group_id"])[0]["inner_sent_id"] + ] + cur_sent_ids = inner_group_sent_ids + else: + cur_sent_ids = ianno_info["sent_id"] + + for isent_id in cur_sent_ids: + if isent_id not in sent_ids: + continue + + seg = ianno_info["segmentation"][0].copy() + if isinstance(seg, dict): # RLE + counts = seg["counts"] + if not isinstance(counts, str): + # make it json-serializable + seg["counts"] = counts.decode("ascii") + + coco_dict["annotations"].append( + { + "id": item_id, + "image_id": iimg_info["id"], + "category_id": isent_id, + "segmentation": seg, + "area": int(ianno_info["area"][0]), + "bbox": [ + int(cord) for cord in ianno_info["bbox"][0].tolist() + ], + "iscrowd": 0, # TODO: ianno_info["iscrowd"] + } + ) + item_id += 1 + + with open(osp.join(out_root, "coco_annotations.json"), "w") as f: + json.dump(coco_dict, f, indent=4) + + def sent_analyse(self, save_dir, with_rev=False): + """analyze word info in D-cube and generate word length histograms, word clouds, etc. + + Args: + save_dir (str): path to save the visualized results. + with_rev (bool, optional): consider absence descriptions or not. Defaults to False. + """ + sent_ids = self.get_sent_ids() + if not with_rev: + sent_ids = [sent_id for sent_id in sent_ids if not self.is_revsent(sent_id)] + + sent_lens, sent_raws = [], [] + sent_infos = self.load_sents(sent_ids) + for isent_info in sent_infos: + sent_raws.append(isent_info["raw_sent"]) + sent_lens.append(len(isent_info["raw_sent"].split())) + + os.makedirs(save_dir, exist_ok=True) + # plot_hist( + # sent_lens, + # bins=max(sent_lens) - min(sent_lens) + 1, + # save_path=osp.join(save_dir, "words_hist.pdf"), + # x="Lengths of descriptions", + # ) + # generate_wordclouds(sent_raws, osp.join(save_dir, "word_clouds")) + + def group_analysis(self, save_dir, with_rev=False): + group_infos = self.load_groups() + scene_tree = defaultdict(dict) + + for group_info in group_infos: + scene_tree[group_info["scene"]][group_info["group_name"]] = {"nimg": 0.1} + + # vis_group_tree(scene_tree, osp.join(save_dir, 'scene_tree.png')) # the visualized result is ugly + + def bbox_num_analyze(self): + n_cat = len(self.data["sentences"].keys()) + all_img_ids = self.data["images"].keys() + n_img = len(all_img_ids) + cat_obj_count = np.zeros((n_cat, n_img), dtype=int) + for img_id in all_img_ids: + # img_cat_ids = self.get_sent_ids(img_ids=img_id) + anno_ids = self.get_anno_ids(img_ids=img_id) + anno_infos = self.load_annos(anno_ids=anno_ids) + for anno in anno_infos: + for sid in anno["sent_id"]: + cat_obj_count[sid - 1, img_id] += 1 + return cat_obj_count diff --git a/approach/ovod/d-cube/d_cube/data_util.py b/approach/ovod/d-cube/d_cube/data_util.py new file mode 100644 index 0000000000000000000000000000000000000000..8f5d026aca06f296d42509499d487aa6e9462871 --- /dev/null +++ b/approach/ovod/d-cube/d_cube/data_util.py @@ -0,0 +1,269 @@ +# -*- coding: utf-8 -*- +__author__ = "Chi Xie and Zhao Zhang" +__maintainer__ = "Chi Xie" +# data utility functions are defined in the script +import json +import pickle +import shutil + +# from io import StringIO +# import string + +import numpy as np +import cv2 +from pycocotools import mask as cocomask + +VOC_COLORMAP = [ + [128, 0, 0], + [0, 128, 0], + [128, 128, 0], + [0, 0, 128], + [128, 0, 128], + [0, 128, 128], + [128, 128, 128], + [64, 0, 0], + [192, 0, 0], + [64, 128, 0], + [192, 128, 0], + [64, 0, 128], + [192, 0, 128], + [64, 128, 128], + [192, 128, 128], + [0, 64, 0], + [128, 64, 0], + [0, 192, 0], + [128, 192, 0], + [0, 64, 128], +] + + +def visualize_bbox_on_image(img, bbox_list, save_path=None, thickness=3): + img_copy = img.copy() + for i, bbox in enumerate(bbox_list): + color = tuple(VOC_COLORMAP[i % len(VOC_COLORMAP)]) + x, y, w, h = bbox + img_copy = cv2.rectangle( + img_copy, (int(x), int(y)), (int((x + w)), int(y + h)), color, thickness + ) + if save_path: + cv2.imwrite(save_path, img_copy) + return img_copy + + +def rle2bmask(rle): + bm = cocomask.decode(rle) + if len(bm.shape) == 3: + bm = np.sum( + bm, axis=2 + ) # sometimes there are multiple binary map (corresponding to multiple segs) + bm = bm.astype(np.uint8) # convert to np.uint8 + return bm + + +def merge_rle(rle_list, is_instance=True, on_image=False): + if is_instance: + cm_list = [] + for rle_idx, rle in enumerate(rle_list): + color = VOC_COLORMAP[rle_idx] + bm = rle2bmask(rle) + cm = cv2.cvtColor(bm, cv2.COLOR_GRAY2BGR) + cm_list.append(cm * color) + merge_map = np.sum(cm_list, axis=0, dtype=np.uint8) + else: + bm_list = [rle2bmask(rle) for rle in rle_list] + merge_map = np.sum(bm_list, axis=0, dtype=np.uint8) + merge_map[merge_map >= 1] = 1 + if not on_image: + color = VOC_COLORMAP[0] + merge_map = cv2.cvtColor(merge_map, cv2.COLOR_GRAY2BGR) + merge_map *= np.array(color, dtype=np.uint8) + + merge_map[merge_map > 255] = 255 + + if not on_image: + tmp_sum_map = np.sum(merge_map, axis=-1) + merge_map[tmp_sum_map == 0] = 220 + return merge_map + + +def merge2bin(rle_list, img_h, img_w): + if rle_list: + bm_list = [rle2bmask(rle) for rle in rle_list] + merge_map = np.sum(bm_list, axis=0, dtype=np.uint8) + merge_map[merge_map >= 1] = 255 + merge_map = np.expand_dims(merge_map, axis=-1) + return merge_map + else: + return np.zeros([img_h, img_w, 1], dtype=np.uint8) + + +def paste_text(img, text): + fontFace = cv2.FONT_HERSHEY_COMPLEX_SMALL + overlay = img.copy() + # fontFace = cv2.FONT_HERSHEY_TRIPLEX + fontScale = 1 + thickness = 1 + backgroud_alpha = 0.8 + + retval, baseLine = cv2.getTextSize( + text, fontFace=fontFace, fontScale=fontScale, thickness=thickness + ) + topleft = (0, 0) + # bottomright = (topleft[0] + retval[0], topleft[1] + retval[1]+10) + bottomright = (img.shape[1], topleft[1] + retval[1] + 10) + + cv2.rectangle(overlay, topleft, bottomright, thickness=-1, color=(250, 250, 250)) + img = cv2.addWeighted(overlay, backgroud_alpha, img, 1 - backgroud_alpha, 0) + + cv2.putText( + img, + text, + (0, baseLine + 10), + fontScale=fontScale, + fontFace=fontFace, + thickness=thickness, + color=(10, 10, 10), + ) + return img + + +def load_json(json_path, to_int=False): + clean_res_dic = {} + with open(json_path, "r", encoding="utf-8") as f_in: + res_dic = json.load(f_in) + + for ikey, iv in res_dic.items(): + ikey = int(ikey.strip()) if to_int else ikey.strip() + clean_res_dic[ikey] = iv + + return clean_res_dic + + +def path_map(src_path, obj_path): + def inner_map(full_path): + return full_path.replace(src_path, obj_path) + + +def save_pkl(src, obj_path): + with open(obj_path, "wb") as f_out: + pickle.dump(src, f_out) + + +def load_pkl(src_path): + with open(src_path, "rb") as f_in: + in_pkl = pickle.load(f_in) + return in_pkl + + +def copy_file(src_path, obj_path): + shutil.copy(src_path, obj_path) + + +def sentence_analysis(): + return 0 + + +def add_checkerboard_bg(image, mask, save_path=None): + # Create a new image with the same size as the original image + new_image = np.zeros_like(image) + + # Define the size of the checkerboard pattern + checkerboard_size = 24 + + # Loop over each pixel in the mask + for x in range(mask.shape[1]): + for y in range(mask.shape[0]): + # If the pixel is transparent, draw a checkerboard pattern + if mask[y, x] == 0: + if (x // checkerboard_size) % 2 == (y // checkerboard_size) % 2: + new_image[y, x] = (255, 255, 255) + else: + new_image[y, x] = (128, 128, 128) + # Otherwise, copy the corresponding pixel from the original image + else: + new_image[y, x] = image[y, x] + + # Save the new image with the checkerboard background + if save_path: + cv2.imwrite(save_path, new_image) + return new_image + + +def visualize_mask_on_image( + img, mask, save_path=None, add_edge=False, dark_background=False +): + # Convert the mask to a binary mask if it's not already + if mask.max() > 1: + mask = mask.astype(np.uint8) // 255 + + # Convert the mask to a 3-channel mask if it's not already + if len(mask.shape) == 2: + mask = np.expand_dims(mask, axis=2) + mask = np.tile(mask, (1, 1, 3)) + + # Create a color map for the mask + cmap = np.array([255, 117, 44], dtype=np.uint8) + mask_colors = mask * cmap + + # Add an opaque white edge to the mask if desired + if add_edge: + if len(mask.shape) == 2: + mask = np.expand_dims(mask, axis=2) + mask = np.tile(mask, (1, 1, 3)) + + kernel = np.ones((5, 5), dtype=np.uint8) + mask_edge = cv2.erode(mask, kernel, iterations=1) + mask_edge = mask - mask_edge + + # mask_edge = np.tile(mask_edge[:, :, np.newaxis], [1, 1, 3]) + mask_colors[mask_edge > 0] = 255 + + # Overlay the mask on the masked image + if dark_background: + masked_img = cv2.addWeighted(img, 0.4, mask_colors, 0.6, 0) + else: + masked_img = img.copy() + masked_img[mask > 0] = cv2.addWeighted(img, 0.4, mask_colors, 0.6, 0)[mask > 0] + + # Save the result to the specified path if provided + if save_path is not None: + cv2.imwrite(save_path, masked_img) + + return masked_img + + +# def visualize_mask_on_image(img, mask, save_path=None, add_edge=False): +# # Convert the mask to a binary mask if it's not already +# if mask.max() > 1: +# mask = mask.astype(np.uint8) // 255 + +# # Convert the mask to a 3-channel mask if it's not already +# if len(mask.shape) == 2: +# mask = np.expand_dims(mask, axis=2) +# mask = np.tile(mask, (1, 1, 3)) + +# # Create a color map for the mask +# cmap = np.array([255, 117, 44], dtype=np.uint8) +# mask_colors = mask * cmap + +# # Add an opaque white edge to the mask if desired +# if add_edge: +# if len(mask.shape) == 2: +# mask = np.expand_dims(mask, axis=2) +# mask = np.tile(mask, (1, 1, 3)) + +# kernel = np.ones((5, 5), dtype=np.uint8) +# mask_edge = cv2.erode(mask, kernel, iterations=1) +# mask_edge = mask - mask_edge + +# # mask_edge = np.tile(mask_edge[:, :, np.newaxis], [1, 1, 3]) +# mask_colors[mask_edge > 0] = 255 + +# # Overlay the mask on the masked image +# masked_img = cv2.addWeighted(img, 0.5, mask_colors, 0.5, 0) + +# # Save the result to the specified path if provided +# if save_path is not None: +# cv2.imwrite(save_path, masked_img) + +# return masked_img diff --git a/approach/ovod/d-cube/d_cube/vis_util.py b/approach/ovod/d-cube/d_cube/vis_util.py new file mode 100644 index 0000000000000000000000000000000000000000..6d35390e136ab9d5ca80a2eb0f8fe6b57356c422 --- /dev/null +++ b/approach/ovod/d-cube/d_cube/vis_util.py @@ -0,0 +1,199 @@ +# -*- coding: utf-8 -*- +__author__ = "Chi Xie and Zhao Zhang" +__maintainer__ = "Chi Xie" +import os +from collections import Counter + +import spacy +import matplotlib.pyplot as plt +import seaborn as sns +from wordcloud import WordCloud + +# from pycirclize import Circos +# from Bio.Phylo.BaseTree import Tree +# from Bio import Phylo +# from newick import Node + + +def plot_hist(data, bins=10, is_norm=False, save_path=None, x=None): + sns.set_theme(style="whitegrid", font_scale=2.0) + ax = sns.histplot(data, bins=bins, common_norm=is_norm, kde=False) + ax.set_xlabel(x) + plt.tight_layout() + plt.savefig(save_path) + plt.close() + + +def plot_bars(names, nums, is_sort, save_path=None): + sns.set(style="whitegrid") + + if is_sort: + zipped = zip(nums, names) + sort_zipped = sorted(zipped, key=lambda x: (x[0], x[1])) + result = zip(*sort_zipped) + nums, names = [list(x) for x in result] + + fontx = {"family": "Times New Roman", "size": 10} + fig, ax = plt.subplots() + fig = plt.figure(figsize=(16, 4)) + # sns.set_palette("PuBuGn_d") + sns.barplot(names, nums, palette=sns.cubehelix_palette(80, start=0.5, rot=-0.75)) + fig.autofmt_xdate(rotation=90) + plt.tick_params(axis="x", labelsize=10) + labels = ax.get_xticklabels() + ax.get_yticklabels() + [label.set_fontname("Times New Roman") for label in labels] + plt.tight_layout() + plt.savefig(save_path) + + +def generate_wordclouds(sentences, save_dir): + """Generates word clouds for different parts of speech in a list of sentences. + + Args: + sentences: A list of sentences. + save_dir: The directory to save the word cloud images. + """ + + os.makedirs(save_dir, exist_ok=True) + # Load the spacy model + nlp = spacy.load("en_core_web_sm") + + # Define the parts of speech to include in the word clouds + pos_to_include = ["NOUN", "VERB", "ADJ", "ADV"] + + # Process each sentence and collect the relevant words for each part of speech + words_by_pos = {pos: [] for pos in pos_to_include} + for sent in sentences: + doc = nlp(sent) + for token in doc: + if token.pos_ in pos_to_include: + words_by_pos[token.pos_].append(token.lemma_.lower()) + + # Generate a word cloud for each part of speech + for pos, words in words_by_pos.items(): + if len(words) == 0: + continue # skip parts of speech with no words + + # Count the frequency of each word + word_counts = Counter(words) + + # Generate the word cloud + wordcloud = WordCloud( + width=800, + height=800, + background_color="white", + max_words=200, + colormap="Set2", + max_font_size=150, + ).generate_from_frequencies(word_counts) + + # Save the word cloud image + filename = f"{pos.lower()}_wordcloud.png" + filepath = os.path.join(save_dir, filename) + wordcloud.to_file(filepath) + + +# def vis_group_tree(data_dict, save_path): + +# # Create 3 randomized trees +# tree_size_list = [60, 40, 50] +# trees = [Tree.randomized(string.ascii_uppercase, branch_stdev=0.5) for size in tree_size_list] + +# # Initialize circos sector with 3 randomized tree size +# sectors = {name: size for name, size in zip(list("ABC"), tree_size_list)} +# circos = Circos(sectors, space=5) + +# colors = ["tomato", "skyblue", "limegreen"] +# cmaps = ["bwr", "viridis", "Spectral"] +# for idx, sector in enumerate(circos.sectors): +# sector.text(sector.name, r=120, size=12) +# # Plot randomized tree +# tree = trees[idx] +# tree_track = sector.add_track((30, 70)) +# tree_track.axis(fc=colors[idx], alpha=0.2) +# tree_track.tree(tree, leaf_label_size=3, leaf_label_margin=21) +# # Plot randomized bar +# bar_track = sector.add_track((70, 90)) +# x = np.arange(0, int(sector.size)) + 0.5 +# height = np.random.randint(1, 10, int(sector.size)) +# bar_track.bar(x, height, facecolor=colors[idx], ec="grey", lw=0.5, hatch="//") + +# circos.savefig(save_path, dpi=600) + +# def clean_newick_key(in_str): +# bad_chars = [':', ';', ',', '(', ')'] +# for bad_char in bad_chars: +# in_str = in_str.replace(bad_char, ' ') +# return in_str + +# def build_tree_from_dict(data): +# root = Node() # create the root node +# for key, value in data.items(): +# node = Node(name=clean_newick_key(key)) # name doesn't need to be cleaned +# if value is not None: +# child_node = build_tree_from_dict(value) +# node.add_descendant(child_node) +# root.add_descendant(node) + +# return root + + +def replace_chars_in_dict_keys(d): + """ + Replaces the characters ':', ';', ',', '(', and ')' in the keys of a nested dictionary with '_'. + """ + new_dict = {} + for k, v in d.items(): + if isinstance(v, dict): + v = replace_chars_in_dict_keys(v) + new_key = k.translate(str.maketrans(":;,()", "_____")) + new_dict[new_key] = v + return new_dict + + +def build_newick_tree(tree_dict): + newick_tree = "" + if isinstance(tree_dict, dict): + for key, value in tree_dict.items(): + if isinstance(value, dict): + subtree = build_newick_tree(value) + if subtree: + newick_tree += "(" + subtree + ")" + key + "," + else: + newick_tree += key + "," + else: + newick_tree += key + ":" + str(value) + "," + newick_tree = newick_tree.rstrip(",") + ")" + return newick_tree + else: + return None + + +# def vis_group_tree(data_dict, save_path): +# data_dic = replace_chars_in_dict_keys(data_dict) +# super_group_names = data_dict.keys() + +# # Create 3 randomized trees +# tree_size_list = [60, 40, 50] +# trees = [Phylo.read(StringIO(build_newick_tree(data_dict[super_group_name])), "newick") for super_group_name in super_group_names] + +# # Initialize circos sector with 3 randomized tree size +# sectors = {name: size for name, size in zip(list("ABC"), tree_size_list)} +# circos = Circos(sectors, space=5) + +# colors = ["tomato", "skyblue", "limegreen"] +# cmaps = ["bwr", "viridis", "Spectral"] +# for idx, sector in enumerate(circos.sectors): +# sector.text(sector.name, r=120, size=12) +# # Plot randomized tree +# tree = trees[idx] +# tree_track = sector.add_track((30, 70)) +# tree_track.axis(fc=colors[idx], alpha=0.2) +# tree_track.tree(tree, leaf_label_size=3, leaf_label_margin=21) +# # Plot randomized bar +# bar_track = sector.add_track((70, 90)) +# x = np.arange(0, int(sector.size)) + 0.5 +# height = np.random.randint(1, 10, int(sector.size)) +# bar_track.bar(x, height, facecolor=colors[idx], ec="grey", lw=0.5, hatch="//") + +# circos.savefig(save_path, dpi=600) diff --git a/approach/ovod/d-cube/eval_sota/README.md b/approach/ovod/d-cube/eval_sota/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e72f60cb2f5312db12463611d5ee8dae8397b81b --- /dev/null +++ b/approach/ovod/d-cube/eval_sota/README.md @@ -0,0 +1,27 @@ +# Evaluting SOTA Methods on $D^3$ + +## Leaderboard + +In this directory, we keep the scripts or github links (official or custom) to evaluate SOTA methods (REC/OVD/DOD/MLLM) on $D^3$: + +| Name | Paper | Original Tasks | Training Data | Evaluation Code | Intra-FULL/PRES/ABS/Inter-FULL/PRES/ABS | Source | Note | +|:-----|:-----:|:----:|:-----:|:-----:|:-----:|:-----:|:-----:| +| OFA-large | [OFA: Unifying Architectures, Tasks, and Modalities Through a Simple Sequence-to-Sequence Learning Framework (ICML 2022)](https://arxiv.org/abs/2202.03052) | REC | - | - | 4.2/4.1/4.6/0.1/0.1/0.1 | [DOD paper](https://arxiv.org/abs/2307.12813) | - | +| CORA-R50 | [CORA: Adapting CLIP for Open-Vocabulary Detection with Region Prompting and Anchor Pre-Matching (CVPR 2023)](https://openaccess.thecvf.com/content/CVPR2023/papers/Wu_CORA_Adapting_CLIP_for_Open-Vocabulary_Detection_With_Region_Prompting_and_CVPR_2023_paper.pdf) | OVD | - | - | 6.2/6.7/5.0/2.0/2.2/1.3 | [DOD paper](https://arxiv.org/abs/2307.12813) | - | +| OWL-ViT-large | [Simple Open-Vocabulary Object Detection with Vision Transformers (ECCV 2022)](https://www.ecva.net/papers/eccv_2022/papers_ECCV/papers/136700714.pdf) | OVD | - | [DOD official](./owl_vit.py) | 9.6/10.7/6.4/2.5/2.9/2.1 | [DOD paper](https://arxiv.org/abs/2307.12813) | Post-processing hyper-parameters may affect the performance and the result may not exactly match the paper | +| SPHINX-7B | [SPHINX: The Joint Mixing of Weights, Tasks, and Visual Embeddings for Multi-modal Large Language Models (arxiv 2023)](https://arxiv.org/abs/2311.07575) | **MLLM** capable of REC | - | [DOD official](./sphinx.py) | 10.6/11.4/7.9/-/-/- | DOD authors | A lot of contribution from [Jie Li](https://github.com/theFool32) | +| GLIP-T | [Grounded Language-Image Pre-training (CVPR 2022)](https://arxiv.org/abs/2112.03857) | OVD & PG | - | - | 19.1/18.3/21.5/-/-/- | GEN paper | - | +| UNINEXT-huge | [Universal Instance Perception as Object Discovery and Retrieval (CVPR 2023)](https://arxiv.org/abs/2303.06674v2) | OVD & REC | - | [DOD official](https://github.com/Charles-Xie/UNINEXT_D3) | 20.0/20.6/18.1/3.3/3.9/1.6 | [DOD paper](https://arxiv.org/abs/2307.12813) | - | +| Grounding-DINO-base | [Grounding DINO: Marrying DINO with Grounded Pre-Training for Open-Set Object Detection (arxiv 2023)](https://arxiv.org/abs/2303.05499) | OVD & REC | - | [DOD official](./groundingdino.py) | 20.7/20.1/22.5/2.7/2.4/3.5 | [DOD paper](https://arxiv.org/abs/2307.12813) | Post-processing hyper-parameters may affect the performance and the result may not exactly match the paper | +| OFA-DOD-base | [Described Object Detection: Liberating Object Detection with Flexible Expressions (NeurIPS 2023)](https://arxiv.org/abs/2307.12813) | DOD | - | - | 21.6/23.7/15.4/5.7/6.9/2.3 | [DOD paper](https://arxiv.org/abs/2307.12813) | - | +| FIBER-B | [Coarse-to-Fine Vision-Language Pre-training with Fusion in the Backbone (NeurIPS 2022)](https://arxiv.org/abs/2206.07643) | OVD & REC | - | - | 22.7/21.5/26.0/-/-/- | GEN paper | - | +| MM-Grounding-DINO | [An Open and Comprehensive Pipeline for Unified Object Grounding and Detection (arxiv 2024)](https://arxiv.org/abs/2401.02361) | DOD & OVD & REC | O365, GoldG, GRIT, V3Det | [MM-GDINO official](https://github.com/open-mmlab/mmdetection/tree/main/configs/mm_grounding_dino#zero-shot-description-detection-datasetdod) | 22.9/21.9/26.0/-/-/- | MM-GDINO paper | - | +| GEN (FIBER-B) | [Generating Enhanced Negatives for Training Language-Based Object Detectors (arxiv 2024](https://arxiv.org/abs/2401.00094) | DOD | - | - | 26.0/25.2/28.1/-/-/- | GEN paper | Enhancement based on FIBER-B | +| APE-large (D) | [Aligning and Prompting Everything All at Once for Universal Visual Perception (arxiv 2023)](https://arxiv.org/abs/2312.02153) | DOD & OVD & REC | COCO, LVIS, O365, OpenImages, Visual Genome, RefCOCO/+/g, SA-1B, GQA, PhraseCut, Flickr30k | [APE official](https://github.com/shenyunhang/APE) | 37.5/38.8/33.9/21.0/22.0/17.9 | APE paper | Extra training data helps for this amazing performance | + + +Some extra notes: +- Each method is currently recorded by *the variant with the highest performance* in this table, if there are multiple variants available, so it's only a leaderboard, not meant for fair comparison. +- Methods like GLIP, FIBER, etc. are actually not evaluated on OVD benchmarks. For zero-shot eval on DOD, We currently do not distinguish between methods for OVD benchmarks and methods for ZS-OD, as long as it is verified with open-set detection capability. + +For other variants (e.g. for a fair comparison regarding data, backbone, etc.), please refer to the papers. diff --git a/approach/ovod/d-cube/eval_sota/groundingdino.py b/approach/ovod/d-cube/eval_sota/groundingdino.py new file mode 100644 index 0000000000000000000000000000000000000000..f9879257327dd98ea3c20c888b4c1e3f33029a86 --- /dev/null +++ b/approach/ovod/d-cube/eval_sota/groundingdino.py @@ -0,0 +1,304 @@ +# -*- coding: utf-8 -*- +__author__ = "Chi Xie" +__maintainer__ = "Chi Xie" + +# An example for how to run this script: +# CUDA_VISIBLE_DEVICES=0 +# python groundingdino.py \ +# -c ./groundingdino/config/GroundingDINO_SwinB.cfg.py \ +# -p ./ckpt/groundingdino_swinb_cogcoor.pth \ +# -o "outputs/gdino_d3" \ +# --box_threshold 0.05 \ +# --text_threshold 0.05 \ +# --img-top1 + +import argparse +import json +import os + +import numpy as np +import torch +from PIL import Image, ImageDraw, ImageFont +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval +from tqdm import tqdm + +import groundingdino.datasets.transforms as T +from groundingdino.models import build_model +from groundingdino.util.slconfig import SLConfig +from groundingdino.util.utils import clean_state_dict, get_phrases_from_posmap +from d_cube import D3 + + +def plot_boxes_to_image(image_pil, tgt): + H, W = tgt["size"] + boxes = tgt["boxes"] + labels = tgt["labels"] + assert len(boxes) == len(labels), "boxes and labels must have same length" + + draw = ImageDraw.Draw(image_pil) + mask = Image.new("L", image_pil.size, 0) + mask_draw = ImageDraw.Draw(mask) + + # draw boxes and masks + for box, label in zip(boxes, labels): + # from 0..1 to 0..W, 0..H + box = box * torch.Tensor([W, H, W, H]) + # from xywh to xyxy + box[:2] -= box[2:] / 2 + box[2:] += box[:2] + # random color + color = tuple(np.random.randint(0, 255, size=3).tolist()) + # draw + x0, y0, x1, y1 = box + x0, y0, x1, y1 = int(x0), int(y0), int(x1), int(y1) + + draw.rectangle([x0, y0, x1, y1], outline=color, width=6) + # draw.text((x0, y0), str(label), fill=color) + + font = ImageFont.load_default() + if hasattr(font, "getbbox"): + bbox = draw.textbbox((x0, y0), str(label), font) + else: + w, h = draw.textsize(str(label), font) + bbox = (x0, y0, w + x0, y0 + h) + # bbox = draw.textbbox((x0, y0), str(label)) + draw.rectangle(bbox, fill=color) + draw.text((x0, y0), str(label), fill="white") + + mask_draw.rectangle([x0, y0, x1, y1], fill=255, width=6) + return image_pil, mask + + +def load_image(image_path): + # load image + image_pil = Image.open(image_path).convert("RGB") # load image + + transform = T.Compose( + [ + T.RandomResize([800], max_size=1333), + T.ToTensor(), + T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), + ] + ) + image, _ = transform(image_pil, None) # 3, h, w + return image_pil, image + + +def load_model(model_config_path, model_checkpoint_path, cpu_only=False): + args = SLConfig.fromfile(model_config_path) + args.device = "cuda" if not cpu_only else "cpu" + model = build_model(args) + checkpoint = torch.load(model_checkpoint_path, map_location="cpu") + load_res = model.load_state_dict(clean_state_dict(checkpoint["model"]), strict=False) + print(load_res) + _ = model.eval() + return model + + +def get_grounding_output(model, image, caption, box_threshold, text_threshold, with_logits=True, cpu_only=False): + caption = caption.lower() + caption = caption.strip() + if not caption.endswith("."): + caption = caption + "." + device = "cuda" if not cpu_only else "cpu" + model = model.to(device) + image = image.to(device) + with torch.no_grad(): + outputs = model(image[None], captions=[caption]) + logits = outputs["pred_logits"].cpu().sigmoid()[0] # (nq, 256) + boxes = outputs["pred_boxes"].cpu()[0] # (nq, 4) + logits.shape[0] + + # filter output + logits_filt = logits.clone() + boxes_filt = boxes.clone() + filt_mask = logits_filt.max(dim=1)[0] > box_threshold + logits_filt = logits_filt[filt_mask] # num_filt, 256 + boxes_filt = boxes_filt[filt_mask] # num_filt, 4 + logits_filt.shape[0] + + # get phrase + tokenlizer = model.tokenizer + tokenized = tokenlizer(caption) + # build pred + pred_phrases = [] + logits_list = [] + for logit, box in zip(logits_filt, boxes_filt): + pred_phrase = get_phrases_from_posmap(logit > text_threshold, tokenized, tokenlizer) + logits_list.append(logit.max().item()) + if with_logits: + pred_phrases.append(pred_phrase + f"({str(logit.max().item())[:4]})") + else: + pred_phrases.append(pred_phrase) + + return boxes_filt, pred_phrases, logits_list + + +def get_dataset_iter(coco): + img_ids = coco.get_img_ids() + for img_id in img_ids: + img_info = coco.load_imgs(img_id)[0] + file_name = img_info["file_name"] + img_path = os.path.join(IMG_ROOT, file_name) + yield img_id, img_path + + +def eval_on_d3(pred_path, mode="pn"): + assert mode in ("pn", "p", "n") + if mode == "pn": + gt_path = os.path.join(JSON_ANNO_PATH, "d3_full_annotations.json") + elif mode == "p": + gt_path = os.path.join(JSON_ANNO_PATH, "d3_pres_annotations.json") + else: + gt_path = os.path.join(JSON_ANNO_PATH, "d3_abs_annotations.json") + coco = COCO(gt_path) + d3_res = coco.loadRes(pred_path) + cocoEval = COCOeval(coco, d3_res, "bbox") + cocoEval.evaluate() + cocoEval.accumulate() + cocoEval.summarize() + + # comment the following if u only need intra/inter map for full/pres/abs + # ===================== uncomment this if u need detailed analysis ===================== + # aps = cocoEval.eval["precision"][:, :, :, 0, -1] + # category_ids = coco.getCatIds() + # category_names = [cat["name"] for cat in coco.loadCats(category_ids)] + + # aps_lens = defaultdict(list) + # counter_lens = defaultdict(int) + # for i in range(len(category_names)): + # ap = aps[:, :, i] + # ap_value = ap[ap > -1].mean() + # if not np.isnan(ap_value): + # len_ref = len(category_names[i].split(" ")) + # aps_lens[len_ref].append(ap_value) + # counter_lens[len_ref] += 1 + + # ap_sum_short = sum([sum(aps_lens[i]) for i in range(0, 4)]) + # ap_sum_mid = sum([sum(aps_lens[i]) for i in range(4, 7)]) + # ap_sum_long = sum([sum(aps_lens[i]) for i in range(7, 10)]) + # ap_sum_very_long = sum( + # [sum(aps_lens[i]) for i in range(10, max(counter_lens.keys()) + 1)] + # ) + # c_sum_short = sum([counter_lens[i] for i in range(1, 4)]) + # c_sum_mid = sum([counter_lens[i] for i in range(4, 7)]) + # c_sum_long = sum([counter_lens[i] for i in range(7, 10)]) + # c_sum_very_long = sum( + # [counter_lens[i] for i in range(10, max(counter_lens.keys()) + 1)] + # ) + # map_short = ap_sum_short / c_sum_short + # map_mid = ap_sum_mid / c_sum_mid + # map_long = ap_sum_long / c_sum_long + # map_very_long = ap_sum_very_long / c_sum_very_long + # print( + # f"mAP over reference length: short - {map_short:.4f}, mid - {map_mid:.4f}, long - {map_long:.4f}, very long - {map_very_long:.4f}" + # ) + # ===================== uncomment this if u need detailed analysis ===================== + + +def inference_on_d3(data_iter, model, args, box_threshold, text_threshold): + pred = [] + for idx, (img_id, image_path) in enumerate(tqdm(data_iter)): + # load image + image_pil, image = load_image(image_path) + size = image_pil.size + W, H = size + + group_ids = d3.get_group_ids(img_ids=[img_id]) + sent_ids = d3.get_sent_ids(group_ids=group_ids) + sent_list = d3.load_sents(sent_ids=sent_ids) + text_list = [sent['raw_sent'] for sent in sent_list] + + for sent_id, text_prompt in zip(sent_ids, text_list): + # run model + boxes_filt, pred_phrases, logit_list = get_grounding_output( + model, image, text_prompt, box_threshold, text_threshold, cpu_only=args.cpu_only, with_logits=False, + ) + if args.vis: + pred_dict = { + "boxes": boxes_filt, # [x_center, y_center, w, h] + "size": [size[1], size[0]], + "labels": [f"{phrase}({str(logit)[:4]})" for phrase, logit in zip(pred_phrases, logit_list)], + } + image_with_box = plot_boxes_to_image(image_pil.copy(), pred_dict)[0] + image_with_box.save(os.path.join(output_dir, f"{img_id}_{text_prompt}.jpg")) + if not logit_list: + continue + if args.img_top1: + max_score_idx = logit_list.index(max(logit_list)) + bboxes, phrases, logits = [boxes_filt[max_score_idx]], [pred_phrases[max_score_idx]], [logit_list[max_score_idx]] + else: + bboxes, phrases, logits = boxes_filt, pred_phrases, logit_list + for box, phrase, logit in zip(bboxes, phrases, logits): + if len(phrase) > args.overlap_percent * len(text_prompt) or phrase == text_prompt: + x1, y1, w, h = box.tolist() + x0, y0 = x1 - w / 2, y1 - h / 2 + pred_item = { + "image_id": img_id, + "category_id": sent_id, + "bbox": [x0 * W, y0 * H, w * W, h * H], + "score": float(logit), + } + pred.append(pred_item) + + return pred + + +if __name__ == "__main__": + IMG_ROOT = None # set here + JSON_ANNO_PATH = None # set here + PKL_ANNO_PATH = None # set here + assert IMG_ROOT is not None, "Please set IMG_ROOT in the script first" + assert JSON_ANNO_PATH is not None, "Please set JSON_ANNO_PATH in the script first" + assert PKL_ANNO_PATH is not None, "Please set PKL_ANNO_PATH in the script first" + + d3 = D3(IMG_ROOT, PKL_ANNO_PATH) + + parser = argparse.ArgumentParser("Grounding DINO evaluation on D-cube (https://arxiv.org/abs/2307.12813)", add_help=True) + parser.add_argument("--config_file", "-c", type=str, required=True, help="path to config file") + parser.add_argument( + "--checkpoint_path", "-p", type=str, required=True, help="path to checkpoint file" + ) + # parser.add_argument("--image_path", "-i", type=str, required=True, help="path to image file") + # parser.add_argument("--text_prompt", "-t", type=str, required=True, help="text prompt") + parser.add_argument( + "--output_dir", "-o", type=str, default="outputs", required=True, help="output directory" + ) + parser.add_argument("--vis", action="store_true", help="visualization on D3") + + parser.add_argument("--box_threshold", type=float, default=0.3, help="box threshold") + parser.add_argument("--text_threshold", type=float, default=0.25, help="text threshold") + + parser.add_argument("--cpu-only", action="store_true", help="running on cpu only!, default=False") + parser.add_argument("--img-top1", action="store_true", help="select only the box with top max score") + # parser.add_argument("--overlap-percent", type=float, default=1.0, help="overlapping percentage between input prompt and output label") + # this overlapping percentage denotes an additional post-processing technique we designed. if you turn this on, you may get higher performance by tuning this parameter. + args = parser.parse_args() + args.overlap_percent = 1 # by default, we do not use this technique. + print(args) + + # cfg + config_file = args.config_file # change the path of the model config file + checkpoint_path = args.checkpoint_path # change the path of the model + # image_path = args.image_path + # text_prompt = args.text_prompt + output_dir = args.output_dir + box_threshold = args.box_threshold + text_threshold = args.text_threshold + + # make dir + os.makedirs(output_dir, exist_ok=True) + # load model + model = load_model(config_file, checkpoint_path, cpu_only=args.cpu_only) + + data_iter = get_dataset_iter(d3) + + pred = inference_on_d3(data_iter, model, args, box_threshold=box_threshold, text_threshold=text_threshold) + + pred_path = os.path.join(output_dir, f"prediction.json") + with open(pred_path, "w") as f_: + json.dump(pred, f_) + eval_on_d3(pred_path, mode='pn') + eval_on_d3(pred_path, mode='p') + eval_on_d3(pred_path, mode='n') diff --git a/approach/ovod/d-cube/eval_sota/owl_vit.py b/approach/ovod/d-cube/eval_sota/owl_vit.py new file mode 100644 index 0000000000000000000000000000000000000000..bc28f6f4d2a9693732bb787a43259a82e9370723 --- /dev/null +++ b/approach/ovod/d-cube/eval_sota/owl_vit.py @@ -0,0 +1,192 @@ +import json +import os +from collections import defaultdict + +from tqdm import tqdm +from PIL import Image +import numpy as np +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval +import torch +from transformers import OwlViTProcessor, OwlViTForObjectDetection + +from d_cube import D3 + + +def write_json(json_path, json_data): + with open(json_path, "w") as f_: + json.dump(json_data, f_) + + +def read_json(json_path): + with open(json_path, "r") as f_: + json_data = json.load(f_) + return json_data + + +def load_image_general(image_path): + image_pil = Image.open(image_path) + return image_pil + + +def get_prediction(model, image, captions, cpu_only=False): + for i in range(len(captions)): + captions[i] = captions[i].lower() + captions[i] = captions[i].strip() + if not captions[i].endswith("."): + captions[i] = captions[i] + "." + device = "cuda" if not cpu_only else "cpu" + model = model.to(device) + with torch.no_grad(): + inputs = processor(text=[captions], images=image, return_tensors="pt").to( + device + ) + outputs = model(**inputs) + target_size = torch.Tensor([image.size[::-1]]).to(device) + results = processor.post_process_object_detection( + outputs=outputs, target_sizes=target_size, threshold=0.1 + # the post precessing threshold will affect the performance obviously + # you may tune it to get better performance, e.g., 0.05 + ) + boxes, scores, labels = ( + results[0]["boxes"], + results[0]["scores"], + results[0]["labels"], + ) + return boxes, scores, labels + + +def get_dataset_iter(coco): + img_ids = coco.get_img_ids() + for img_id in img_ids: + img_info = coco.load_imgs(img_id)[0] + file_name = img_info["file_name"] + img_path = os.path.join(IMG_ROOT, file_name) + yield img_id, img_path + + +def eval_on_d3(pred_path, mode="pn"): + assert mode in ("pn", "p", "n") + if mode == "pn": + gt_path = os.path.join(JSON_ANNO_PATH, "d3_full_annotations.json") + elif mode == "p": + gt_path = os.path.join(JSON_ANNO_PATH, "d3_pres_annotations.json") + else: + gt_path = os.path.join(JSON_ANNO_PATH, "d3_abs_annotations.json") + coco = COCO(gt_path) + d3_res = coco.loadRes(pred_path) + cocoEval = COCOeval(coco, d3_res, "bbox") + cocoEval.evaluate() + cocoEval.accumulate() + cocoEval.summarize() + + # comment the following if u only need intra/inter map for full/pres/abs + # ===================== uncomment this if u need detailed analysis ===================== + # aps = cocoEval.eval["precision"][:, :, :, 0, -1] + # category_ids = coco.getCatIds() + # category_names = [cat["name"] for cat in coco.loadCats(category_ids)] + + # aps_lens = defaultdict(list) + # counter_lens = defaultdict(int) + # for i in range(len(category_names)): + # ap = aps[:, :, i] + # ap_value = ap[ap > -1].mean() + # if not np.isnan(ap_value): + # len_ref = len(category_names[i].split(" ")) + # aps_lens[len_ref].append(ap_value) + # counter_lens[len_ref] += 1 + + # ap_sum_short = sum([sum(aps_lens[i]) for i in range(0, 4)]) + # ap_sum_mid = sum([sum(aps_lens[i]) for i in range(4, 7)]) + # ap_sum_long = sum([sum(aps_lens[i]) for i in range(7, 10)]) + # ap_sum_very_long = sum( + # [sum(aps_lens[i]) for i in range(10, max(counter_lens.keys()) + 1)] + # ) + # c_sum_short = sum([counter_lens[i] for i in range(1, 4)]) + # c_sum_mid = sum([counter_lens[i] for i in range(4, 7)]) + # c_sum_long = sum([counter_lens[i] for i in range(7, 10)]) + # c_sum_very_long = sum( + # [counter_lens[i] for i in range(10, max(counter_lens.keys()) + 1)] + # ) + # map_short = ap_sum_short / c_sum_short + # map_mid = ap_sum_mid / c_sum_mid + # map_long = ap_sum_long / c_sum_long + # map_very_long = ap_sum_very_long / c_sum_very_long + # print( + # f"mAP over reference length: short - {map_short:.4f}, mid - {map_mid:.4f}, long - {map_long:.4f}, very long - {map_very_long:.4f}" + # ) + # ===================== uncomment this if u need detailed analysis ===================== + + +def inference_on_d3(data_iter, model): + pred = [] + error = [] + for img_id, image_path in tqdm(data_iter): + image = load_image_general(image_path) + + # ==================================== intra-group setting ==================================== + # each image is evaluated with the categories in its group (usually 4) + group_ids = d3.get_group_ids(img_ids=[img_id]) + sent_ids = d3.get_sent_ids(group_ids=group_ids) + # ==================================== intra-group setting ==================================== + # ==================================== inter-group setting ==================================== + # each image is evaluated with all categories in the dataset (422 for the first version of the dataset) + # sent_ids = d3.get_sent_ids() + # ==================================== inter-group setting ==================================== + sent_list = d3.load_sents(sent_ids=sent_ids) + text_list = [sent["raw_sent"] for sent in sent_list] + + try: + boxes, scores, labels = get_prediction(model, image, text_list, cpu_only=False) + for box, score, label in zip(boxes, scores, labels): + pred_item = { + "image_id": img_id, + "category_id": sent_ids[label], + "bbox": convert_to_xywh(box.tolist()), # use xywh + "score": float(score), + } + pred.append(pred_item) # the output to be saved to JSON. + except: + print("error!!!") + return pred, error + + +def convert_to_xywh(bbox_xyxy): + """ + Convert top-left and bottom-right corner coordinates to [x, y, width, height] format. + """ + x1, y1, x2, y2 = bbox_xyxy + width = x2 - x1 + height = y2 - y1 + return [x1, y1, width, height] + + +if __name__ == "__main__": + IMG_ROOT = None # set here + JSON_ANNO_PATH = None # set here + PKL_ANNO_PATH = None # set here + assert IMG_ROOT is not None, "Please set IMG_ROOT in the script first" + assert JSON_ANNO_PATH is not None, "Please set JSON_ANNO_PATH in the script first" + assert PKL_ANNO_PATH is not None, "Please set PKL_ANNO_PATH in the script first" + + d3 = D3(IMG_ROOT, PKL_ANNO_PATH) + + output_dir = "ovd/owlvit/" + os.makedirs(output_dir, exist_ok=True) + + # model prediction + processor = OwlViTProcessor.from_pretrained("owl-vit") + model = OwlViTForObjectDetection.from_pretrained("owl-vit") + data_iter = get_dataset_iter(d3) + pred, error = inference_on_d3(data_iter, model) + + pred_path = os.path.join(output_dir, f"prediction.json") + pred_path_error = os.path.join(output_dir, "error.json") + write_json(pred_path, pred) + write_json(pred_path_error, error) + # see https://github.com/shikras/d-cube/blob/main/doc.md#output-format for the output format + # the output format is identical to COCO. + + eval_on_d3(pred_path, mode="pn") # the FULL setting + eval_on_d3(pred_path, mode="p") # the PRES setting + eval_on_d3(pred_path, mode="n") # the ABS setting diff --git a/approach/ovod/d-cube/eval_sota/sphinx.py b/approach/ovod/d-cube/eval_sota/sphinx.py new file mode 100644 index 0000000000000000000000000000000000000000..1d9a3e482bb35bc6bae20f0388ef1d3d4efa08ab --- /dev/null +++ b/approach/ovod/d-cube/eval_sota/sphinx.py @@ -0,0 +1,184 @@ +# -*- coding: utf-8 -*- +__author__ = "Chi Xie and Jie Li" +__maintainer__ = "Chi Xie" + +import json +import os +from collections import defaultdict +import re + +from PIL import Image +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval + +from d_cube import D3 + + +def write_json(json_path, json_data): + with open(json_path, "w") as f_: + json.dump(json_data, f_) + + +def read_json(json_path): + with open(json_path, "r") as f_: + json_data = json.load(f_) + return json_data + + +def load_image_general(image_path): + image_pil = Image.open(image_path) + return image_pil + + +def extract_boxes(input_string): + # if input_string.startswith("None"): + # return [] + # Define the pattern using regular expression + pattern = r'\[([\d.,; ]+)\]' + + # Search for the pattern in the input string + match = re.search(pattern, input_string) + + # If a match is found, extract and return the boxes as a list + if match: + boxes_str = match.group(1) + boxes_list = [list(map(float, box.split(','))) for box in boxes_str.split(';')] + return boxes_list + else: + return [] + + +def get_prediction(mllm_res, image, captions, cpu_only=False): + boxes, scores, labels = [], [], [] + width, height = image.size + for idx, res_item in enumerate(mllm_res): + boxes_list = extract_boxes(res_item["answer"]) + for bbox in boxes_list: + bbox_rescaled = get_true_bbox(image.size, bbox) + boxes.append(bbox_rescaled) + scores.append(1.0) + labels.append(idx) + return boxes, scores, labels + + +def get_dataset_iter(coco): + img_ids = coco.get_img_ids() + for img_id in img_ids: + img_info = coco.load_imgs(img_id)[0] + file_name = img_info["file_name"] + img_path = os.path.join(IMG_ROOT, file_name) + yield img_id, file_name, img_path + + +def eval_on_d3(pred_path, mode="pn"): + assert mode in ("pn", "p", "n") + if mode == "pn": + gt_path = os.path.join(JSON_ANNO_PATH, "d3_full_annotations.json") + elif mode == "p": + gt_path = os.path.join(JSON_ANNO_PATH, "d3_pres_annotations.json") + else: + gt_path = os.path.join(JSON_ANNO_PATH, "d3_abs_annotations.json") + coco = COCO(gt_path) + d3_res = coco.loadRes(pred_path) + cocoEval = COCOeval(coco, d3_res, "bbox") + cocoEval.evaluate() + cocoEval.accumulate() + cocoEval.summarize() + + +def group_sphinx_res_by_img(inference_res): + inference_res_by_img = defaultdict(list) + for res_item in inference_res: + img_path = "/".join(res_item["image_path"].split("/")[-2:]) + inference_res_by_img[img_path].append(res_item) + inference_res_by_img = dict(inference_res_by_img) + return inference_res_by_img + + +def get_true_bbox(img_size, bbox): + width, height = img_size + max_edge = max(height, width) + bbox = [v * max_edge for v in bbox] + diff = abs(width - height) // 2 + if height < width: + bbox[1] -= diff + bbox[3] -= diff + else: + bbox[0] -= diff + bbox[2] -= diff + return bbox + + +def inference_on_d3(data_iter, inference_res): + pred = [] + inf_res_by_img = group_sphinx_res_by_img(inference_res) + for idx, (img_id, img_name, img_path) in enumerate(data_iter): + image = load_image_general(img_path) + + # ==================================== intra-group setting ==================================== + # each image is evaluated with the categories in its group (usually 4) + group_ids = d3.get_group_ids(img_ids=[img_id]) + sent_ids = d3.get_sent_ids(group_ids=group_ids) + # ==================================== intra-group setting ==================================== + # ==================================== inter-group setting ==================================== + # each image is evaluated with all categories in the dataset (422 for the first version of the dataset) + # sent_ids = d3.get_sent_ids() + # ==================================== inter-group setting ==================================== + sent_list = d3.load_sents(sent_ids=sent_ids) + text_list = [sent["raw_sent"] for sent in sent_list] + + boxes, scores, labels = get_prediction(inf_res_by_img[img_name], image, text_list, cpu_only=False) + for box, score, label in zip(boxes, scores, labels): + pred_item = { + "image_id": img_id, + "category_id": sent_ids[label], + "bbox": convert_to_xywh(box), # use xywh + "score": float(score), + } + pred.append(pred_item) # the output to be saved to JSON. + return pred + + +def convert_to_xywh(bbox_xyxy): + """ + Convert top-left and bottom-right corner coordinates to [x, y, width, height] format. + """ + x1, y1, x2, y2 = bbox_xyxy + width = x2 - x1 + height = y2 - y1 + return [x1, y1, width, height] + + +if __name__ == "__main__": + IMG_ROOT = None # set here + JSON_ANNO_PATH = None # set here + PKL_ANNO_PATH = None # set here + # ============================== SPHINX inference result file =============== + SPHINX_INFERENCE_RES_PATH = None + # You can download the SPHINX d3 inference result example from: + # https://github.com/shikras/d-cube/files/14276682/sphinx_d3_result.json + # For the inference process, please refer to SPHINX official repo (https://github.com/Alpha-VLLM/LLaMA2-Accessory) + # the prompts we used are available in this JSON file + # Thanks for the contribution from Jie Li (https://github.com/theFool32) + # ============================== SPHINX inference result file =============== + assert IMG_ROOT is not None, "Please set IMG_ROOT in the script first" + assert JSON_ANNO_PATH is not None, "Please set JSON_ANNO_PATH in the script first" + assert PKL_ANNO_PATH is not None, "Please set PKL_ANNO_PATH in the script first" + + d3 = D3(IMG_ROOT, PKL_ANNO_PATH) + + output_dir = "mllm/sphinx/" # or whatever you prefer + inference_res = read_json(SPHINX_INFERENCE_RES_PATH) + + # model prediction + data_iter = get_dataset_iter(d3) + pred = inference_on_d3(data_iter, inference_res) + + pred_path = os.path.join(output_dir, f"prediction.json") + write_json(pred_path, pred) + # see https://github.com/shikras/d-cube/blob/main/doc.md#output-format for the output format + # the output format is identical to COCO. + + eval_on_d3(pred_path, mode="pn") # the FULL setting + eval_on_d3(pred_path, mode="p") # the PRES setting + eval_on_d3(pred_path, mode="n") # the ABS setting diff --git a/approach/ovod/d-cube/scripts/eval_and_analysis_json.py b/approach/ovod/d-cube/scripts/eval_and_analysis_json.py new file mode 100644 index 0000000000000000000000000000000000000000..1d488c9ee63ab0b8070ac9506342023265049fed --- /dev/null +++ b/approach/ovod/d-cube/scripts/eval_and_analysis_json.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- +__author__ = "Chi Xie and Zhao Zhang" +__maintainer__ = "Chi Xie" +# this script takes the result json in, and print evaluation and analysis result on D-cube (FULL/PRES/ABS, etc.) +import os +import json +import argparse +from collections import defaultdict + +import numpy as np +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval + +from d_cube import D3 + +def eval_on_d3(pred_path, mode="pn", nbox_partition=None, lref_partition=False): + assert mode in ("pn", "p", "n") + if mode == "pn": + gt_path = os.path.join(JSON_ANNO_PATH, "d3_full_annotations.json") + elif mode == "p": + gt_path = os.path.join(JSON_ANNO_PATH, "d3_pres_annotations.json") + else: + gt_path = os.path.join(JSON_ANNO_PATH, "d3_abs_annotations.json") + + if nbox_partition: + gt_path, pred_path = nbox_partition_json(gt_path, pred_path, nbox_partition) + + # Eval results + coco = COCO(gt_path) + d3_res = coco.loadRes(pred_path) + cocoEval = COCOeval(coco, d3_res, "bbox") + cocoEval.evaluate() + cocoEval.accumulate() + cocoEval.summarize() + + aps = cocoEval.eval["precision"][:, :, :, 0, -1] + category_ids = coco.getCatIds() + category_names = [cat["name"] for cat in coco.loadCats(category_ids)] + + if lref_partition: + aps_lens = defaultdict(list) + counter_lens = defaultdict(int) + for i in range(len(category_names)): + ap = aps[:, :, i] + ap_value = ap[ap > -1].mean() + if not np.isnan(ap_value): + len_ref = len(category_names[i].split(" ")) + aps_lens[len_ref].append(ap_value) + counter_lens[len_ref] += 1 + + ap_sum_short = sum([sum(aps_lens[i]) for i in range(0, 4)]) + ap_sum_mid = sum([sum(aps_lens[i]) for i in range(4, 7)]) + ap_sum_long = sum([sum(aps_lens[i]) for i in range(7, 10)]) + ap_sum_very_long = sum( + [sum(aps_lens[i]) for i in range(10, max(counter_lens.keys()) + 1)] + ) + c_sum_short = sum([counter_lens[i] for i in range(1, 4)]) + c_sum_mid = sum([counter_lens[i] for i in range(4, 7)]) + c_sum_long = sum([counter_lens[i] for i in range(7, 10)]) + c_sum_very_long = sum( + [counter_lens[i] for i in range(10, max(counter_lens.keys()) + 1)] + ) + map_short = ap_sum_short / c_sum_short + map_mid = ap_sum_mid / c_sum_mid + map_long = ap_sum_long / c_sum_long + map_very_long = ap_sum_very_long / c_sum_very_long + print( + f"mAP over reference length: short - {map_short:.4f}, mid - {map_mid:.4f}, long - {map_long:.4f}, very long - {map_very_long:.4f}" + ) + + +def nbox_partition_json(gt_path, pred_path, nbox_partition): + with open(gt_path, "r") as f_gt: + gts = json.load(f_gt) + with open(pred_path, "r") as f_pred: + preds = json.load(f_pred) + + cat_obj_count = d3.bbox_num_analyze() + annos = gts["annotations"] + new_annos = [] + for ann in annos: + img_id = ann["image_id"] + category_id = ann["category_id"] + if nbox_partition == "one" and cat_obj_count[category_id - 1, img_id] == 1: + new_annos.append(ann) + if nbox_partition == "multi" and cat_obj_count[category_id - 1, img_id] > 1: + new_annos.append(ann) + if nbox_partition == "two" and cat_obj_count[category_id - 1, img_id] == 2: + new_annos.append(ann) + if nbox_partition == "three" and cat_obj_count[category_id - 1, img_id] == 3: + new_annos.append(ann) + if nbox_partition == "four" and cat_obj_count[category_id - 1, img_id] == 4: + new_annos.append(ann) + if nbox_partition == "four_more" and cat_obj_count[category_id - 1, img_id] > 4: + new_annos.append(ann) + gts["annotations"] = new_annos + new_gts = gts + new_preds = [] + for prd in preds: + img_id = prd["image_id"] + category_id = prd["category_id"] + if nbox_partition == "no" and cat_obj_count[category_id - 1, img_id] == 0: + new_preds.append(prd) + if nbox_partition == "one" and cat_obj_count[category_id - 1, img_id] == 1: + new_preds.append(prd) + if nbox_partition == "multi" and cat_obj_count[category_id - 1, img_id] > 1: + new_preds.append(prd) + if nbox_partition == "two" and cat_obj_count[category_id - 1, img_id] == 2: + new_preds.append(prd) + if nbox_partition == "three" and cat_obj_count[category_id - 1, img_id] == 3: + new_preds.append(prd) + if nbox_partition == "four" and cat_obj_count[category_id - 1, img_id] == 4: + new_preds.append(prd) + if nbox_partition == "four_more" and cat_obj_count[category_id - 1, img_id] > 4: + new_preds.append(prd) + + new_gt_path = gt_path.replace(".json", f".{nbox_partition}-instance.json") + new_pred_path = pred_path.replace(".json", f".{nbox_partition}-instance.json") + with open(new_gt_path, "w") as fo_gt: + json.dump(new_gts, fo_gt) + with open(new_pred_path, "w") as fo_pred: + json.dump(new_preds, fo_pred) + return new_gt_path, new_pred_path + + +def convert_to_xywh(x1, y1, x2, y2): + """ + Convert top-left and bottom-right corner coordinates to [x,y,width,height] format. + """ + width = x2 - x1 + height = y2 - y1 + return x1, y1, width, height + + +def transform_json_boxes(pred_path): + with open(pred_path, "r") as f_: + res = json.load(f_) + for item in res: + item["bbox"] = convert_to_xywh(*item["bbox"]) + res_path = pred_path.replace(".json", ".xywh.json") + with open(res_path, "w") as f_w: + json.dump(res, f_w) + return res_path + + +if __name__ == "__main__": + D3_DATASET_ROOT = os.environ.get("D3_DATASET_ROOT") + if not D3_DATASET_ROOT: + raise RuntimeError("Set D3_DATASET_ROOT to the extracted D3 dataset directory.") + IMG_ROOT = D3_DATASET_ROOT + JSON_ANNO_PATH = os.path.join(D3_DATASET_ROOT, "d3_json") + PKL_ANNO_PATH = os.path.join(D3_DATASET_ROOT, "d3_pkl") + d3 = D3(IMG_ROOT, PKL_ANNO_PATH) + + parser = argparse.ArgumentParser( + "An example script for D-cube evaluation with prediction file (JSON)", + add_help=True, + ) + parser.add_argument("pred_path", type=str, help="path to the prediction JSON file") + parser.add_argument( + "--partition-by-nbox", + action="store_true", + help="divide the images by num of boxes for each ref", + ) + parser.add_argument( + "--partition-by-lens", + action="store_true", + help="divide the references by their lengths", + ) + parser.add_argument( + "--xyxy2xywh", + action="store_true", + help="transform box coords from xyxy to xywh", + ) + args = parser.parse_args() + if args.xyxy2xywh: + pred_path = transform_json_boxes(args.pred_path) + else: + pred_path = args.pred_path + pred_path = args.pred_path + if args.partition_by_nbox: + # partiton: no-instance, one-instance, multi-instance + for mode in ("pn", "p", "n"): + # for ptt in ('no', 'one', 'multi'): + for ptt in ("no", "one", "two", "three", "four", "four_more"): + eval_on_d3(pred_path, mode=mode, nbox_partition=ptt) + else: + eval_on_d3(pred_path, mode="pn", lref_partition=args.partition_by_lens) + eval_on_d3(pred_path, mode="p", lref_partition=args.partition_by_lens) + eval_on_d3(pred_path, mode="n", lref_partition=args.partition_by_lens) diff --git a/approach/ovod/d-cube/scripts/eval_json_example.py b/approach/ovod/d-cube/scripts/eval_json_example.py new file mode 100644 index 0000000000000000000000000000000000000000..3c2fb164c923152fb722671794f42aa38ffaac87 --- /dev/null +++ b/approach/ovod/d-cube/scripts/eval_json_example.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +__author__ = "Chi Xie and Zhao Zhang" +__maintainer__ = "Chi Xie" +# this script takes the result json in, and print evaluation and analysis result on D-cube (FULL/PRES/ABS, etc.) +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval + +# Eval results with COCOAPI +gt_path = "./d3_full_annotations.json" # FULL, PRES or ABS +pred_path = None # set your prediction JSON path +coco = COCO(gt_path) +d3_res = coco.loadRes(pred_path) +cocoEval = COCOeval(coco, d3_res, "bbox") +cocoEval.evaluate() +cocoEval.accumulate() +cocoEval.summarize() diff --git a/approach/ovod/d-cube/scripts/get_d3_stat.py b/approach/ovod/d-cube/scripts/get_d3_stat.py new file mode 100644 index 0000000000000000000000000000000000000000..a7801c7a3d66958d0014d20248bb7c9f61156d03 --- /dev/null +++ b/approach/ovod/d-cube/scripts/get_d3_stat.py @@ -0,0 +1,98 @@ +import numpy as np + +from d_cube.vis_util import plot_hist +from d_cube import D3 + + +def vis_num_instance(cat_obj_count): + # Assuming `cat_obj_count` is your numpy array of shape [n_cat, n_img] + + # Calculate the total number of instances in each image + total_instances_per_image = np.sum(cat_obj_count, axis=0) + + # # Plot the histogram + # plt.hist(total_instances_per_image, bins=20) + # plt.xlabel('Number of Instances') + # plt.ylabel('Frequency') + # plt.title('Distribution of Number of Instances on a Image') + + # # Save the figure + # plt.savefig('vis_fig/instance_distribution.png', bbox_inches='tight') + # plt.close() + plot_hist( + total_instances_per_image, + bins=max(total_instances_per_image) - min(total_instances_per_image) + 1, + save_path="vis_fig/instance_dist_hist.pdf", + ) + + +def vis_num_category(cat_obj_count): + # Assuming `cat_obj_count` is your numpy array of shape [n_cat, n_img] + + # Calculate the number of categories in each image + num_categories_per_image = np.sum(cat_obj_count > 0, axis=0) + + # # Plot the histogram + # plt.hist(num_categories_per_image, bins=20) + # plt.xlabel('Number of Categories') + # plt.ylabel('Frequency') + # plt.title('Distribution of Number of Categories on a Image') + + # # Save the figure + # plt.savefig('vis_fig/category_distribution.png', bbox_inches='tight') + # plt.close() + plot_hist( + num_categories_per_image, + bins=max(num_categories_per_image) - min(num_categories_per_image) + 1, + save_path="vis_fig/category_dist_hist.pdf", + ) + + +def vis_num_img_per_cat(cat_obj_count): + num_img_per_cat = np.sum(cat_obj_count > 0, axis=1) + plot_hist( + num_img_per_cat, + bins=20, + save_path="vis_fig/nimg_pcat_hist.pdf", + x="Num. of images", + ) + + +def vis_num_box_per_cat(cat_obj_count): + num_box_per_cat = np.sum(cat_obj_count, axis=1) + plot_hist( + num_box_per_cat, + bins=20, + save_path="vis_fig/nbox_pcat_hist.pdf", + x="Num. of instances", + ) + + +def vis_num_box_per_cat_per_img(cat_obj_count): + img_obj_count = cat_obj_count.reshape(-1) + plot_hist( + img_obj_count[img_obj_count > 0], + bins=max(img_obj_count) - min(img_obj_count) + 1, + save_path="vis_fig/nbox_pcat_pimg_hist.pdf", + x="Num. of instances on a image", + ) + + +if __name__ == "__main__": + IMG_ROOT = None # set here + PKL_ANNO_PATH = None # set here + assert IMG_ROOT is not None, "Please set IMG_ROOT in the script first" + assert PKL_ANNO_PATH is not None, "Please set PKL_ANNO_PATH in the script first" + d3 = D3(IMG_ROOT, PKL_ANNO_PATH) + + cat_obj_count = d3.bbox_num_analyze() + vis_num_instance(cat_obj_count) + vis_num_category(cat_obj_count) + vis_num_img_per_cat(cat_obj_count) + vis_num_box_per_cat(cat_obj_count) + vis_num_box_per_cat_per_img(cat_obj_count) + + d3.stat_description(with_rev=False) + d3.stat_description(with_rev=True) + d3.stat_description(with_rev=False, inter_group=True) + d3.stat_description(with_rev=True, inter_group=True) diff --git a/approach/ovod/detectron2/.circleci/config.yml b/approach/ovod/detectron2/.circleci/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..33358b52f1734163fd97a4e7a0f4c542e66da147 --- /dev/null +++ b/approach/ovod/detectron2/.circleci/config.yml @@ -0,0 +1,270 @@ +version: 2.1 + +# ------------------------------------------------------------------------------------- +# Environments to run the jobs in +# ------------------------------------------------------------------------------------- +cpu: &cpu + machine: + image: ubuntu-2004:202107-02 + resource_class: medium + +gpu: &gpu + machine: + # NOTE: use a cuda version that's supported by all our pytorch versions + image: ubuntu-1604-cuda-11.1:202012-01 + resource_class: gpu.nvidia.small + +windows-cpu: &windows_cpu + machine: + resource_class: windows.medium + image: windows-server-2019-vs2019:stable + shell: powershell.exe + +# windows-gpu: &windows_gpu +# machine: +# resource_class: windows.gpu.nvidia.medium +# image: windows-server-2019-nvidia:stable + +version_parameters: &version_parameters + parameters: + pytorch_version: + type: string + torchvision_version: + type: string + pytorch_index: + type: string + # use test wheels index to have access to RC wheels + # https://download.pytorch.org/whl/test/torch_test.html + default: "https://download.pytorch.org/whl/torch_stable.html" + python_version: # NOTE: only affect linux + type: string + default: '3.7.9' + + environment: + PYTORCH_VERSION: << parameters.pytorch_version >> + TORCHVISION_VERSION: << parameters.torchvision_version >> + PYTORCH_INDEX: << parameters.pytorch_index >> + PYTHON_VERSION: << parameters.python_version>> + # point datasets to ~/.torch so it's cached in CI + DETECTRON2_DATASETS: ~/.torch/datasets + +# ------------------------------------------------------------------------------------- +# Re-usable commands +# ------------------------------------------------------------------------------------- +# install_nvidia_driver: &install_nvidia_driver +# - run: +# name: Install nvidia driver +# working_directory: ~/ +# command: | +# wget -q 'https://s3.amazonaws.com/ossci-linux/nvidia_driver/NVIDIA-Linux-x86_64-430.40.run' +# sudo /bin/bash ./NVIDIA-Linux-x86_64-430.40.run -s --no-drm +# nvidia-smi + +add_ssh_keys: &add_ssh_keys + # https://circleci.com/docs/2.0/add-ssh-key/ + - add_ssh_keys: + fingerprints: + - "e4:13:f2:22:d4:49:e8:e4:57:5a:ac:20:2f:3f:1f:ca" + +install_python: &install_python + - run: + name: Install Python + working_directory: ~/ + command: | + # upgrade pyenv + cd /opt/circleci/.pyenv/plugins/python-build/../.. && git pull && cd - + pyenv install -s $PYTHON_VERSION + pyenv global $PYTHON_VERSION + python --version + which python + pip install --upgrade pip + +setup_venv: &setup_venv + - run: + name: Setup Virtual Env + working_directory: ~/ + command: | + python -m venv ~/venv + echo ". ~/venv/bin/activate" >> $BASH_ENV + . ~/venv/bin/activate + python --version + which python + which pip + pip install --upgrade pip + +setup_venv_win: &setup_venv_win + - run: + name: Setup Virtual Env for Windows + command: | + pip install virtualenv + python -m virtualenv env + .\env\Scripts\activate + python --version + which python + which pip + +install_linux_dep: &install_linux_dep + - run: + name: Install Dependencies + command: | + # disable crash coredump, so unittests fail fast + sudo systemctl stop apport.service + # install from github to get latest; install iopath first since fvcore depends on it + pip install --progress-bar off -U 'git+https://github.com/facebookresearch/iopath' + pip install --progress-bar off -U 'git+https://github.com/facebookresearch/fvcore' + # Don't use pytest-xdist: cuda tests are unstable under multi-process workers. + pip install --progress-bar off ninja opencv-python-headless pytest tensorboard pycocotools onnx + pip install --progress-bar off torch==$PYTORCH_VERSION -f $PYTORCH_INDEX + if [[ "$TORCHVISION_VERSION" == "master" ]]; then + pip install git+https://github.com/pytorch/vision.git + else + pip install --progress-bar off torchvision==$TORCHVISION_VERSION -f $PYTORCH_INDEX + fi + + python -c 'import torch; print("CUDA:", torch.cuda.is_available())' + gcc --version + +install_detectron2: &install_detectron2 + - run: + name: Install Detectron2 + command: | + # Remove first, in case it's in the CI cache + pip uninstall -y detectron2 + + pip install --progress-bar off -e .[all] + python -m detectron2.utils.collect_env + ./datasets/prepare_for_tests.sh + +run_unittests: &run_unittests + - run: + name: Run Unit Tests + command: | + pytest -sv --durations=15 tests # parallel causes some random failures + +uninstall_tests: &uninstall_tests + - run: + name: Run Tests After Uninstalling + command: | + pip uninstall -y detectron2 + # Remove built binaries + rm -rf build/ detectron2/*.so + # Tests that code is importable without installation + PYTHONPATH=. ./.circleci/import-tests.sh + + +# ------------------------------------------------------------------------------------- +# Jobs to run +# ------------------------------------------------------------------------------------- +jobs: + linux_cpu_tests: + <<: *cpu + <<: *version_parameters + + working_directory: ~/detectron2 + + steps: + - checkout + + # Cache the venv directory that contains python, dependencies, and checkpoints + # Refresh the key when dependencies should be updated (e.g. when pytorch releases) + - restore_cache: + keys: + - cache-{{ arch }}-<< parameters.pytorch_version >>-{{ .Branch }}-20210827 + + - <<: *install_python + - <<: *install_linux_dep + - <<: *install_detectron2 + - <<: *run_unittests + - <<: *uninstall_tests + + - save_cache: + paths: + - /opt/circleci/.pyenv + - ~/.torch + key: cache-{{ arch }}-<< parameters.pytorch_version >>-{{ .Branch }}-20210827 + + + linux_gpu_tests: + <<: *gpu + <<: *version_parameters + + working_directory: ~/detectron2 + + steps: + - checkout + + - restore_cache: + keys: + - cache-{{ arch }}-<< parameters.pytorch_version >>-{{ .Branch }}-20210827 + + - <<: *install_python + - <<: *install_linux_dep + - <<: *install_detectron2 + - <<: *run_unittests + - <<: *uninstall_tests + + - save_cache: + paths: + - /opt/circleci/.pyenv + - ~/.torch + key: cache-{{ arch }}-<< parameters.pytorch_version >>-{{ .Branch }}-20210827 + + windows_cpu_build: + <<: *windows_cpu + <<: *version_parameters + steps: + - <<: *add_ssh_keys + - checkout + - <<: *setup_venv_win + + # Cache the env directory that contains dependencies + - restore_cache: + keys: + - cache-{{ arch }}-<< parameters.pytorch_version >>-{{ .Branch }}-20210404 + + - run: + name: Install Dependencies + command: | + pip install certifi --ignore-installed # required on windows to workaround some cert issue + pip install numpy cython # required on windows before pycocotools + pip install opencv-python-headless pytest-xdist pycocotools tensorboard onnx + pip install -U git+https://github.com/facebookresearch/iopath + pip install -U git+https://github.com/facebookresearch/fvcore + pip install torch==$env:PYTORCH_VERSION torchvision==$env:TORCHVISION_VERSION -f $env:PYTORCH_INDEX + + - save_cache: + paths: + - env + key: cache-{{ arch }}-<< parameters.pytorch_version >>-{{ .Branch }}-20210404 + + - <<: *install_detectron2 + # TODO: unittest fails for now + +workflows: + version: 2 + regular_test: + jobs: + - linux_cpu_tests: + name: linux_cpu_tests_pytorch1.10 + pytorch_version: '1.10.0+cpu' + torchvision_version: '0.11.1+cpu' + - linux_gpu_tests: + name: linux_gpu_tests_pytorch1.8 + pytorch_version: '1.8.1+cu111' + torchvision_version: '0.9.1+cu111' + - linux_gpu_tests: + name: linux_gpu_tests_pytorch1.9 + pytorch_version: '1.9+cu111' + torchvision_version: '0.10+cu111' + - linux_gpu_tests: + name: linux_gpu_tests_pytorch1.10 + pytorch_version: '1.10+cu111' + torchvision_version: '0.11.1+cu111' + - linux_gpu_tests: + name: linux_gpu_tests_pytorch1.10_python39 + pytorch_version: '1.10+cu111' + torchvision_version: '0.11.1+cu111' + python_version: '3.9.6' + - windows_cpu_build: + pytorch_version: '1.10+cpu' + torchvision_version: '0.11.1+cpu' diff --git a/approach/ovod/detectron2/.circleci/import-tests.sh b/approach/ovod/detectron2/.circleci/import-tests.sh new file mode 100644 index 0000000000000000000000000000000000000000..8e8deb6ad699fd673fea0f66b91aa3ec6e3c7c7c --- /dev/null +++ b/approach/ovod/detectron2/.circleci/import-tests.sh @@ -0,0 +1,16 @@ +#!/bin/bash -e +# Copyright (c) Facebook, Inc. and its affiliates. + +# Test that import works without building detectron2. + +# Check that _C is not importable +python -c "from detectron2 import _C" > /dev/null 2>&1 && { + echo "This test should be run without building detectron2." + exit 1 +} + +# Check that other modules are still importable, even when _C is not importable +python -c "from detectron2 import modeling" +python -c "from detectron2 import modeling, data" +python -c "from detectron2 import evaluation, export, checkpoint" +python -c "from detectron2 import utils, engine" diff --git a/approach/ovod/detectron2/.github/CODE_OF_CONDUCT.md b/approach/ovod/detectron2/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000000000000000000000000000000000..0f7ad8bfc173eac554f0b6ef7c684861e8014bbe --- /dev/null +++ b/approach/ovod/detectron2/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,5 @@ +# Code of Conduct + +Facebook has adopted a Code of Conduct that we expect project participants to adhere to. +Please read the [full text](https://code.fb.com/codeofconduct/) +so that you can understand what actions will and will not be tolerated. diff --git a/approach/ovod/detectron2/.github/CONTRIBUTING.md b/approach/ovod/detectron2/.github/CONTRIBUTING.md new file mode 100644 index 0000000000000000000000000000000000000000..9bab709cae689ba3b92dd52f7fbcc0c6926f4a38 --- /dev/null +++ b/approach/ovod/detectron2/.github/CONTRIBUTING.md @@ -0,0 +1,68 @@ +# Contributing to detectron2 + +## Issues +We use GitHub issues to track public bugs and questions. +Please make sure to follow one of the +[issue templates](https://github.com/facebookresearch/detectron2/issues/new/choose) +when reporting any issues. + +Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe +disclosure of security bugs. In those cases, please go through the process +outlined on that page and do not file a public issue. + +## Pull Requests +We actively welcome pull requests. + +However, if you're adding any significant features (e.g. > 50 lines), please +make sure to discuss with maintainers about your motivation and proposals in an issue +before sending a PR. This is to save your time so you don't spend time on a PR that we'll not accept. + +We do not always accept new features, and we take the following +factors into consideration: + +1. Whether the same feature can be achieved without modifying detectron2. + Detectron2 is designed so that you can implement many extensions from the outside, e.g. + those in [projects](https://github.com/facebookresearch/detectron2/tree/master/projects). + * If some part of detectron2 is not extensible enough, you can also bring up a more general issue to + improve it. Such feature request may be useful to more users. +2. Whether the feature is potentially useful to a large audience (e.g. an impactful detection paper, a popular dataset, + a significant speedup, a widely useful utility), + or only to a small portion of users (e.g., a less-known paper, an improvement not in the object + detection field, a trick that's not very popular in the community, code to handle a non-standard type of data) + * Adoption of additional models, datasets, new task are by default not added to detectron2 before they + receive significant popularity in the community. + We sometimes accept such features in `projects/`, or as a link in `projects/README.md`. +3. Whether the proposed solution has a good design / interface. This can be discussed in the issue prior to PRs, or + in the form of a draft PR. +4. Whether the proposed solution adds extra mental/practical overhead to users who don't + need such feature. +5. Whether the proposed solution breaks existing APIs. + +To add a feature to an existing function/class `Func`, there are always two approaches: +(1) add new arguments to `Func`; (2) write a new `Func_with_new_feature`. +To meet the above criteria, we often prefer approach (2), because: + +1. It does not involve modifying or potentially breaking existing code. +2. It does not add overhead to users who do not need the new feature. +3. Adding new arguments to a function/class is not scalable w.r.t. all the possible new research ideas in the future. + +When sending a PR, please do: + +1. If a PR contains multiple orthogonal changes, split it to several PRs. +2. If you've added code that should be tested, add tests. +3. For PRs that need experiments (e.g. adding a new model or new methods), + you don't need to update model zoo, but do provide experiment results in the description of the PR. +4. If APIs are changed, update the documentation. +5. We use the [Google style docstrings](https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html) in python. +6. Make sure your code lints with `./dev/linter.sh`. + + +## Contributor License Agreement ("CLA") +In order to accept your pull request, we need you to submit a CLA. You only need +to do this once to work on any of Facebook's open source projects. + +Complete your CLA here: + +## License +By contributing to detectron2, you agree that your contributions will be licensed +under the LICENSE file in the root directory of this source tree. diff --git a/approach/ovod/detectron2/.github/Detectron2-Logo-Horz.svg b/approach/ovod/detectron2/.github/Detectron2-Logo-Horz.svg new file mode 100644 index 0000000000000000000000000000000000000000..eb2d643ddd940cd8bdb5eaad093029969ff2364c --- /dev/null +++ b/approach/ovod/detectron2/.github/Detectron2-Logo-Horz.svg @@ -0,0 +1 @@ +Detectron2-Logo-Horz \ No newline at end of file diff --git a/approach/ovod/detectron2/.github/ISSUE_TEMPLATE.md b/approach/ovod/detectron2/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000000000000000000000000000000000000..5e8aaa2d3722e7e73a3d94b2b7dfc4f751d7a240 --- /dev/null +++ b/approach/ovod/detectron2/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,5 @@ + +Please select an issue template from +https://github.com/facebookresearch/detectron2/issues/new/choose . + +Otherwise your issue will be closed. diff --git a/approach/ovod/detectron2/.github/pull_request_template.md b/approach/ovod/detectron2/.github/pull_request_template.md new file mode 100644 index 0000000000000000000000000000000000000000..d71729baee1ec324ab9db6e7562965cf9e2a091b --- /dev/null +++ b/approach/ovod/detectron2/.github/pull_request_template.md @@ -0,0 +1,10 @@ +Thanks for your contribution! + +If you're sending a large PR (e.g., >100 lines), +please open an issue first about the feature / bug, and indicate how you want to contribute. + +We do not always accept features. +See https://detectron2.readthedocs.io/notes/contributing.html#pull-requests about how we handle PRs. + +Before submitting a PR, please run `dev/linter.sh` to lint the code. + diff --git a/approach/ovod/detectron2/configs/Base-RCNN-C4.yaml b/approach/ovod/detectron2/configs/Base-RCNN-C4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fbf34a0ea57a587e09997edd94c4012d69d0b6ad --- /dev/null +++ b/approach/ovod/detectron2/configs/Base-RCNN-C4.yaml @@ -0,0 +1,18 @@ +MODEL: + META_ARCHITECTURE: "GeneralizedRCNN" + RPN: + PRE_NMS_TOPK_TEST: 6000 + POST_NMS_TOPK_TEST: 1000 + ROI_HEADS: + NAME: "Res5ROIHeads" +DATASETS: + TRAIN: ("coco_2017_train",) + TEST: ("coco_2017_val",) +SOLVER: + IMS_PER_BATCH: 16 + BASE_LR: 0.02 + STEPS: (60000, 80000) + MAX_ITER: 90000 +INPUT: + MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800) +VERSION: 2 diff --git a/approach/ovod/detectron2/configs/Base-RCNN-DilatedC5.yaml b/approach/ovod/detectron2/configs/Base-RCNN-DilatedC5.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0d6d16bdaf532f09e4976f0aa240a49e748da27 --- /dev/null +++ b/approach/ovod/detectron2/configs/Base-RCNN-DilatedC5.yaml @@ -0,0 +1,31 @@ +MODEL: + META_ARCHITECTURE: "GeneralizedRCNN" + RESNETS: + OUT_FEATURES: ["res5"] + RES5_DILATION: 2 + RPN: + IN_FEATURES: ["res5"] + PRE_NMS_TOPK_TEST: 6000 + POST_NMS_TOPK_TEST: 1000 + ROI_HEADS: + NAME: "StandardROIHeads" + IN_FEATURES: ["res5"] + ROI_BOX_HEAD: + NAME: "FastRCNNConvFCHead" + NUM_FC: 2 + POOLER_RESOLUTION: 7 + ROI_MASK_HEAD: + NAME: "MaskRCNNConvUpsampleHead" + NUM_CONV: 4 + POOLER_RESOLUTION: 14 +DATASETS: + TRAIN: ("coco_2017_train",) + TEST: ("coco_2017_val",) +SOLVER: + IMS_PER_BATCH: 16 + BASE_LR: 0.02 + STEPS: (60000, 80000) + MAX_ITER: 90000 +INPUT: + MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800) +VERSION: 2 diff --git a/approach/ovod/detectron2/configs/Base-RCNN-FPN.yaml b/approach/ovod/detectron2/configs/Base-RCNN-FPN.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e020f2e7b2f26765be317f907126a1556621abf --- /dev/null +++ b/approach/ovod/detectron2/configs/Base-RCNN-FPN.yaml @@ -0,0 +1,42 @@ +MODEL: + META_ARCHITECTURE: "GeneralizedRCNN" + BACKBONE: + NAME: "build_resnet_fpn_backbone" + RESNETS: + OUT_FEATURES: ["res2", "res3", "res4", "res5"] + FPN: + IN_FEATURES: ["res2", "res3", "res4", "res5"] + ANCHOR_GENERATOR: + SIZES: [[32], [64], [128], [256], [512]] # One size for each in feature map + ASPECT_RATIOS: [[0.5, 1.0, 2.0]] # Three aspect ratios (same for all in feature maps) + RPN: + IN_FEATURES: ["p2", "p3", "p4", "p5", "p6"] + PRE_NMS_TOPK_TRAIN: 2000 # Per FPN level + PRE_NMS_TOPK_TEST: 1000 # Per FPN level + # Detectron1 uses 2000 proposals per-batch, + # (See "modeling/rpn/rpn_outputs.py" for details of this legacy issue) + # which is approximately 1000 proposals per-image since the default batch size for FPN is 2. + POST_NMS_TOPK_TRAIN: 1000 + POST_NMS_TOPK_TEST: 1000 + ROI_HEADS: + NAME: "StandardROIHeads" + IN_FEATURES: ["p2", "p3", "p4", "p5"] + ROI_BOX_HEAD: + NAME: "FastRCNNConvFCHead" + NUM_FC: 2 + POOLER_RESOLUTION: 7 + ROI_MASK_HEAD: + NAME: "MaskRCNNConvUpsampleHead" + NUM_CONV: 4 + POOLER_RESOLUTION: 14 +DATASETS: + TRAIN: ("coco_2017_train",) + TEST: ("coco_2017_val",) +SOLVER: + IMS_PER_BATCH: 16 + BASE_LR: 0.02 + STEPS: (60000, 80000) + MAX_ITER: 90000 +INPUT: + MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800) +VERSION: 2 diff --git a/approach/ovod/detectron2/configs/Base-RetinaNet.yaml b/approach/ovod/detectron2/configs/Base-RetinaNet.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b45b982bbf84b34d2a6a172ab0a946b1029f7c8 --- /dev/null +++ b/approach/ovod/detectron2/configs/Base-RetinaNet.yaml @@ -0,0 +1,25 @@ +MODEL: + META_ARCHITECTURE: "RetinaNet" + BACKBONE: + NAME: "build_retinanet_resnet_fpn_backbone" + RESNETS: + OUT_FEATURES: ["res3", "res4", "res5"] + ANCHOR_GENERATOR: + SIZES: !!python/object/apply:eval ["[[x, x * 2**(1.0/3), x * 2**(2.0/3) ] for x in [32, 64, 128, 256, 512 ]]"] + FPN: + IN_FEATURES: ["res3", "res4", "res5"] + RETINANET: + IOU_THRESHOLDS: [0.4, 0.5] + IOU_LABELS: [0, -1, 1] + SMOOTH_L1_LOSS_BETA: 0.0 +DATASETS: + TRAIN: ("coco_2017_train",) + TEST: ("coco_2017_val",) +SOLVER: + IMS_PER_BATCH: 16 + BASE_LR: 0.01 # Note that RetinaNet uses a different default learning rate + STEPS: (60000, 80000) + MAX_ITER: 90000 +INPUT: + MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800) +VERSION: 2 diff --git a/approach/ovod/detectron2/demo/README.md b/approach/ovod/detectron2/demo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..133d8d38e5e9f5f44aca92c59f73309e166d7132 --- /dev/null +++ b/approach/ovod/detectron2/demo/README.md @@ -0,0 +1,8 @@ + +## Detectron2 Demo + +We provide a command line tool to run a simple demo of builtin configs. +The usage is explained in [GETTING_STARTED.md](../GETTING_STARTED.md). + +See our [blog post](https://ai.facebook.com/blog/-detectron2-a-pytorch-based-modular-object-detection-library-) +for a high-quality demo generated with this tool. diff --git a/approach/ovod/detectron2/demo/demo.py b/approach/ovod/detectron2/demo/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..4baa8767f7b299f18253aadb15a9bac5b9cc07fc --- /dev/null +++ b/approach/ovod/detectron2/demo/demo.py @@ -0,0 +1,188 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import argparse +import glob +import multiprocessing as mp +import numpy as np +import os +import tempfile +import time +import warnings +import cv2 +import tqdm + +from detectron2.config import get_cfg +from detectron2.data.detection_utils import read_image +from detectron2.utils.logger import setup_logger + +from predictor import VisualizationDemo + +# constants +WINDOW_NAME = "COCO detections" + + +def setup_cfg(args): + # load config from file and command-line arguments + cfg = get_cfg() + # To use demo for Panoptic-DeepLab, please uncomment the following two lines. + # from detectron2.projects.panoptic_deeplab import add_panoptic_deeplab_config # noqa + # add_panoptic_deeplab_config(cfg) + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + # Set score_threshold for builtin models + cfg.MODEL.RETINANET.SCORE_THRESH_TEST = args.confidence_threshold + cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = args.confidence_threshold + cfg.MODEL.PANOPTIC_FPN.COMBINE.INSTANCES_CONFIDENCE_THRESH = args.confidence_threshold + cfg.freeze() + return cfg + + +def get_parser(): + parser = argparse.ArgumentParser(description="Detectron2 demo for builtin configs") + parser.add_argument( + "--config-file", + default="configs/quick_schedules/mask_rcnn_R_50_FPN_inference_acc_test.yaml", + metavar="FILE", + help="path to config file", + ) + parser.add_argument("--webcam", action="store_true", help="Take inputs from webcam.") + parser.add_argument("--video-input", help="Path to video file.") + parser.add_argument( + "--input", + nargs="+", + help="A list of space separated input images; " + "or a single glob pattern such as 'directory/*.jpg'", + ) + parser.add_argument( + "--output", + help="A file or directory to save output visualizations. " + "If not given, will show output in an OpenCV window.", + ) + + parser.add_argument( + "--confidence-threshold", + type=float, + default=0.5, + help="Minimum score for instance predictions to be shown", + ) + parser.add_argument( + "--opts", + help="Modify config options using the command-line 'KEY VALUE' pairs", + default=[], + nargs=argparse.REMAINDER, + ) + return parser + + +def test_opencv_video_format(codec, file_ext): + with tempfile.TemporaryDirectory(prefix="video_format_test") as dir: + filename = os.path.join(dir, "test_file" + file_ext) + writer = cv2.VideoWriter( + filename=filename, + fourcc=cv2.VideoWriter_fourcc(*codec), + fps=float(30), + frameSize=(10, 10), + isColor=True, + ) + [writer.write(np.zeros((10, 10, 3), np.uint8)) for _ in range(30)] + writer.release() + if os.path.isfile(filename): + return True + return False + + +if __name__ == "__main__": + mp.set_start_method("spawn", force=True) + args = get_parser().parse_args() + setup_logger(name="fvcore") + logger = setup_logger() + logger.info("Arguments: " + str(args)) + + cfg = setup_cfg(args) + + demo = VisualizationDemo(cfg) + + if args.input: + if len(args.input) == 1: + args.input = glob.glob(os.path.expanduser(args.input[0])) + assert args.input, "The input path(s) was not found" + for path in tqdm.tqdm(args.input, disable=not args.output): + # use PIL, to be consistent with evaluation + img = read_image(path, format="BGR") + start_time = time.time() + predictions, visualized_output = demo.run_on_image(img) + logger.info( + "{}: {} in {:.2f}s".format( + path, + "detected {} instances".format(len(predictions["instances"])) + if "instances" in predictions + else "finished", + time.time() - start_time, + ) + ) + + if args.output: + if os.path.isdir(args.output): + assert os.path.isdir(args.output), args.output + out_filename = os.path.join(args.output, os.path.basename(path)) + else: + assert len(args.input) == 1, "Please specify a directory with args.output" + out_filename = args.output + visualized_output.save(out_filename) + else: + cv2.namedWindow(WINDOW_NAME, cv2.WINDOW_NORMAL) + cv2.imshow(WINDOW_NAME, visualized_output.get_image()[:, :, ::-1]) + if cv2.waitKey(0) == 27: + break # esc to quit + elif args.webcam: + assert args.input is None, "Cannot have both --input and --webcam!" + assert args.output is None, "output not yet supported with --webcam!" + cam = cv2.VideoCapture(0) + for vis in tqdm.tqdm(demo.run_on_video(cam)): + cv2.namedWindow(WINDOW_NAME, cv2.WINDOW_NORMAL) + cv2.imshow(WINDOW_NAME, vis) + if cv2.waitKey(1) == 27: + break # esc to quit + cam.release() + cv2.destroyAllWindows() + elif args.video_input: + video = cv2.VideoCapture(args.video_input) + width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH)) + height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT)) + frames_per_second = video.get(cv2.CAP_PROP_FPS) + num_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) + basename = os.path.basename(args.video_input) + codec, file_ext = ( + ("x264", ".mkv") if test_opencv_video_format("x264", ".mkv") else ("mp4v", ".mp4") + ) + if codec == ".mp4v": + warnings.warn("x264 codec not available, switching to mp4v") + if args.output: + if os.path.isdir(args.output): + output_fname = os.path.join(args.output, basename) + output_fname = os.path.splitext(output_fname)[0] + file_ext + else: + output_fname = args.output + assert not os.path.isfile(output_fname), output_fname + output_file = cv2.VideoWriter( + filename=output_fname, + # some installation of opencv may not support x264 (due to its license), + # you can try other format (e.g. MPEG) + fourcc=cv2.VideoWriter_fourcc(*codec), + fps=float(frames_per_second), + frameSize=(width, height), + isColor=True, + ) + assert os.path.isfile(args.video_input) + for vis_frame in tqdm.tqdm(demo.run_on_video(video), total=num_frames): + if args.output: + output_file.write(vis_frame) + else: + cv2.namedWindow(basename, cv2.WINDOW_NORMAL) + cv2.imshow(basename, vis_frame) + if cv2.waitKey(1) == 27: + break # esc to quit + video.release() + if args.output: + output_file.release() + else: + cv2.destroyAllWindows() diff --git a/approach/ovod/detectron2/demo/predictor.py b/approach/ovod/detectron2/demo/predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..7b7ebd3f846850172c1f560f8492d51e5667f76d --- /dev/null +++ b/approach/ovod/detectron2/demo/predictor.py @@ -0,0 +1,220 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import atexit +import bisect +import multiprocessing as mp +from collections import deque +import cv2 +import torch + +from detectron2.data import MetadataCatalog +from detectron2.engine.defaults import DefaultPredictor +from detectron2.utils.video_visualizer import VideoVisualizer +from detectron2.utils.visualizer import ColorMode, Visualizer + + +class VisualizationDemo(object): + def __init__(self, cfg, instance_mode=ColorMode.IMAGE, parallel=False): + """ + Args: + cfg (CfgNode): + instance_mode (ColorMode): + parallel (bool): whether to run the model in different processes from visualization. + Useful since the visualization logic can be slow. + """ + self.metadata = MetadataCatalog.get( + cfg.DATASETS.TEST[0] if len(cfg.DATASETS.TEST) else "__unused" + ) + self.cpu_device = torch.device("cpu") + self.instance_mode = instance_mode + + self.parallel = parallel + if parallel: + num_gpu = torch.cuda.device_count() + self.predictor = AsyncPredictor(cfg, num_gpus=num_gpu) + else: + self.predictor = DefaultPredictor(cfg) + + def run_on_image(self, image): + """ + Args: + image (np.ndarray): an image of shape (H, W, C) (in BGR order). + This is the format used by OpenCV. + + Returns: + predictions (dict): the output of the model. + vis_output (VisImage): the visualized image output. + """ + vis_output = None + predictions = self.predictor(image) + # Convert image from OpenCV BGR format to Matplotlib RGB format. + image = image[:, :, ::-1] + visualizer = Visualizer(image, self.metadata, instance_mode=self.instance_mode) + if "panoptic_seg" in predictions: + panoptic_seg, segments_info = predictions["panoptic_seg"] + vis_output = visualizer.draw_panoptic_seg_predictions( + panoptic_seg.to(self.cpu_device), segments_info + ) + else: + if "sem_seg" in predictions: + vis_output = visualizer.draw_sem_seg( + predictions["sem_seg"].argmax(dim=0).to(self.cpu_device) + ) + if "instances" in predictions: + instances = predictions["instances"].to(self.cpu_device) + vis_output = visualizer.draw_instance_predictions(predictions=instances) + + return predictions, vis_output + + def _frame_from_video(self, video): + while video.isOpened(): + success, frame = video.read() + if success: + yield frame + else: + break + + def run_on_video(self, video): + """ + Visualizes predictions on frames of the input video. + + Args: + video (cv2.VideoCapture): a :class:`VideoCapture` object, whose source can be + either a webcam or a video file. + + Yields: + ndarray: BGR visualizations of each video frame. + """ + video_visualizer = VideoVisualizer(self.metadata, self.instance_mode) + + def process_predictions(frame, predictions): + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + if "panoptic_seg" in predictions: + panoptic_seg, segments_info = predictions["panoptic_seg"] + vis_frame = video_visualizer.draw_panoptic_seg_predictions( + frame, panoptic_seg.to(self.cpu_device), segments_info + ) + elif "instances" in predictions: + predictions = predictions["instances"].to(self.cpu_device) + vis_frame = video_visualizer.draw_instance_predictions(frame, predictions) + elif "sem_seg" in predictions: + vis_frame = video_visualizer.draw_sem_seg( + frame, predictions["sem_seg"].argmax(dim=0).to(self.cpu_device) + ) + + # Converts Matplotlib RGB format to OpenCV BGR format + vis_frame = cv2.cvtColor(vis_frame.get_image(), cv2.COLOR_RGB2BGR) + return vis_frame + + frame_gen = self._frame_from_video(video) + if self.parallel: + buffer_size = self.predictor.default_buffer_size + + frame_data = deque() + + for cnt, frame in enumerate(frame_gen): + frame_data.append(frame) + self.predictor.put(frame) + + if cnt >= buffer_size: + frame = frame_data.popleft() + predictions = self.predictor.get() + yield process_predictions(frame, predictions) + + while len(frame_data): + frame = frame_data.popleft() + predictions = self.predictor.get() + yield process_predictions(frame, predictions) + else: + for frame in frame_gen: + yield process_predictions(frame, self.predictor(frame)) + + +class AsyncPredictor: + """ + A predictor that runs the model asynchronously, possibly on >1 GPUs. + Because rendering the visualization takes considerably amount of time, + this helps improve throughput a little bit when rendering videos. + """ + + class _StopToken: + pass + + class _PredictWorker(mp.Process): + def __init__(self, cfg, task_queue, result_queue): + self.cfg = cfg + self.task_queue = task_queue + self.result_queue = result_queue + super().__init__() + + def run(self): + predictor = DefaultPredictor(self.cfg) + + while True: + task = self.task_queue.get() + if isinstance(task, AsyncPredictor._StopToken): + break + idx, data = task + result = predictor(data) + self.result_queue.put((idx, result)) + + def __init__(self, cfg, num_gpus: int = 1): + """ + Args: + cfg (CfgNode): + num_gpus (int): if 0, will run on CPU + """ + num_workers = max(num_gpus, 1) + self.task_queue = mp.Queue(maxsize=num_workers * 3) + self.result_queue = mp.Queue(maxsize=num_workers * 3) + self.procs = [] + for gpuid in range(max(num_gpus, 1)): + cfg = cfg.clone() + cfg.defrost() + cfg.MODEL.DEVICE = "cuda:{}".format(gpuid) if num_gpus > 0 else "cpu" + self.procs.append( + AsyncPredictor._PredictWorker(cfg, self.task_queue, self.result_queue) + ) + + self.put_idx = 0 + self.get_idx = 0 + self.result_rank = [] + self.result_data = [] + + for p in self.procs: + p.start() + atexit.register(self.shutdown) + + def put(self, image): + self.put_idx += 1 + self.task_queue.put((self.put_idx, image)) + + def get(self): + self.get_idx += 1 # the index needed for this request + if len(self.result_rank) and self.result_rank[0] == self.get_idx: + res = self.result_data[0] + del self.result_data[0], self.result_rank[0] + return res + + while True: + # make sure the results are returned in the correct order + idx, res = self.result_queue.get() + if idx == self.get_idx: + return res + insert = bisect.bisect(self.result_rank, idx) + self.result_rank.insert(insert, idx) + self.result_data.insert(insert, res) + + def __len__(self): + return self.put_idx - self.get_idx + + def __call__(self, image): + self.put(image) + return self.get() + + def shutdown(self): + for _ in self.procs: + self.task_queue.put(AsyncPredictor._StopToken()) + + @property + def default_buffer_size(self): + return len(self.procs) * 5 diff --git a/approach/ovod/detectron2/detectron2/__init__.py b/approach/ovod/detectron2/detectron2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bdd994b49294485c27610772f97f177741f5518f --- /dev/null +++ b/approach/ovod/detectron2/detectron2/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from .utils.env import setup_environment + +setup_environment() + + +# This line will be programatically read/write by setup.py. +# Leave them at the bottom of this file and don't touch them. +__version__ = "0.6" diff --git a/approach/ovod/detectron2/dev/README.md b/approach/ovod/detectron2/dev/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bec811ad002a016f2137d9d0ea61c27ee5e78992 --- /dev/null +++ b/approach/ovod/detectron2/dev/README.md @@ -0,0 +1,7 @@ + +## Some scripts for developers to use, include: + +- `linter.sh`: lint the codebase before commit. +- `run_{inference,instant}_tests.sh`: run inference/training for a few iterations. + Note that these tests require 2 GPUs. +- `parse_results.sh`: parse results from a log file. diff --git a/approach/ovod/detectron2/dev/linter.sh b/approach/ovod/detectron2/dev/linter.sh new file mode 100644 index 0000000000000000000000000000000000000000..55793e01819853987e81e6a14a5905ce0b40bf81 --- /dev/null +++ b/approach/ovod/detectron2/dev/linter.sh @@ -0,0 +1,42 @@ +#!/bin/bash -e +# Copyright (c) Facebook, Inc. and its affiliates. + +# cd to detectron2 project root +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +{ + black --version | grep -E "22\." > /dev/null +} || { + echo "Linter requires 'black==22.*' !" + exit 1 +} + +ISORT_VERSION=$(isort --version-number) +if [[ "$ISORT_VERSION" != 4.3* ]]; then + echo "Linter requires isort==4.3.21 !" + exit 1 +fi + +set -v + +echo "Running isort ..." +isort -y -sp . --atomic + +echo "Running black ..." +black -l 100 . + +echo "Running flake8 ..." +if [ -x "$(command -v flake8)" ]; then + flake8 . +else + python3 -m flake8 . +fi + +# echo "Running mypy ..." +# Pytorch does not have enough type annotations +# mypy detectron2/solver detectron2/structures detectron2/config + +echo "Running clang-format ..." +find . -regex ".*\.\(cpp\|c\|cc\|cu\|cxx\|h\|hh\|hpp\|hxx\|tcc\|mm\|m\)" -print0 | xargs -0 clang-format -i + +command -v arc > /dev/null && arc lint diff --git a/approach/ovod/detectron2/dev/parse_results.sh b/approach/ovod/detectron2/dev/parse_results.sh new file mode 100644 index 0000000000000000000000000000000000000000..80768a4005753447c49339790fe66c9b82a80aaf --- /dev/null +++ b/approach/ovod/detectron2/dev/parse_results.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. + +# A shell script that parses metrics from the log file. +# Make it easier for developers to track performance of models. + +LOG="$1" + +if [[ -z "$LOG" ]]; then + echo "Usage: $0 /path/to/log/file" + exit 1 +fi + +# [12/15 11:47:32] trainer INFO: Total training time: 12:15:04.446477 (0.4900 s / it) +# [12/15 11:49:03] inference INFO: Total inference time: 0:01:25.326167 (0.13652186737060548 s / img per device, on 8 devices) +# [12/15 11:49:03] inference INFO: Total inference pure compute time: ..... + +# training time +trainspeed=$(grep -o 'Overall training.*' "$LOG" | grep -Eo '\(.*\)' | grep -o '[0-9\.]*') +echo "Training speed: $trainspeed s/it" + +# inference time: there could be multiple inference during training +inferencespeed=$(grep -o 'Total inference pure.*' "$LOG" | tail -n1 | grep -Eo '\(.*\)' | grep -o '[0-9\.]*' | head -n1) +echo "Inference speed: $inferencespeed s/it" + +# [12/15 11:47:18] trainer INFO: eta: 0:00:00 iter: 90000 loss: 0.5407 (0.7256) loss_classifier: 0.1744 (0.2446) loss_box_reg: 0.0838 (0.1160) loss_mask: 0.2159 (0.2722) loss_objectness: 0.0244 (0.0429) loss_rpn_box_reg: 0.0279 (0.0500) time: 0.4487 (0.4899) data: 0.0076 (0.0975) lr: 0.000200 max mem: 4161 +memory=$(grep -o 'max[_ ]mem: [0-9]*' "$LOG" | tail -n1 | grep -o '[0-9]*') +echo "Training memory: $memory MB" + +echo "Easy to copypaste:" +echo "$trainspeed","$inferencespeed","$memory" + +echo "------------------------------" + +# [12/26 17:26:32] engine.coco_evaluation: copypaste: Task: bbox +# [12/26 17:26:32] engine.coco_evaluation: copypaste: AP,AP50,AP75,APs,APm,APl +# [12/26 17:26:32] engine.coco_evaluation: copypaste: 0.0017,0.0024,0.0017,0.0005,0.0019,0.0011 +# [12/26 17:26:32] engine.coco_evaluation: copypaste: Task: segm +# [12/26 17:26:32] engine.coco_evaluation: copypaste: AP,AP50,AP75,APs,APm,APl +# [12/26 17:26:32] engine.coco_evaluation: copypaste: 0.0014,0.0021,0.0016,0.0005,0.0016,0.0011 + +echo "COCO Results:" +num_tasks=$(grep -o 'copypaste:.*Task.*' "$LOG" | sort -u | wc -l) +# each task has 3 lines +grep -o 'copypaste:.*' "$LOG" | cut -d ' ' -f 2- | tail -n $((num_tasks * 3)) diff --git a/approach/ovod/detectron2/dev/run_inference_tests.sh b/approach/ovod/detectron2/dev/run_inference_tests.sh new file mode 100644 index 0000000000000000000000000000000000000000..bc9dcc56f06f79fc5efa42c04ffdc07c2787e3ac --- /dev/null +++ b/approach/ovod/detectron2/dev/run_inference_tests.sh @@ -0,0 +1,44 @@ +#!/bin/bash -e +# Copyright (c) Facebook, Inc. and its affiliates. + +BIN="python tools/train_net.py" +OUTPUT="inference_test_output" +NUM_GPUS=2 + +CFG_LIST=( "${@:1}" ) + +if [ ${#CFG_LIST[@]} -eq 0 ]; then + CFG_LIST=( ./configs/quick_schedules/*inference_acc_test.yaml ) +fi + +echo "========================================================================" +echo "Configs to run:" +echo "${CFG_LIST[@]}" +echo "========================================================================" + + +for cfg in "${CFG_LIST[@]}"; do + echo "========================================================================" + echo "Running $cfg ..." + echo "========================================================================" + $BIN \ + --eval-only \ + --num-gpus $NUM_GPUS \ + --config-file "$cfg" \ + OUTPUT_DIR $OUTPUT + rm -rf $OUTPUT +done + + +echo "========================================================================" +echo "Running demo.py ..." +echo "========================================================================" +DEMO_BIN="python demo/demo.py" +COCO_DIR=datasets/coco/val2014 +mkdir -pv $OUTPUT + +set -v + +$DEMO_BIN --config-file ./configs/quick_schedules/panoptic_fpn_R_50_inference_acc_test.yaml \ + --input $COCO_DIR/COCO_val2014_0000001933* --output $OUTPUT +rm -rf $OUTPUT diff --git a/approach/ovod/detectron2/dev/run_instant_tests.sh b/approach/ovod/detectron2/dev/run_instant_tests.sh new file mode 100644 index 0000000000000000000000000000000000000000..9fd9ba0c239d3e982c17711c9db872de3730decf --- /dev/null +++ b/approach/ovod/detectron2/dev/run_instant_tests.sh @@ -0,0 +1,27 @@ +#!/bin/bash -e +# Copyright (c) Facebook, Inc. and its affiliates. + +BIN="python tools/train_net.py" +OUTPUT="instant_test_output" +NUM_GPUS=2 + +CFG_LIST=( "${@:1}" ) +if [ ${#CFG_LIST[@]} -eq 0 ]; then + CFG_LIST=( ./configs/quick_schedules/*instant_test.yaml ) +fi + +echo "========================================================================" +echo "Configs to run:" +echo "${CFG_LIST[@]}" +echo "========================================================================" + +for cfg in "${CFG_LIST[@]}"; do + echo "========================================================================" + echo "Running $cfg ..." + echo "========================================================================" + $BIN --num-gpus $NUM_GPUS --config-file "$cfg" \ + SOLVER.IMS_PER_BATCH $(($NUM_GPUS * 2)) \ + OUTPUT_DIR "$OUTPUT" + rm -rf "$OUTPUT" +done + diff --git a/approach/ovod/detectron2/docs/.gitignore b/approach/ovod/detectron2/docs/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..e35d8850c9688b1ce82711694692cc574a799396 --- /dev/null +++ b/approach/ovod/detectron2/docs/.gitignore @@ -0,0 +1 @@ +_build diff --git a/approach/ovod/detectron2/docs/Makefile b/approach/ovod/detectron2/docs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..718eddce170fe13b67216baf9d4d25b20e860506 --- /dev/null +++ b/approach/ovod/detectron2/docs/Makefile @@ -0,0 +1,19 @@ +# Minimal makefile for Sphinx documentation +# Copyright (c) Facebook, Inc. and its affiliates. + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/approach/ovod/detectron2/docs/README.md b/approach/ovod/detectron2/docs/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8531cafd4d1aae0267f4fc5e7212f7db5ed90686 --- /dev/null +++ b/approach/ovod/detectron2/docs/README.md @@ -0,0 +1,15 @@ +# Read the docs: + +The latest documentation built from this directory is available at [detectron2.readthedocs.io](https://detectron2.readthedocs.io/). +Documents in this directory are not meant to be read on github. + +# Build the docs: + +1. Install detectron2 according to [INSTALL.md](../INSTALL.md). +2. Install additional libraries required to build docs: + - docutils==0.16 + - Sphinx==3.2.0 + - recommonmark==0.6.0 + - sphinx_rtd_theme + +3. Run `make html` from this directory. diff --git a/approach/ovod/detectron2/docs/conf.py b/approach/ovod/detectron2/docs/conf.py new file mode 100644 index 0000000000000000000000000000000000000000..83250e834f00208a11438c5b608fb19ecbbd8b6e --- /dev/null +++ b/approach/ovod/detectron2/docs/conf.py @@ -0,0 +1,391 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +# flake8: noqa + +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +from unittest import mock +from sphinx.domains import Domain +from typing import Dict, List, Tuple + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +import sphinx_rtd_theme + + +class GithubURLDomain(Domain): + """ + Resolve certain links in markdown files to github source. + """ + + name = "githuburl" + ROOT = "https://github.com/facebookresearch/detectron2/blob/main/" + LINKED_DOC = ["tutorials/install", "tutorials/getting_started"] + + def resolve_any_xref(self, env, fromdocname, builder, target, node, contnode): + github_url = None + if not target.endswith("html") and target.startswith("../../"): + url = target.replace("../", "") + github_url = url + if fromdocname in self.LINKED_DOC: + # unresolved links in these docs are all github links + github_url = target + + if github_url is not None: + if github_url.endswith("MODEL_ZOO") or github_url.endswith("README"): + # bug of recommonmark. + # https://github.com/readthedocs/recommonmark/blob/ddd56e7717e9745f11300059e4268e204138a6b1/recommonmark/parser.py#L152-L155 + github_url += ".md" + print("Ref {} resolved to github:{}".format(target, github_url)) + contnode["refuri"] = self.ROOT + github_url + return [("githuburl:any", contnode)] + else: + return [] + + +# to support markdown +from recommonmark.parser import CommonMarkParser + +sys.path.insert(0, os.path.abspath("../")) +os.environ["_DOC_BUILDING"] = "True" +DEPLOY = os.environ.get("READTHEDOCS") == "True" + + +# -- Project information ----------------------------------------------------- + +# fmt: off +try: + import torch # noqa +except ImportError: + for m in [ + "torch", "torchvision", "torch.nn", "torch.nn.parallel", "torch.distributed", "torch.multiprocessing", "torch.autograd", + "torch.autograd.function", "torch.nn.modules", "torch.nn.modules.utils", "torch.utils", "torch.utils.data", "torch.onnx", + "torchvision", "torchvision.ops", + ]: + sys.modules[m] = mock.Mock(name=m) + sys.modules['torch'].__version__ = "1.7" # fake version + HAS_TORCH = False +else: + try: + torch.ops.detectron2 = mock.Mock(name="torch.ops.detectron2") + except: + pass + HAS_TORCH = True + +for m in [ + "cv2", "scipy", "portalocker", "detectron2._C", + "pycocotools", "pycocotools.mask", "pycocotools.coco", "pycocotools.cocoeval", + "google", "google.protobuf", "google.protobuf.internal", "onnx", + "caffe2", "caffe2.proto", "caffe2.python", "caffe2.python.utils", "caffe2.python.onnx", "caffe2.python.onnx.backend", +]: + sys.modules[m] = mock.Mock(name=m) +# fmt: on +sys.modules["cv2"].__version__ = "3.4" + +import detectron2 # isort: skip + +if HAS_TORCH: + from detectron2.utils.env import fixup_module_metadata + + fixup_module_metadata("torch.nn", torch.nn.__dict__) + fixup_module_metadata("torch.utils.data", torch.utils.data.__dict__) + + +project = "detectron2" +copyright = "2019-2020, detectron2 contributors" +author = "detectron2 contributors" + +# The short X.Y version +version = detectron2.__version__ +# The full version, including alpha/beta/rc tags +release = version + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = "3.0" + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "recommonmark", + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.mathjax", + "sphinx.ext.viewcode", + "sphinx.ext.githubpages", +] + +# -- Configurations for plugins ------------ +napoleon_google_docstring = True +napoleon_include_init_with_doc = True +napoleon_include_special_with_doc = True +napoleon_numpy_docstring = False +napoleon_use_rtype = False +autodoc_inherit_docstrings = False +autodoc_member_order = "bysource" + +if DEPLOY: + intersphinx_timeout = 10 +else: + # skip this when building locally + intersphinx_timeout = 0.5 +intersphinx_mapping = { + "python": ("https://docs.python.org/3.7", None), + "numpy": ("https://docs.scipy.org/doc/numpy/", None), + "torch": ("https://pytorch.org/docs/master/", None), +} +# ------------------------- + + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +source_suffix = [".rst", ".md"] + +# The master toctree document. +master_doc = "index" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "build", "README.md", "tutorials/README.md"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + + +# -- Options for HTML output ------------------------------------------------- + +html_theme = "sphinx_rtd_theme" +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] +html_css_files = ["css/custom.css"] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = "detectron2doc" + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, "detectron2.tex", "detectron2 Documentation", "detectron2 contributors", "manual") +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, "detectron2", "detectron2 Documentation", [author], 1)] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "detectron2", + "detectron2 Documentation", + author, + "detectron2", + "One line description of project.", + "Miscellaneous", + ) +] + + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +def autodoc_skip_member(app, what, name, obj, skip, options): + # we hide something deliberately + if getattr(obj, "__HIDE_SPHINX_DOC__", False): + return True + + # Hide some that are deprecated or not intended to be used + HIDDEN = { + "ResNetBlockBase", + "GroupedBatchSampler", + "build_transform_gen", + "apply_transform_gens", + "TransformGen", + "apply_augmentations", + "StandardAugInput", + "build_batch_data_loader", + "draw_panoptic_seg_predictions", + "WarmupCosineLR", + "WarmupMultiStepLR", + "downgrade_config", + "upgrade_config", + "add_export_config", + } + try: + if name in HIDDEN or ( + hasattr(obj, "__doc__") and obj.__doc__.lower().strip().startswith("deprecated") + ): + print("Skipping deprecated object: {}".format(name)) + return True + except: + pass + return skip + + +_PAPER_DATA = { + "resnet": ("1512.03385", "Deep Residual Learning for Image Recognition"), + "fpn": ("1612.03144", "Feature Pyramid Networks for Object Detection"), + "mask r-cnn": ("1703.06870", "Mask R-CNN"), + "faster r-cnn": ( + "1506.01497", + "Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks", + ), + "deformconv": ("1703.06211", "Deformable Convolutional Networks"), + "deformconv2": ("1811.11168", "Deformable ConvNets v2: More Deformable, Better Results"), + "panopticfpn": ("1901.02446", "Panoptic Feature Pyramid Networks"), + "retinanet": ("1708.02002", "Focal Loss for Dense Object Detection"), + "cascade r-cnn": ("1712.00726", "Cascade R-CNN: Delving into High Quality Object Detection"), + "lvis": ("1908.03195", "LVIS: A Dataset for Large Vocabulary Instance Segmentation"), + "rrpn": ("1703.01086", "Arbitrary-Oriented Scene Text Detection via Rotation Proposals"), + "imagenet in 1h": ("1706.02677", "Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour"), + "xception": ("1610.02357", "Xception: Deep Learning with Depthwise Separable Convolutions"), + "mobilenet": ( + "1704.04861", + "MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications", + ), + "deeplabv3+": ( + "1802.02611", + "Encoder-Decoder with Atrous Separable Convolution for Semantic Image Segmentation", + ), + "dds": ("2003.13678", "Designing Network Design Spaces"), + "scaling": ("2103.06877", "Fast and Accurate Model Scaling"), + "fcos": ("2006.09214", "FCOS: A Simple and Strong Anchor-free Object Detector"), + "rethinking-batchnorm": ("2105.07576", 'Rethinking "Batch" in BatchNorm'), + "vitdet": ("2203.16527", "Exploring Plain Vision Transformer Backbones for Object Detection"), + "mvitv2": ( + "2112.01526", + "MViTv2: Improved Multiscale Vision Transformers for Classification and Detection", + ), + "swin": ( + "2103.14030", + "Swin Transformer: Hierarchical Vision Transformer using Shifted Windows", + ), +} + + +def paper_ref_role( + typ: str, + rawtext: str, + text: str, + lineno: int, + inliner, + options: Dict = {}, + content: List[str] = [], +): + """ + Parse :paper:`xxx`. Similar to the "extlinks" sphinx extension. + """ + from docutils import nodes, utils + from sphinx.util.nodes import split_explicit_title + + text = utils.unescape(text) + has_explicit_title, title, link = split_explicit_title(text) + link = link.lower() + if link not in _PAPER_DATA: + inliner.reporter.warning("Cannot find paper " + link) + paper_url, paper_title = "#", link + else: + paper_url, paper_title = _PAPER_DATA[link] + if "/" not in paper_url: + paper_url = "https://arxiv.org/abs/" + paper_url + if not has_explicit_title: + title = paper_title + pnode = nodes.reference(title, title, internal=False, refuri=paper_url) + return [pnode], [] + + +def setup(app): + from recommonmark.transform import AutoStructify + + app.add_domain(GithubURLDomain) + app.connect("autodoc-skip-member", autodoc_skip_member) + app.add_role("paper", paper_ref_role) + app.add_config_value( + "recommonmark_config", + {"enable_math": True, "enable_inline_math": True, "enable_eval_rst": True}, + True, + ) + app.add_transform(AutoStructify) diff --git a/approach/ovod/detectron2/docs/index.rst b/approach/ovod/detectron2/docs/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..8634b7b12ab906c10a78d6053428029799282ffd --- /dev/null +++ b/approach/ovod/detectron2/docs/index.rst @@ -0,0 +1,14 @@ +.. detectron2 documentation master file, created by + sphinx-quickstart on Sat Sep 21 13:46:45 2019. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to detectron2's documentation! +====================================== + +.. toctree:: + :maxdepth: 2 + + tutorials/index + notes/index + modules/index diff --git a/approach/ovod/detectron2/docs/notes/benchmarks.md b/approach/ovod/detectron2/docs/notes/benchmarks.md new file mode 100644 index 0000000000000000000000000000000000000000..b41588daf3a039b9034e80366c2710e90ba3e056 --- /dev/null +++ b/approach/ovod/detectron2/docs/notes/benchmarks.md @@ -0,0 +1,196 @@ + +# Benchmarks + +Here we benchmark the training speed of a Mask R-CNN in detectron2, +with some other popular open source Mask R-CNN implementations. + + +### Settings + +* Hardware: 8 NVIDIA V100s with NVLink. +* Software: Python 3.7, CUDA 10.1, cuDNN 7.6.5, PyTorch 1.5, + TensorFlow 1.15.0rc2, Keras 2.2.5, MxNet 1.6.0b20190820. +* Model: an end-to-end R-50-FPN Mask-RCNN model, using the same hyperparameter as the + [Detectron baseline config](https://github.com/facebookresearch/Detectron/blob/master/configs/12_2017_baselines/e2e_mask_rcnn_R-50-FPN_1x.yaml) + (it does not have scale augmentation). +* Metrics: We use the average throughput in iterations 100-500 to skip GPU warmup time. + Note that for R-CNN-style models, the throughput of a model typically changes during training, because + it depends on the predictions of the model. Therefore this metric is not directly comparable with + "train speed" in model zoo, which is the average speed of the entire training run. + + +### Main Results + +```eval_rst ++-------------------------------+--------------------+ +| Implementation | Throughput (img/s) | ++===============================+====================+ +| |D2| |PT| | 62 | ++-------------------------------+--------------------+ +| mmdetection_ |PT| | 53 | ++-------------------------------+--------------------+ +| maskrcnn-benchmark_ |PT| | 53 | ++-------------------------------+--------------------+ +| tensorpack_ |TF| | 50 | ++-------------------------------+--------------------+ +| simpledet_ |mxnet| | 39 | ++-------------------------------+--------------------+ +| Detectron_ |C2| | 19 | ++-------------------------------+--------------------+ +| `matterport/Mask_RCNN`__ |TF| | 14 | ++-------------------------------+--------------------+ + +.. _maskrcnn-benchmark: https://github.com/facebookresearch/maskrcnn-benchmark/ +.. _tensorpack: https://github.com/tensorpack/tensorpack/tree/master/examples/FasterRCNN +.. _mmdetection: https://github.com/open-mmlab/mmdetection/ +.. _simpledet: https://github.com/TuSimple/simpledet/ +.. _Detectron: https://github.com/facebookresearch/Detectron +__ https://github.com/matterport/Mask_RCNN/ + +.. |D2| image:: https://github.com/facebookresearch/detectron2/raw/main/.github/Detectron2-Logo-Horz.svg?sanitize=true + :height: 15pt + :target: https://github.com/facebookresearch/detectron2/ +.. |PT| image:: https://pytorch.org/assets/images/logo-icon.svg + :width: 15pt + :height: 15pt + :target: https://pytorch.org +.. |TF| image:: https://static.nvidiagrid.net/ngc/containers/tensorflow.png + :width: 15pt + :height: 15pt + :target: https://tensorflow.org +.. |mxnet| image:: https://github.com/dmlc/web-data/raw/master/mxnet/image/mxnet_favicon.png + :width: 15pt + :height: 15pt + :target: https://mxnet.apache.org/ +.. |C2| image:: https://caffe2.ai/static/logo.svg + :width: 15pt + :height: 15pt + :target: https://caffe2.ai +``` + + +Details for each implementation: + +* __Detectron2__: with release v0.1.2, run: + ``` + python tools/train_net.py --config-file configs/Detectron1-Comparisons/mask_rcnn_R_50_FPN_noaug_1x.yaml --num-gpus 8 + ``` + +* __mmdetection__: at commit `b0d845f`, run + ``` + ./tools/dist_train.sh configs/mask_rcnn/mask_rcnn_r50_caffe_fpn_1x_coco.py 8 + ``` + +* __maskrcnn-benchmark__: use commit `0ce8f6f` with `sed -i 's/torch.uint8/torch.bool/g' **/*.py; sed -i 's/AT_CHECK/TORCH_CHECK/g' **/*.cu` + to make it compatible with PyTorch 1.5. Then, run training with + ``` + python -m torch.distributed.launch --nproc_per_node=8 tools/train_net.py --config-file configs/e2e_mask_rcnn_R_50_FPN_1x.yaml + ``` + The speed we observed is faster than its model zoo, likely due to different software versions. + +* __tensorpack__: at commit `caafda`, `export TF_CUDNN_USE_AUTOTUNE=0`, then run + ``` + mpirun -np 8 ./train.py --config DATA.BASEDIR=/data/coco TRAINER=horovod BACKBONE.STRIDE_1X1=True TRAIN.STEPS_PER_EPOCH=50 --load ImageNet-R50-AlignPadding.npz + ``` + +* __SimpleDet__: at commit `9187a1`, run + ``` + python detection_train.py --config config/mask_r50v1_fpn_1x.py + ``` + +* __Detectron__: run + ``` + python tools/train_net.py --cfg configs/12_2017_baselines/e2e_mask_rcnn_R-50-FPN_1x.yaml + ``` + Note that many of its ops run on CPUs, therefore the performance is limited. + +* __matterport/Mask_RCNN__: at commit `3deaec`, apply the following diff, `export TF_CUDNN_USE_AUTOTUNE=0`, then run + ``` + python coco.py train --dataset=/data/coco/ --model=imagenet + ``` + Note that many small details in this implementation might be different + from Detectron's standards. + +
+ + (diff to make it use the same hyperparameters - click to expand) + + + ```diff + diff --git i/mrcnn/model.py w/mrcnn/model.py + index 62cb2b0..61d7779 100644 + --- i/mrcnn/model.py + +++ w/mrcnn/model.py + @@ -2367,8 +2367,8 @@ class MaskRCNN(): + epochs=epochs, + steps_per_epoch=self.config.STEPS_PER_EPOCH, + callbacks=callbacks, + - validation_data=val_generator, + - validation_steps=self.config.VALIDATION_STEPS, + + #validation_data=val_generator, + + #validation_steps=self.config.VALIDATION_STEPS, + max_queue_size=100, + workers=workers, + use_multiprocessing=True, + diff --git i/mrcnn/parallel_model.py w/mrcnn/parallel_model.py + index d2bf53b..060172a 100644 + --- i/mrcnn/parallel_model.py + +++ w/mrcnn/parallel_model.py + @@ -32,6 +32,7 @@ class ParallelModel(KM.Model): + keras_model: The Keras model to parallelize + gpu_count: Number of GPUs. Must be > 1 + """ + + super().__init__() + self.inner_model = keras_model + self.gpu_count = gpu_count + merged_outputs = self.make_parallel() + diff --git i/samples/coco/coco.py w/samples/coco/coco.py + index 5d172b5..239ed75 100644 + --- i/samples/coco/coco.py + +++ w/samples/coco/coco.py + @@ -81,7 +81,10 @@ class CocoConfig(Config): + IMAGES_PER_GPU = 2 + + # Uncomment to train on 8 GPUs (default is 1) + - # GPU_COUNT = 8 + + GPU_COUNT = 8 + + BACKBONE = "resnet50" + + STEPS_PER_EPOCH = 50 + + TRAIN_ROIS_PER_IMAGE = 512 + + # Number of classes (including background) + NUM_CLASSES = 1 + 80 # COCO has 80 classes + @@ -496,29 +499,10 @@ if __name__ == '__main__': + # *** This training schedule is an example. Update to your needs *** + + # Training - Stage 1 + - print("Training network heads") + model.train(dataset_train, dataset_val, + learning_rate=config.LEARNING_RATE, + epochs=40, + - layers='heads', + - augmentation=augmentation) + - + - # Training - Stage 2 + - # Finetune layers from ResNet stage 4 and up + - print("Fine tune Resnet stage 4 and up") + - model.train(dataset_train, dataset_val, + - learning_rate=config.LEARNING_RATE, + - epochs=120, + - layers='4+', + - augmentation=augmentation) + - + - # Training - Stage 3 + - # Fine tune all layers + - print("Fine tune all layers") + - model.train(dataset_train, dataset_val, + - learning_rate=config.LEARNING_RATE / 10, + - epochs=160, + - layers='all', + + layers='3+', + augmentation=augmentation) + + elif args.command == "evaluate": + ``` + +
diff --git a/approach/ovod/detectron2/docs/notes/changelog.md b/approach/ovod/detectron2/docs/notes/changelog.md new file mode 100644 index 0000000000000000000000000000000000000000..000e9f8898dba53f54121a5325ba5165e45ddea2 --- /dev/null +++ b/approach/ovod/detectron2/docs/notes/changelog.md @@ -0,0 +1,48 @@ +# Change Log and Backward Compatibility + +### Releases +See release logs at +[https://github.com/facebookresearch/detectron2/releases](https://github.com/facebookresearch/detectron2/releases) +for new updates. + +### Backward Compatibility + +Due to the research nature of what the library does, there might be backward incompatible changes. +But we try to reduce users' disruption by the following ways: +* APIs listed in [API documentation](https://detectron2.readthedocs.io/modules/index.html), including + function/class names, their arguments, and documented class attributes, are considered *stable* unless + otherwise noted in the documentation. + They are less likely to be broken, but if needed, will trigger a deprecation warning for a reasonable period + before getting broken, and will be documented in release logs. +* Others functions/classses/attributes are considered internal, and are more likely to change. + However, we're aware that some of them may be already used by other projects, and in particular we may + use them for convenience among projects under `detectron2/projects`. + For such APIs, we may treat them as stable APIs and also apply the above strategies. + They may be promoted to stable when we're ready. +* Projects under "detectron2/projects" or imported with "detectron2.projects" are research projects + and are all considered experimental. +* Classes/functions that contain the word "default" or are explicitly documented to produce + "default behavior" may change their behaviors when new features are added. + +Despite of the possible breakage, if a third-party project would like to keep up with the latest updates +in detectron2, using it as a library will still be less disruptive than forking, because +the frequency and scope of API changes will be much smaller than code changes. + +To see such changes, search for "incompatible changes" in [release logs](https://github.com/facebookresearch/detectron2/releases). + +### Config Version Change Log + +Detectron2's config version has not been changed since open source. +There is no need for an open source user to worry about this. + +* v1: Rename `RPN_HEAD.NAME` to `RPN.HEAD_NAME`. +* v2: A batch of rename of many configurations before release. + +### Silent Regressions in Historical Versions: + +We list a few silent regressions, since they may silently produce incorrect results and will be hard to debug. + +* 04/01/2020 - 05/11/2020: Bad accuracy if `TRAIN_ON_PRED_BOXES` is set to True. +* 03/30/2020 - 04/01/2020: ResNets are not correctly built. +* 12/19/2019 - 12/26/2019: Using aspect ratio grouping causes a drop in accuracy. +* - 11/9/2019: Test time augmentation does not predict the last category. diff --git a/approach/ovod/detectron2/docs/notes/compatibility.md b/approach/ovod/detectron2/docs/notes/compatibility.md new file mode 100644 index 0000000000000000000000000000000000000000..83d93f51c056c598c1209f9a21a4e04407b827f0 --- /dev/null +++ b/approach/ovod/detectron2/docs/notes/compatibility.md @@ -0,0 +1,84 @@ +# Compatibility with Other Libraries + +## Compatibility with Detectron (and maskrcnn-benchmark) + +Detectron2 addresses some legacy issues left in Detectron. As a result, their models +are not compatible: +running inference with the same model weights will produce different results in the two code bases. + +The major differences regarding inference are: + +- The height and width of a box with corners (x1, y1) and (x2, y2) is now computed more naturally as + width = x2 - x1 and height = y2 - y1; + In Detectron, a "+ 1" was added both height and width. + + Note that the relevant ops in Caffe2 have [adopted this change of convention](https://github.com/pytorch/pytorch/pull/20550) + with an extra option. + So it is still possible to run inference with a Detectron2-trained model in Caffe2. + + The change in height/width calculations most notably changes: + - encoding/decoding in bounding box regression. + - non-maximum suppression. The effect here is very negligible, though. + +- RPN now uses simpler anchors with fewer quantization artifacts. + + In Detectron, the anchors were quantized and + [do not have accurate areas](https://github.com/facebookresearch/Detectron/issues/227). + In Detectron2, the anchors are center-aligned to feature grid points and not quantized. + +- Classification layers have a different ordering of class labels. + + This involves any trainable parameter with shape (..., num_categories + 1, ...). + In Detectron2, integer labels [0, K-1] correspond to the K = num_categories object categories + and the label "K" corresponds to the special "background" category. + In Detectron, label "0" means background, and labels [1, K] correspond to the K categories. + +- ROIAlign is implemented differently. The new implementation is [available in Caffe2](https://github.com/pytorch/pytorch/pull/23706). + + 1. All the ROIs are shifted by half a pixel compared to Detectron in order to create better image-feature-map alignment. + See `layers/roi_align.py` for details. + To enable the old behavior, use `ROIAlign(aligned=False)`, or `POOLER_TYPE=ROIAlign` instead of + `ROIAlignV2` (the default). + + 1. The ROIs are not required to have a minimum size of 1. + This will lead to tiny differences in the output, but should be negligible. + +- Mask inference function is different. + + In Detectron2, the "paste_mask" function is different and should be more accurate than in Detectron. This change + can improve mask AP on COCO by ~0.5% absolute. + +There are some other differences in training as well, but they won't affect +model-level compatibility. The major ones are: + +- We fixed a [bug](https://github.com/facebookresearch/Detectron/issues/459) in + Detectron, by making `RPN.POST_NMS_TOPK_TRAIN` per-image, rather than per-batch. + The fix may lead to a small accuracy drop for a few models (e.g. keypoint + detection) and will require some parameter tuning to match the Detectron results. +- For simplicity, we change the default loss in bounding box regression to L1 loss, instead of smooth L1 loss. + We have observed that this tends to slightly decrease box AP50 while improving box AP for higher + overlap thresholds (and leading to a slight overall improvement in box AP). +- We interpret the coordinates in COCO bounding box and segmentation annotations + as coordinates in range `[0, width]` or `[0, height]`. The coordinates in + COCO keypoint annotations are interpreted as pixel indices in range `[0, width - 1]` or `[0, height - 1]`. + Note that this affects how flip augmentation is implemented. + + +[This article](https://ppwwyyxx.com/blog/2021/Where-are-Pixels/) +explains more details on the above mentioned issues +about pixels, coordinates, and "+1"s. + + +## Compatibility with Caffe2 + +As mentioned above, despite the incompatibilities with Detectron, the relevant +ops have been implemented in Caffe2. +Therefore, models trained with detectron2 can be converted in Caffe2. +See [Deployment](../tutorials/deployment.md) for the tutorial. + +## Compatibility with TensorFlow + +Most ops are available in TensorFlow, although some tiny differences in +the implementation of resize / ROIAlign / padding need to be addressed. +A working conversion script is provided by [tensorpack Faster R-CNN](https://github.com/tensorpack/tensorpack/tree/master/examples/FasterRCNN/convert_d2) +to run a standard detectron2 model in TensorFlow. diff --git a/approach/ovod/detectron2/docs/notes/contributing.md b/approach/ovod/detectron2/docs/notes/contributing.md new file mode 100644 index 0000000000000000000000000000000000000000..9bab709cae689ba3b92dd52f7fbcc0c6926f4a38 --- /dev/null +++ b/approach/ovod/detectron2/docs/notes/contributing.md @@ -0,0 +1,68 @@ +# Contributing to detectron2 + +## Issues +We use GitHub issues to track public bugs and questions. +Please make sure to follow one of the +[issue templates](https://github.com/facebookresearch/detectron2/issues/new/choose) +when reporting any issues. + +Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe +disclosure of security bugs. In those cases, please go through the process +outlined on that page and do not file a public issue. + +## Pull Requests +We actively welcome pull requests. + +However, if you're adding any significant features (e.g. > 50 lines), please +make sure to discuss with maintainers about your motivation and proposals in an issue +before sending a PR. This is to save your time so you don't spend time on a PR that we'll not accept. + +We do not always accept new features, and we take the following +factors into consideration: + +1. Whether the same feature can be achieved without modifying detectron2. + Detectron2 is designed so that you can implement many extensions from the outside, e.g. + those in [projects](https://github.com/facebookresearch/detectron2/tree/master/projects). + * If some part of detectron2 is not extensible enough, you can also bring up a more general issue to + improve it. Such feature request may be useful to more users. +2. Whether the feature is potentially useful to a large audience (e.g. an impactful detection paper, a popular dataset, + a significant speedup, a widely useful utility), + or only to a small portion of users (e.g., a less-known paper, an improvement not in the object + detection field, a trick that's not very popular in the community, code to handle a non-standard type of data) + * Adoption of additional models, datasets, new task are by default not added to detectron2 before they + receive significant popularity in the community. + We sometimes accept such features in `projects/`, or as a link in `projects/README.md`. +3. Whether the proposed solution has a good design / interface. This can be discussed in the issue prior to PRs, or + in the form of a draft PR. +4. Whether the proposed solution adds extra mental/practical overhead to users who don't + need such feature. +5. Whether the proposed solution breaks existing APIs. + +To add a feature to an existing function/class `Func`, there are always two approaches: +(1) add new arguments to `Func`; (2) write a new `Func_with_new_feature`. +To meet the above criteria, we often prefer approach (2), because: + +1. It does not involve modifying or potentially breaking existing code. +2. It does not add overhead to users who do not need the new feature. +3. Adding new arguments to a function/class is not scalable w.r.t. all the possible new research ideas in the future. + +When sending a PR, please do: + +1. If a PR contains multiple orthogonal changes, split it to several PRs. +2. If you've added code that should be tested, add tests. +3. For PRs that need experiments (e.g. adding a new model or new methods), + you don't need to update model zoo, but do provide experiment results in the description of the PR. +4. If APIs are changed, update the documentation. +5. We use the [Google style docstrings](https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html) in python. +6. Make sure your code lints with `./dev/linter.sh`. + + +## Contributor License Agreement ("CLA") +In order to accept your pull request, we need you to submit a CLA. You only need +to do this once to work on any of Facebook's open source projects. + +Complete your CLA here: + +## License +By contributing to detectron2, you agree that your contributions will be licensed +under the LICENSE file in the root directory of this source tree. diff --git a/approach/ovod/detectron2/docs/notes/index.rst b/approach/ovod/detectron2/docs/notes/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..63cf907be7bb15f5316af6d44a46df601755a86b --- /dev/null +++ b/approach/ovod/detectron2/docs/notes/index.rst @@ -0,0 +1,10 @@ +Notes +====================================== + +.. toctree:: + :maxdepth: 2 + + benchmarks + compatibility + contributing + changelog diff --git a/approach/ovod/detectron2/docs/requirements.txt b/approach/ovod/detectron2/docs/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..2deb839ef648a90bb67d9b73c65a90e0bfd0cce9 --- /dev/null +++ b/approach/ovod/detectron2/docs/requirements.txt @@ -0,0 +1,24 @@ +docutils==0.16 +# https://github.com/sphinx-doc/sphinx/commit/7acd3ada3f38076af7b2b5c9f3b60bb9c2587a3d +sphinx==3.2.0 +recommonmark==0.6.0 +sphinx_rtd_theme +# Dependencies here are only those required by import +termcolor +numpy +tqdm +matplotlib +termcolor +yacs +tabulate +cloudpickle +Pillow +future +git+https://github.com/facebookresearch/fvcore.git +https://download.pytorch.org/whl/cpu/torch-1.8.1%2Bcpu-cp37-cp37m-linux_x86_64.whl +https://download.pytorch.org/whl/cpu/torchvision-0.9.1%2Bcpu-cp37-cp37m-linux_x86_64.whl +omegaconf>=2.1.0.dev24 +hydra-core>=1.1.0.dev5 +scipy +timm +fairscale diff --git a/approach/ovod/detectron2/docs/tutorials/README.md b/approach/ovod/detectron2/docs/tutorials/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1ca9c94d042ef838143a45490fe6b4556c19f3c9 --- /dev/null +++ b/approach/ovod/detectron2/docs/tutorials/README.md @@ -0,0 +1,4 @@ +# Read the docs: + +The latest documentation built from this directory is available at [detectron2.readthedocs.io](https://detectron2.readthedocs.io/). +Documents in this directory are not meant to be read on github. diff --git a/approach/ovod/detectron2/docs/tutorials/builtin_datasets.md b/approach/ovod/detectron2/docs/tutorials/builtin_datasets.md new file mode 100644 index 0000000000000000000000000000000000000000..0eb44cc3b23beeb1755ab8d12002d26f13434235 --- /dev/null +++ b/approach/ovod/detectron2/docs/tutorials/builtin_datasets.md @@ -0,0 +1,140 @@ +# Use Builtin Datasets + +A dataset can be used by accessing [DatasetCatalog](https://detectron2.readthedocs.io/modules/data.html#detectron2.data.DatasetCatalog) +for its data, or [MetadataCatalog](https://detectron2.readthedocs.io/modules/data.html#detectron2.data.MetadataCatalog) for its metadata (class names, etc). +This document explains how to setup the builtin datasets so they can be used by the above APIs. +[Use Custom Datasets](https://detectron2.readthedocs.io/tutorials/datasets.html) gives a deeper dive on how to use `DatasetCatalog` and `MetadataCatalog`, +and how to add new datasets to them. + +Detectron2 has builtin support for a few datasets. +The datasets are assumed to exist in a directory specified by the environment variable +`DETECTRON2_DATASETS`. +Under this directory, detectron2 will look for datasets in the structure described below, if needed. +``` +$DETECTRON2_DATASETS/ + coco/ + lvis/ + cityscapes/ + VOC20{07,12}/ +``` + +You can set the location for builtin datasets by `export DETECTRON2_DATASETS=/path/to/datasets`. +If left unset, the default is `./datasets` relative to your current working directory. + +The [model zoo](https://github.com/facebookresearch/detectron2/blob/master/MODEL_ZOO.md) +contains configs and models that use these builtin datasets. + +## Expected dataset structure for [COCO instance/keypoint detection](https://cocodataset.org/#download): + +``` +coco/ + annotations/ + instances_{train,val}2017.json + person_keypoints_{train,val}2017.json + {train,val}2017/ + # image files that are mentioned in the corresponding json +``` + +You can use the 2014 version of the dataset as well. + +Some of the builtin tests (`dev/run_*_tests.sh`) uses a tiny version of the COCO dataset, +which you can download with `./datasets/prepare_for_tests.sh`. + +## Expected dataset structure for PanopticFPN: + +Extract panoptic annotations from [COCO website](https://cocodataset.org/#download) +into the following structure: +``` +coco/ + annotations/ + panoptic_{train,val}2017.json + panoptic_{train,val}2017/ # png annotations + panoptic_stuff_{train,val}2017/ # generated by the script mentioned below +``` + +Install panopticapi by: +``` +pip install git+https://github.com/cocodataset/panopticapi.git +``` +Then, run `python datasets/prepare_panoptic_fpn.py`, to extract semantic annotations from panoptic annotations. + +## Expected dataset structure for [LVIS instance segmentation](https://www.lvisdataset.org/dataset): +``` +coco/ + {train,val,test}2017/ +lvis/ + lvis_v0.5_{train,val}.json + lvis_v0.5_image_info_test.json + lvis_v1_{train,val}.json + lvis_v1_image_info_test{,_challenge}.json +``` + +Install lvis-api by: +``` +pip install git+https://github.com/lvis-dataset/lvis-api.git +``` + +To evaluate models trained on the COCO dataset using LVIS annotations, +run `python datasets/prepare_cocofied_lvis.py` to prepare "cocofied" LVIS annotations. + +## Expected dataset structure for [cityscapes](https://www.cityscapes-dataset.com/downloads/): +``` +cityscapes/ + gtFine/ + train/ + aachen/ + color.png, instanceIds.png, labelIds.png, polygons.json, + labelTrainIds.png + ... + val/ + test/ + # below are generated Cityscapes panoptic annotation + cityscapes_panoptic_train.json + cityscapes_panoptic_train/ + cityscapes_panoptic_val.json + cityscapes_panoptic_val/ + cityscapes_panoptic_test.json + cityscapes_panoptic_test/ + leftImg8bit/ + train/ + val/ + test/ +``` +Install cityscapes scripts by: +``` +pip install git+https://github.com/mcordts/cityscapesScripts.git +``` + +Note: to create labelTrainIds.png, first prepare the above structure, then run cityscapesescript with: +``` +CITYSCAPES_DATASET=/path/to/abovementioned/cityscapes python cityscapesscripts/preparation/createTrainIdLabelImgs.py +``` +These files are not needed for instance segmentation. + +Note: to generate Cityscapes panoptic dataset, run cityscapesescript with: +``` +CITYSCAPES_DATASET=/path/to/abovementioned/cityscapes python cityscapesscripts/preparation/createPanopticImgs.py +``` +These files are not needed for semantic and instance segmentation. + +## Expected dataset structure for [Pascal VOC](http://host.robots.ox.ac.uk/pascal/VOC/index.html): +``` +VOC20{07,12}/ + Annotations/ + ImageSets/ + Main/ + trainval.txt + test.txt + # train.txt or val.txt, if you use these splits + JPEGImages/ +``` + +## Expected dataset structure for [ADE20k Scene Parsing](http://sceneparsing.csail.mit.edu/): +``` +ADEChallengeData2016/ + annotations/ + annotations_detectron2/ + images/ + objectInfo150.txt +``` +The directory `annotations_detectron2` is generated by running `python datasets/prepare_ade20k_sem_seg.py`. diff --git a/approach/ovod/detectron2/docs/tutorials/configs.md b/approach/ovod/detectron2/docs/tutorials/configs.md new file mode 100644 index 0000000000000000000000000000000000000000..49538d0532994664584460560f4f809ff3a6e6df --- /dev/null +++ b/approach/ovod/detectron2/docs/tutorials/configs.md @@ -0,0 +1,62 @@ +# Yacs Configs + +Detectron2 provides a key-value based config system that can be +used to obtain standard, common behaviors. + +This system uses YAML and [yacs](https://github.com/rbgirshick/yacs). +Yaml is a very limited language, +so we do not expect all features in detectron2 to be available through configs. +If you need something that's not available in the config space, +please write code using detectron2's API. + +With the introduction of a more powerful [LazyConfig system](lazyconfigs.md), +we no longer add functionality / new keys to the Yacs/Yaml-based config system. + +### Basic Usage + +Some basic usage of the `CfgNode` object is shown here. See more in [documentation](../modules/config.html#detectron2.config.CfgNode). +```python +from detectron2.config import get_cfg +cfg = get_cfg() # obtain detectron2's default config +cfg.xxx = yyy # add new configs for your own custom components +cfg.merge_from_file("my_cfg.yaml") # load values from a file + +cfg.merge_from_list(["MODEL.WEIGHTS", "weights.pth"]) # can also load values from a list of str +print(cfg.dump()) # print formatted configs +with open("output.yaml", "w") as f: + f.write(cfg.dump()) # save config to file +``` + +In addition to the basic Yaml syntax, the config file can +define a `_BASE_: base.yaml` field, which will load a base config file first. +Values in the base config will be overwritten in sub-configs, if there are any conflicts. +We provided several base configs for standard model architectures. + +Many builtin tools in detectron2 accept command line config overwrite: +Key-value pairs provided in the command line will overwrite the existing values in the config file. +For example, [demo.py](../../demo/demo.py) can be used with +```sh +./demo.py --config-file config.yaml [--other-options] \ + --opts MODEL.WEIGHTS /path/to/weights INPUT.MIN_SIZE_TEST 1000 +``` + +To see a list of available configs in detectron2 and what they mean, +check [Config References](../modules/config.html#config-references) + +### Configs in Projects + +A project that lives outside the detectron2 library may define its own configs, which will need to be added +for the project to be functional, e.g.: +```python +from detectron2.projects.point_rend import add_pointrend_config +cfg = get_cfg() # obtain detectron2's default config +add_pointrend_config(cfg) # add pointrend's default config +# ... ... +``` + +### Best Practice with Configs + +1. Treat the configs you write as "code": avoid copying them or duplicating them; use `_BASE_` + to share common parts between configs. + +2. Keep the configs you write simple: don't include keys that do not affect the experimental setting. diff --git a/approach/ovod/detectron2/docs/tutorials/data_loading.md b/approach/ovod/detectron2/docs/tutorials/data_loading.md new file mode 100644 index 0000000000000000000000000000000000000000..1d2769fc513abb0981a140f3a6b6432538704261 --- /dev/null +++ b/approach/ovod/detectron2/docs/tutorials/data_loading.md @@ -0,0 +1,95 @@ + +# Dataloader + +Dataloader is the component that provides data to models. +A dataloader usually (but not necessarily) takes raw information from [datasets](./datasets.md), +and process them into a format needed by the model. + +## How the Existing Dataloader Works + +Detectron2 contains a builtin data loading pipeline. +It's good to understand how it works, in case you need to write a custom one. + +Detectron2 provides two functions +[build_detection_{train,test}_loader](../modules/data.html#detectron2.data.build_detection_train_loader) +that create a default data loader from a given config. +Here is how `build_detection_{train,test}_loader` work: + +1. It takes the name of a registered dataset (e.g., "coco_2017_train") and loads a `list[dict]` representing the dataset items + in a lightweight format. These dataset items are not yet ready to be used by the model (e.g., images are + not loaded into memory, random augmentations have not been applied, etc.). + Details about the dataset format and dataset registration can be found in + [datasets](./datasets.md). +2. Each dict in this list is mapped by a function ("mapper"): + * Users can customize this mapping function by specifying the "mapper" argument in + `build_detection_{train,test}_loader`. The default mapper is [DatasetMapper](../modules/data.html#detectron2.data.DatasetMapper). + * The output format of the mapper can be arbitrary, as long as it is accepted by the consumer of this data loader (usually the model). + The outputs of the default mapper, after batching, follow the default model input format documented in + [Use Models](./models.html#model-input-format). + * The role of the mapper is to transform the lightweight representation of a dataset item into a format + that is ready for the model to consume (including, e.g., read images, perform random data augmentation and convert to torch Tensors). + If you would like to perform custom transformations to data, you often want a custom mapper. +3. The outputs of the mapper are batched (simply into a list). +4. This batched data is the output of the data loader. Typically, it's also the input of + `model.forward()`. + + +## Write a Custom Dataloader + +Using a different "mapper" with `build_detection_{train,test}_loader(mapper=)` works for most use cases +of custom data loading. +For example, if you want to resize all images to a fixed size for training, use: + +```python +import detectron2.data.transforms as T +from detectron2.data import DatasetMapper # the default mapper +dataloader = build_detection_train_loader(cfg, + mapper=DatasetMapper(cfg, is_train=True, augmentations=[ + T.Resize((800, 800)) + ])) +# use this dataloader instead of the default +``` +If the arguments of the default [DatasetMapper](../modules/data.html#detectron2.data.DatasetMapper) +does not provide what you need, you may write a custom mapper function and use it instead, e.g.: + +```python +from detectron2.data import detection_utils as utils + # Show how to implement a minimal mapper, similar to the default DatasetMapper +def mapper(dataset_dict): + dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below + # can use other ways to read image + image = utils.read_image(dataset_dict["file_name"], format="BGR") + # See "Data Augmentation" tutorial for details usage + auginput = T.AugInput(image) + transform = T.Resize((800, 800))(auginput) + image = torch.from_numpy(auginput.image.transpose(2, 0, 1)) + annos = [ + utils.transform_instance_annotations(annotation, [transform], image.shape[1:]) + for annotation in dataset_dict.pop("annotations") + ] + return { + # create the format that the model expects + "image": image, + "instances": utils.annotations_to_instances(annos, image.shape[1:]) + } +dataloader = build_detection_train_loader(cfg, mapper=mapper) +``` + +If you want to change not only the mapper (e.g., in order to implement different sampling or batching logic), +`build_detection_train_loader` won't work and you will need to write a different data loader. +The data loader is simply a +python iterator that produces [the format](./models.md) that the model accepts. +You can implement it using any tools you like. + +No matter what to implement, it's recommended to +check out [API documentation of detectron2.data](../modules/data) to learn more about the APIs of +these functions. + +## Use a Custom Dataloader + +If you use [DefaultTrainer](../modules/engine.html#detectron2.engine.defaults.DefaultTrainer), +you can overwrite its `build_{train,test}_loader` method to use your own dataloader. +See the [deeplab dataloader](../../projects/DeepLab/train_net.py) +for an example. + +If you write your own training loop, you can plug in your data loader easily. diff --git a/approach/ovod/detectron2/docs/tutorials/datasets.md b/approach/ovod/detectron2/docs/tutorials/datasets.md new file mode 100644 index 0000000000000000000000000000000000000000..91103f64264aa6f3059611c5fe06ecd65bcb986f --- /dev/null +++ b/approach/ovod/detectron2/docs/tutorials/datasets.md @@ -0,0 +1,290 @@ +# Use Custom Datasets + +This document explains how the dataset APIs +([DatasetCatalog](../modules/data.html#detectron2.data.DatasetCatalog), [MetadataCatalog](../modules/data.html#detectron2.data.MetadataCatalog)) +work, and how to use them to add custom datasets. + +Datasets that have builtin support in detectron2 are listed in [builtin datasets](builtin_datasets.md). +If you want to use a custom dataset while also reusing detectron2's data loaders, +you will need to: + +1. __Register__ your dataset (i.e., tell detectron2 how to obtain your dataset). +2. Optionally, __register metadata__ for your dataset. + +Next, we explain the above two concepts in detail. + +The [Colab tutorial](https://colab.research.google.com/drive/16jcaJoc6bCFAQ96jDe2HwtXj7BMD_-m5) +has a live example of how to register and train on a dataset of custom formats. + +### Register a Dataset + +To let detectron2 know how to obtain a dataset named "my_dataset", users need to implement +a function that returns the items in your dataset and then tell detectron2 about this +function: +```python +def my_dataset_function(): + ... + return list[dict] in the following format + +from detectron2.data import DatasetCatalog +DatasetCatalog.register("my_dataset", my_dataset_function) +# later, to access the data: +data: List[Dict] = DatasetCatalog.get("my_dataset") +``` + +Here, the snippet associates a dataset named "my_dataset" with a function that returns the data. +The function must return the same data (with same order) if called multiple times. +The registration stays effective until the process exits. + +The function can do arbitrary things and should return the data in `list[dict]`, each dict in either +of the following formats: +1. Detectron2's standard dataset dict, described below. This will make it work with many other builtin + features in detectron2, so it's recommended to use it when it's sufficient. +2. Any custom format. You can also return arbitrary dicts in your own format, + such as adding extra keys for new tasks. + Then you will need to handle them properly downstream as well. + See below for more details. + +#### Standard Dataset Dicts + +For standard tasks +(instance detection, instance/semantic/panoptic segmentation, keypoint detection), +we load the original dataset into `list[dict]` with a specification similar to COCO's annotations. +This is our standard representation for a dataset. + +Each dict contains information about one image. +The dict may have the following fields, +and the required fields vary based on what the dataloader or the task needs (see more below). + +```eval_rst +.. list-table:: + :header-rows: 1 + + * - Task + - Fields + * - Common + - file_name, height, width, image_id + + * - Instance detection/segmentation + - annotations + + * - Semantic segmentation + - sem_seg_file_name + + * - Panoptic segmentation + - pan_seg_file_name, segments_info +``` + ++ `file_name`: the full path to the image file. ++ `height`, `width`: integer. The shape of the image. ++ `image_id` (str or int): a unique id that identifies this image. Required by many + evaluators to identify the images, but a dataset may use it for different purposes. ++ `annotations` (list[dict]): Required by __instance detection/segmentation or keypoint detection__ tasks. + Each dict corresponds to annotations of one instance in this image, and + may contain the following keys: + + `bbox` (list[float], required): list of 4 numbers representing the bounding box of the instance. + + `bbox_mode` (int, required): the format of bbox. It must be a member of + [structures.BoxMode](../modules/structures.html#detectron2.structures.BoxMode). + Currently supports: `BoxMode.XYXY_ABS`, `BoxMode.XYWH_ABS`. + + `category_id` (int, required): an integer in the range [0, num_categories-1] representing the category label. + The value num_categories is reserved to represent the "background" category, if applicable. + + `segmentation` (list[list[float]] or dict): the segmentation mask of the instance. + + If `list[list[float]]`, it represents a list of polygons, one for each connected component + of the object. Each `list[float]` is one simple polygon in the format of `[x1, y1, ..., xn, yn]` (n≥3). + The Xs and Ys are absolute coordinates in unit of pixels. + + If `dict`, it represents the per-pixel segmentation mask in COCO's compressed RLE format. + The dict should have keys "size" and "counts". You can convert a uint8 segmentation mask of 0s and + 1s into such dict by `pycocotools.mask.encode(np.asarray(mask, order="F"))`. + `cfg.INPUT.MASK_FORMAT` must be set to `bitmask` if using the default data loader with such format. + + `keypoints` (list[float]): in the format of [x1, y1, v1,..., xn, yn, vn]. + v[i] means the [visibility](http://cocodataset.org/#format-data) of this keypoint. + `n` must be equal to the number of keypoint categories. + The Xs and Ys are absolute real-value coordinates in range [0, W or H]. + + (Note that the keypoint coordinates in COCO format are integers in range [0, W-1 or H-1], which is different + from our standard format. Detectron2 adds 0.5 to COCO keypoint coordinates to convert them from discrete + pixel indices to floating point coordinates.) + + `iscrowd`: 0 (default) or 1. Whether this instance is labeled as COCO's "crowd + region". Don't include this field if you don't know what it means. + + If `annotations` is an empty list, it means the image is labeled to have no objects. + Such images will by default be removed from training, + but can be included using `DATALOADER.FILTER_EMPTY_ANNOTATIONS`. + ++ `sem_seg_file_name` (str): + The full path to the semantic segmentation ground truth file. + It should be a grayscale image whose pixel values are integer labels. ++ `pan_seg_file_name` (str): + The full path to panoptic segmentation ground truth file. + It should be an RGB image whose pixel values are integer ids encoded using the + [panopticapi.utils.id2rgb](https://github.com/cocodataset/panopticapi/) function. + The ids are defined by `segments_info`. + If an id does not appear in `segments_info`, the pixel is considered unlabeled + and is usually ignored in training & evaluation. ++ `segments_info` (list[dict]): defines the meaning of each id in panoptic segmentation ground truth. + Each dict has the following keys: + + `id` (int): integer that appears in the ground truth image. + + `category_id` (int): an integer in the range [0, num_categories-1] representing the category label. + + `iscrowd`: 0 (default) or 1. Whether this instance is labeled as COCO's "crowd region". + + +```eval_rst + +.. note:: + + The PanopticFPN model does not use the panoptic segmentation + format defined here, but a combination of both instance segmentation and semantic segmentation data + format. See :doc:`builtin_datasets` for instructions on COCO. + +``` + +Fast R-CNN (with pre-computed proposals) models are rarely used today. +To train a Fast R-CNN, the following extra keys are needed: + ++ `proposal_boxes` (array): 2D numpy array with shape (K, 4) representing K precomputed proposal boxes for this image. ++ `proposal_objectness_logits` (array): numpy array with shape (K, ), which corresponds to the objectness + logits of proposals in 'proposal_boxes'. ++ `proposal_bbox_mode` (int): the format of the precomputed proposal bbox. + It must be a member of + [structures.BoxMode](../modules/structures.html#detectron2.structures.BoxMode). + Default is `BoxMode.XYXY_ABS`. + + + +#### Custom Dataset Dicts for New Tasks + +In the `list[dict]` that your dataset function returns, the dictionary can also have __arbitrary custom data__. +This will be useful for a new task that needs extra information not covered +by the standard dataset dicts. In this case, you need to make sure the downstream code can handle your data +correctly. Usually this requires writing a new `mapper` for the dataloader (see [Use Custom Dataloaders](./data_loading.md)). + +When designing a custom format, note that all dicts are stored in memory +(sometimes serialized and with multiple copies). +To save memory, each dict is meant to contain __small__ but sufficient information +about each sample, such as file names and annotations. +Loading full samples typically happens in the data loader. + +For attributes shared among the entire dataset, use `Metadata` (see below). +To avoid extra memory, do not save such information inside each sample. + +### "Metadata" for Datasets + +Each dataset is associated with some metadata, accessible through +`MetadataCatalog.get(dataset_name).some_metadata`. +Metadata is a key-value mapping that contains information that's shared among +the entire dataset, and usually is used to interpret what's in the dataset, e.g., +names of classes, colors of classes, root of files, etc. +This information will be useful for augmentation, evaluation, visualization, logging, etc. +The structure of metadata depends on what is needed from the corresponding downstream code. + +If you register a new dataset through `DatasetCatalog.register`, +you may also want to add its corresponding metadata through +`MetadataCatalog.get(dataset_name).some_key = some_value`, to enable any features that need the metadata. +You can do it like this (using the metadata key "thing_classes" as an example): + +```python +from detectron2.data import MetadataCatalog +MetadataCatalog.get("my_dataset").thing_classes = ["person", "dog"] +``` + +Here is a list of metadata keys that are used by builtin features in detectron2. +If you add your own dataset without these metadata, some features may be +unavailable to you: + +* `thing_classes` (list[str]): Used by all instance detection/segmentation tasks. + A list of names for each instance/thing category. + If you load a COCO format dataset, it will be automatically set by the function `load_coco_json`. + +* `thing_colors` (list[tuple(r, g, b)]): Pre-defined color (in [0, 255]) for each thing category. + Used for visualization. If not given, random colors will be used. + +* `stuff_classes` (list[str]): Used by semantic and panoptic segmentation tasks. + A list of names for each stuff category. + +* `stuff_colors` (list[tuple(r, g, b)]): Pre-defined color (in [0, 255]) for each stuff category. + Used for visualization. If not given, random colors are used. + +* `ignore_label` (int): Used by semantic and panoptic segmentation tasks. Pixels in ground-truth + annotations with this category label should be ignored in evaluation. Typically these are "unlabeled" + pixels. + +* `keypoint_names` (list[str]): Used by keypoint detection. A list of names for each keypoint. + +* `keypoint_flip_map` (list[tuple[str]]): Used by keypoint detection. A list of pairs of names, + where each pair are the two keypoints that should be flipped if the image is + flipped horizontally during augmentation. +* `keypoint_connection_rules`: list[tuple(str, str, (r, g, b))]. Each tuple specifies a pair of keypoints + that are connected and the color (in [0, 255]) to use for the line between them when visualized. + +Some additional metadata that are specific to the evaluation of certain datasets (e.g. COCO): + +* `thing_dataset_id_to_contiguous_id` (dict[int->int]): Used by all instance detection/segmentation tasks in the COCO format. + A mapping from instance class ids in the dataset to contiguous ids in range [0, #class). + Will be automatically set by the function `load_coco_json`. + +* `stuff_dataset_id_to_contiguous_id` (dict[int->int]): Used when generating prediction json files for + semantic/panoptic segmentation. + A mapping from semantic segmentation class ids in the dataset + to contiguous ids in [0, num_categories). It is useful for evaluation only. + +* `json_file`: The COCO annotation json file. Used by COCO evaluation for COCO-format datasets. +* `panoptic_root`, `panoptic_json`: Used by COCO-format panoptic evaluation. +* `evaluator_type`: Used by the builtin main training script to select + evaluator. Don't use it in a new training script. + You can just provide the [DatasetEvaluator](../modules/evaluation.html#detectron2.evaluation.DatasetEvaluator) + for your dataset directly in your main script. + +```eval_rst +.. note:: + + In recognition, sometimes we use the term "thing" for instance-level tasks, + and "stuff" for semantic segmentation tasks. + Both are used in panoptic segmentation tasks. + For background on the concept of "thing" and "stuff", see + `On Seeing Stuff: The Perception of Materials by Humans and Machines + `_. +``` + +### Register a COCO Format Dataset + +If your instance-level (detection, segmentation, keypoint) dataset is already a json file in the COCO format, +the dataset and its associated metadata can be registered easily with: +```python +from detectron2.data.datasets import register_coco_instances +register_coco_instances("my_dataset", {}, "json_annotation.json", "path/to/image/dir") +``` + +If your dataset is in COCO format but need to be further processed, or has extra custom per-instance annotations, +the [load_coco_json](../modules/data.html#detectron2.data.datasets.load_coco_json) +function might be useful. + +### Update the Config for New Datasets + +Once you've registered the dataset, you can use the name of the dataset (e.g., "my_dataset" in +example above) in `cfg.DATASETS.{TRAIN,TEST}`. +There are other configs you might want to change to train or evaluate on new datasets: + +* `MODEL.ROI_HEADS.NUM_CLASSES` and `MODEL.RETINANET.NUM_CLASSES` are the number of thing classes + for R-CNN and RetinaNet models, respectively. +* `MODEL.ROI_KEYPOINT_HEAD.NUM_KEYPOINTS` sets the number of keypoints for Keypoint R-CNN. + You'll also need to set [Keypoint OKS](http://cocodataset.org/#keypoints-eval) + with `TEST.KEYPOINT_OKS_SIGMAS` for evaluation. +* `MODEL.SEM_SEG_HEAD.NUM_CLASSES` sets the number of stuff classes for Semantic FPN & Panoptic FPN. +* `TEST.DETECTIONS_PER_IMAGE` controls the maximum number of objects to be detected. + Set it to a larger number if test images may contain >100 objects. +* If you're training Fast R-CNN (with precomputed proposals), `DATASETS.PROPOSAL_FILES_{TRAIN,TEST}` + need to match the datasets. The format of proposal files are documented + [here](../modules/data.html#detectron2.data.load_proposals_into_dataset). + +New models +(e.g. [TensorMask](../../projects/TensorMask), +[PointRend](../../projects/PointRend)) +often have similar configs of their own that need to be changed as well. + +```eval_rst +.. tip:: + + After changing the number of classes, certain layers in a pre-trained model will become incompatible + and therefore cannot be loaded to the new model. + This is expected, and loading such pre-trained models will produce warnings about such layers. +``` diff --git a/approach/ovod/detectron2/docs/tutorials/deployment.md b/approach/ovod/detectron2/docs/tutorials/deployment.md new file mode 100644 index 0000000000000000000000000000000000000000..f7598880a9946402848301123d2889cfec2359e5 --- /dev/null +++ b/approach/ovod/detectron2/docs/tutorials/deployment.md @@ -0,0 +1,137 @@ +# Deployment + +Models written in Python need to go through an export process to become a deployable artifact. +A few basic concepts about this process: + +__"Export method"__ is how a Python model is fully serialized to a deployable format. +We support the following export methods: + +* `tracing`: see [pytorch documentation](https://pytorch.org/tutorials/beginner/Intro_to_TorchScript_tutorial.html) to learn about it +* `scripting`: see [pytorch documentation](https://pytorch.org/tutorials/beginner/Intro_to_TorchScript_tutorial.html) to learn about it +* `caffe2_tracing`: replace parts of the model by caffe2 operators, then use tracing. + +__"Format"__ is how a serialized model is described in a file, e.g. +TorchScript, Caffe2 protobuf, ONNX format. +__"Runtime"__ is an engine that loads a serialized model and executes it, +e.g., PyTorch, Caffe2, TensorFlow, onnxruntime, TensorRT, etc. +A runtime is often tied to a specific format +(e.g. PyTorch needs TorchScript format, Caffe2 needs protobuf format). +We currently support the following combination and each has some limitations: + +```eval_rst ++----------------------------+-------------+-------------+-----------------------------+ +| Export Method | tracing | scripting | caffe2_tracing | ++============================+=============+=============+=============================+ +| **Formats** | TorchScript | TorchScript | Caffe2, TorchScript, ONNX | ++----------------------------+-------------+-------------+-----------------------------+ +| **Runtime** | PyTorch | PyTorch | Caffe2, PyTorch | ++----------------------------+-------------+-------------+-----------------------------+ +| C++/Python inference | ✅ | ✅ | ✅ | ++----------------------------+-------------+-------------+-----------------------------+ +| Dynamic resolution | ✅ | ✅ | ✅ | ++----------------------------+-------------+-------------+-----------------------------+ +| Batch size requirement | Constant | Dynamic | Batch inference unsupported | ++----------------------------+-------------+-------------+-----------------------------+ +| Extra runtime deps | torchvision | torchvision | Caffe2 ops (usually already | +| | | | | +| | | | included in PyTorch) | ++----------------------------+-------------+-------------+-----------------------------+ +| Faster/Mask/Keypoint R-CNN | ✅ | ✅ | ✅ | ++----------------------------+-------------+-------------+-----------------------------+ +| RetinaNet | ✅ | ✅ | ✅ | ++----------------------------+-------------+-------------+-----------------------------+ +| PointRend R-CNN | ✅ | ❌ | ❌ | ++----------------------------+-------------+-------------+-----------------------------+ +| Cascade R-CNN | ✅ | ❌ | ❌ | ++----------------------------+-------------+-------------+-----------------------------+ + +``` + +`caffe2_tracing` is going to be deprecated. +We don't plan to work on additional support for other formats/runtime, but contributions are welcome. + + +## Deployment with Tracing or Scripting + +Models can be exported to TorchScript format, by either +[tracing or scripting](https://pytorch.org/tutorials/beginner/Intro_to_TorchScript_tutorial.html). +The output model file can be loaded without detectron2 dependency in either Python or C++. +The exported model often requires torchvision (or its C++ library) dependency for some custom ops. + +This feature requires PyTorch ≥ 1.8. + +### Coverage +Most official models under the meta architectures `GeneralizedRCNN` and `RetinaNet` +are supported in both tracing and scripting mode. +Cascade R-CNN and PointRend are currently supported in tracing. +Users' custom extensions are supported if they are also scriptable or traceable. + +For models exported with tracing, dynamic input resolution is allowed, but batch size +(number of input images) must be fixed. +Scripting can support dynamic batch size. + +### Usage + +The main export APIs for tracing and scripting are [TracingAdapter](../modules/export.html#detectron2.export.TracingAdapter) +and [scripting_with_instances](../modules/export.html#detectron2.export.scripting_with_instances). +Their usage is currently demonstrated in [test_export_torchscript.py](../../tests/test_export_torchscript.py) +(see `TestScripting` and `TestTracing`) +as well as the [deployment example](../../tools/deploy). +Please check that these examples can run, and then modify for your use cases. +The usage now requires some user effort and necessary knowledge for each model to workaround the limitation of scripting and tracing. +In the future we plan to wrap these under simpler APIs to lower the bar to use them. + +## Deployment with Caffe2-tracing +We provide [Caffe2Tracer](../modules/export.html#detectron2.export.Caffe2Tracer) +that performs the export logic. +It replaces parts of the model with Caffe2 operators, +and then export the model into Caffe2, TorchScript or ONNX format. + +The converted model is able to run in either Python or C++ without detectron2/torchvision dependency, on CPU or GPUs. +It has a runtime optimized for CPU & mobile inference, but not optimized for GPU inference. + +This feature requires ONNX ≥ 1.6. + +### Coverage + +Most official models under these 3 common meta architectures: `GeneralizedRCNN`, `RetinaNet`, `PanopticFPN` +are supported. Cascade R-CNN is not supported. Batch inference is not supported. + +Users' custom extensions under these architectures (added through registration) are supported +as long as they do not contain control flow or operators not available in Caffe2 (e.g. deformable convolution). +For example, custom backbones and heads are often supported out of the box. + +### Usage + +The APIs are listed at [the API documentation](../modules/export). +We provide [export_model.py](../../tools/deploy/) as an example that uses +these APIs to convert a standard model. For custom models/datasets, you can add them to this script. + +### Use the model in C++/Python + +The model can be loaded in C++ and deployed with +either Caffe2 or Pytorch runtime.. [C++ examples](../../tools/deploy/) for Mask R-CNN +are given as a reference. Note that: + +* Models exported with `caffe2_tracing` method take a special input format + described in [documentation](../modules/export.html#detectron2.export.Caffe2Tracer). + This was taken care of in the C++ example. + +* The converted models do not contain post-processing operations that + transform raw layer outputs into formatted predictions. + For example, the C++ examples only produce raw outputs (28x28 masks) from the final + layers that are not post-processed, because in actual deployment, an application often needs + its custom lightweight post-processing, so this step is left for users. + +To help use the Caffe2-format model in python, +we provide a python wrapper around the converted model, in the +[Caffe2Model.\_\_call\_\_](../modules/export.html#detectron2.export.Caffe2Model.__call__) method. +This method has an interface that's identical to the [pytorch versions of models](./models.md), +and it internally applies pre/post-processing code to match the formats. +This wrapper can serve as a reference for how to use Caffe2's python API, +or for how to implement pre/post-processing in actual deployment. + +## Conversion to TensorFlow +[tensorpack Faster R-CNN](https://github.com/tensorpack/tensorpack/tree/master/examples/FasterRCNN/convert_d2) +provides scripts to convert a few standard detectron2 R-CNN models to TensorFlow's pb format. +It works by translating configs and weights, therefore only support a few models. diff --git a/approach/ovod/detectron2/docs/tutorials/extend.md b/approach/ovod/detectron2/docs/tutorials/extend.md new file mode 100644 index 0000000000000000000000000000000000000000..a6af550fdb2aa79c818cef54b009f2fe816d46a9 --- /dev/null +++ b/approach/ovod/detectron2/docs/tutorials/extend.md @@ -0,0 +1,141 @@ +# Extend Detectron2's Defaults + +__Research is about doing things in new ways__. +This brings a tension in how to create abstractions in code, +which is a challenge for any research engineering project of a significant size: + +1. On one hand, it needs to have very thin abstractions to allow for the possibility of doing + everything in new ways. It should be reasonably easy to break existing + abstractions and replace them with new ones. + +2. On the other hand, such a project also needs reasonably high-level + abstractions, so that users can easily do things in standard ways, + without worrying too much about the details that only certain researchers care about. + +In detectron2, there are two types of interfaces that address this tension together: + +1. Functions and classes that take a config (`cfg`) argument + created from a yaml file + (sometimes with few extra arguments). + + Such functions and classes implement + the "standard default" behavior: it will read what it needs from a given + config and do the "standard" thing. + Users only need to load an expert-made config and pass it around, without having to worry about + which arguments are used and what they all mean. + + See [Yacs Configs](configs.md) for a detailed tutorial. + +2. Functions and classes that have well-defined explicit arguments. + + Each of these is a small building block of the entire system. + They require users' expertise to understand what each argument should be, + and require more effort to stitch together to a larger system. + But they can be stitched together in more flexible ways. + + When you need to implement something not supported by the "standard defaults" + included in detectron2, these well-defined components can be reused. + + The [LazyConfig system](lazyconfigs.md) relies on such functions and classes. + +3. A few functions and classes are implemented with the + [@configurable](../modules/config.html#detectron2.config.configurable) + decorator - they can be called with either a config, or with explicit arguments, or a mixture of both. + Their explicit argument interfaces are currently experimental. + + As an example, a Mask R-CNN model can be built in the following ways: + + 1. Config-only: + ```python + # load proper yaml config file, then + model = build_model(cfg) + ``` + + 2. Mixture of config and additional argument overrides: + ```python + model = GeneralizedRCNN( + cfg, + roi_heads=StandardROIHeads(cfg, batch_size_per_image=666), + pixel_std=[57.0, 57.0, 57.0]) + ``` + + 3. Full explicit arguments: +
+ + (click to expand) + + + ```python + model = GeneralizedRCNN( + backbone=FPN( + ResNet( + BasicStem(3, 64, norm="FrozenBN"), + ResNet.make_default_stages(50, stride_in_1x1=True, norm="FrozenBN"), + out_features=["res2", "res3", "res4", "res5"], + ).freeze(2), + ["res2", "res3", "res4", "res5"], + 256, + top_block=LastLevelMaxPool(), + ), + proposal_generator=RPN( + in_features=["p2", "p3", "p4", "p5", "p6"], + head=StandardRPNHead(in_channels=256, num_anchors=3), + anchor_generator=DefaultAnchorGenerator( + sizes=[[32], [64], [128], [256], [512]], + aspect_ratios=[0.5, 1.0, 2.0], + strides=[4, 8, 16, 32, 64], + offset=0.0, + ), + anchor_matcher=Matcher([0.3, 0.7], [0, -1, 1], allow_low_quality_matches=True), + box2box_transform=Box2BoxTransform([1.0, 1.0, 1.0, 1.0]), + batch_size_per_image=256, + positive_fraction=0.5, + pre_nms_topk=(2000, 1000), + post_nms_topk=(1000, 1000), + nms_thresh=0.7, + ), + roi_heads=StandardROIHeads( + num_classes=80, + batch_size_per_image=512, + positive_fraction=0.25, + proposal_matcher=Matcher([0.5], [0, 1], allow_low_quality_matches=False), + box_in_features=["p2", "p3", "p4", "p5"], + box_pooler=ROIPooler(7, (1.0 / 4, 1.0 / 8, 1.0 / 16, 1.0 / 32), 0, "ROIAlignV2"), + box_head=FastRCNNConvFCHead( + ShapeSpec(channels=256, height=7, width=7), conv_dims=[], fc_dims=[1024, 1024] + ), + box_predictor=FastRCNNOutputLayers( + ShapeSpec(channels=1024), + test_score_thresh=0.05, + box2box_transform=Box2BoxTransform((10, 10, 5, 5)), + num_classes=80, + ), + mask_in_features=["p2", "p3", "p4", "p5"], + mask_pooler=ROIPooler(14, (1.0 / 4, 1.0 / 8, 1.0 / 16, 1.0 / 32), 0, "ROIAlignV2"), + mask_head=MaskRCNNConvUpsampleHead( + ShapeSpec(channels=256, width=14, height=14), + num_classes=80, + conv_dims=[256, 256, 256, 256, 256], + ), + ), + pixel_mean=[103.530, 116.280, 123.675], + pixel_std=[1.0, 1.0, 1.0], + input_format="BGR", + ) + ``` + +
+ + +If you only need the standard behavior, the [Beginner's Tutorial](./getting_started.md) +should suffice. If you need to extend detectron2 to your own needs, +see the following tutorials for more details: + +* Detectron2 includes a few standard datasets. To use custom ones, see + [Use Custom Datasets](./datasets.md). +* Detectron2 contains the standard logic that creates a data loader for training/testing from a + dataset, but you can write your own as well. See [Use Custom Data Loaders](./data_loading.md). +* Detectron2 implements many standard detection models, and provide ways for you + to overwrite their behaviors. See [Use Models](./models.md) and [Write Models](./write-models.md). +* Detectron2 provides a default training loop that is good for common training tasks. + You can customize it with hooks, or write your own loop instead. See [training](./training.md). diff --git a/approach/ovod/detectron2/docs/tutorials/getting_started.md b/approach/ovod/detectron2/docs/tutorials/getting_started.md new file mode 100644 index 0000000000000000000000000000000000000000..404b0c8f467264d1adf61e8274e5f864e24018e8 --- /dev/null +++ b/approach/ovod/detectron2/docs/tutorials/getting_started.md @@ -0,0 +1,79 @@ +## Getting Started with Detectron2 + +This document provides a brief intro of the usage of builtin command-line tools in detectron2. + +For a tutorial that involves actual coding with the API, +see our [Colab Notebook](https://colab.research.google.com/drive/16jcaJoc6bCFAQ96jDe2HwtXj7BMD_-m5) +which covers how to run inference with an +existing model, and how to train a builtin model on a custom dataset. + + +### Inference Demo with Pre-trained Models + +1. Pick a model and its config file from + [model zoo](MODEL_ZOO.md), + for example, `mask_rcnn_R_50_FPN_3x.yaml`. +2. We provide `demo.py` that is able to demo builtin configs. Run it with: +``` +cd demo/ +python demo.py --config-file ../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml \ + --input input1.jpg input2.jpg \ + [--other-options] + --opts MODEL.WEIGHTS detectron2://COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/model_final_f10217.pkl +``` +The configs are made for training, therefore we need to specify `MODEL.WEIGHTS` to a model from model zoo for evaluation. +This command will run the inference and show visualizations in an OpenCV window. + +For details of the command line arguments, see `demo.py -h` or look at its source code +to understand its behavior. Some common arguments are: +* To run __on your webcam__, replace `--input files` with `--webcam`. +* To run __on a video__, replace `--input files` with `--video-input video.mp4`. +* To run __on cpu__, add `MODEL.DEVICE cpu` after `--opts`. +* To save outputs to a directory (for images) or a file (for webcam or video), use `--output`. + + +### Training & Evaluation in Command Line + +We provide two scripts in "tools/plain_train_net.py" and "tools/train_net.py", +that are made to train all the configs provided in detectron2. You may want to +use it as a reference to write your own training script. + +Compared to "train_net.py", "plain_train_net.py" supports fewer default +features. It also includes fewer abstraction, therefore is easier to add custom +logic. + +To train a model with "train_net.py", first +setup the corresponding datasets following +[datasets/README.md](./datasets/README.md), +then run: +``` +cd tools/ +./train_net.py --num-gpus 8 \ + --config-file ../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml +``` + +The configs are made for 8-GPU training. +To train on 1 GPU, you may need to [change some parameters](https://arxiv.org/abs/1706.02677), e.g.: +``` +./train_net.py \ + --config-file ../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \ + --num-gpus 1 SOLVER.IMS_PER_BATCH 2 SOLVER.BASE_LR 0.0025 +``` + +To evaluate a model's performance, use +``` +./train_net.py \ + --config-file ../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \ + --eval-only MODEL.WEIGHTS /path/to/checkpoint_file +``` +For more options, see `./train_net.py -h`. + +### Use Detectron2 APIs in Your Code + +See our [Colab Notebook](https://colab.research.google.com/drive/16jcaJoc6bCFAQ96jDe2HwtXj7BMD_-m5) +to learn how to use detectron2 APIs to: +1. run inference with an existing model +2. train a builtin model on a custom dataset + +See [detectron2/projects](https://github.com/facebookresearch/detectron2/tree/main/projects) +for more ways to build your project on detectron2. diff --git a/approach/ovod/detectron2/docs/tutorials/index.rst b/approach/ovod/detectron2/docs/tutorials/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..850b95cfa873ffa0ba2d6f6e4263ad0895c08be8 --- /dev/null +++ b/approach/ovod/detectron2/docs/tutorials/index.rst @@ -0,0 +1,20 @@ +Tutorials +====================================== + +.. toctree:: + :maxdepth: 2 + + install + getting_started + builtin_datasets + extend + datasets + data_loading + augmentation + models + write-models + training + evaluation + configs + lazyconfigs + deployment diff --git a/approach/ovod/detectron2/docs/tutorials/install.md b/approach/ovod/detectron2/docs/tutorials/install.md new file mode 100644 index 0000000000000000000000000000000000000000..f522e6f624372f39ee5366f5b032c0cd1ebcf5c8 --- /dev/null +++ b/approach/ovod/detectron2/docs/tutorials/install.md @@ -0,0 +1,261 @@ +## Installation + +### Requirements +- Linux or macOS with Python ≥ 3.7 +- PyTorch ≥ 1.8 and [torchvision](https://github.com/pytorch/vision/) that matches the PyTorch installation. + Install them together at [pytorch.org](https://pytorch.org) to make sure of this +- OpenCV is optional but needed by demo and visualization + + +### Build Detectron2 from Source + +gcc & g++ ≥ 5.4 are required. [ninja](https://ninja-build.org/) is optional but recommended for faster build. +After having them, run: +``` +python -m pip install 'git+https://github.com/facebookresearch/detectron2.git' +# (add --user if you don't have permission) + +# Or, to install it from a local clone: +git clone https://github.com/facebookresearch/detectron2.git +python -m pip install -e detectron2 + +# On macOS, you may need to prepend the above commands with a few environment variables: +CC=clang CXX=clang++ ARCHFLAGS="-arch x86_64" python -m pip install ... +``` + +To __rebuild__ detectron2 that's built from a local clone, use `rm -rf build/ **/*.so` to clean the +old build first. You often need to rebuild detectron2 after reinstalling PyTorch. + +### Install Pre-Built Detectron2 (Linux only) + +Choose from this table to install [v0.6 (Oct 2021)](https://github.com/facebookresearch/detectron2/releases): + +
CUDA torch 1.10torch 1.9torch 1.8
11.3
install 
python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cu113/torch1.10/index.html
+
11.1
install 
python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cu111/torch1.10/index.html
+
install 
python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cu111/torch1.9/index.html
+
install 
python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cu111/torch1.8/index.html
+
10.2
install 
python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.10/index.html
+
install 
python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.9/index.html
+
install 
python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.8/index.html
+
10.1
install 
python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cu101/torch1.8/index.html
+
cpu
install 
python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cpu/torch1.10/index.html
+
install 
python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cpu/torch1.9/index.html
+
install 
python -m pip install detectron2 -f \
+  https://dl.fbaipublicfiles.com/detectron2/wheels/cpu/torch1.8/index.html
+
+ +Note that: +1. The pre-built packages have to be used with corresponding version of CUDA and the official package of PyTorch. + Otherwise, please build detectron2 from source. +2. New packages are released every few months. Therefore, packages may not contain latest features in the main + branch and may not be compatible with the main branch of a research project that uses detectron2 + (e.g. those in [projects](projects)). + +### Common Installation Issues + +Click each issue for its solutions: + +
+ +Undefined symbols that looks like "TH..","at::Tensor...","torch..." + +
+ +This usually happens when detectron2 or torchvision is not +compiled with the version of PyTorch you're running. + +If the error comes from a pre-built torchvision, uninstall torchvision and pytorch and reinstall them +following [pytorch.org](http://pytorch.org). So the versions will match. + +If the error comes from a pre-built detectron2, check [release notes](https://github.com/facebookresearch/detectron2/releases), +uninstall and reinstall the correct pre-built detectron2 that matches pytorch version. + +If the error comes from detectron2 or torchvision that you built manually from source, +remove files you built (`build/`, `**/*.so`) and rebuild it so it can pick up the version of pytorch currently in your environment. + +If the above instructions do not resolve this problem, please provide an environment (e.g. a dockerfile) that can reproduce the issue. +
+ +
+ +Missing torch dynamic libraries, OR segmentation fault immediately when using detectron2. + +This usually happens when detectron2 or torchvision is not +compiled with the version of PyTorch you're running. See the previous common issue for the solution. +
+ +
+ +Undefined C++ symbols (e.g. "GLIBCXX..") or C++ symbols not found. + +
+Usually it's because the library is compiled with a newer C++ compiler but run with an old C++ runtime. + +This often happens with old anaconda. +It may help to run `conda update libgcc` to upgrade its runtime. + +The fundamental solution is to avoid the mismatch, either by compiling using older version of C++ +compiler, or run the code with proper C++ runtime. +To run the code with a specific C++ runtime, you can use environment variable `LD_PRELOAD=/path/to/libstdc++.so`. + +
+ +
+ +"nvcc not found" or "Not compiled with GPU support" or "Detectron2 CUDA Compiler: not available". + +
+CUDA is not found when building detectron2. +You should make sure + +``` +python -c 'import torch; from torch.utils.cpp_extension import CUDA_HOME; print(torch.cuda.is_available(), CUDA_HOME)' +``` + +print `(True, a directory with cuda)` at the time you build detectron2. + +Most models can run inference (but not training) without GPU support. To use CPUs, set `MODEL.DEVICE='cpu'` in the config. +
+ +
+ +"invalid device function" or "no kernel image is available for execution". + +
+Two possibilities: + +* You build detectron2 with one version of CUDA but run it with a different version. + + To check whether it is the case, + use `python -m detectron2.utils.collect_env` to find out inconsistent CUDA versions. + In the output of this command, you should expect "Detectron2 CUDA Compiler", "CUDA_HOME", "PyTorch built with - CUDA" + to contain cuda libraries of the same version. + + When they are inconsistent, + you need to either install a different build of PyTorch (or build by yourself) + to match your local CUDA installation, or install a different version of CUDA to match PyTorch. + +* PyTorch/torchvision/Detectron2 is not built for the correct GPU SM architecture (aka. compute capability). + + The architecture included by PyTorch/detectron2/torchvision is available in the "architecture flags" in + `python -m detectron2.utils.collect_env`. It must include + the architecture of your GPU, which can be found at [developer.nvidia.com/cuda-gpus](https://developer.nvidia.com/cuda-gpus). + + If you're using pre-built PyTorch/detectron2/torchvision, they have included support for most popular GPUs already. + If not supported, you need to build them from source. + + When building detectron2/torchvision from source, they detect the GPU device and build for only the device. + This means the compiled code may not work on a different GPU device. + To recompile them for the correct architecture, remove all installed/compiled files, + and rebuild them with the `TORCH_CUDA_ARCH_LIST` environment variable set properly. + For example, `export TORCH_CUDA_ARCH_LIST="6.0;7.0"` makes it compile for both P100s and V100s. +
+ +
+ +Undefined CUDA symbols; Cannot open libcudart.so + +
+The version of NVCC you use to build detectron2 or torchvision does +not match the version of CUDA you are running with. +This often happens when using anaconda's CUDA runtime. + +Use `python -m detectron2.utils.collect_env` to find out inconsistent CUDA versions. +In the output of this command, you should expect "Detectron2 CUDA Compiler", "CUDA_HOME", "PyTorch built with - CUDA" +to contain cuda libraries of the same version. + +When they are inconsistent, +you need to either install a different build of PyTorch (or build by yourself) +to match your local CUDA installation, or install a different version of CUDA to match PyTorch. +
+ + +
+ +C++ compilation errors from NVCC / NVRTC, or "Unsupported gpu architecture" + +
+A few possibilities: + +1. Local CUDA/NVCC version has to match the CUDA version of your PyTorch. Both can be found in `python collect_env.py` + (download from [here](./detectron2/utils/collect_env.py)). + When they are inconsistent, you need to either install a different build of PyTorch (or build by yourself) + to match your local CUDA installation, or install a different version of CUDA to match PyTorch. + +2. Local CUDA/NVCC version shall support the SM architecture (a.k.a. compute capability) of your GPU. + The capability of your GPU can be found at [developer.nvidia.com/cuda-gpus](https://developer.nvidia.com/cuda-gpus). + The capability supported by NVCC is listed at [here](https://gist.github.com/ax3l/9489132). + If your NVCC version is too old, this can be workaround by setting environment variable + `TORCH_CUDA_ARCH_LIST` to a lower, supported capability. + +3. The combination of NVCC and GCC you use is incompatible. You need to change one of their versions. + See [here](https://gist.github.com/ax3l/9489132) for some valid combinations. + Notably, CUDA<=10.1.105 doesn't support GCC>7.3. + + The CUDA/GCC version used by PyTorch can be found by `print(torch.__config__.show())`. + +
+ + +
+ +"ImportError: cannot import name '_C'". + +
+Please build and install detectron2 following the instructions above. + +Or, if you are running code from detectron2's root directory, `cd` to a different one. +Otherwise you may not import the code that you installed. +
+ + +
+ +Any issue on windows. + +
+ +Detectron2 is continuously built on windows with [CircleCI](https://app.circleci.com/pipelines/github/facebookresearch/detectron2?branch=main). +However we do not provide official support for it. +PRs that improves code compatibility on windows are welcome. +
+ +
+ +ONNX conversion segfault after some "TraceWarning". + +
+The ONNX package is compiled with a too old compiler. + +Please build and install ONNX from its source code using a compiler +whose version is closer to what's used by PyTorch (available in `torch.__config__.show()`). +
+ + +
+ +"library not found for -lstdc++" on older version of MacOS + +
+ +See [this stackoverflow answer](https://stackoverflow.com/questions/56083725/macos-build-issues-lstdc-not-found-while-building-python-package). + +
+ + +### Installation inside specific environments: + +* __Colab__: see our [Colab Tutorial](https://colab.research.google.com/drive/16jcaJoc6bCFAQ96jDe2HwtXj7BMD_-m5) + which has step-by-step instructions. + +* __Docker__: The official [Dockerfile](docker) installs detectron2 with a few simple commands. diff --git a/approach/ovod/detectron2/docs/tutorials/lazyconfigs.md b/approach/ovod/detectron2/docs/tutorials/lazyconfigs.md new file mode 100644 index 0000000000000000000000000000000000000000..a01101ae40ec12d25d5a3d96892b60ef32dca21e --- /dev/null +++ b/approach/ovod/detectron2/docs/tutorials/lazyconfigs.md @@ -0,0 +1,170 @@ +# Lazy Configs + +The traditional yacs-based config system provides basic, standard functionalities. +However, it does not offer enough flexibility for many new projects. +We develop an alternative, non-intrusive config system that can be used with +detectron2 or potentially any other complex projects. + +## Python Syntax + +Our config objects are still dictionaries. Instead of using Yaml to define dictionaries, +we create dictionaries in Python directly. This gives users the following power that +doesn't exist in Yaml: + +* Easily manipulate the dictionary (addition & deletion) using Python. +* Write simple arithmetics or call simple functions. +* Use more data types / objects. +* Import / compose other config files, using the familiar Python import syntax. + +A Python config file can be loaded like this: +```python +# config.py: +a = dict(x=1, y=2, z=dict(xx=1)) +b = dict(x=3, y=4) + +# my_code.py: +from detectron2.config import LazyConfig +cfg = LazyConfig.load("path/to/config.py") # an omegaconf dictionary +assert cfg.a.z.xx == 1 +``` + +After [LazyConfig.load](../modules/config.html#detectron2.config.LazyConfig.load), `cfg` will be a dictionary that contains all dictionaries +defined in the global scope of the config file. Note that: +* All dictionaries are turned to an [omegaconf](https://omegaconf.readthedocs.io/) + config object during loading. This enables access to omegaconf features, + such as its [access syntax](https://omegaconf.readthedocs.io/en/2.1_branch/usage.html#access-and-manipulation) + and [interpolation](https://omegaconf.readthedocs.io/en/2.1_branch/usage.html#variable-interpolation). +* Absolute imports in `config.py` works the same as in regular Python. +* Relative imports can only import dictionaries from config files. + They are simply a syntax sugar for [LazyConfig.load_rel](../modules/config.html#detectron2.config.LazyConfig.load_rel). + They can load Python files at relative path without requiring `__init__.py`. + +[LazyConfig.save](../modules/config.html#detectron2.config.LazyConfig.save) can save a config object to yaml. +Note that this is not always successful if non-serializable objects appear in the config file (e.g. lambdas). +It is up to users whether to sacrifice the ability to save in exchange for flexibility. + +## Recursive Instantiation + +The LazyConfig system heavily uses recursive instantiation, which is a pattern that +uses a dictionary to describe a +call to a function/class. The dictionary consists of: + +1. A "\_target\_" key which contains path to the callable, such as "module.submodule.class_name". +2. Other keys that represent arguments to pass to the callable. Arguments themselves can be defined + using recursive instantiation. + +We provide a helper function [LazyCall](../modules/config.html#detectron2.config.LazyCall) that helps create such dictionaries. +The following code using `LazyCall` +```python +from detectron2.config import LazyCall as L +from my_app import Trainer, Optimizer +cfg = L(Trainer)( + optimizer=L(Optimizer)( + lr=0.01, + algo="SGD" + ) +) +``` +creates a dictionary like this: +```python +cfg = { + "_target_": "my_app.Trainer", + "optimizer": { + "_target_": "my_app.Optimizer", + "lr": 0.01, "algo": "SGD" + } +} +``` + +By representing objects using such dictionaries, a general +[instantiate](../modules/config.html#detectron2.config.instantiate) +function can turn them into actual objects, i.e.: +```python +from detectron2.config import instantiate +trainer = instantiate(cfg) +# equivalent to: +# from my_app import Trainer, Optimizer +# trainer = Trainer(optimizer=Optimizer(lr=0.01, algo="SGD")) +``` + +This pattern is powerful enough to describe very complex objects, e.g.: + +
+ +A Full Mask R-CNN described in recursive instantiation (click to expand) + + +```eval_rst +.. literalinclude:: ../../configs/common/models/mask_rcnn_fpn.py + :language: python + :linenos: +``` + +
+ +There are also objects or logic that cannot be described simply by a dictionary, +such as reused objects or method calls. They may require some refactoring +to work with recursive instantiation. + +## Using Model Zoo LazyConfigs + +We provide some configs in the model zoo using the LazyConfig system, for example: + +* [common baselines](../../configs/common/). +* [new Mask R-CNN baselines](../../configs/new_baselines/) + +After installing detectron2, they can be loaded by the model zoo API +[model_zoo.get_config](../modules/model_zoo.html#detectron2.model_zoo.get_config). + +Using these as references, you're free to define custom config structure / fields for your own +project, as long as your training script can understand them. +Despite of this, our model zoo configs still follow some simple conventions for consistency, e.g. +`cfg.model` defines a model object, `cfg.dataloader.{train,test}` defines dataloader objects, +and `cfg.train` contains training options in key-value form. +In addition to `print()`, a better way to view the structure of a config is like this: +```python +from detectron2.model_zoo import get_config +from detectron2.config import LazyConfig +print(LazyConfig.to_py(get_config("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.py"))) +``` +From the output it's easier to find relevant options to change, e.g. +`dataloader.train.total_batch_size` for the batch size, or `optimizer.lr` for base learning rate. + +We provide a reference training script +[tools/lazyconfig_train_net.py](../../tools/lazyconfig_train_net.py), +that can train/eval our model zoo configs. +It also shows how to support command line value overrides. + +To demonstrate the power and flexibility of the new system, we show that +[a simple config file](../../configs/Misc/torchvision_imagenet_R_50.py) +can let detectron2 train an ImageNet classification model from torchvision, even though +detectron2 contains no features about ImageNet classification. +This can serve as a reference for using detectron2 in other deep learning tasks. + +## Summary + +By using recursive instantiation to create objects, +we avoid passing a giant config to many places, because `cfg` is only passed to `instantiate`. +This has the following benefits: + +* It's __non-intrusive__: objects to be constructed are config-agnostic, regular Python + functions/classes. + They can even live in other libraries. For example, + `{"_target_": "torch.nn.Conv2d", "in_channels": 10, "out_channels": 10, "kernel_size": 1}` + defines a conv layer. +* __Clarity__ of what function/classes will be called, and what arguments they use. +* `cfg` doesn't need pre-defined keys and structures. It's valid as long as it translates to valid + code. This gives a lot more __flexibility__. +* You can still pass huge dictionaries as arguments, just like the old way. + +Recursive instantiation and Python syntax are orthogonal: you can use one without the other. +But by putting them together, the config file looks a lot like the code that will be executed: + +![img](./lazyconfig.jpg) + +However, the config file just defines dictionaries, which can be easily manipulated further +by composition or overrides. +The corresponding code will only be executed +later when `instantiate` is called. In some way, +in config files we're writing "editable code" that will be "lazily executed" later when needed. +That's why we call this system "LazyConfig". diff --git a/approach/ovod/detectron2/docs/tutorials/models.md b/approach/ovod/detectron2/docs/tutorials/models.md new file mode 100644 index 0000000000000000000000000000000000000000..a2def5c715ac793e6269cbb84ef4792f91a774c1 --- /dev/null +++ b/approach/ovod/detectron2/docs/tutorials/models.md @@ -0,0 +1,180 @@ +# Use Models + +## Build Models from Yacs Config +From a yacs config object, +models (and their sub-models) can be built by +functions such as `build_model`, `build_backbone`, `build_roi_heads`: +```python +from detectron2.modeling import build_model +model = build_model(cfg) # returns a torch.nn.Module +``` + +`build_model` only builds the model structure and fills it with random parameters. +See below for how to load an existing checkpoint to the model and how to use the `model` object. + +### Load/Save a Checkpoint +```python +from detectron2.checkpoint import DetectionCheckpointer +DetectionCheckpointer(model).load(file_path_or_url) # load a file, usually from cfg.MODEL.WEIGHTS + +checkpointer = DetectionCheckpointer(model, save_dir="output") +checkpointer.save("model_999") # save to output/model_999.pth +``` + +Detectron2's checkpointer recognizes models in pytorch's `.pth` format, as well as the `.pkl` files +in our model zoo. +See [API doc](../modules/checkpoint.html#detectron2.checkpoint.DetectionCheckpointer) +for more details about its usage. + +The model files can be arbitrarily manipulated using `torch.{load,save}` for `.pth` files or +`pickle.{dump,load}` for `.pkl` files. + +### Use a Model + +A model can be called by `outputs = model(inputs)`, where `inputs` is a `list[dict]`. +Each dict corresponds to one image and the required keys +depend on the type of model, and whether the model is in training or evaluation mode. +For example, in order to do inference, +all existing models expect the "image" key, and optionally "height" and "width". +The detailed format of inputs and outputs of existing models are explained below. + +__Training__: When in training mode, all models are required to be used under an `EventStorage`. +The training statistics will be put into the storage: +```python +from detectron2.utils.events import EventStorage +with EventStorage() as storage: + losses = model(inputs) +``` + +__Inference__: If you only want to do simple inference using an existing model, +[DefaultPredictor](../modules/engine.html#detectron2.engine.defaults.DefaultPredictor) +is a wrapper around model that provides such basic functionality. +It includes default behavior including model loading, preprocessing, +and operates on single image rather than batches. See its documentation for usage. + +You can also run inference directly like this: +```python +model.eval() +with torch.no_grad(): + outputs = model(inputs) +``` + +### Model Input Format + +Users can implement custom models that support any arbitrary input format. +Here we describe the standard input format that all builtin models support in detectron2. +They all take a `list[dict]` as the inputs. Each dict +corresponds to information about one image. + +The dict may contain the following keys: + +* "image": `Tensor` in (C, H, W) format. The meaning of channels are defined by `cfg.INPUT.FORMAT`. + Image normalization, if any, will be performed inside the model using + `cfg.MODEL.PIXEL_{MEAN,STD}`. +* "height", "width": the **desired** output height and width **in inference**, which is not necessarily the same + as the height or width of the `image` field. + For example, the `image` field contains the resized image, if resize is used as a preprocessing step. + But you may want the outputs to be in **original** resolution. + If provided, the model will produce output in this resolution, + rather than in the resolution of the `image` as input into the model. This is more efficient and accurate. +* "instances": an [Instances](../modules/structures.html#detectron2.structures.Instances) + object for training, with the following fields: + + "gt_boxes": a [Boxes](../modules/structures.html#detectron2.structures.Boxes) object storing N boxes, one for each instance. + + "gt_classes": `Tensor` of long type, a vector of N labels, in range [0, num_categories). + + "gt_masks": a [PolygonMasks](../modules/structures.html#detectron2.structures.PolygonMasks) + or [BitMasks](../modules/structures.html#detectron2.structures.BitMasks) object storing N masks, one for each instance. + + "gt_keypoints": a [Keypoints](../modules/structures.html#detectron2.structures.Keypoints) + object storing N keypoint sets, one for each instance. +* "sem_seg": `Tensor[int]` in (H, W) format. The semantic segmentation ground truth for training. + Values represent category labels starting from 0. +* "proposals": an [Instances](../modules/structures.html#detectron2.structures.Instances) + object used only in Fast R-CNN style models, with the following fields: + + "proposal_boxes": a [Boxes](../modules/structures.html#detectron2.structures.Boxes) object storing P proposal boxes. + + "objectness_logits": `Tensor`, a vector of P scores, one for each proposal. + +For inference of builtin models, only "image" key is required, and "width/height" are optional. + +We currently don't define standard input format for panoptic segmentation training, +because models now use custom formats produced by custom data loaders. + +#### How it connects to data loader: + +The output of the default [DatasetMapper]( ../modules/data.html#detectron2.data.DatasetMapper) is a dict +that follows the above format. +After the data loader performs batching, it becomes `list[dict]` which the builtin models support. + + +### Model Output Format + +When in training mode, the builtin models output a `dict[str->ScalarTensor]` with all the losses. + +When in inference mode, the builtin models output a `list[dict]`, one dict for each image. +Based on the tasks the model is doing, each dict may contain the following fields: + +* "instances": [Instances](../modules/structures.html#detectron2.structures.Instances) + object with the following fields: + * "pred_boxes": [Boxes](../modules/structures.html#detectron2.structures.Boxes) object storing N boxes, one for each detected instance. + * "scores": `Tensor`, a vector of N confidence scores. + * "pred_classes": `Tensor`, a vector of N labels in range [0, num_categories). + + "pred_masks": a `Tensor` of shape (N, H, W), masks for each detected instance. + + "pred_keypoints": a `Tensor` of shape (N, num_keypoint, 3). + Each row in the last dimension is (x, y, score). Confidence scores are larger than 0. +* "sem_seg": `Tensor` of (num_categories, H, W), the semantic segmentation prediction. +* "proposals": [Instances](../modules/structures.html#detectron2.structures.Instances) + object with the following fields: + * "proposal_boxes": [Boxes](../modules/structures.html#detectron2.structures.Boxes) + object storing N boxes. + * "objectness_logits": a torch vector of N confidence scores. +* "panoptic_seg": A tuple of `(pred: Tensor, segments_info: Optional[list[dict]])`. + The `pred` tensor has shape (H, W), containing the segment id of each pixel. + + * If `segments_info` exists, each dict describes one segment id in `pred` and has the following fields: + + * "id": the segment id + * "isthing": whether the segment is a thing or stuff + * "category_id": the category id of this segment. + + If a pixel's id does not exist in `segments_info`, it is considered to be void label + defined in [Panoptic Segmentation](https://arxiv.org/abs/1801.00868). + + * If `segments_info` is None, all pixel values in `pred` must be ≥ -1. + Pixels with value -1 are assigned void labels. + Otherwise, the category id of each pixel is obtained by + `category_id = pixel // metadata.label_divisor`. + + +### Partially execute a model: + +Sometimes you may want to obtain an intermediate tensor inside a model, +such as the input of certain layer, the output before post-processing. +Since there are typically hundreds of intermediate tensors, there isn't an API that provides you +the intermediate result you need. +You have the following options: + +1. Write a (sub)model. Following the [tutorial](./write-models.md), you can + rewrite a model component (e.g. a head of a model), such that it + does the same thing as the existing component, but returns the output + you need. +2. Partially execute a model. You can create the model as usual, + but use custom code to execute it instead of its `forward()`. For example, + the following code obtains mask features before mask head. + + ```python + images = ImageList.from_tensors(...) # preprocessed input tensor + model = build_model(cfg) + model.eval() + features = model.backbone(images.tensor) + proposals, _ = model.proposal_generator(images, features) + instances, _ = model.roi_heads(images, features, proposals) + mask_features = [features[f] for f in model.roi_heads.in_features] + mask_features = model.roi_heads.mask_pooler(mask_features, [x.pred_boxes for x in instances]) + ``` + +3. Use [forward hooks](https://pytorch.org/tutorials/beginner/former_torchies/nnft_tutorial.html#forward-and-backward-function-hooks). + Forward hooks can help you obtain inputs or outputs of a certain module. + If they are not exactly what you want, they can at least be used together with partial execution + to obtain other tensors. + +All options require you to read documentation and sometimes code +of the existing models to understand the internal logic, +in order to write code to obtain the internal tensors. diff --git a/approach/ovod/detectron2/docs/tutorials/training.md b/approach/ovod/detectron2/docs/tutorials/training.md new file mode 100644 index 0000000000000000000000000000000000000000..83a6cb0a8e38ca06bbf96201ac2595d2116523c3 --- /dev/null +++ b/approach/ovod/detectron2/docs/tutorials/training.md @@ -0,0 +1,67 @@ +# Training + +From the previous tutorials, you may now have a custom model and a data loader. +To run training, users typically have a preference in one of the following two styles: + +### Custom Training Loop + +With a model and a data loader ready, everything else needed to write a training loop can +be found in PyTorch, and you are free to write the training loop yourself. +This style allows researchers to manage the entire training logic more clearly and have full control. +One such example is provided in [tools/plain_train_net.py](../../tools/plain_train_net.py). + +Any customization on the training logic is then easily controlled by the user. + +### Trainer Abstraction + +We also provide a standardized "trainer" abstraction with a +hook system that helps simplify the standard training behavior. +It includes the following two instantiations: + +* [SimpleTrainer](../modules/engine.html#detectron2.engine.SimpleTrainer) + provides a minimal training loop for single-cost single-optimizer single-data-source training, with nothing else. + Other tasks (checkpointing, logging, etc) can be implemented using + [the hook system](../modules/engine.html#detectron2.engine.HookBase). +* [DefaultTrainer](../modules/engine.html#detectron2.engine.defaults.DefaultTrainer) is a `SimpleTrainer` initialized from a + yacs config, used by + [tools/train_net.py](../../tools/train_net.py) and many scripts. + It includes more standard default behaviors that one might want to opt in, + including default configurations for optimizer, learning rate schedule, + logging, evaluation, checkpointing etc. + +To customize a `DefaultTrainer`: + +1. For simple customizations (e.g. change optimizer, evaluator, LR scheduler, data loader, etc.), overwrite [its methods](../modules/engine.html#detectron2.engine.defaults.DefaultTrainer) in a subclass, just like [tools/train_net.py](../../tools/train_net.py). +2. For extra tasks during training, check the + [hook system](../modules/engine.html#detectron2.engine.HookBase) to see if it's supported. + + As an example, to print hello during training: + ```python + class HelloHook(HookBase): + def after_step(self): + if self.trainer.iter % 100 == 0: + print(f"Hello at iteration {self.trainer.iter}!") + ``` +3. Using a trainer+hook system means there will always be some non-standard behaviors that cannot be supported, especially in research. + For this reason, we intentionally keep the trainer & hook system minimal, rather than powerful. + If anything cannot be achieved by such a system, it's easier to start from [tools/plain_train_net.py](../../tools/plain_train_net.py) to implement custom training logic manually. + +### Logging of Metrics + +During training, detectron2 models and trainer put metrics to a centralized [EventStorage](../modules/utils.html#detectron2.utils.events.EventStorage). +You can use the following code to access it and log metrics to it: +```python +from detectron2.utils.events import get_event_storage + +# inside the model: +if self.training: + value = # compute the value from inputs + storage = get_event_storage() + storage.put_scalar("some_accuracy", value) +``` + +Refer to its documentation for more details. + +Metrics are then written to various destinations with [EventWriter](../modules/utils.html#module-detectron2.utils.events). +DefaultTrainer enables a few `EventWriter` with default configurations. +See above for how to customize them. diff --git a/approach/ovod/detectron2/docs/tutorials/write-models.md b/approach/ovod/detectron2/docs/tutorials/write-models.md new file mode 100644 index 0000000000000000000000000000000000000000..967d126503c71b419bca94615cb1090e1a79cb49 --- /dev/null +++ b/approach/ovod/detectron2/docs/tutorials/write-models.md @@ -0,0 +1,90 @@ +# Write Models + +If you are trying to do something completely new, you may wish to implement +a model entirely from scratch. However, in many situations you may +be interested in modifying or extending some components of an existing model. +Therefore, we also provide mechanisms that let users override the +behavior of certain internal components of standard models. + + +## Register New Components + +For common concepts that users often want to customize, such as "backbone feature extractor", "box head", +we provide a registration mechanism for users to inject custom implementation that +will be immediately available to use in config files. + +For example, to add a new backbone, import this code in your code: +```python +from detectron2.modeling import BACKBONE_REGISTRY, Backbone, ShapeSpec + +@BACKBONE_REGISTRY.register() +class ToyBackbone(Backbone): + def __init__(self, cfg, input_shape): + super().__init__() + # create your own backbone + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=16, padding=3) + + def forward(self, image): + return {"conv1": self.conv1(image)} + + def output_shape(self): + return {"conv1": ShapeSpec(channels=64, stride=16)} +``` + +In this code, we implement a new backbone following the interface of the +[Backbone](../modules/modeling.html#detectron2.modeling.Backbone) class, +and register it into the [BACKBONE_REGISTRY](../modules/modeling.html#detectron2.modeling.BACKBONE_REGISTRY) +which requires subclasses of `Backbone`. +After importing this code, detectron2 can link the name of the class to its implementation. Therefore you can write the following code: + +```python +cfg = ... # read a config +cfg.MODEL.BACKBONE.NAME = 'ToyBackbone' # or set it in the config file +model = build_model(cfg) # it will find `ToyBackbone` defined above +``` + +As another example, to add new abilities to the ROI heads in the Generalized R-CNN meta-architecture, +you can implement a new +[ROIHeads](../modules/modeling.html#detectron2.modeling.ROIHeads) subclass and put it in the `ROI_HEADS_REGISTRY`. +[DensePose](../../projects/DensePose) +and [MeshRCNN](https://github.com/facebookresearch/meshrcnn) +are two examples that implement new ROIHeads to perform new tasks. +And [projects/](../../projects/) +contains more examples that implement different architectures. + +A complete list of registries can be found in [API documentation](../modules/modeling.html#model-registries). +You can register components in these registries to customize different parts of a model, or the +entire model. + +## Construct Models with Explicit Arguments + +Registry is a bridge to connect names in config files to the actual code. +They are meant to cover a few main components that users frequently need to replace. +However, the capability of a text-based config file is sometimes limited and +some deeper customization may be available only through writing code. + +Most model components in detectron2 have a clear `__init__` interface that documents +what input arguments it needs. Calling them with custom arguments will give you a custom variant +of the model. + +As an example, to use __custom loss function__ in the box head of a Faster R-CNN, we can do the following: + +1. Losses are currently computed in [FastRCNNOutputLayers](../modules/modeling.html#detectron2.modeling.FastRCNNOutputLayers). + We need to implement a variant or a subclass of it, with custom loss functions, named `MyRCNNOutput`. +2. Call `StandardROIHeads` with `box_predictor=MyRCNNOutput()` argument instead of the builtin `FastRCNNOutputLayers`. + If all other arguments should stay unchanged, this can be easily achieved by using the [configurable `__init__`](../modules/config.html#detectron2.config.configurable) mechanism: + + ```python + roi_heads = StandardROIHeads( + cfg, backbone.output_shape(), + box_predictor=MyRCNNOutput(...) + ) + ``` +3. (optional) If we want to enable this new model from a config file, registration is needed: + ```python + @ROI_HEADS_REGISTRY.register() + class MyStandardROIHeads(StandardROIHeads): + def __init__(self, cfg, input_shape): + super().__init__(cfg, input_shape, + box_predictor=MyRCNNOutput(...)) + ``` diff --git a/approach/ovod/detectron2/projects/DeepLab/README.md b/approach/ovod/detectron2/projects/DeepLab/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bd03cf1c41f7b0358fb6988d6a387effbb328a50 --- /dev/null +++ b/approach/ovod/detectron2/projects/DeepLab/README.md @@ -0,0 +1,100 @@ +# DeepLab in Detectron2 + +In this repository, we implement DeepLabV3 and DeepLabV3+ in Detectron2. + +## Installation +Install Detectron2 following [the instructions](https://detectron2.readthedocs.io/tutorials/install.html). + +## Training + +To train a model with 8 GPUs run: +```bash +cd /path/to/detectron2/projects/DeepLab +python train_net.py --config-file configs/Cityscapes-SemanticSegmentation/deeplab_v3_plus_R_103_os16_mg124_poly_90k_bs16.yaml --num-gpus 8 +``` + +## Evaluation + +Model evaluation can be done similarly: +```bash +cd /path/to/detectron2/projects/DeepLab +python train_net.py --config-file configs/Cityscapes-SemanticSegmentation/deeplab_v3_plus_R_103_os16_mg124_poly_90k_bs16.yaml --eval-only MODEL.WEIGHTS /path/to/model_checkpoint +``` + +## Cityscapes Semantic Segmentation +Cityscapes models are trained with ImageNet pretraining. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MethodBackboneOutput
resolution		mIoUmodel iddownload
DeepLabV3R101-DC51024×2048 76.7 - -  |  -
DeepLabV3R103-DC51024×2048 78.5 28041665 model | metrics
DeepLabV3+R101-DC51024×2048 78.1 - -  |  -
DeepLabV3+R103-DC51024×2048 80.0 28054032model | metrics
+ +Note: +- [R103](https://dl.fbaipublicfiles.com/detectron2/DeepLab/R-103.pkl): a ResNet-101 with its first 7x7 convolution replaced by 3 3x3 convolutions. +This modification has been used in most semantic segmentation papers. We pre-train this backbone on ImageNet using the default recipe of [pytorch examples](https://github.com/pytorch/examples/tree/master/imagenet). +- DC5 means using dilated convolution in `res5`. + +## Citing DeepLab + +If you use DeepLab, please use the following BibTeX entry. + +* DeepLabv3+: + +``` +@inproceedings{deeplabv3plus2018, + title={Encoder-Decoder with Atrous Separable Convolution for Semantic Image Segmentation}, + author={Liang-Chieh Chen and Yukun Zhu and George Papandreou and Florian Schroff and Hartwig Adam}, + booktitle={ECCV}, + year={2018} +} +``` + +* DeepLabv3: + +``` +@article{deeplabv32018, + title={Rethinking atrous convolution for semantic image segmentation}, + author={Chen, Liang-Chieh and Papandreou, George and Schroff, Florian and Adam, Hartwig}, + journal={arXiv:1706.05587}, + year={2017} +} +``` diff --git a/approach/ovod/detectron2/projects/DeepLab/train_net.py b/approach/ovod/detectron2/projects/DeepLab/train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..d3414ddf8e7af49640dd1372d75df7acb0b8bb49 --- /dev/null +++ b/approach/ovod/detectron2/projects/DeepLab/train_net.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. + +""" +DeepLab Training Script. + +This script is a simplified version of the training script in detectron2/tools. +""" + +import os + +import detectron2.data.transforms as T +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import get_cfg +from detectron2.data import DatasetMapper, MetadataCatalog, build_detection_train_loader +from detectron2.engine import DefaultTrainer, default_argument_parser, default_setup, launch +from detectron2.evaluation import CityscapesSemSegEvaluator, DatasetEvaluators, SemSegEvaluator +from detectron2.projects.deeplab import add_deeplab_config, build_lr_scheduler + + +def build_sem_seg_train_aug(cfg): + augs = [ + T.ResizeShortestEdge( + cfg.INPUT.MIN_SIZE_TRAIN, cfg.INPUT.MAX_SIZE_TRAIN, cfg.INPUT.MIN_SIZE_TRAIN_SAMPLING + ) + ] + if cfg.INPUT.CROP.ENABLED: + augs.append( + T.RandomCrop_CategoryAreaConstraint( + cfg.INPUT.CROP.TYPE, + cfg.INPUT.CROP.SIZE, + cfg.INPUT.CROP.SINGLE_CATEGORY_MAX_AREA, + cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE, + ) + ) + augs.append(T.RandomFlip()) + return augs + + +class Trainer(DefaultTrainer): + """ + We use the "DefaultTrainer" which contains a number pre-defined logic for + standard training workflow. They may not work for you, especially if you + are working on a new research project. In that case you can use the cleaner + "SimpleTrainer", or write your own training loop. + """ + + @classmethod + def build_evaluator(cls, cfg, dataset_name, output_folder=None): + """ + Create evaluator(s) for a given dataset. + This uses the special metadata "evaluator_type" associated with each builtin dataset. + For your own dataset, you can simply create an evaluator manually in your + script and do not have to worry about the hacky if-else logic here. + """ + if output_folder is None: + output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") + evaluator_list = [] + evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type + if evaluator_type == "sem_seg": + return SemSegEvaluator( + dataset_name, + distributed=True, + output_dir=output_folder, + ) + if evaluator_type == "cityscapes_sem_seg": + return CityscapesSemSegEvaluator(dataset_name) + if len(evaluator_list) == 0: + raise NotImplementedError( + "no Evaluator for the dataset {} with the type {}".format( + dataset_name, evaluator_type + ) + ) + if len(evaluator_list) == 1: + return evaluator_list[0] + return DatasetEvaluators(evaluator_list) + + @classmethod + def build_train_loader(cls, cfg): + if "SemanticSegmentor" in cfg.MODEL.META_ARCHITECTURE: + mapper = DatasetMapper(cfg, is_train=True, augmentations=build_sem_seg_train_aug(cfg)) + else: + mapper = None + return build_detection_train_loader(cfg, mapper=mapper) + + @classmethod + def build_lr_scheduler(cls, cfg, optimizer): + """ + It now calls :func:`detectron2.solver.build_lr_scheduler`. + Overwrite it if you'd like a different scheduler. + """ + return build_lr_scheduler(cfg, optimizer) + + +def setup(args): + """ + Create configs and perform basic setups. + """ + cfg = get_cfg() + add_deeplab_config(cfg) + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + cfg.freeze() + default_setup(cfg, args) + return cfg + + +def main(args): + cfg = setup(args) + + if args.eval_only: + model = Trainer.build_model(cfg) + DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( + cfg.MODEL.WEIGHTS, resume=args.resume + ) + res = Trainer.test(cfg, model) + return res + + trainer = Trainer(cfg) + trainer.resume_or_load(resume=args.resume) + return trainer.train() + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + print("Command Line Args:", args) + launch( + main, + args.num_gpus, + num_machines=args.num_machines, + machine_rank=args.machine_rank, + dist_url=args.dist_url, + args=(args,), + ) diff --git a/approach/ovod/detectron2/projects/DensePose/README.md b/approach/ovod/detectron2/projects/DensePose/README.md new file mode 100644 index 0000000000000000000000000000000000000000..38f4f834adfcd5490a790a715b24c9ad26ab4dde --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/README.md @@ -0,0 +1,64 @@ +# DensePose in Detectron2 + +DensePose aims at learning and establishing dense correspondences between image pixels +and 3D object geometry for deformable objects, such as humans or animals. +In this repository, we provide the code to train and evaluate DensePose R-CNN and +various tools to visualize DensePose annotations and results. + +There are two main paradigms that are used within DensePose project. + +## [Chart-based Dense Pose Estimation for Humans and Animals](doc/DENSEPOSE_IUV.md) + +
+ +
+ +For chart-based estimation, 3D object mesh is split into charts and +for each pixel the model estimates chart index `I` and local chart coordinates `(U, V)`. +Please follow the link above to find a [detailed overview](doc/DENSEPOSE_IUV.md#Overview) +of the method, links to trained models along with their performance evaluation in the +[Model Zoo](doc/DENSEPOSE_IUV.md#ModelZoo) and +[references](doc/DENSEPOSE_IUV.md#References) to the corresponding papers. + +## [Continuous Surface Embeddings for Dense Pose Estimation for Humans and Animals](doc/DENSEPOSE_CSE.md) + +
+ +
+ +To establish continuous surface embeddings, the model simultaneously learns +descriptors for mesh vertices and for image pixels. +The embeddings are put into correspondence, thus the location +of each pixel on the 3D model is derived. +Please follow the link above to find a [detailed overview](doc/DENSEPOSE_CSE.md#Overview) +of the method, links to trained models along with their performance evaluation in the +[Model Zoo](doc/DENSEPOSE_CSE.md#ModelZoo) and +[references](doc/DENSEPOSE_CSE.md#References) to the corresponding papers. + +# Quick Start + +See [ Getting Started ](doc/GETTING_STARTED.md) + +# Model Zoo + +Please check the dedicated pages +for [chart-based model zoo](doc/DENSEPOSE_IUV.md#ModelZoo) +and for [continuous surface embeddings model zoo](doc/DENSEPOSE_CSE.md#ModelZoo). + +# What's New + +* June 2021: [DensePose CSE with Cycle Losses](doc/RELEASE_2021_06.md) +* March 2021: [DensePose CSE (a framework to extend DensePose to various categories using 3D models) + and DensePose Evolution (a framework to bootstrap DensePose on unlabeled data) released](doc/RELEASE_2021_03.md) +* April 2020: [DensePose Confidence Estimation and Model Zoo Improvements](doc/RELEASE_2020_04.md) + +# License + +Detectron2 is released under the [Apache 2.0 license](../../LICENSE) + +## Citing DensePose + +If you use DensePose, please refer to the BibTeX entries +for [chart-based models](doc/DENSEPOSE_IUV.md#References) +and for [continuous surface embeddings](doc/DENSEPOSE_CSE.md#References). + diff --git a/approach/ovod/detectron2/projects/DensePose/apply_net.py b/approach/ovod/detectron2/projects/DensePose/apply_net.py new file mode 100644 index 0000000000000000000000000000000000000000..6d3a51ee7befdde9c6aa066dffe0faeba89e3fb9 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/apply_net.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. + +import argparse +import glob +import logging +import os +import sys +from typing import Any, ClassVar, Dict, List +import torch + +from detectron2.config import CfgNode, get_cfg +from detectron2.data.detection_utils import read_image +from detectron2.engine.defaults import DefaultPredictor +from detectron2.structures.instances import Instances +from detectron2.utils.logger import setup_logger + +from densepose import add_densepose_config +from densepose.structures import DensePoseChartPredictorOutput, DensePoseEmbeddingPredictorOutput +from densepose.utils.logger import verbosity_to_level +from densepose.vis.base import CompoundVisualizer +from densepose.vis.bounding_box import ScoredBoundingBoxVisualizer +from densepose.vis.densepose_outputs_vertex import ( + DensePoseOutputsTextureVisualizer, + DensePoseOutputsVertexVisualizer, + get_texture_atlases, +) +from densepose.vis.densepose_results import ( + DensePoseResultsContourVisualizer, + DensePoseResultsFineSegmentationVisualizer, + DensePoseResultsUVisualizer, + DensePoseResultsVVisualizer, +) +from densepose.vis.densepose_results_textures import ( + DensePoseResultsVisualizerWithTexture, + get_texture_atlas, +) +from densepose.vis.extractor import ( + CompoundExtractor, + DensePoseOutputsExtractor, + DensePoseResultExtractor, + create_extractor, +) + +DOC = """Apply Net - a tool to print / visualize DensePose results +""" + +LOGGER_NAME = "apply_net" +logger = logging.getLogger(LOGGER_NAME) + +_ACTION_REGISTRY: Dict[str, "Action"] = {} + + +class Action(object): + @classmethod + def add_arguments(cls: type, parser: argparse.ArgumentParser): + parser.add_argument( + "-v", + "--verbosity", + action="count", + help="Verbose mode. Multiple -v options increase the verbosity.", + ) + + +def register_action(cls: type): + """ + Decorator for action classes to automate action registration + """ + global _ACTION_REGISTRY + _ACTION_REGISTRY[cls.COMMAND] = cls + return cls + + +class InferenceAction(Action): + @classmethod + def add_arguments(cls: type, parser: argparse.ArgumentParser): + super(InferenceAction, cls).add_arguments(parser) + parser.add_argument("cfg", metavar="", help="Config file") + parser.add_argument("model", metavar="", help="Model file") + parser.add_argument("input", metavar="", help="Input data") + parser.add_argument( + "--opts", + help="Modify config options using the command-line 'KEY VALUE' pairs", + default=[], + nargs=argparse.REMAINDER, + ) + + @classmethod + def execute(cls: type, args: argparse.Namespace): + logger.info(f"Loading config from {args.cfg}") + opts = [] + cfg = cls.setup_config(args.cfg, args.model, args, opts) + logger.info(f"Loading model from {args.model}") + predictor = DefaultPredictor(cfg) + logger.info(f"Loading data from {args.input}") + file_list = cls._get_input_file_list(args.input) + if len(file_list) == 0: + logger.warning(f"No input images for {args.input}") + return + context = cls.create_context(args, cfg) + for file_name in file_list: + img = read_image(file_name, format="BGR") # predictor expects BGR image. + with torch.no_grad(): + outputs = predictor(img)["instances"] + cls.execute_on_outputs(context, {"file_name": file_name, "image": img}, outputs) + cls.postexecute(context) + + @classmethod + def setup_config( + cls: type, config_fpath: str, model_fpath: str, args: argparse.Namespace, opts: List[str] + ): + cfg = get_cfg() + add_densepose_config(cfg) + cfg.merge_from_file(config_fpath) + cfg.merge_from_list(args.opts) + if opts: + cfg.merge_from_list(opts) + cfg.MODEL.WEIGHTS = model_fpath + cfg.freeze() + return cfg + + @classmethod + def _get_input_file_list(cls: type, input_spec: str): + if os.path.isdir(input_spec): + file_list = [ + os.path.join(input_spec, fname) + for fname in os.listdir(input_spec) + if os.path.isfile(os.path.join(input_spec, fname)) + ] + elif os.path.isfile(input_spec): + file_list = [input_spec] + else: + file_list = glob.glob(input_spec) + return file_list + + +@register_action +class DumpAction(InferenceAction): + """ + Dump action that outputs results to a pickle file + """ + + COMMAND: ClassVar[str] = "dump" + + @classmethod + def add_parser(cls: type, subparsers: argparse._SubParsersAction): + parser = subparsers.add_parser(cls.COMMAND, help="Dump model outputs to a file.") + cls.add_arguments(parser) + parser.set_defaults(func=cls.execute) + + @classmethod + def add_arguments(cls: type, parser: argparse.ArgumentParser): + super(DumpAction, cls).add_arguments(parser) + parser.add_argument( + "--output", + metavar="", + default="results.pkl", + help="File name to save dump to", + ) + + @classmethod + def execute_on_outputs( + cls: type, context: Dict[str, Any], entry: Dict[str, Any], outputs: Instances + ): + image_fpath = entry["file_name"] + logger.info(f"Processing {image_fpath}") + result = {"file_name": image_fpath} + if outputs.has("scores"): + result["scores"] = outputs.get("scores").cpu() + if outputs.has("pred_boxes"): + result["pred_boxes_XYXY"] = outputs.get("pred_boxes").tensor.cpu() + if outputs.has("pred_densepose"): + if isinstance(outputs.pred_densepose, DensePoseChartPredictorOutput): + extractor = DensePoseResultExtractor() + elif isinstance(outputs.pred_densepose, DensePoseEmbeddingPredictorOutput): + extractor = DensePoseOutputsExtractor() + result["pred_densepose"] = extractor(outputs)[0] + context["results"].append(result) + + @classmethod + def create_context(cls: type, args: argparse.Namespace, cfg: CfgNode): + context = {"results": [], "out_fname": args.output} + return context + + @classmethod + def postexecute(cls: type, context: Dict[str, Any]): + out_fname = context["out_fname"] + out_dir = os.path.dirname(out_fname) + if len(out_dir) > 0 and not os.path.exists(out_dir): + os.makedirs(out_dir) + with open(out_fname, "wb") as hFile: + torch.save(context["results"], hFile) + logger.info(f"Output saved to {out_fname}") + + +@register_action +class ShowAction(InferenceAction): + """ + Show action that visualizes selected entries on an image + """ + + COMMAND: ClassVar[str] = "show" + VISUALIZERS: ClassVar[Dict[str, object]] = { + "dp_contour": DensePoseResultsContourVisualizer, + "dp_segm": DensePoseResultsFineSegmentationVisualizer, + "dp_u": DensePoseResultsUVisualizer, + "dp_v": DensePoseResultsVVisualizer, + "dp_iuv_texture": DensePoseResultsVisualizerWithTexture, + "dp_cse_texture": DensePoseOutputsTextureVisualizer, + "dp_vertex": DensePoseOutputsVertexVisualizer, + "bbox": ScoredBoundingBoxVisualizer, + } + + @classmethod + def add_parser(cls: type, subparsers: argparse._SubParsersAction): + parser = subparsers.add_parser(cls.COMMAND, help="Visualize selected entries") + cls.add_arguments(parser) + parser.set_defaults(func=cls.execute) + + @classmethod + def add_arguments(cls: type, parser: argparse.ArgumentParser): + super(ShowAction, cls).add_arguments(parser) + parser.add_argument( + "visualizations", + metavar="", + help="Comma separated list of visualizations, possible values: " + "[{}]".format(",".join(sorted(cls.VISUALIZERS.keys()))), + ) + parser.add_argument( + "--min_score", + metavar="", + default=0.8, + type=float, + help="Minimum detection score to visualize", + ) + parser.add_argument( + "--nms_thresh", metavar="", default=None, type=float, help="NMS threshold" + ) + parser.add_argument( + "--texture_atlas", + metavar="", + default=None, + help="Texture atlas file (for IUV texture transfer)", + ) + parser.add_argument( + "--texture_atlases_map", + metavar="", + default=None, + help="JSON string of a dict containing texture atlas files for each mesh", + ) + parser.add_argument( + "--output", + metavar="", + default="outputres.png", + help="File name to save output to", + ) + + @classmethod + def setup_config( + cls: type, config_fpath: str, model_fpath: str, args: argparse.Namespace, opts: List[str] + ): + opts.append("MODEL.ROI_HEADS.SCORE_THRESH_TEST") + opts.append(str(args.min_score)) + if args.nms_thresh is not None: + opts.append("MODEL.ROI_HEADS.NMS_THRESH_TEST") + opts.append(str(args.nms_thresh)) + cfg = super(ShowAction, cls).setup_config(config_fpath, model_fpath, args, opts) + return cfg + + @classmethod + def execute_on_outputs( + cls: type, context: Dict[str, Any], entry: Dict[str, Any], outputs: Instances + ): + import cv2 + import numpy as np + + visualizer = context["visualizer"] + extractor = context["extractor"] + image_fpath = entry["file_name"] + logger.info(f"Processing {image_fpath}") + image = cv2.cvtColor(entry["image"], cv2.COLOR_BGR2GRAY) + image = np.tile(image[:, :, np.newaxis], [1, 1, 3]) + data = extractor(outputs) + image_vis = visualizer.visualize(image, data) + entry_idx = context["entry_idx"] + 1 + out_fname = cls._get_out_fname(entry_idx, context["out_fname"]) + out_dir = os.path.dirname(out_fname) + if len(out_dir) > 0 and not os.path.exists(out_dir): + os.makedirs(out_dir) + cv2.imwrite(out_fname, image_vis) + logger.info(f"Output saved to {out_fname}") + context["entry_idx"] += 1 + + @classmethod + def postexecute(cls: type, context: Dict[str, Any]): + pass + + @classmethod + def _get_out_fname(cls: type, entry_idx: int, fname_base: str): + base, ext = os.path.splitext(fname_base) + return base + ".{0:04d}".format(entry_idx) + ext + + @classmethod + def create_context(cls: type, args: argparse.Namespace, cfg: CfgNode) -> Dict[str, Any]: + vis_specs = args.visualizations.split(",") + visualizers = [] + extractors = [] + for vis_spec in vis_specs: + texture_atlas = get_texture_atlas(args.texture_atlas) + texture_atlases_dict = get_texture_atlases(args.texture_atlases_map) + vis = cls.VISUALIZERS[vis_spec]( + cfg=cfg, + texture_atlas=texture_atlas, + texture_atlases_dict=texture_atlases_dict, + ) + visualizers.append(vis) + extractor = create_extractor(vis) + extractors.append(extractor) + visualizer = CompoundVisualizer(visualizers) + extractor = CompoundExtractor(extractors) + context = { + "extractor": extractor, + "visualizer": visualizer, + "out_fname": args.output, + "entry_idx": 0, + } + return context + + +def create_argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=DOC, + formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=120), + ) + parser.set_defaults(func=lambda _: parser.print_help(sys.stdout)) + subparsers = parser.add_subparsers(title="Actions") + for _, action in _ACTION_REGISTRY.items(): + action.add_parser(subparsers) + return parser + + +def main(): + parser = create_argument_parser() + args = parser.parse_args() + verbosity = args.verbosity if hasattr(args, "verbosity") else None + global logger + logger = setup_logger(name=LOGGER_NAME) + logger.setLevel(verbosity_to_level(verbosity)) + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/approach/ovod/detectron2/projects/DensePose/configs/Base-DensePose-RCNN-FPN.yaml b/approach/ovod/detectron2/projects/DensePose/configs/Base-DensePose-RCNN-FPN.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1579187a7004e716eb3a86dbbfebb092d7aca84b --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/Base-DensePose-RCNN-FPN.yaml @@ -0,0 +1,48 @@ +VERSION: 2 +MODEL: + META_ARCHITECTURE: "GeneralizedRCNN" + BACKBONE: + NAME: "build_resnet_fpn_backbone" + RESNETS: + OUT_FEATURES: ["res2", "res3", "res4", "res5"] + FPN: + IN_FEATURES: ["res2", "res3", "res4", "res5"] + ANCHOR_GENERATOR: + SIZES: [[32], [64], [128], [256], [512]] # One size for each in feature map + ASPECT_RATIOS: [[0.5, 1.0, 2.0]] # Three aspect ratios (same for all in feature maps) + RPN: + IN_FEATURES: ["p2", "p3", "p4", "p5", "p6"] + PRE_NMS_TOPK_TRAIN: 2000 # Per FPN level + PRE_NMS_TOPK_TEST: 1000 # Per FPN level + # Detectron1 uses 2000 proposals per-batch, + # (See "modeling/rpn/rpn_outputs.py" for details of this legacy issue) + # which is approximately 1000 proposals per-image since the default batch size for FPN is 2. + POST_NMS_TOPK_TRAIN: 1000 + POST_NMS_TOPK_TEST: 1000 + + DENSEPOSE_ON: True + ROI_HEADS: + NAME: "DensePoseROIHeads" + IN_FEATURES: ["p2", "p3", "p4", "p5"] + NUM_CLASSES: 1 + ROI_BOX_HEAD: + NAME: "FastRCNNConvFCHead" + NUM_FC: 2 + POOLER_RESOLUTION: 7 + POOLER_SAMPLING_RATIO: 2 + POOLER_TYPE: "ROIAlign" + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseV1ConvXHead" + POOLER_TYPE: "ROIAlign" + NUM_COARSE_SEGM_CHANNELS: 2 +DATASETS: + TRAIN: ("densepose_coco_2014_train", "densepose_coco_2014_valminusminival") + TEST: ("densepose_coco_2014_minival",) +SOLVER: + IMS_PER_BATCH: 16 + BASE_LR: 0.01 + STEPS: (60000, 80000) + MAX_ITER: 90000 + WARMUP_FACTOR: 0.1 +INPUT: + MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/HRNet/densepose_rcnn_HRFPN_HRNet_w32_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/HRNet/densepose_rcnn_HRFPN_HRNet_w32_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36eabfed984b360907f5782d4e8b0232784f8a40 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/HRNet/densepose_rcnn_HRFPN_HRNet_w32_s1x.yaml @@ -0,0 +1,16 @@ +_BASE_: "../Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "https://1drv.ms/u/s!Aus8VCZ_C_33dYBMemi9xOUFR0w" + BACKBONE: + NAME: "build_hrfpn_backbone" + RPN: + IN_FEATURES: ['p1', 'p2', 'p3', 'p4', 'p5'] + ROI_HEADS: + IN_FEATURES: ['p1', 'p2', 'p3', 'p4', 'p5'] +SOLVER: + MAX_ITER: 130000 + STEPS: (100000, 120000) + CLIP_GRADIENTS: + ENABLED: True + CLIP_TYPE: "norm" + BASE_LR: 0.03 diff --git a/approach/ovod/detectron2/projects/DensePose/configs/HRNet/densepose_rcnn_HRFPN_HRNet_w40_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/HRNet/densepose_rcnn_HRFPN_HRNet_w40_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ca8085e154c40a5b0f42a17575d2d48328619f0 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/HRNet/densepose_rcnn_HRFPN_HRNet_w40_s1x.yaml @@ -0,0 +1,23 @@ +_BASE_: "../Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "https://1drv.ms/u/s!Aus8VCZ_C_33ck0gvo5jfoWBOPo" + BACKBONE: + NAME: "build_hrfpn_backbone" + RPN: + IN_FEATURES: ['p1', 'p2', 'p3', 'p4', 'p5'] + ROI_HEADS: + IN_FEATURES: ['p1', 'p2', 'p3', 'p4', 'p5'] + HRNET: + STAGE2: + NUM_CHANNELS: [40, 80] + STAGE3: + NUM_CHANNELS: [40, 80, 160] + STAGE4: + NUM_CHANNELS: [40, 80, 160, 320] +SOLVER: + MAX_ITER: 130000 + STEPS: (100000, 120000) + CLIP_GRADIENTS: + ENABLED: True + CLIP_TYPE: "norm" + BASE_LR: 0.03 diff --git a/approach/ovod/detectron2/projects/DensePose/configs/HRNet/densepose_rcnn_HRFPN_HRNet_w48_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/HRNet/densepose_rcnn_HRFPN_HRNet_w48_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3f437ab57ae0ff48cd4a97cbda987346f9a5a24 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/HRNet/densepose_rcnn_HRFPN_HRNet_w48_s1x.yaml @@ -0,0 +1,23 @@ +_BASE_: "../Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "https://1drv.ms/u/s!Aus8VCZ_C_33dKvqI6pBZlifgJk" + BACKBONE: + NAME: "build_hrfpn_backbone" + RPN: + IN_FEATURES: ['p1', 'p2', 'p3', 'p4', 'p5'] + ROI_HEADS: + IN_FEATURES: ['p1', 'p2', 'p3', 'p4', 'p5'] + HRNET: + STAGE2: + NUM_CHANNELS: [48, 96] + STAGE3: + NUM_CHANNELS: [48, 96, 192] + STAGE4: + NUM_CHANNELS: [48, 96, 192, 384] +SOLVER: + MAX_ITER: 130000 + STEPS: (100000, 120000) + CLIP_GRADIENTS: + ENABLED: True + CLIP_TYPE: "norm" + BASE_LR: 0.03 diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/Base-DensePose-RCNN-FPN-Human.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/Base-DensePose-RCNN-FPN-Human.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e92340ee0cdba2abd0a35114cbf3e78b04435dfe --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/Base-DensePose-RCNN-FPN-Human.yaml @@ -0,0 +1,20 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + ROI_DENSEPOSE_HEAD: + CSE: + EMBEDDERS: + "smpl_27554": + TYPE: vertex_feature + NUM_VERTICES: 27554 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_smpl_27554_256.pkl" +DATASETS: + TRAIN: + - "densepose_coco_2014_train_cse" + - "densepose_coco_2014_valminusminival_cse" + TEST: + - "densepose_coco_2014_minival_cse" + CLASS_TO_MESH_NAME_MAPPING: + "0": "smpl_27554" diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/Base-DensePose-RCNN-FPN.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/Base-DensePose-RCNN-FPN.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de3b26009bdee95666248f99cd243fe37e7fd8bd --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/Base-DensePose-RCNN-FPN.yaml @@ -0,0 +1,60 @@ +VERSION: 2 +MODEL: + META_ARCHITECTURE: "GeneralizedRCNN" + BACKBONE: + NAME: "build_resnet_fpn_backbone" + RESNETS: + OUT_FEATURES: ["res2", "res3", "res4", "res5"] + FPN: + IN_FEATURES: ["res2", "res3", "res4", "res5"] + ANCHOR_GENERATOR: + SIZES: [[32], [64], [128], [256], [512]] # One size for each in feature map + ASPECT_RATIOS: [[0.5, 1.0, 2.0]] # Three aspect ratios (same for all in feature maps) + RPN: + IN_FEATURES: ["p2", "p3", "p4", "p5", "p6"] + PRE_NMS_TOPK_TRAIN: 2000 # Per FPN level + PRE_NMS_TOPK_TEST: 1000 # Per FPN level + # Detectron1 uses 2000 proposals per-batch, + # (See "modeling/rpn/rpn_outputs.py" for details of this legacy issue) + # which is approximately 1000 proposals per-image since the default batch size for FPN is 2. + POST_NMS_TOPK_TRAIN: 1000 + POST_NMS_TOPK_TEST: 1000 + + DENSEPOSE_ON: True + ROI_HEADS: + NAME: "DensePoseROIHeads" + IN_FEATURES: ["p2", "p3", "p4", "p5"] + NUM_CLASSES: 1 + ROI_BOX_HEAD: + NAME: "FastRCNNConvFCHead" + NUM_FC: 2 + POOLER_RESOLUTION: 7 + POOLER_SAMPLING_RATIO: 2 + POOLER_TYPE: "ROIAlign" + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseV1ConvXHead" + POOLER_TYPE: "ROIAlign" + NUM_COARSE_SEGM_CHANNELS: 2 + PREDICTOR_NAME: "DensePoseEmbeddingPredictor" + LOSS_NAME: "DensePoseCseLoss" + CSE: + # embedding loss, possible values: + # - "EmbeddingLoss" + # - "SoftEmbeddingLoss" + EMBED_LOSS_NAME: "EmbeddingLoss" +SOLVER: + IMS_PER_BATCH: 16 + BASE_LR: 0.01 + STEPS: (60000, 80000) + MAX_ITER: 90000 + WARMUP_FACTOR: 0.1 + CLIP_GRADIENTS: + CLIP_TYPE: norm + CLIP_VALUE: 1.0 + ENABLED: true + NORM_TYPE: 2.0 +INPUT: + MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800) +DENSEPOSE_EVALUATION: + TYPE: cse + STORAGE: file diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_101_FPN_DL_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_101_FPN_DL_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69d858902671e683b884b32c3c1448a44dc3995e --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_101_FPN_DL_s1x.yaml @@ -0,0 +1,12 @@ +_BASE_: "Base-DensePose-RCNN-FPN-Human.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-101.pkl" + RESNETS: + DEPTH: 101 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + CSE: + EMBED_LOSS_NAME: "EmbeddingLoss" +SOLVER: + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_101_FPN_DL_soft_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_101_FPN_DL_soft_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..141657cdab24a2f591eeef763aef29543c43108e --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_101_FPN_DL_soft_s1x.yaml @@ -0,0 +1,12 @@ +_BASE_: "Base-DensePose-RCNN-FPN-Human.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-101.pkl" + RESNETS: + DEPTH: 101 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + CSE: + EMBED_LOSS_NAME: "SoftEmbeddingLoss" +SOLVER: + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_101_FPN_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_101_FPN_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d2eea1e2c3cecc7bba1bfd6f2332227bd3d0f5ed --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_101_FPN_s1x.yaml @@ -0,0 +1,12 @@ +_BASE_: "Base-DensePose-RCNN-FPN-Human.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-101.pkl" + RESNETS: + DEPTH: 101 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseV1ConvXHead" + CSE: + EMBED_LOSS_NAME: "EmbeddingLoss" +SOLVER: + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_101_FPN_soft_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_101_FPN_soft_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1c362e1f9e93f9b9b458532f5318518396404d9f --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_101_FPN_soft_s1x.yaml @@ -0,0 +1,12 @@ +_BASE_: "Base-DensePose-RCNN-FPN-Human.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-101.pkl" + RESNETS: + DEPTH: 101 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseV1ConvXHead" + CSE: + EMBED_LOSS_NAME: "SoftEmbeddingLoss" +SOLVER: + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_DL_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_DL_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26684deaa9c72aab1408dbe3abb6ac3a9b6a17ac --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_DL_s1x.yaml @@ -0,0 +1,12 @@ +_BASE_: "Base-DensePose-RCNN-FPN-Human.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + CSE: + EMBED_LOSS_NAME: "EmbeddingLoss" +SOLVER: + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_DL_soft_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_DL_soft_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b53501d29b84e9ff4088ce98bc83688e89e546ed --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_DL_soft_s1x.yaml @@ -0,0 +1,12 @@ +_BASE_: "Base-DensePose-RCNN-FPN-Human.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + CSE: + EMBED_LOSS_NAME: "SoftEmbeddingLoss" +SOLVER: + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c186625a86cc76441b9edeefeabd7caf44af7755 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_s1x.yaml @@ -0,0 +1,12 @@ +_BASE_: "Base-DensePose-RCNN-FPN-Human.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseV1ConvXHead" + CSE: + EMBED_LOSS_NAME: "EmbeddingLoss" +SOLVER: + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_CA_finetune_16k.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_CA_finetune_16k.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69ab22669e2176b6ec661fc982be7412abb5e0e8 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_CA_finetune_16k.yaml @@ -0,0 +1,133 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "https://dl.fbaipublicfiles.com/densepose/cse/densepose_rcnn_R_50_FPN_soft_s1x/250533982/model_final_2c4512.pkl" + RESNETS: + DEPTH: 50 + ROI_HEADS: + NUM_CLASSES: 1 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseV1ConvXHead" + COARSE_SEGM_TRAINED_BY_MASKS: True + CSE: + EMBED_LOSS_NAME: "SoftEmbeddingLoss" + EMBEDDING_DIST_GAUSS_SIGMA: 0.1 + GEODESIC_DIST_GAUSS_SIGMA: 0.1 + EMBEDDERS: + "cat_7466": + TYPE: vertex_feature + NUM_VERTICES: 7466 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cat_7466_256.pkl" + "dog_7466": + TYPE: vertex_feature + NUM_VERTICES: 7466 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_dog_7466_256.pkl" + "sheep_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_sheep_5004_256.pkl" + "horse_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_horse_5004_256.pkl" + "zebra_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_zebra_5002_256.pkl" + "giraffe_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_giraffe_5002_256.pkl" + "elephant_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_elephant_5002_256.pkl" + "cow_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cow_5002_256.pkl" + "bear_4936": + TYPE: vertex_feature + NUM_VERTICES: 4936 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_bear_4936_256.pkl" +DATASETS: + TRAIN: + - "densepose_lvis_v1_ds2_train_v1" + TEST: + - "densepose_lvis_v1_ds2_val_v1" + WHITELISTED_CATEGORIES: + "densepose_lvis_v1_ds2_train_v1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + "densepose_lvis_v1_ds2_val_v1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + CATEGORY_MAPS: + "densepose_lvis_v1_ds2_train_v1": + "1202": 943 # zebra -> sheep + "569": 943 # horse -> sheep + "496": 943 # giraffe -> sheep + "422": 943 # elephant -> sheep + "80": 943 # cow -> sheep + "76": 943 # bear -> sheep + "225": 943 # cat -> sheep + "378": 943 # dog -> sheep + "densepose_lvis_v1_ds2_val_v1": + "1202": 943 # zebra -> sheep + "569": 943 # horse -> sheep + "496": 943 # giraffe -> sheep + "422": 943 # elephant -> sheep + "80": 943 # cow -> sheep + "76": 943 # bear -> sheep + "225": 943 # cat -> sheep + "378": 943 # dog -> sheep + CLASS_TO_MESH_NAME_MAPPING: + # Note: different classes are mapped to a single class + # mesh is chosen based on GT data, so this is just some + # value which has no particular meaning + "0": "sheep_5004" +SOLVER: + MAX_ITER: 16000 + STEPS: (12000, 14000) +DENSEPOSE_EVALUATION: + EVALUATE_MESH_ALIGNMENT: True diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_CA_finetune_4k.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_CA_finetune_4k.yaml new file mode 100644 index 0000000000000000000000000000000000000000..921a9c125d9da982fb88172acc7825ba3c583370 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_CA_finetune_4k.yaml @@ -0,0 +1,133 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "https://dl.fbaipublicfiles.com/densepose/cse/densepose_rcnn_R_50_FPN_soft_s1x/250533982/model_final_2c4512.pkl" + RESNETS: + DEPTH: 50 + ROI_HEADS: + NUM_CLASSES: 1 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseV1ConvXHead" + COARSE_SEGM_TRAINED_BY_MASKS: True + CSE: + EMBED_LOSS_NAME: "SoftEmbeddingLoss" + EMBEDDING_DIST_GAUSS_SIGMA: 0.1 + GEODESIC_DIST_GAUSS_SIGMA: 0.1 + EMBEDDERS: + "cat_5001": + TYPE: vertex_feature + NUM_VERTICES: 5001 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cat_5001_256.pkl" + "dog_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_dog_5002_256.pkl" + "sheep_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_sheep_5004_256.pkl" + "horse_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_horse_5004_256.pkl" + "zebra_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_zebra_5002_256.pkl" + "giraffe_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_giraffe_5002_256.pkl" + "elephant_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_elephant_5002_256.pkl" + "cow_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cow_5002_256.pkl" + "bear_4936": + TYPE: vertex_feature + NUM_VERTICES: 4936 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_bear_4936_256.pkl" +DATASETS: + TRAIN: + - "densepose_lvis_v1_ds1_train_v1" + TEST: + - "densepose_lvis_v1_ds1_val_v1" + WHITELISTED_CATEGORIES: + "densepose_lvis_v1_ds1_train_v1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + "densepose_lvis_v1_ds1_val_v1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + CATEGORY_MAPS: + "densepose_lvis_v1_ds1_train_v1": + "1202": 943 # zebra -> sheep + "569": 943 # horse -> sheep + "496": 943 # giraffe -> sheep + "422": 943 # elephant -> sheep + "80": 943 # cow -> sheep + "76": 943 # bear -> sheep + "225": 943 # cat -> sheep + "378": 943 # dog -> sheep + "densepose_lvis_v1_ds1_val_v1": + "1202": 943 # zebra -> sheep + "569": 943 # horse -> sheep + "496": 943 # giraffe -> sheep + "422": 943 # elephant -> sheep + "80": 943 # cow -> sheep + "76": 943 # bear -> sheep + "225": 943 # cat -> sheep + "378": 943 # dog -> sheep + CLASS_TO_MESH_NAME_MAPPING: + # Note: different classes are mapped to a single class + # mesh is chosen based on GT data, so this is just some + # value which has no particular meaning + "0": "sheep_5004" +SOLVER: + MAX_ITER: 4000 + STEPS: (3000, 3500) +DENSEPOSE_EVALUATION: + EVALUATE_MESH_ALIGNMENT: True diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_I0_finetune_16k.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_I0_finetune_16k.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b5a098d171e508fcb9dd8088ecc1799c3068efc --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_I0_finetune_16k.yaml @@ -0,0 +1,119 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "https://dl.fbaipublicfiles.com/densepose/cse/densepose_rcnn_R_50_FPN_soft_animals_finetune_maskonly_24k/270668502/model_final_21b1d2.pkl" + RESNETS: + DEPTH: 50 + ROI_HEADS: + NUM_CLASSES: 9 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseV1ConvXHead" + COARSE_SEGM_TRAINED_BY_MASKS: True + CSE: + EMBED_LOSS_NAME: "SoftEmbeddingLoss" + EMBEDDING_DIST_GAUSS_SIGMA: 0.1 + GEODESIC_DIST_GAUSS_SIGMA: 0.1 + EMBEDDERS: + "cat_7466": + TYPE: vertex_feature + NUM_VERTICES: 7466 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cat_7466_256.pkl" + "dog_7466": + TYPE: vertex_feature + NUM_VERTICES: 7466 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_dog_7466_256.pkl" + "sheep_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_sheep_5004_256.pkl" + "horse_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_horse_5004_256.pkl" + "zebra_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_zebra_5002_256.pkl" + "giraffe_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_giraffe_5002_256.pkl" + "elephant_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_elephant_5002_256.pkl" + "cow_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cow_5002_256.pkl" + "bear_4936": + TYPE: vertex_feature + NUM_VERTICES: 4936 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_bear_4936_256.pkl" +DATASETS: + TRAIN: + - "densepose_lvis_v1_ds2_train_v1" + TEST: + - "densepose_lvis_v1_ds2_val_v1" + WHITELISTED_CATEGORIES: + "densepose_lvis_v1_ds2_train_v1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + "densepose_lvis_v1_ds2_val_v1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + CLASS_TO_MESH_NAME_MAPPING: + "0": "bear_4936" + "1": "cow_5002" + "2": "cat_7466" + "3": "dog_7466" + "4": "elephant_5002" + "5": "giraffe_5002" + "6": "horse_5004" + "7": "sheep_5004" + "8": "zebra_5002" +SOLVER: + MAX_ITER: 16000 + STEPS: (12000, 14000) +DENSEPOSE_EVALUATION: + EVALUATE_MESH_ALIGNMENT: True diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_I0_finetune_i2m_16k.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_I0_finetune_i2m_16k.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18d6dacf4b62e609aa85735a87daa8d2506000d7 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_I0_finetune_i2m_16k.yaml @@ -0,0 +1,121 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "https://dl.fbaipublicfiles.com/densepose/cse/densepose_rcnn_R_50_FPN_soft_animals_finetune_maskonly_24k/270668502/model_final_21b1d2.pkl" + RESNETS: + DEPTH: 50 + ROI_HEADS: + NUM_CLASSES: 9 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseV1ConvXHead" + COARSE_SEGM_TRAINED_BY_MASKS: True + CSE: + EMBED_LOSS_NAME: "SoftEmbeddingLoss" + EMBEDDING_DIST_GAUSS_SIGMA: 0.1 + GEODESIC_DIST_GAUSS_SIGMA: 0.1 + PIX_TO_SHAPE_CYCLE_LOSS: + ENABLED: True + EMBEDDERS: + "cat_7466": + TYPE: vertex_feature + NUM_VERTICES: 7466 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cat_7466_256.pkl" + "dog_7466": + TYPE: vertex_feature + NUM_VERTICES: 7466 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_dog_7466_256.pkl" + "sheep_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_sheep_5004_256.pkl" + "horse_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_horse_5004_256.pkl" + "zebra_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_zebra_5002_256.pkl" + "giraffe_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_giraffe_5002_256.pkl" + "elephant_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_elephant_5002_256.pkl" + "cow_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cow_5002_256.pkl" + "bear_4936": + TYPE: vertex_feature + NUM_VERTICES: 4936 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_bear_4936_256.pkl" +DATASETS: + TRAIN: + - "densepose_lvis_v1_ds2_train_v1" + TEST: + - "densepose_lvis_v1_ds2_val_v1" + WHITELISTED_CATEGORIES: + "densepose_lvis_v1_ds2_train_v1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + "densepose_lvis_v1_ds2_val_v1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + CLASS_TO_MESH_NAME_MAPPING: + "0": "bear_4936" + "1": "cow_5002" + "2": "cat_7466" + "3": "dog_7466" + "4": "elephant_5002" + "5": "giraffe_5002" + "6": "horse_5004" + "7": "sheep_5004" + "8": "zebra_5002" +SOLVER: + MAX_ITER: 16000 + STEPS: (12000, 14000) +DENSEPOSE_EVALUATION: + EVALUATE_MESH_ALIGNMENT: True diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_I0_finetune_m2m_16k.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_I0_finetune_m2m_16k.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b798ae21204b9310adae33040c870253edc68ee --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_I0_finetune_m2m_16k.yaml @@ -0,0 +1,138 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "https://dl.fbaipublicfiles.com/densepose/cse/densepose_rcnn_R_50_FPN_soft_animals_finetune_maskonly_24k/267687159/model_final_354e61.pkl" + RESNETS: + DEPTH: 50 + ROI_HEADS: + NUM_CLASSES: 9 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseV1ConvXHead" + COARSE_SEGM_TRAINED_BY_MASKS: True + CSE: + EMBED_LOSS_NAME: "SoftEmbeddingLoss" + EMBEDDING_DIST_GAUSS_SIGMA: 0.1 + GEODESIC_DIST_GAUSS_SIGMA: 0.1 + SHAPE_TO_SHAPE_CYCLE_LOSS: + ENABLED: True + EMBEDDERS: + "cat_7466": + TYPE: vertex_feature + NUM_VERTICES: 7466 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cat_7466_256.pkl" + "dog_7466": + TYPE: vertex_feature + NUM_VERTICES: 7466 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_dog_7466_256.pkl" + "sheep_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_sheep_5004_256.pkl" + "horse_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_horse_5004_256.pkl" + "zebra_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_zebra_5002_256.pkl" + "giraffe_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_giraffe_5002_256.pkl" + "elephant_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_elephant_5002_256.pkl" + "cow_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cow_5002_256.pkl" + "bear_4936": + TYPE: vertex_feature + NUM_VERTICES: 4936 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_bear_4936_256.pkl" + "smpl_27554": + TYPE: vertex_feature + NUM_VERTICES: 27554 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_smpl_27554_256.pkl" +DATASETS: + TRAIN: + - "densepose_lvis_v1_ds2_train_v1" + TEST: + - "densepose_lvis_v1_ds2_val_v1" + WHITELISTED_CATEGORIES: + "densepose_lvis_v1_ds2_train_v1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + "densepose_lvis_v1_ds2_val_v1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + CLASS_TO_MESH_NAME_MAPPING: + "0": "bear_4936" + "1": "cow_5002" + "2": "cat_7466" + "3": "dog_7466" + "4": "elephant_5002" + "5": "giraffe_5002" + "6": "horse_5004" + "7": "sheep_5004" + "8": "zebra_5002" +SOLVER: + MAX_ITER: 16000 + STEPS: (12000, 14000) +DENSEPOSE_EVALUATION: + EVALUATE_MESH_ALIGNMENT: True + MESH_ALIGNMENT_MESH_NAMES: + - bear_4936 + - cow_5002 + - cat_7466 + - dog_7466 + - elephant_5002 + - giraffe_5002 + - horse_5004 + - sheep_5004 + - zebra_5002 diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_finetune_16k.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_finetune_16k.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1462e374377fbf448e176951794face175b5002 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_finetune_16k.yaml @@ -0,0 +1,119 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "https://dl.fbaipublicfiles.com/densepose/cse/densepose_rcnn_R_50_FPN_soft_s1x/250533982/model_final_2c4512.pkl" + RESNETS: + DEPTH: 50 + ROI_HEADS: + NUM_CLASSES: 9 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseV1ConvXHead" + COARSE_SEGM_TRAINED_BY_MASKS: True + CSE: + EMBED_LOSS_NAME: "SoftEmbeddingLoss" + EMBEDDING_DIST_GAUSS_SIGMA: 0.1 + GEODESIC_DIST_GAUSS_SIGMA: 0.1 + EMBEDDERS: + "cat_7466": + TYPE: vertex_feature + NUM_VERTICES: 7466 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cat_7466_256.pkl" + "dog_7466": + TYPE: vertex_feature + NUM_VERTICES: 7466 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_dog_7466_256.pkl" + "sheep_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_sheep_5004_256.pkl" + "horse_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_horse_5004_256.pkl" + "zebra_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_zebra_5002_256.pkl" + "giraffe_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_giraffe_5002_256.pkl" + "elephant_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_elephant_5002_256.pkl" + "cow_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cow_5002_256.pkl" + "bear_4936": + TYPE: vertex_feature + NUM_VERTICES: 4936 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_bear_4936_256.pkl" +DATASETS: + TRAIN: + - "densepose_lvis_v1_ds2_train_v1" + TEST: + - "densepose_lvis_v1_ds2_val_v1" + WHITELISTED_CATEGORIES: + "densepose_lvis_v1_ds2_train_v1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + "densepose_lvis_v1_ds2_val_v1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + CLASS_TO_MESH_NAME_MAPPING: + "0": "bear_4936" + "1": "cow_5002" + "2": "cat_7466" + "3": "dog_7466" + "4": "elephant_5002" + "5": "giraffe_5002" + "6": "horse_5004" + "7": "sheep_5004" + "8": "zebra_5002" +SOLVER: + MAX_ITER: 16000 + STEPS: (12000, 14000) +DENSEPOSE_EVALUATION: + EVALUATE_MESH_ALIGNMENT: True diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_finetune_4k.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_finetune_4k.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba4b81dde2ef53749b096f137ac658563fdad857 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_finetune_4k.yaml @@ -0,0 +1,119 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "https://dl.fbaipublicfiles.com/densepose/cse/densepose_rcnn_R_50_FPN_soft_s1x/250533982/model_final_2c4512.pkl" + RESNETS: + DEPTH: 50 + ROI_HEADS: + NUM_CLASSES: 9 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseV1ConvXHead" + COARSE_SEGM_TRAINED_BY_MASKS: True + CSE: + EMBED_LOSS_NAME: "SoftEmbeddingLoss" + EMBEDDING_DIST_GAUSS_SIGMA: 0.1 + GEODESIC_DIST_GAUSS_SIGMA: 0.1 + EMBEDDERS: + "cat_5001": + TYPE: vertex_feature + NUM_VERTICES: 5001 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cat_5001_256.pkl" + "dog_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_dog_5002_256.pkl" + "sheep_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_sheep_5004_256.pkl" + "horse_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_horse_5004_256.pkl" + "zebra_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_zebra_5002_256.pkl" + "giraffe_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_giraffe_5002_256.pkl" + "elephant_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_elephant_5002_256.pkl" + "cow_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cow_5002_256.pkl" + "bear_4936": + TYPE: vertex_feature + NUM_VERTICES: 4936 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_bear_4936_256.pkl" +DATASETS: + TRAIN: + - "densepose_lvis_v1_ds1_train_v1" + TEST: + - "densepose_lvis_v1_ds1_val_v1" + WHITELISTED_CATEGORIES: + "densepose_lvis_v1_ds1_train_v1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + "densepose_lvis_v1_ds1_val_v1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + CLASS_TO_MESH_NAME_MAPPING: + "0": "bear_4936" + "1": "cow_5002" + "2": "cat_5001" + "3": "dog_5002" + "4": "elephant_5002" + "5": "giraffe_5002" + "6": "horse_5004" + "7": "sheep_5004" + "8": "zebra_5002" +SOLVER: + MAX_ITER: 4000 + STEPS: (3000, 3500) +DENSEPOSE_EVALUATION: + EVALUATE_MESH_ALIGNMENT: True diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_finetune_maskonly_24k.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_finetune_maskonly_24k.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb6136e274ca64aa2285698664d3243519d1979f --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_animals_finetune_maskonly_24k.yaml @@ -0,0 +1,118 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "https://dl.fbaipublicfiles.com/densepose/cse/densepose_rcnn_R_50_FPN_soft_s1x/250533982/model_final_2c4512.pkl" + RESNETS: + DEPTH: 50 + ROI_HEADS: + NUM_CLASSES: 9 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseV1ConvXHead" + COARSE_SEGM_TRAINED_BY_MASKS: True + CSE: + EMBED_LOSS_NAME: "SoftEmbeddingLoss" + EMBED_LOSS_WEIGHT: 0.0 + EMBEDDING_DIST_GAUSS_SIGMA: 0.1 + GEODESIC_DIST_GAUSS_SIGMA: 0.1 + EMBEDDERS: + "cat_7466": + TYPE: vertex_feature + NUM_VERTICES: 7466 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cat_7466_256.pkl" + "dog_7466": + TYPE: vertex_feature + NUM_VERTICES: 7466 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_dog_7466_256.pkl" + "sheep_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_sheep_5004_256.pkl" + "horse_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_horse_5004_256.pkl" + "zebra_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_zebra_5002_256.pkl" + "giraffe_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_giraffe_5002_256.pkl" + "elephant_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_elephant_5002_256.pkl" + "cow_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cow_5002_256.pkl" + "bear_4936": + TYPE: vertex_feature + NUM_VERTICES: 4936 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_bear_4936_256.pkl" +DATASETS: + TRAIN: + - "densepose_lvis_v1_ds2_train_v1" + TEST: + - "densepose_lvis_v1_ds2_val_v1" + WHITELISTED_CATEGORIES: + "densepose_lvis_v1_ds2_train_v1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + "densepose_lvis_v1_ds2_val_v1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + CLASS_TO_MESH_NAME_MAPPING: + "0": "bear_4936" + "1": "cow_5002" + "2": "cat_7466" + "3": "dog_7466" + "4": "elephant_5002" + "5": "giraffe_5002" + "6": "horse_5004" + "7": "sheep_5004" + "8": "zebra_5002" +SOLVER: + MAX_ITER: 24000 + STEPS: (20000, 22000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_chimps_finetune_4k.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_chimps_finetune_4k.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3bccb7837a2e4b905b4e3c7af465c3be3a44452d --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_chimps_finetune_4k.yaml @@ -0,0 +1,29 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "https://dl.fbaipublicfiles.com/densepose/cse/densepose_rcnn_R_50_FPN_soft_s1x/250533982/model_final_2c4512.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseV1ConvXHead" + CSE: + EMBED_LOSS_NAME: "SoftEmbeddingLoss" + EMBEDDING_DIST_GAUSS_SIGMA: 0.1 + GEODESIC_DIST_GAUSS_SIGMA: 0.1 + EMBEDDERS: + "chimp_5029": + TYPE: vertex_feature + NUM_VERTICES: 5029 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_chimp_5029_256.pkl" +DATASETS: + TRAIN: + - "densepose_chimps_cse_train" + TEST: + - "densepose_chimps_cse_val" + CLASS_TO_MESH_NAME_MAPPING: + "0": "chimp_5029" +SOLVER: + MAX_ITER: 4000 + STEPS: (3000, 3500) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9662fb8f8a4e9f7b01f41ddb79a3469ecab7032b --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/cse/densepose_rcnn_R_50_FPN_soft_s1x.yaml @@ -0,0 +1,12 @@ +_BASE_: "Base-DensePose-RCNN-FPN-Human.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseV1ConvXHead" + CSE: + EMBED_LOSS_NAME: "SoftEmbeddingLoss" +SOLVER: + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_DL_WC1M_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_DL_WC1M_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c16763c532499c1a0c62fb8c81a2ab97be3a1ec --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_DL_WC1M_s1x.yaml @@ -0,0 +1,18 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-101.pkl" + RESNETS: + DEPTH: 101 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + UV_CONFIDENCE: + ENABLED: True + TYPE: "iid_iso" + SEGM_CONFIDENCE: + ENABLED: True + POINT_REGRESSION_WEIGHTS: 0.0005 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_DL_WC1_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_DL_WC1_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15475b1ac3bb7272a7ebc0061a55119ffd2591b9 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_DL_WC1_s1x.yaml @@ -0,0 +1,16 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-101.pkl" + RESNETS: + DEPTH: 101 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + UV_CONFIDENCE: + ENABLED: True + TYPE: "iid_iso" + POINT_REGRESSION_WEIGHTS: 0.0005 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_DL_WC2M_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_DL_WC2M_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0cbe07f3bb0027bb7ecdc86f96d60790382b477b --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_DL_WC2M_s1x.yaml @@ -0,0 +1,18 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-101.pkl" + RESNETS: + DEPTH: 101 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + UV_CONFIDENCE: + ENABLED: True + TYPE: "indep_aniso" + SEGM_CONFIDENCE: + ENABLED: True + POINT_REGRESSION_WEIGHTS: 0.0005 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_DL_WC2_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_DL_WC2_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7546b967ab89129c9a276f19b1cf2d6b59f1a462 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_DL_WC2_s1x.yaml @@ -0,0 +1,16 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-101.pkl" + RESNETS: + DEPTH: 101 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + UV_CONFIDENCE: + ENABLED: True + TYPE: "indep_aniso" + POINT_REGRESSION_WEIGHTS: 0.0005 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_DL_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_DL_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..045f7f02f1b4eb0c0ef1733c3ac65e3aa70168de --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_DL_s1x.yaml @@ -0,0 +1,10 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-101.pkl" + RESNETS: + DEPTH: 101 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" +SOLVER: + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_WC1M_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_WC1M_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9334e18655d4451457a58c6ce945e01855f95105 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_WC1M_s1x.yaml @@ -0,0 +1,18 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-101.pkl" + RESNETS: + DEPTH: 101 + ROI_DENSEPOSE_HEAD: + UV_CONFIDENCE: + ENABLED: True + TYPE: "iid_iso" + SEGM_CONFIDENCE: + ENABLED: True + POINT_REGRESSION_WEIGHTS: 0.0005 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 130000 + STEPS: (100000, 120000) + WARMUP_FACTOR: 0.025 diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_WC1_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_WC1_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ace62094fbc4ce2024810333c11c7a955d8eeb22 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_WC1_s1x.yaml @@ -0,0 +1,16 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-101.pkl" + RESNETS: + DEPTH: 101 + ROI_DENSEPOSE_HEAD: + UV_CONFIDENCE: + ENABLED: True + TYPE: "iid_iso" + POINT_REGRESSION_WEIGHTS: 0.0005 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 130000 + STEPS: (100000, 120000) + WARMUP_FACTOR: 0.025 diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_WC2M_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_WC2M_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90f0be2805cd04e83c25d041d35ae66c90ce2b95 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_WC2M_s1x.yaml @@ -0,0 +1,18 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-101.pkl" + RESNETS: + DEPTH: 101 + ROI_DENSEPOSE_HEAD: + UV_CONFIDENCE: + ENABLED: True + TYPE: "indep_aniso" + SEGM_CONFIDENCE: + ENABLED: True + POINT_REGRESSION_WEIGHTS: 0.0005 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 130000 + STEPS: (100000, 120000) + WARMUP_FACTOR: 0.025 diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_WC2_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_WC2_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..766c098f6dcdd1fb3f67957d7d1d982b37747b96 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_WC2_s1x.yaml @@ -0,0 +1,16 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-101.pkl" + RESNETS: + DEPTH: 101 + ROI_DENSEPOSE_HEAD: + UV_CONFIDENCE: + ENABLED: True + TYPE: "indep_aniso" + POINT_REGRESSION_WEIGHTS: 0.0005 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 130000 + STEPS: (100000, 120000) + WARMUP_FACTOR: 0.025 diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af44fb767edf9bf093463e62f93e070d0d019c5a --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_s1x.yaml @@ -0,0 +1,8 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-101.pkl" + RESNETS: + DEPTH: 101 +SOLVER: + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_s1x_legacy.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_s1x_legacy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e79a1b9549cf19ed4a43cf9caf3dc88f6133310 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_101_FPN_s1x_legacy.yaml @@ -0,0 +1,17 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-101.pkl" + RESNETS: + DEPTH: 101 + ROI_DENSEPOSE_HEAD: + NUM_COARSE_SEGM_CHANNELS: 15 + POOLER_RESOLUTION: 14 + HEATMAP_SIZE: 56 + INDEX_WEIGHTS: 2.0 + PART_WEIGHTS: 0.3 + POINT_REGRESSION_WEIGHTS: 0.1 + DECODER_ON: False +SOLVER: + BASE_LR: 0.002 + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_DL_WC1M_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_DL_WC1M_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18a417a9a76d388810d46d1ee738d8b19abf0db0 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_DL_WC1M_s1x.yaml @@ -0,0 +1,18 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + UV_CONFIDENCE: + ENABLED: True + TYPE: "iid_iso" + SEGM_CONFIDENCE: + ENABLED: True + POINT_REGRESSION_WEIGHTS: 0.0005 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_DL_WC1_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_DL_WC1_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3720eff56ce042a68da6c99f484b963cae2c7d9 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_DL_WC1_s1x.yaml @@ -0,0 +1,16 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + UV_CONFIDENCE: + ENABLED: True + TYPE: "iid_iso" + POINT_REGRESSION_WEIGHTS: 0.0005 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_DL_WC2M_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_DL_WC2M_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a413d2a0d1549702fb45a2e50056fe0abde941f --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_DL_WC2M_s1x.yaml @@ -0,0 +1,18 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + UV_CONFIDENCE: + ENABLED: True + TYPE: "indep_aniso" + SEGM_CONFIDENCE: + ENABLED: True + POINT_REGRESSION_WEIGHTS: 0.0005 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_DL_WC2_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_DL_WC2_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a47cc05e6e9dc882778c6b502d93cbcec88fb88 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_DL_WC2_s1x.yaml @@ -0,0 +1,16 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + UV_CONFIDENCE: + ENABLED: True + TYPE: "indep_aniso" + POINT_REGRESSION_WEIGHTS: 0.0005 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_DL_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_DL_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52a170b4a28289ad943314f77256e34800d23121 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_DL_s1x.yaml @@ -0,0 +1,10 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" +SOLVER: + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_WC1M_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_WC1M_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a81f2a143cbfcd2dbc92f0fc5c86f951b9b7adf --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_WC1M_s1x.yaml @@ -0,0 +1,20 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + UV_CONFIDENCE: + ENABLED: True + TYPE: "iid_iso" + SEGM_CONFIDENCE: + ENABLED: True + POINT_REGRESSION_WEIGHTS: 0.0005 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + CLIP_TYPE: norm + CLIP_VALUE: 100.0 + MAX_ITER: 130000 + STEPS: (100000, 120000) + WARMUP_FACTOR: 0.025 diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_WC1_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_WC1_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d36e54256ac22f1b01604e54430da24972f06eeb --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_WC1_s1x.yaml @@ -0,0 +1,16 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + UV_CONFIDENCE: + ENABLED: True + TYPE: "iid_iso" + POINT_REGRESSION_WEIGHTS: 0.0005 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 130000 + STEPS: (100000, 120000) + WARMUP_FACTOR: 0.025 diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_WC2M_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_WC2M_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cf29eacd57626c676ed4c960a3e97e552b6dbdf --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_WC2M_s1x.yaml @@ -0,0 +1,18 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + UV_CONFIDENCE: + ENABLED: True + TYPE: "indep_aniso" + SEGM_CONFIDENCE: + ENABLED: True + POINT_REGRESSION_WEIGHTS: 0.0005 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 130000 + STEPS: (100000, 120000) + WARMUP_FACTOR: 0.025 diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_WC2_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_WC2_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e880d469564a3757ba3f4d708054074cefda49b6 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_WC2_s1x.yaml @@ -0,0 +1,16 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + UV_CONFIDENCE: + ENABLED: True + TYPE: "indep_aniso" + POINT_REGRESSION_WEIGHTS: 0.0005 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 130000 + STEPS: (100000, 120000) + WARMUP_FACTOR: 0.025 diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_s1x.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_s1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d2dd14c6f92f3850b99e6f1c828c0fcee52120e1 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_s1x.yaml @@ -0,0 +1,8 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 +SOLVER: + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_s1x_legacy.yaml b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_s1x_legacy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c5391f3b3c3d437312a290d29b0656cb3804b25 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/densepose_rcnn_R_50_FPN_s1x_legacy.yaml @@ -0,0 +1,17 @@ +_BASE_: "Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + NUM_COARSE_SEGM_CHANNELS: 15 + POOLER_RESOLUTION: 14 + HEATMAP_SIZE: 56 + INDEX_WEIGHTS: 2.0 + PART_WEIGHTS: 0.3 + POINT_REGRESSION_WEIGHTS: 0.1 + DECODER_ON: False +SOLVER: + BASE_LR: 0.002 + MAX_ITER: 130000 + STEPS: (100000, 120000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/evolution/Base-RCNN-FPN-Atop10P_CA.yaml b/approach/ovod/detectron2/projects/DensePose/configs/evolution/Base-RCNN-FPN-Atop10P_CA.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f09d723f3cb9eef94223c5926dbb7731397304c9 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/evolution/Base-RCNN-FPN-Atop10P_CA.yaml @@ -0,0 +1,91 @@ +MODEL: + META_ARCHITECTURE: "GeneralizedRCNN" + BACKBONE: + NAME: "build_resnet_fpn_backbone" + RESNETS: + OUT_FEATURES: ["res2", "res3", "res4", "res5"] + FPN: + IN_FEATURES: ["res2", "res3", "res4", "res5"] + ANCHOR_GENERATOR: + SIZES: [[32], [64], [128], [256], [512]] # One size for each in feature map + ASPECT_RATIOS: [[0.5, 1.0, 2.0]] # Three aspect ratios (same for all in feature maps) + RPN: + IN_FEATURES: ["p2", "p3", "p4", "p5", "p6"] + PRE_NMS_TOPK_TRAIN: 2000 # Per FPN level + PRE_NMS_TOPK_TEST: 1000 # Per FPN level + # Detectron1 uses 2000 proposals per-batch, + # (See "modeling/rpn/rpn_outputs.py" for details of this legacy issue) + # which is approximately 1000 proposals per-image since the default batch size for FPN is 2. + POST_NMS_TOPK_TRAIN: 1000 + POST_NMS_TOPK_TEST: 1000 + ROI_HEADS: + NAME: "StandardROIHeads" + IN_FEATURES: ["p2", "p3", "p4", "p5"] + NUM_CLASSES: 1 + ROI_BOX_HEAD: + NAME: "FastRCNNConvFCHead" + NUM_FC: 2 + POOLER_RESOLUTION: 7 + ROI_MASK_HEAD: + NAME: "MaskRCNNConvUpsampleHead" + NUM_CONV: 4 + POOLER_RESOLUTION: 14 +DATASETS: + TRAIN: ("base_coco_2017_train", "densepose_coco_2014_train") + TEST: ("densepose_chimps",) + CATEGORY_MAPS: + "base_coco_2017_train": + "16": 1 # bird -> person + "17": 1 # cat -> person + "18": 1 # dog -> person + "19": 1 # horse -> person + "20": 1 # sheep -> person + "21": 1 # cow -> person + "22": 1 # elephant -> person + "23": 1 # bear -> person + "24": 1 # zebra -> person + "25": 1 # girafe -> person + "base_coco_2017_val": + "16": 1 # bird -> person + "17": 1 # cat -> person + "18": 1 # dog -> person + "19": 1 # horse -> person + "20": 1 # sheep -> person + "21": 1 # cow -> person + "22": 1 # elephant -> person + "23": 1 # bear -> person + "24": 1 # zebra -> person + "25": 1 # girafe -> person + WHITELISTED_CATEGORIES: + "base_coco_2017_train": + - 1 # person + - 16 # bird + - 17 # cat + - 18 # dog + - 19 # horse + - 20 # sheep + - 21 # cow + - 22 # elephant + - 23 # bear + - 24 # zebra + - 25 # girafe + "base_coco_2017_val": + - 1 # person + - 16 # bird + - 17 # cat + - 18 # dog + - 19 # horse + - 20 # sheep + - 21 # cow + - 22 # elephant + - 23 # bear + - 24 # zebra + - 25 # girafe +SOLVER: + IMS_PER_BATCH: 16 + BASE_LR: 0.02 + STEPS: (60000, 80000) + MAX_ITER: 90000 +INPUT: + MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800) +VERSION: 2 diff --git a/approach/ovod/detectron2/projects/DensePose/configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA.yaml b/approach/ovod/detectron2/projects/DensePose/configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6296692d5ff15da24f87adb6327a62d9f4a34892 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA.yaml @@ -0,0 +1,28 @@ +_BASE_: "Base-RCNN-FPN-Atop10P_CA.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + DENSEPOSE_ON: True + ROI_HEADS: + NAME: "DensePoseROIHeads" + IN_FEATURES: ["p2", "p3", "p4", "p5"] + NUM_CLASSES: 1 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + UV_CONFIDENCE: + ENABLED: True + TYPE: "iid_iso" + SEGM_CONFIDENCE: + ENABLED: True + POINT_REGRESSION_WEIGHTS: 0.0005 + POOLER_TYPE: "ROIAlign" + NUM_COARSE_SEGM_CHANNELS: 2 + COARSE_SEGM_TRAINED_BY_MASKS: True + INDEX_WEIGHTS: 1.0 +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + WARMUP_FACTOR: 0.025 + MAX_ITER: 270000 + STEPS: (210000, 250000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_coarsesegm.yaml b/approach/ovod/detectron2/projects/DensePose/configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_coarsesegm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..033918e0daec8c225306dafac3a5fe9923189e53 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_coarsesegm.yaml @@ -0,0 +1,56 @@ +_BASE_: "Base-RCNN-FPN-Atop10P_CA.yaml" +MODEL: + WEIGHTS: https://dl.fbaipublicfiles.com/densepose/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA/217578784/model_final_9fe1cc.pkl + RESNETS: + DEPTH: 50 + DENSEPOSE_ON: True + ROI_HEADS: + NAME: "DensePoseROIHeads" + IN_FEATURES: ["p2", "p3", "p4", "p5"] + NUM_CLASSES: 1 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + UV_CONFIDENCE: + ENABLED: True + TYPE: "iid_iso" + SEGM_CONFIDENCE: + ENABLED: True + POINT_REGRESSION_WEIGHTS: 0.0005 + POOLER_TYPE: "ROIAlign" + NUM_COARSE_SEGM_CHANNELS: 2 + COARSE_SEGM_TRAINED_BY_MASKS: True +BOOTSTRAP_DATASETS: + - DATASET: "chimpnsee" + RATIO: 1.0 + IMAGE_LOADER: + TYPE: "video_keyframe" + SELECT: + STRATEGY: "random_k" + NUM_IMAGES: 4 + TRANSFORM: + TYPE: "resize" + MIN_SIZE: 800 + MAX_SIZE: 1333 + BATCH_SIZE: 8 + NUM_WORKERS: 1 + INFERENCE: + INPUT_BATCH_SIZE: 1 + OUTPUT_BATCH_SIZE: 1 + DATA_SAMPLER: + # supported types: + # densepose_uniform + # densepose_UV_confidence + # densepose_fine_segm_confidence + # densepose_coarse_segm_confidence + TYPE: "densepose_coarse_segm_confidence" + COUNT_PER_CLASS: 8 + FILTER: + TYPE: "detection_score" + MIN_VALUE: 0.8 +BOOTSTRAP_MODEL: + WEIGHTS: https://dl.fbaipublicfiles.com/densepose/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA/217578784/model_final_9fe1cc.pkl +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 270000 + STEPS: (210000, 250000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_finesegm.yaml b/approach/ovod/detectron2/projects/DensePose/configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_finesegm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5814a4a01fd772674fa40c0cba34666aed87b33a --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_finesegm.yaml @@ -0,0 +1,56 @@ +_BASE_: "Base-RCNN-FPN-Atop10P_CA.yaml" +MODEL: + WEIGHTS: https://dl.fbaipublicfiles.com/densepose/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA/217578784/model_final_9fe1cc.pkl + RESNETS: + DEPTH: 50 + DENSEPOSE_ON: True + ROI_HEADS: + NAME: "DensePoseROIHeads" + IN_FEATURES: ["p2", "p3", "p4", "p5"] + NUM_CLASSES: 1 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + UV_CONFIDENCE: + ENABLED: True + TYPE: "iid_iso" + SEGM_CONFIDENCE: + ENABLED: True + POINT_REGRESSION_WEIGHTS: 0.0005 + POOLER_TYPE: "ROIAlign" + NUM_COARSE_SEGM_CHANNELS: 2 + COARSE_SEGM_TRAINED_BY_MASKS: True +BOOTSTRAP_DATASETS: + - DATASET: "chimpnsee" + RATIO: 1.0 + IMAGE_LOADER: + TYPE: "video_keyframe" + SELECT: + STRATEGY: "random_k" + NUM_IMAGES: 4 + TRANSFORM: + TYPE: "resize" + MIN_SIZE: 800 + MAX_SIZE: 1333 + BATCH_SIZE: 8 + NUM_WORKERS: 1 + INFERENCE: + INPUT_BATCH_SIZE: 1 + OUTPUT_BATCH_SIZE: 1 + DATA_SAMPLER: + # supported types: + # densepose_uniform + # densepose_UV_confidence + # densepose_fine_segm_confidence + # densepose_coarse_segm_confidence + TYPE: "densepose_fine_segm_confidence" + COUNT_PER_CLASS: 8 + FILTER: + TYPE: "detection_score" + MIN_VALUE: 0.8 +BOOTSTRAP_MODEL: + WEIGHTS: https://dl.fbaipublicfiles.com/densepose/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA/217578784/model_final_9fe1cc.pkl +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 270000 + STEPS: (210000, 250000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_uniform.yaml b/approach/ovod/detectron2/projects/DensePose/configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_uniform.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d591ea6e22282f43fff0b44131e0913aa7261276 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_uniform.yaml @@ -0,0 +1,56 @@ +_BASE_: "Base-RCNN-FPN-Atop10P_CA.yaml" +MODEL: + WEIGHTS: https://dl.fbaipublicfiles.com/densepose/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA/217578784/model_final_9fe1cc.pkl + RESNETS: + DEPTH: 50 + DENSEPOSE_ON: True + ROI_HEADS: + NAME: "DensePoseROIHeads" + IN_FEATURES: ["p2", "p3", "p4", "p5"] + NUM_CLASSES: 1 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + UV_CONFIDENCE: + ENABLED: True + TYPE: "iid_iso" + SEGM_CONFIDENCE: + ENABLED: True + POINT_REGRESSION_WEIGHTS: 0.0005 + POOLER_TYPE: "ROIAlign" + NUM_COARSE_SEGM_CHANNELS: 2 + COARSE_SEGM_TRAINED_BY_MASKS: True +BOOTSTRAP_DATASETS: + - DATASET: "chimpnsee" + RATIO: 1.0 + IMAGE_LOADER: + TYPE: "video_keyframe" + SELECT: + STRATEGY: "random_k" + NUM_IMAGES: 4 + TRANSFORM: + TYPE: "resize" + MIN_SIZE: 800 + MAX_SIZE: 1333 + BATCH_SIZE: 8 + NUM_WORKERS: 1 + INFERENCE: + INPUT_BATCH_SIZE: 1 + OUTPUT_BATCH_SIZE: 1 + DATA_SAMPLER: + # supported types: + # densepose_uniform + # densepose_UV_confidence + # densepose_fine_segm_confidence + # densepose_coarse_segm_confidence + TYPE: "densepose_uniform" + COUNT_PER_CLASS: 8 + FILTER: + TYPE: "detection_score" + MIN_VALUE: 0.8 +BOOTSTRAP_MODEL: + WEIGHTS: https://dl.fbaipublicfiles.com/densepose/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA/217578784/model_final_9fe1cc.pkl +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 270000 + STEPS: (210000, 250000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_uv.yaml b/approach/ovod/detectron2/projects/DensePose/configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_uv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..110acff5a54247abb7b344672038b71e24167f33 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_uv.yaml @@ -0,0 +1,56 @@ +_BASE_: "Base-RCNN-FPN-Atop10P_CA.yaml" +MODEL: + WEIGHTS: https://dl.fbaipublicfiles.com/densepose/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA/217578784/model_final_9fe1cc.pkl + RESNETS: + DEPTH: 50 + DENSEPOSE_ON: True + ROI_HEADS: + NAME: "DensePoseROIHeads" + IN_FEATURES: ["p2", "p3", "p4", "p5"] + NUM_CLASSES: 1 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" + UV_CONFIDENCE: + ENABLED: True + TYPE: "iid_iso" + SEGM_CONFIDENCE: + ENABLED: True + POINT_REGRESSION_WEIGHTS: 0.0005 + POOLER_TYPE: "ROIAlign" + NUM_COARSE_SEGM_CHANNELS: 2 + COARSE_SEGM_TRAINED_BY_MASKS: True +BOOTSTRAP_DATASETS: + - DATASET: "chimpnsee" + RATIO: 1.0 + IMAGE_LOADER: + TYPE: "video_keyframe" + SELECT: + STRATEGY: "random_k" + NUM_IMAGES: 4 + TRANSFORM: + TYPE: "resize" + MIN_SIZE: 800 + MAX_SIZE: 1333 + BATCH_SIZE: 8 + NUM_WORKERS: 1 + INFERENCE: + INPUT_BATCH_SIZE: 1 + OUTPUT_BATCH_SIZE: 1 + DATA_SAMPLER: + # supported types: + # densepose_uniform + # densepose_UV_confidence + # densepose_fine_segm_confidence + # densepose_coarse_segm_confidence + TYPE: "densepose_UV_confidence" + COUNT_PER_CLASS: 8 + FILTER: + TYPE: "detection_score" + MIN_VALUE: 0.8 +BOOTSTRAP_MODEL: + WEIGHTS: https://dl.fbaipublicfiles.com/densepose/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA/217578784/model_final_9fe1cc.pkl +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 270000 + STEPS: (210000, 250000) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/cse/densepose_rcnn_R_50_FPN_DL_instant_test.yaml b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/cse/densepose_rcnn_R_50_FPN_DL_instant_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b43f75da549a9e5148c8528b5d375317680d738 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/cse/densepose_rcnn_R_50_FPN_DL_instant_test.yaml @@ -0,0 +1,11 @@ +_BASE_: "../../cse/Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" +DATASETS: + TRAIN: ("densepose_coco_2014_minival_100_cse",) + TEST: ("densepose_coco_2014_minival_100_cse",) +SOLVER: + MAX_ITER: 40 + STEPS: (30,) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/cse/densepose_rcnn_R_50_FPN_soft_animals_finetune_instant_test.yaml b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/cse/densepose_rcnn_R_50_FPN_soft_animals_finetune_instant_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2c49a2d14e5665af117972d126e25422e37b2b9 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/cse/densepose_rcnn_R_50_FPN_soft_animals_finetune_instant_test.yaml @@ -0,0 +1,126 @@ +_BASE_: "../../cse/Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_HEADS: + NUM_CLASSES: 9 + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseV1ConvXHead" + CSE: + EMBED_LOSS_NAME: "SoftEmbeddingLoss" + EMBEDDING_DIST_GAUSS_SIGMA: 0.1 + EMBEDDERS: + "cat_5001": + TYPE: vertex_feature + NUM_VERTICES: 5001 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cat_5001_256.pkl" + "dog_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_dog_5002_256.pkl" + "sheep_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_sheep_5004_256.pkl" + "horse_5004": + TYPE: vertex_feature + NUM_VERTICES: 5004 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_horse_5004_256.pkl" + "zebra_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_zebra_5002_256.pkl" + "giraffe_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_giraffe_5002_256.pkl" + "elephant_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_elephant_5002_256.pkl" + "cow_5002": + TYPE: vertex_feature + NUM_VERTICES: 5002 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_cow_5002_256.pkl" + "bear_4936": + TYPE: vertex_feature + NUM_VERTICES: 4936 + FEATURE_DIM: 256 + FEATURES_TRAINABLE: False + IS_TRAINABLE: True + INIT_FILE: "https://dl.fbaipublicfiles.com/densepose/data/cse/lbo/phi_bear_4936_256.pkl" +DATASETS: + TRAIN: + - "densepose_lvis_v1_train1" + - "densepose_lvis_v1_train2" + TEST: + - "densepose_lvis_v1_val_animals_100" + WHITELISTED_CATEGORIES: + "densepose_lvis_v1_train1": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + "densepose_lvis_v1_train2": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + "densepose_lvis_v1_val_animals_100": + - 943 # sheep + - 1202 # zebra + - 569 # horse + - 496 # giraffe + - 422 # elephant + - 80 # cow + - 76 # bear + - 225 # cat + - 378 # dog + CLASS_TO_MESH_NAME_MAPPING: + "0": "bear_4936" + "1": "cow_5002" + "2": "cat_5001" + "3": "dog_5002" + "4": "elephant_5002" + "5": "giraffe_5002" + "6": "horse_5004" + "7": "sheep_5004" + "8": "zebra_5002" +SOLVER: + MAX_ITER: 40 + STEPS: (30,) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_HRFPN_HRNet_w32_instant_test.yaml b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_HRFPN_HRNet_w32_instant_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95677ce9a7ff426a9051737876e7424908b1423f --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_HRFPN_HRNet_w32_instant_test.yaml @@ -0,0 +1,8 @@ +_BASE_: "../HRNet/densepose_rcnn_HRFPN_HRNet_w32_s1x.yaml" +DATASETS: + TRAIN: ("densepose_coco_2014_minival_100",) + TEST: ("densepose_coco_2014_minival_100",) +SOLVER: + MAX_ITER: 40 + STEPS: (30,) + IMS_PER_BATCH: 2 diff --git a/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_DL_instant_test.yaml b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_DL_instant_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b90989eef81e27d23119d2cd4627e8cea211ac51 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_DL_instant_test.yaml @@ -0,0 +1,11 @@ +_BASE_: "../Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + ROI_DENSEPOSE_HEAD: + NAME: "DensePoseDeepLabHead" +DATASETS: + TRAIN: ("densepose_coco_2014_minival_100",) + TEST: ("densepose_coco_2014_minival_100",) +SOLVER: + MAX_ITER: 40 + STEPS: (30,) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_TTA_inference_acc_test.yaml b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_TTA_inference_acc_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b124da19140f564258b583ec109eeeeaff8fd78a --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_TTA_inference_acc_test.yaml @@ -0,0 +1,13 @@ +_BASE_: "../densepose_rcnn_R_50_FPN_s1x.yaml" +MODEL: + WEIGHTS: "https://dl.fbaipublicfiles.com/densepose/densepose_rcnn_R_50_FPN_s1x/165712039/model_final_162be9.pkl" +DATASETS: + TRAIN: () + TEST: ("densepose_coco_2014_minival_100",) +TEST: + AUG: + ENABLED: True + MIN_SIZES: (400, 500, 600, 700, 800, 900, 1000, 1100, 1200) + MAX_SIZE: 4000 + FLIP: True + EXPECTED_RESULTS: [["bbox_TTA", "AP", 61.74, 0.03], ["densepose_gps_TTA", "AP", 60.22, 0.03], ["densepose_gpsm_TTA", "AP", 63.59, 0.03]] diff --git a/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_WC1_instant_test.yaml b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_WC1_instant_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0fe61151adf255baba717f3e65ff6fab52829a6 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_WC1_instant_test.yaml @@ -0,0 +1,19 @@ +_BASE_: "../Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + UV_CONFIDENCE: + ENABLED: True + TYPE: "iid_iso" + POINT_REGRESSION_WEIGHTS: 0.0005 +DATASETS: + TRAIN: ("densepose_coco_2014_minival_100",) + TEST: ("densepose_coco_2014_minival_100",) +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 40 + STEPS: (30,) + WARMUP_FACTOR: 0.025 diff --git a/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_WC2_instant_test.yaml b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_WC2_instant_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f0d9358c8846452314697a19b5e2ea9e075ddaeb --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_WC2_instant_test.yaml @@ -0,0 +1,19 @@ +_BASE_: "../Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + ROI_DENSEPOSE_HEAD: + UV_CONFIDENCE: + ENABLED: True + TYPE: "indep_aniso" + POINT_REGRESSION_WEIGHTS: 0.0005 +DATASETS: + TRAIN: ("densepose_coco_2014_minival_100",) + TEST: ("densepose_coco_2014_minival_100",) +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + MAX_ITER: 40 + STEPS: (30,) + WARMUP_FACTOR: 0.025 diff --git a/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_inference_acc_test.yaml b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_inference_acc_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d607c98813d045c1e19875bdfe45fbc1c3fdb292 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_inference_acc_test.yaml @@ -0,0 +1,8 @@ +_BASE_: "../densepose_rcnn_R_50_FPN_s1x.yaml" +MODEL: + WEIGHTS: "https://dl.fbaipublicfiles.com/densepose/densepose_rcnn_R_50_FPN_s1x/165712039/model_final_162be9.pkl" +DATASETS: + TRAIN: () + TEST: ("densepose_coco_2014_minival_100",) +TEST: + EXPECTED_RESULTS: [["bbox", "AP", 59.27, 0.025], ["densepose_gps", "AP", 60.11, 0.02], ["densepose_gpsm", "AP", 64.09, 0.02]] diff --git a/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_instant_test.yaml b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_instant_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..057c8768186e8a818228aa2f028ba3007374c571 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_instant_test.yaml @@ -0,0 +1,9 @@ +_BASE_: "../Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" +DATASETS: + TRAIN: ("densepose_coco_2014_minival_100",) + TEST: ("densepose_coco_2014_minival_100",) +SOLVER: + MAX_ITER: 40 + STEPS: (30,) diff --git a/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_training_acc_test.yaml b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_training_acc_test.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0053c9d7d41af0ee7262804838d8edcde10ed40d --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/configs/quick_schedules/densepose_rcnn_R_50_FPN_training_acc_test.yaml @@ -0,0 +1,18 @@ +_BASE_: "../Base-DensePose-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + ROI_HEADS: + NUM_CLASSES: 1 +DATASETS: + TRAIN: ("densepose_coco_2014_minival",) + TEST: ("densepose_coco_2014_minival",) +SOLVER: + CLIP_GRADIENTS: + ENABLED: True + CLIP_TYPE: norm + CLIP_VALUE: 1.0 + MAX_ITER: 6000 + STEPS: (5500, 5800) +TEST: + EXPECTED_RESULTS: [["bbox", "AP", 76.2477, 1.0], ["densepose_gps", "AP", 79.6090, 1.5], ["densepose_gpsm", "AP", 80.0061, 1.5]] + diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/__init__.py b/approach/ovod/detectron2/projects/DensePose/densepose/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b50a3da91dd0d2a69502af9d5d62f2f4280d973f --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .data.datasets import builtin # just to register data +from .converters import builtin as builtin_converters # register converters +from .config import ( + add_densepose_config, + add_densepose_head_config, + add_hrnet_config, + add_dataset_category_config, + add_bootstrap_config, + load_bootstrap_config, +) +from .structures import DensePoseDataRelative, DensePoseList, DensePoseTransformData +from .evaluation import DensePoseCOCOEvaluator +from .modeling.roi_heads import DensePoseROIHeads +from .modeling.test_time_augmentation import ( + DensePoseGeneralizedRCNNWithTTA, + DensePoseDatasetMapperTTA, +) +from .utils.transform import load_from_cfg +from .modeling.hrfpn import build_hrfpn_backbone diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/config.py b/approach/ovod/detectron2/projects/DensePose/densepose/config.py new file mode 100644 index 0000000000000000000000000000000000000000..2a06a09c80865ab987773511b2acc71e232b26ac --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/config.py @@ -0,0 +1,277 @@ +# -*- coding = utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. +# pyre-ignore-all-errors + +from detectron2.config import CfgNode as CN + + +def add_dataset_category_config(cfg: CN) -> None: + """ + Add config for additional category-related dataset options + - category whitelisting + - category mapping + """ + _C = cfg + _C.DATASETS.CATEGORY_MAPS = CN(new_allowed=True) + _C.DATASETS.WHITELISTED_CATEGORIES = CN(new_allowed=True) + # class to mesh mapping + _C.DATASETS.CLASS_TO_MESH_NAME_MAPPING = CN(new_allowed=True) + + +def add_evaluation_config(cfg: CN) -> None: + _C = cfg + _C.DENSEPOSE_EVALUATION = CN() + # evaluator type, possible values: + # - "iou": evaluator for models that produce iou data + # - "cse": evaluator for models that produce cse data + _C.DENSEPOSE_EVALUATION.TYPE = "iou" + # storage for DensePose results, possible values: + # - "none": no explicit storage, all the results are stored in the + # dictionary with predictions, memory intensive; + # historically the default storage type + # - "ram": RAM storage, uses per-process RAM storage, which is + # reduced to a single process storage on later stages, + # less memory intensive + # - "file": file storage, uses per-process file-based storage, + # the least memory intensive, but may create bottlenecks + # on file system accesses + _C.DENSEPOSE_EVALUATION.STORAGE = "none" + # minimum threshold for IOU values: the lower its values is, + # the more matches are produced (and the higher the AP score) + _C.DENSEPOSE_EVALUATION.MIN_IOU_THRESHOLD = 0.5 + # Non-distributed inference is slower (at inference time) but can avoid RAM OOM + _C.DENSEPOSE_EVALUATION.DISTRIBUTED_INFERENCE = True + # evaluate mesh alignment based on vertex embeddings, only makes sense in CSE context + _C.DENSEPOSE_EVALUATION.EVALUATE_MESH_ALIGNMENT = False + # meshes to compute mesh alignment for + _C.DENSEPOSE_EVALUATION.MESH_ALIGNMENT_MESH_NAMES = [] + + +def add_bootstrap_config(cfg: CN) -> None: + """ """ + _C = cfg + _C.BOOTSTRAP_DATASETS = [] + _C.BOOTSTRAP_MODEL = CN() + _C.BOOTSTRAP_MODEL.WEIGHTS = "" + _C.BOOTSTRAP_MODEL.DEVICE = "cuda" + + +def get_bootstrap_dataset_config() -> CN: + _C = CN() + _C.DATASET = "" + # ratio used to mix data loaders + _C.RATIO = 0.1 + # image loader + _C.IMAGE_LOADER = CN(new_allowed=True) + _C.IMAGE_LOADER.TYPE = "" + _C.IMAGE_LOADER.BATCH_SIZE = 4 + _C.IMAGE_LOADER.NUM_WORKERS = 4 + _C.IMAGE_LOADER.CATEGORIES = [] + _C.IMAGE_LOADER.MAX_COUNT_PER_CATEGORY = 1_000_000 + _C.IMAGE_LOADER.CATEGORY_TO_CLASS_MAPPING = CN(new_allowed=True) + # inference + _C.INFERENCE = CN() + # batch size for model inputs + _C.INFERENCE.INPUT_BATCH_SIZE = 4 + # batch size to group model outputs + _C.INFERENCE.OUTPUT_BATCH_SIZE = 2 + # sampled data + _C.DATA_SAMPLER = CN(new_allowed=True) + _C.DATA_SAMPLER.TYPE = "" + _C.DATA_SAMPLER.USE_GROUND_TRUTH_CATEGORIES = False + # filter + _C.FILTER = CN(new_allowed=True) + _C.FILTER.TYPE = "" + return _C + + +def load_bootstrap_config(cfg: CN) -> None: + """ + Bootstrap datasets are given as a list of `dict` that are not automatically + converted into CfgNode. This method processes all bootstrap dataset entries + and ensures that they are in CfgNode format and comply with the specification + """ + if not cfg.BOOTSTRAP_DATASETS: + return + + bootstrap_datasets_cfgnodes = [] + for dataset_cfg in cfg.BOOTSTRAP_DATASETS: + _C = get_bootstrap_dataset_config().clone() + _C.merge_from_other_cfg(CN(dataset_cfg)) + bootstrap_datasets_cfgnodes.append(_C) + cfg.BOOTSTRAP_DATASETS = bootstrap_datasets_cfgnodes + + +def add_densepose_head_cse_config(cfg: CN) -> None: + """ + Add configuration options for Continuous Surface Embeddings (CSE) + """ + _C = cfg + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE = CN() + # Dimensionality D of the embedding space + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBED_SIZE = 16 + # Embedder specifications for various mesh IDs + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBEDDERS = CN(new_allowed=True) + # normalization coefficient for embedding distances + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBEDDING_DIST_GAUSS_SIGMA = 0.01 + # normalization coefficient for geodesic distances + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.GEODESIC_DIST_GAUSS_SIGMA = 0.01 + # embedding loss weight + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBED_LOSS_WEIGHT = 0.6 + # embedding loss name, currently the following options are supported: + # - EmbeddingLoss: cross-entropy on vertex labels + # - SoftEmbeddingLoss: cross-entropy on vertex label combined with + # Gaussian penalty on distance between vertices + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBED_LOSS_NAME = "EmbeddingLoss" + # optimizer hyperparameters + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.FEATURES_LR_FACTOR = 1.0 + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBEDDING_LR_FACTOR = 1.0 + # Shape to shape cycle consistency loss parameters: + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.SHAPE_TO_SHAPE_CYCLE_LOSS = CN({"ENABLED": False}) + # shape to shape cycle consistency loss weight + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.SHAPE_TO_SHAPE_CYCLE_LOSS.WEIGHT = 0.025 + # norm type used for loss computation + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.SHAPE_TO_SHAPE_CYCLE_LOSS.NORM_P = 2 + # normalization term for embedding similarity matrices + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.SHAPE_TO_SHAPE_CYCLE_LOSS.TEMPERATURE = 0.05 + # maximum number of vertices to include into shape to shape cycle loss + # if negative or zero, all vertices are considered + # if positive, random subset of vertices of given size is considered + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.SHAPE_TO_SHAPE_CYCLE_LOSS.MAX_NUM_VERTICES = 4936 + # Pixel to shape cycle consistency loss parameters: + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.PIX_TO_SHAPE_CYCLE_LOSS = CN({"ENABLED": False}) + # pixel to shape cycle consistency loss weight + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.PIX_TO_SHAPE_CYCLE_LOSS.WEIGHT = 0.0001 + # norm type used for loss computation + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.PIX_TO_SHAPE_CYCLE_LOSS.NORM_P = 2 + # map images to all meshes and back (if false, use only gt meshes from the batch) + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.PIX_TO_SHAPE_CYCLE_LOSS.USE_ALL_MESHES_NOT_GT_ONLY = False + # Randomly select at most this number of pixels from every instance + # if negative or zero, all vertices are considered + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.PIX_TO_SHAPE_CYCLE_LOSS.NUM_PIXELS_TO_SAMPLE = 100 + # normalization factor for pixel to pixel distances (higher value = smoother distribution) + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.PIX_TO_SHAPE_CYCLE_LOSS.PIXEL_SIGMA = 5.0 + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.PIX_TO_SHAPE_CYCLE_LOSS.TEMPERATURE_PIXEL_TO_VERTEX = 0.05 + _C.MODEL.ROI_DENSEPOSE_HEAD.CSE.PIX_TO_SHAPE_CYCLE_LOSS.TEMPERATURE_VERTEX_TO_PIXEL = 0.05 + + +def add_densepose_head_config(cfg: CN) -> None: + """ + Add config for densepose head. + """ + _C = cfg + + _C.MODEL.DENSEPOSE_ON = True + + _C.MODEL.ROI_DENSEPOSE_HEAD = CN() + _C.MODEL.ROI_DENSEPOSE_HEAD.NAME = "" + _C.MODEL.ROI_DENSEPOSE_HEAD.NUM_STACKED_CONVS = 8 + # Number of parts used for point labels + _C.MODEL.ROI_DENSEPOSE_HEAD.NUM_PATCHES = 24 + _C.MODEL.ROI_DENSEPOSE_HEAD.DECONV_KERNEL = 4 + _C.MODEL.ROI_DENSEPOSE_HEAD.CONV_HEAD_DIM = 512 + _C.MODEL.ROI_DENSEPOSE_HEAD.CONV_HEAD_KERNEL = 3 + _C.MODEL.ROI_DENSEPOSE_HEAD.UP_SCALE = 2 + _C.MODEL.ROI_DENSEPOSE_HEAD.HEATMAP_SIZE = 112 + _C.MODEL.ROI_DENSEPOSE_HEAD.POOLER_TYPE = "ROIAlignV2" + _C.MODEL.ROI_DENSEPOSE_HEAD.POOLER_RESOLUTION = 28 + _C.MODEL.ROI_DENSEPOSE_HEAD.POOLER_SAMPLING_RATIO = 2 + _C.MODEL.ROI_DENSEPOSE_HEAD.NUM_COARSE_SEGM_CHANNELS = 2 # 15 or 2 + # Overlap threshold for an RoI to be considered foreground (if >= FG_IOU_THRESHOLD) + _C.MODEL.ROI_DENSEPOSE_HEAD.FG_IOU_THRESHOLD = 0.7 + # Loss weights for annotation masks.(14 Parts) + _C.MODEL.ROI_DENSEPOSE_HEAD.INDEX_WEIGHTS = 5.0 + # Loss weights for surface parts. (24 Parts) + _C.MODEL.ROI_DENSEPOSE_HEAD.PART_WEIGHTS = 1.0 + # Loss weights for UV regression. + _C.MODEL.ROI_DENSEPOSE_HEAD.POINT_REGRESSION_WEIGHTS = 0.01 + # Coarse segmentation is trained using instance segmentation task data + _C.MODEL.ROI_DENSEPOSE_HEAD.COARSE_SEGM_TRAINED_BY_MASKS = False + # For Decoder + _C.MODEL.ROI_DENSEPOSE_HEAD.DECODER_ON = True + _C.MODEL.ROI_DENSEPOSE_HEAD.DECODER_NUM_CLASSES = 256 + _C.MODEL.ROI_DENSEPOSE_HEAD.DECODER_CONV_DIMS = 256 + _C.MODEL.ROI_DENSEPOSE_HEAD.DECODER_NORM = "" + _C.MODEL.ROI_DENSEPOSE_HEAD.DECODER_COMMON_STRIDE = 4 + # For DeepLab head + _C.MODEL.ROI_DENSEPOSE_HEAD.DEEPLAB = CN() + _C.MODEL.ROI_DENSEPOSE_HEAD.DEEPLAB.NORM = "GN" + _C.MODEL.ROI_DENSEPOSE_HEAD.DEEPLAB.NONLOCAL_ON = 0 + # Predictor class name, must be registered in DENSEPOSE_PREDICTOR_REGISTRY + # Some registered predictors: + # "DensePoseChartPredictor": predicts segmentation and UV coordinates for predefined charts + # "DensePoseChartWithConfidencePredictor": predicts segmentation, UV coordinates + # and associated confidences for predefined charts (default) + # "DensePoseEmbeddingWithConfidencePredictor": predicts segmentation, embeddings + # and associated confidences for CSE + _C.MODEL.ROI_DENSEPOSE_HEAD.PREDICTOR_NAME = "DensePoseChartWithConfidencePredictor" + # Loss class name, must be registered in DENSEPOSE_LOSS_REGISTRY + # Some registered losses: + # "DensePoseChartLoss": loss for chart-based models that estimate + # segmentation and UV coordinates + # "DensePoseChartWithConfidenceLoss": loss for chart-based models that estimate + # segmentation, UV coordinates and the corresponding confidences (default) + _C.MODEL.ROI_DENSEPOSE_HEAD.LOSS_NAME = "DensePoseChartWithConfidenceLoss" + # Confidences + # Enable learning UV confidences (variances) along with the actual values + _C.MODEL.ROI_DENSEPOSE_HEAD.UV_CONFIDENCE = CN({"ENABLED": False}) + # UV confidence lower bound + _C.MODEL.ROI_DENSEPOSE_HEAD.UV_CONFIDENCE.EPSILON = 0.01 + # Enable learning segmentation confidences (variances) along with the actual values + _C.MODEL.ROI_DENSEPOSE_HEAD.SEGM_CONFIDENCE = CN({"ENABLED": False}) + # Segmentation confidence lower bound + _C.MODEL.ROI_DENSEPOSE_HEAD.SEGM_CONFIDENCE.EPSILON = 0.01 + # Statistical model type for confidence learning, possible values: + # - "iid_iso": statistically independent identically distributed residuals + # with isotropic covariance + # - "indep_aniso": statistically independent residuals with anisotropic + # covariances + _C.MODEL.ROI_DENSEPOSE_HEAD.UV_CONFIDENCE.TYPE = "iid_iso" + # List of angles for rotation in data augmentation during training + _C.INPUT.ROTATION_ANGLES = [0] + _C.TEST.AUG.ROTATION_ANGLES = () # Rotation TTA + + add_densepose_head_cse_config(cfg) + + +def add_hrnet_config(cfg: CN) -> None: + """ + Add config for HRNet backbone. + """ + _C = cfg + + # For HigherHRNet w32 + _C.MODEL.HRNET = CN() + _C.MODEL.HRNET.STEM_INPLANES = 64 + _C.MODEL.HRNET.STAGE2 = CN() + _C.MODEL.HRNET.STAGE2.NUM_MODULES = 1 + _C.MODEL.HRNET.STAGE2.NUM_BRANCHES = 2 + _C.MODEL.HRNET.STAGE2.BLOCK = "BASIC" + _C.MODEL.HRNET.STAGE2.NUM_BLOCKS = [4, 4] + _C.MODEL.HRNET.STAGE2.NUM_CHANNELS = [32, 64] + _C.MODEL.HRNET.STAGE2.FUSE_METHOD = "SUM" + _C.MODEL.HRNET.STAGE3 = CN() + _C.MODEL.HRNET.STAGE3.NUM_MODULES = 4 + _C.MODEL.HRNET.STAGE3.NUM_BRANCHES = 3 + _C.MODEL.HRNET.STAGE3.BLOCK = "BASIC" + _C.MODEL.HRNET.STAGE3.NUM_BLOCKS = [4, 4, 4] + _C.MODEL.HRNET.STAGE3.NUM_CHANNELS = [32, 64, 128] + _C.MODEL.HRNET.STAGE3.FUSE_METHOD = "SUM" + _C.MODEL.HRNET.STAGE4 = CN() + _C.MODEL.HRNET.STAGE4.NUM_MODULES = 3 + _C.MODEL.HRNET.STAGE4.NUM_BRANCHES = 4 + _C.MODEL.HRNET.STAGE4.BLOCK = "BASIC" + _C.MODEL.HRNET.STAGE4.NUM_BLOCKS = [4, 4, 4, 4] + _C.MODEL.HRNET.STAGE4.NUM_CHANNELS = [32, 64, 128, 256] + _C.MODEL.HRNET.STAGE4.FUSE_METHOD = "SUM" + + _C.MODEL.HRNET.HRFPN = CN() + _C.MODEL.HRNET.HRFPN.OUT_CHANNELS = 256 + + +def add_densepose_config(cfg: CN) -> None: + add_densepose_head_config(cfg) + add_hrnet_config(cfg) + add_bootstrap_config(cfg) + add_dataset_category_config(cfg) + add_evaluation_config(cfg) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/converters/__init__.py b/approach/ovod/detectron2/projects/DensePose/densepose/converters/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..930339e13f408ad46d0504fac557ef8cf0a57a56 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/converters/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from .hflip import HFlipConverter +from .to_mask import ToMaskConverter +from .to_chart_result import ToChartResultConverter, ToChartResultConverterWithConfidences +from .segm_to_mask import ( + predictor_output_with_fine_and_coarse_segm_to_mask, + predictor_output_with_coarse_segm_to_mask, + resample_fine_and_coarse_segm_to_bbox, +) +from .chart_output_to_chart_result import ( + densepose_chart_predictor_output_to_result, + densepose_chart_predictor_output_to_result_with_confidences, +) +from .chart_output_hflip import densepose_chart_predictor_output_hflip diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/converters/base.py b/approach/ovod/detectron2/projects/DensePose/densepose/converters/base.py new file mode 100644 index 0000000000000000000000000000000000000000..c9dbe56cecff6dbbc1a1fda5a89c5f917513dcd8 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/converters/base.py @@ -0,0 +1,93 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from typing import Any, Tuple, Type +import torch + + +class BaseConverter: + """ + Converter base class to be reused by various converters. + Converter allows one to convert data from various source types to a particular + destination type. Each source type needs to register its converter. The + registration for each source type is valid for all descendants of that type. + """ + + @classmethod + def register(cls, from_type: Type, converter: Any = None): + """ + Registers a converter for the specified type. + Can be used as a decorator (if converter is None), or called as a method. + + Args: + from_type (type): type to register the converter for; + all instances of this type will use the same converter + converter (callable): converter to be registered for the given + type; if None, this method is assumed to be a decorator for the converter + """ + + if converter is not None: + cls._do_register(from_type, converter) + + def wrapper(converter: Any) -> Any: + cls._do_register(from_type, converter) + return converter + + return wrapper + + @classmethod + def _do_register(cls, from_type: Type, converter: Any): + cls.registry[from_type] = converter # pyre-ignore[16] + + @classmethod + def _lookup_converter(cls, from_type: Type) -> Any: + """ + Perform recursive lookup for the given type + to find registered converter. If a converter was found for some base + class, it gets registered for this class to save on further lookups. + + Args: + from_type: type for which to find a converter + Return: + callable or None - registered converter or None + if no suitable entry was found in the registry + """ + if from_type in cls.registry: # pyre-ignore[16] + return cls.registry[from_type] + for base in from_type.__bases__: + converter = cls._lookup_converter(base) + if converter is not None: + cls._do_register(from_type, converter) + return converter + return None + + @classmethod + def convert(cls, instance: Any, *args, **kwargs): + """ + Convert an instance to the destination type using some registered + converter. Does recursive lookup for base classes, so there's no need + for explicit registration for derived classes. + + Args: + instance: source instance to convert to the destination type + Return: + An instance of the destination type obtained from the source instance + Raises KeyError, if no suitable converter found + """ + instance_type = type(instance) + converter = cls._lookup_converter(instance_type) + if converter is None: + if cls.dst_type is None: # pyre-ignore[16] + output_type_str = "itself" + else: + output_type_str = cls.dst_type + raise KeyError(f"Could not find converter from {instance_type} to {output_type_str}") + return converter(instance, *args, **kwargs) + + +IntTupleBox = Tuple[int, int, int, int] + + +def make_int_box(box: torch.Tensor) -> IntTupleBox: + int_box = [0, 0, 0, 0] + int_box[0], int_box[1], int_box[2], int_box[3] = tuple(box.long().tolist()) + return int_box[0], int_box[1], int_box[2], int_box[3] diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/converters/builtin.py b/approach/ovod/detectron2/projects/DensePose/densepose/converters/builtin.py new file mode 100644 index 0000000000000000000000000000000000000000..3bd48f8f7afc49cf38bf410f01bc673d446f37d7 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/converters/builtin.py @@ -0,0 +1,31 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from ..structures import DensePoseChartPredictorOutput, DensePoseEmbeddingPredictorOutput +from . import ( + HFlipConverter, + ToChartResultConverter, + ToChartResultConverterWithConfidences, + ToMaskConverter, + densepose_chart_predictor_output_hflip, + densepose_chart_predictor_output_to_result, + densepose_chart_predictor_output_to_result_with_confidences, + predictor_output_with_coarse_segm_to_mask, + predictor_output_with_fine_and_coarse_segm_to_mask, +) + +ToMaskConverter.register( + DensePoseChartPredictorOutput, predictor_output_with_fine_and_coarse_segm_to_mask +) +ToMaskConverter.register( + DensePoseEmbeddingPredictorOutput, predictor_output_with_coarse_segm_to_mask +) + +ToChartResultConverter.register( + DensePoseChartPredictorOutput, densepose_chart_predictor_output_to_result +) + +ToChartResultConverterWithConfidences.register( + DensePoseChartPredictorOutput, densepose_chart_predictor_output_to_result_with_confidences +) + +HFlipConverter.register(DensePoseChartPredictorOutput, densepose_chart_predictor_output_hflip) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/converters/chart_output_hflip.py b/approach/ovod/detectron2/projects/DensePose/densepose/converters/chart_output_hflip.py new file mode 100644 index 0000000000000000000000000000000000000000..17d294841264c248cf7fa9e3d2d2b4efdbb9a5e8 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/converters/chart_output_hflip.py @@ -0,0 +1,71 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from dataclasses import fields +import torch + +from densepose.structures import DensePoseChartPredictorOutput, DensePoseTransformData + + +def densepose_chart_predictor_output_hflip( + densepose_predictor_output: DensePoseChartPredictorOutput, + transform_data: DensePoseTransformData, +) -> DensePoseChartPredictorOutput: + """ + Change to take into account a Horizontal flip. + """ + if len(densepose_predictor_output) > 0: + + PredictorOutput = type(densepose_predictor_output) + output_dict = {} + + for field in fields(densepose_predictor_output): + field_value = getattr(densepose_predictor_output, field.name) + # flip tensors + if isinstance(field_value, torch.Tensor): + setattr(densepose_predictor_output, field.name, torch.flip(field_value, [3])) + + densepose_predictor_output = _flip_iuv_semantics_tensor( + densepose_predictor_output, transform_data + ) + densepose_predictor_output = _flip_segm_semantics_tensor( + densepose_predictor_output, transform_data + ) + + for field in fields(densepose_predictor_output): + output_dict[field.name] = getattr(densepose_predictor_output, field.name) + + return PredictorOutput(**output_dict) + else: + return densepose_predictor_output + + +def _flip_iuv_semantics_tensor( + densepose_predictor_output: DensePoseChartPredictorOutput, + dp_transform_data: DensePoseTransformData, +) -> DensePoseChartPredictorOutput: + point_label_symmetries = dp_transform_data.point_label_symmetries + uv_symmetries = dp_transform_data.uv_symmetries + + N, C, H, W = densepose_predictor_output.u.shape + u_loc = (densepose_predictor_output.u[:, 1:, :, :].clamp(0, 1) * 255).long() + v_loc = (densepose_predictor_output.v[:, 1:, :, :].clamp(0, 1) * 255).long() + Iindex = torch.arange(C - 1, device=densepose_predictor_output.u.device)[ + None, :, None, None + ].expand(N, C - 1, H, W) + densepose_predictor_output.u[:, 1:, :, :] = uv_symmetries["U_transforms"][Iindex, v_loc, u_loc] + densepose_predictor_output.v[:, 1:, :, :] = uv_symmetries["V_transforms"][Iindex, v_loc, u_loc] + + for el in ["fine_segm", "u", "v"]: + densepose_predictor_output.__dict__[el] = densepose_predictor_output.__dict__[el][ + :, point_label_symmetries, :, : + ] + return densepose_predictor_output + + +def _flip_segm_semantics_tensor( + densepose_predictor_output: DensePoseChartPredictorOutput, dp_transform_data +): + if densepose_predictor_output.coarse_segm.shape[1] > 2: + densepose_predictor_output.coarse_segm = densepose_predictor_output.coarse_segm[ + :, dp_transform_data.mask_label_symmetries, :, : + ] + return densepose_predictor_output diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/converters/chart_output_to_chart_result.py b/approach/ovod/detectron2/projects/DensePose/densepose/converters/chart_output_to_chart_result.py new file mode 100644 index 0000000000000000000000000000000000000000..4248f6c91b641a4ad1d00d0316ee82d701f9152f --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/converters/chart_output_to_chart_result.py @@ -0,0 +1,188 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from typing import Dict +import torch +from torch.nn import functional as F + +from detectron2.structures.boxes import Boxes, BoxMode + +from ..structures import ( + DensePoseChartPredictorOutput, + DensePoseChartResult, + DensePoseChartResultWithConfidences, +) +from . import resample_fine_and_coarse_segm_to_bbox +from .base import IntTupleBox, make_int_box + + +def resample_uv_tensors_to_bbox( + u: torch.Tensor, + v: torch.Tensor, + labels: torch.Tensor, + box_xywh_abs: IntTupleBox, +) -> torch.Tensor: + """ + Resamples U and V coordinate estimates for the given bounding box + + Args: + u (tensor [1, C, H, W] of float): U coordinates + v (tensor [1, C, H, W] of float): V coordinates + labels (tensor [H, W] of long): labels obtained by resampling segmentation + outputs for the given bounding box + box_xywh_abs (tuple of 4 int): bounding box that corresponds to predictor outputs + Return: + Resampled U and V coordinates - a tensor [2, H, W] of float + """ + x, y, w, h = box_xywh_abs + w = max(int(w), 1) + h = max(int(h), 1) + u_bbox = F.interpolate(u, (h, w), mode="bilinear", align_corners=False) + v_bbox = F.interpolate(v, (h, w), mode="bilinear", align_corners=False) + uv = torch.zeros([2, h, w], dtype=torch.float32, device=u.device) + for part_id in range(1, u_bbox.size(1)): + uv[0][labels == part_id] = u_bbox[0, part_id][labels == part_id] + uv[1][labels == part_id] = v_bbox[0, part_id][labels == part_id] + return uv + + +def resample_uv_to_bbox( + predictor_output: DensePoseChartPredictorOutput, + labels: torch.Tensor, + box_xywh_abs: IntTupleBox, +) -> torch.Tensor: + """ + Resamples U and V coordinate estimates for the given bounding box + + Args: + predictor_output (DensePoseChartPredictorOutput): DensePose predictor + output to be resampled + labels (tensor [H, W] of long): labels obtained by resampling segmentation + outputs for the given bounding box + box_xywh_abs (tuple of 4 int): bounding box that corresponds to predictor outputs + Return: + Resampled U and V coordinates - a tensor [2, H, W] of float + """ + return resample_uv_tensors_to_bbox( + predictor_output.u, + predictor_output.v, + labels, + box_xywh_abs, + ) + + +def densepose_chart_predictor_output_to_result( + predictor_output: DensePoseChartPredictorOutput, boxes: Boxes +) -> DensePoseChartResult: + """ + Convert densepose chart predictor outputs to results + + Args: + predictor_output (DensePoseChartPredictorOutput): DensePose predictor + output to be converted to results, must contain only 1 output + boxes (Boxes): bounding box that corresponds to the predictor output, + must contain only 1 bounding box + Return: + DensePose chart-based result (DensePoseChartResult) + """ + assert len(predictor_output) == 1 and len(boxes) == 1, ( + f"Predictor output to result conversion can operate only single outputs" + f", got {len(predictor_output)} predictor outputs and {len(boxes)} boxes" + ) + + boxes_xyxy_abs = boxes.tensor.clone() + boxes_xywh_abs = BoxMode.convert(boxes_xyxy_abs, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS) + box_xywh = make_int_box(boxes_xywh_abs[0]) + + labels = resample_fine_and_coarse_segm_to_bbox(predictor_output, box_xywh).squeeze(0) + uv = resample_uv_to_bbox(predictor_output, labels, box_xywh) + return DensePoseChartResult(labels=labels, uv=uv) + + +def resample_confidences_to_bbox( + predictor_output: DensePoseChartPredictorOutput, + labels: torch.Tensor, + box_xywh_abs: IntTupleBox, +) -> Dict[str, torch.Tensor]: + """ + Resamples confidences for the given bounding box + + Args: + predictor_output (DensePoseChartPredictorOutput): DensePose predictor + output to be resampled + labels (tensor [H, W] of long): labels obtained by resampling segmentation + outputs for the given bounding box + box_xywh_abs (tuple of 4 int): bounding box that corresponds to predictor outputs + Return: + Resampled confidences - a dict of [H, W] tensors of float + """ + + x, y, w, h = box_xywh_abs + w = max(int(w), 1) + h = max(int(h), 1) + + confidence_names = [ + "sigma_1", + "sigma_2", + "kappa_u", + "kappa_v", + "fine_segm_confidence", + "coarse_segm_confidence", + ] + confidence_results = {key: None for key in confidence_names} + confidence_names = [ + key for key in confidence_names if getattr(predictor_output, key) is not None + ] + confidence_base = torch.zeros([h, w], dtype=torch.float32, device=predictor_output.u.device) + + # assign data from channels that correspond to the labels + for key in confidence_names: + resampled_confidence = F.interpolate( + getattr(predictor_output, key), + (h, w), + mode="bilinear", + align_corners=False, + ) + result = confidence_base.clone() + for part_id in range(1, predictor_output.u.size(1)): + if resampled_confidence.size(1) != predictor_output.u.size(1): + # confidence is not part-based, don't try to fill it part by part + continue + result[labels == part_id] = resampled_confidence[0, part_id][labels == part_id] + + if resampled_confidence.size(1) != predictor_output.u.size(1): + # confidence is not part-based, fill the data with the first channel + # (targeted for segmentation confidences that have only 1 channel) + result = resampled_confidence[0, 0] + + confidence_results[key] = result + + return confidence_results # pyre-ignore[7] + + +def densepose_chart_predictor_output_to_result_with_confidences( + predictor_output: DensePoseChartPredictorOutput, boxes: Boxes +) -> DensePoseChartResultWithConfidences: + """ + Convert densepose chart predictor outputs to results + + Args: + predictor_output (DensePoseChartPredictorOutput): DensePose predictor + output with confidences to be converted to results, must contain only 1 output + boxes (Boxes): bounding box that corresponds to the predictor output, + must contain only 1 bounding box + Return: + DensePose chart-based result with confidences (DensePoseChartResultWithConfidences) + """ + assert len(predictor_output) == 1 and len(boxes) == 1, ( + f"Predictor output to result conversion can operate only single outputs" + f", got {len(predictor_output)} predictor outputs and {len(boxes)} boxes" + ) + + boxes_xyxy_abs = boxes.tensor.clone() + boxes_xywh_abs = BoxMode.convert(boxes_xyxy_abs, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS) + box_xywh = make_int_box(boxes_xywh_abs[0]) + + labels = resample_fine_and_coarse_segm_to_bbox(predictor_output, box_xywh).squeeze(0) + uv = resample_uv_to_bbox(predictor_output, labels, box_xywh) + confidences = resample_confidences_to_bbox(predictor_output, labels, box_xywh) + return DensePoseChartResultWithConfidences(labels=labels, uv=uv, **confidences) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/converters/hflip.py b/approach/ovod/detectron2/projects/DensePose/densepose/converters/hflip.py new file mode 100644 index 0000000000000000000000000000000000000000..092c50cc67358cc86c3683de71a9633c97667732 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/converters/hflip.py @@ -0,0 +1,32 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from typing import Any + +from .base import BaseConverter + + +class HFlipConverter(BaseConverter): + """ + Converts various DensePose predictor outputs to DensePose results. + Each DensePose predictor output type has to register its convertion strategy. + """ + + registry = {} + dst_type = None + + @classmethod + def convert(cls, predictor_outputs: Any, transform_data: Any, *args, **kwargs): + """ + Performs an horizontal flip on DensePose predictor outputs. + Does recursive lookup for base classes, so there's no need + for explicit registration for derived classes. + + Args: + predictor_outputs: DensePose predictor output to be converted to BitMasks + transform_data: Anything useful for the flip + Return: + An instance of the same type as predictor_outputs + """ + return super(HFlipConverter, cls).convert( + predictor_outputs, transform_data, *args, **kwargs + ) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/converters/segm_to_mask.py b/approach/ovod/detectron2/projects/DensePose/densepose/converters/segm_to_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..6433d5dec75c3d6141252af144b61d8999077bb7 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/converters/segm_to_mask.py @@ -0,0 +1,150 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from typing import Any +import torch +from torch.nn import functional as F + +from detectron2.structures import BitMasks, Boxes, BoxMode + +from .base import IntTupleBox, make_int_box +from .to_mask import ImageSizeType + + +def resample_coarse_segm_tensor_to_bbox(coarse_segm: torch.Tensor, box_xywh_abs: IntTupleBox): + """ + Resample coarse segmentation tensor to the given + bounding box and derive labels for each pixel of the bounding box + + Args: + coarse_segm: float tensor of shape [1, K, Hout, Wout] + box_xywh_abs (tuple of 4 int): bounding box given by its upper-left + corner coordinates, width (W) and height (H) + Return: + Labels for each pixel of the bounding box, a long tensor of size [1, H, W] + """ + x, y, w, h = box_xywh_abs + w = max(int(w), 1) + h = max(int(h), 1) + labels = F.interpolate(coarse_segm, (h, w), mode="bilinear", align_corners=False).argmax(dim=1) + return labels + + +def resample_fine_and_coarse_segm_tensors_to_bbox( + fine_segm: torch.Tensor, coarse_segm: torch.Tensor, box_xywh_abs: IntTupleBox +): + """ + Resample fine and coarse segmentation tensors to the given + bounding box and derive labels for each pixel of the bounding box + + Args: + fine_segm: float tensor of shape [1, C, Hout, Wout] + coarse_segm: float tensor of shape [1, K, Hout, Wout] + box_xywh_abs (tuple of 4 int): bounding box given by its upper-left + corner coordinates, width (W) and height (H) + Return: + Labels for each pixel of the bounding box, a long tensor of size [1, H, W] + """ + x, y, w, h = box_xywh_abs + w = max(int(w), 1) + h = max(int(h), 1) + # coarse segmentation + coarse_segm_bbox = F.interpolate( + coarse_segm, + (h, w), + mode="bilinear", + align_corners=False, + ).argmax(dim=1) + # combined coarse and fine segmentation + labels = ( + F.interpolate(fine_segm, (h, w), mode="bilinear", align_corners=False).argmax(dim=1) + * (coarse_segm_bbox > 0).long() + ) + return labels + + +def resample_fine_and_coarse_segm_to_bbox(predictor_output: Any, box_xywh_abs: IntTupleBox): + """ + Resample fine and coarse segmentation outputs from a predictor to the given + bounding box and derive labels for each pixel of the bounding box + + Args: + predictor_output: DensePose predictor output that contains segmentation + results to be resampled + box_xywh_abs (tuple of 4 int): bounding box given by its upper-left + corner coordinates, width (W) and height (H) + Return: + Labels for each pixel of the bounding box, a long tensor of size [1, H, W] + """ + return resample_fine_and_coarse_segm_tensors_to_bbox( + predictor_output.fine_segm, + predictor_output.coarse_segm, + box_xywh_abs, + ) + + +def predictor_output_with_coarse_segm_to_mask( + predictor_output: Any, boxes: Boxes, image_size_hw: ImageSizeType +) -> BitMasks: + """ + Convert predictor output with coarse and fine segmentation to a mask. + Assumes that predictor output has the following attributes: + - coarse_segm (tensor of size [N, D, H, W]): coarse segmentation + unnormalized scores for N instances; D is the number of coarse + segmentation labels, H and W is the resolution of the estimate + + Args: + predictor_output: DensePose predictor output to be converted to mask + boxes (Boxes): bounding boxes that correspond to the DensePose + predictor outputs + image_size_hw (tuple [int, int]): image height Himg and width Wimg + Return: + BitMasks that contain a bool tensor of size [N, Himg, Wimg] with + a mask of the size of the image for each instance + """ + H, W = image_size_hw + boxes_xyxy_abs = boxes.tensor.clone() + boxes_xywh_abs = BoxMode.convert(boxes_xyxy_abs, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS) + N = len(boxes_xywh_abs) + masks = torch.zeros((N, H, W), dtype=torch.bool, device=boxes.tensor.device) + for i in range(len(boxes_xywh_abs)): + box_xywh = make_int_box(boxes_xywh_abs[i]) + box_mask = resample_coarse_segm_tensor_to_bbox(predictor_output[i].coarse_segm, box_xywh) + x, y, w, h = box_xywh + masks[i, y : y + h, x : x + w] = box_mask + + return BitMasks(masks) + + +def predictor_output_with_fine_and_coarse_segm_to_mask( + predictor_output: Any, boxes: Boxes, image_size_hw: ImageSizeType +) -> BitMasks: + """ + Convert predictor output with coarse and fine segmentation to a mask. + Assumes that predictor output has the following attributes: + - coarse_segm (tensor of size [N, D, H, W]): coarse segmentation + unnormalized scores for N instances; D is the number of coarse + segmentation labels, H and W is the resolution of the estimate + - fine_segm (tensor of size [N, C, H, W]): fine segmentation + unnormalized scores for N instances; C is the number of fine + segmentation labels, H and W is the resolution of the estimate + + Args: + predictor_output: DensePose predictor output to be converted to mask + boxes (Boxes): bounding boxes that correspond to the DensePose + predictor outputs + image_size_hw (tuple [int, int]): image height Himg and width Wimg + Return: + BitMasks that contain a bool tensor of size [N, Himg, Wimg] with + a mask of the size of the image for each instance + """ + H, W = image_size_hw + boxes_xyxy_abs = boxes.tensor.clone() + boxes_xywh_abs = BoxMode.convert(boxes_xyxy_abs, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS) + N = len(boxes_xywh_abs) + masks = torch.zeros((N, H, W), dtype=torch.bool, device=boxes.tensor.device) + for i in range(len(boxes_xywh_abs)): + box_xywh = make_int_box(boxes_xywh_abs[i]) + labels_i = resample_fine_and_coarse_segm_to_bbox(predictor_output[i], box_xywh) + x, y, w, h = box_xywh + masks[i, y : y + h, x : x + w] = labels_i > 0 + return BitMasks(masks) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/converters/to_chart_result.py b/approach/ovod/detectron2/projects/DensePose/densepose/converters/to_chart_result.py new file mode 100644 index 0000000000000000000000000000000000000000..f96da3c2efcc8ab10e6ec12785a83b1af2b867ae --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/converters/to_chart_result.py @@ -0,0 +1,66 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from typing import Any + +from detectron2.structures import Boxes + +from ..structures import DensePoseChartResult, DensePoseChartResultWithConfidences +from .base import BaseConverter + + +class ToChartResultConverter(BaseConverter): + """ + Converts various DensePose predictor outputs to DensePose results. + Each DensePose predictor output type has to register its convertion strategy. + """ + + registry = {} + dst_type = DensePoseChartResult + + @classmethod + def convert(cls, predictor_outputs: Any, boxes: Boxes, *args, **kwargs) -> DensePoseChartResult: + """ + Convert DensePose predictor outputs to DensePoseResult using some registered + converter. Does recursive lookup for base classes, so there's no need + for explicit registration for derived classes. + + Args: + densepose_predictor_outputs: DensePose predictor output to be + converted to BitMasks + boxes (Boxes): bounding boxes that correspond to the DensePose + predictor outputs + Return: + An instance of DensePoseResult. If no suitable converter was found, raises KeyError + """ + return super(ToChartResultConverter, cls).convert(predictor_outputs, boxes, *args, **kwargs) + + +class ToChartResultConverterWithConfidences(BaseConverter): + """ + Converts various DensePose predictor outputs to DensePose results. + Each DensePose predictor output type has to register its convertion strategy. + """ + + registry = {} + dst_type = DensePoseChartResultWithConfidences + + @classmethod + def convert( + cls, predictor_outputs: Any, boxes: Boxes, *args, **kwargs + ) -> DensePoseChartResultWithConfidences: + """ + Convert DensePose predictor outputs to DensePoseResult with confidences + using some registered converter. Does recursive lookup for base classes, + so there's no need for explicit registration for derived classes. + + Args: + densepose_predictor_outputs: DensePose predictor output with confidences + to be converted to BitMasks + boxes (Boxes): bounding boxes that correspond to the DensePose + predictor outputs + Return: + An instance of DensePoseResult. If no suitable converter was found, raises KeyError + """ + return super(ToChartResultConverterWithConfidences, cls).convert( + predictor_outputs, boxes, *args, **kwargs + ) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/converters/to_mask.py b/approach/ovod/detectron2/projects/DensePose/densepose/converters/to_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..9d8152bb1dd3f9218e3db7a78b09ac5a126fbe0e --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/converters/to_mask.py @@ -0,0 +1,47 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from typing import Any, Tuple + +from detectron2.structures import BitMasks, Boxes + +from .base import BaseConverter + +ImageSizeType = Tuple[int, int] + + +class ToMaskConverter(BaseConverter): + """ + Converts various DensePose predictor outputs to masks + in bit mask format (see `BitMasks`). Each DensePose predictor output type + has to register its convertion strategy. + """ + + registry = {} + dst_type = BitMasks + + @classmethod + def convert( + cls, + densepose_predictor_outputs: Any, + boxes: Boxes, + image_size_hw: ImageSizeType, + *args, + **kwargs + ) -> BitMasks: + """ + Convert DensePose predictor outputs to BitMasks using some registered + converter. Does recursive lookup for base classes, so there's no need + for explicit registration for derived classes. + + Args: + densepose_predictor_outputs: DensePose predictor output to be + converted to BitMasks + boxes (Boxes): bounding boxes that correspond to the DensePose + predictor outputs + image_size_hw (tuple [int, int]): image height and width + Return: + An instance of `BitMasks`. If no suitable converter was found, raises KeyError + """ + return super(ToMaskConverter, cls).convert( + densepose_predictor_outputs, boxes, image_size_hw, *args, **kwargs + ) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/engine/__init__.py b/approach/ovod/detectron2/projects/DensePose/densepose/engine/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..539b93a7beca07d229a6b6d387f885469242ad86 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/engine/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from .trainer import Trainer diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/engine/trainer.py b/approach/ovod/detectron2/projects/DensePose/densepose/engine/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..4306a6124d477822ea0fa5fc7c948af2633edf4b --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/engine/trainer.py @@ -0,0 +1,260 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +import logging +import os +from collections import OrderedDict +from typing import List, Optional, Union +import torch +from torch import nn + +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import CfgNode +from detectron2.engine import DefaultTrainer +from detectron2.evaluation import ( + DatasetEvaluator, + DatasetEvaluators, + inference_on_dataset, + print_csv_format, +) +from detectron2.solver.build import get_default_optimizer_params, maybe_add_gradient_clipping +from detectron2.utils import comm +from detectron2.utils.events import EventWriter, get_event_storage + +from densepose import DensePoseDatasetMapperTTA, DensePoseGeneralizedRCNNWithTTA, load_from_cfg +from densepose.data import ( + DatasetMapper, + build_combined_loader, + build_detection_test_loader, + build_detection_train_loader, + build_inference_based_loaders, + has_inference_based_loaders, +) +from densepose.evaluation.d2_evaluator_adapter import Detectron2COCOEvaluatorAdapter +from densepose.evaluation.evaluator import DensePoseCOCOEvaluator, build_densepose_evaluator_storage +from densepose.modeling.cse import Embedder + + +class SampleCountingLoader: + def __init__(self, loader): + self.loader = loader + + def __iter__(self): + it = iter(self.loader) + storage = get_event_storage() + while True: + try: + batch = next(it) + num_inst_per_dataset = {} + for data in batch: + dataset_name = data["dataset"] + if dataset_name not in num_inst_per_dataset: + num_inst_per_dataset[dataset_name] = 0 + num_inst = len(data["instances"]) + num_inst_per_dataset[dataset_name] += num_inst + for dataset_name in num_inst_per_dataset: + storage.put_scalar(f"batch/{dataset_name}", num_inst_per_dataset[dataset_name]) + yield batch + except StopIteration: + break + + +class SampleCountMetricPrinter(EventWriter): + def __init__(self): + self.logger = logging.getLogger(__name__) + + def write(self): + storage = get_event_storage() + batch_stats_strs = [] + for key, buf in storage.histories().items(): + if key.startswith("batch/"): + batch_stats_strs.append(f"{key} {buf.avg(20)}") + self.logger.info(", ".join(batch_stats_strs)) + + +class Trainer(DefaultTrainer): + @classmethod + def extract_embedder_from_model(cls, model: nn.Module) -> Optional[Embedder]: + if isinstance(model, nn.parallel.DistributedDataParallel): + model = model.module + if hasattr(model, "roi_heads") and hasattr(model.roi_heads, "embedder"): + # pyre-fixme[16]: Item `Tensor` of `Union[Tensor, Module]` has no + # attribute `embedder`. + return model.roi_heads.embedder + return None + + # TODO: the only reason to copy the base class code here is to pass the embedder from + # the model to the evaluator; that should be refactored to avoid unnecessary copy-pasting + @classmethod + def test( + cls, + cfg: CfgNode, + model: nn.Module, + evaluators: Optional[Union[DatasetEvaluator, List[DatasetEvaluator]]] = None, + ): + """ + Args: + cfg (CfgNode): + model (nn.Module): + evaluators (DatasetEvaluator, list[DatasetEvaluator] or None): if None, will call + :meth:`build_evaluator`. Otherwise, must have the same length as + ``cfg.DATASETS.TEST``. + + Returns: + dict: a dict of result metrics + """ + logger = logging.getLogger(__name__) + if isinstance(evaluators, DatasetEvaluator): + evaluators = [evaluators] + if evaluators is not None: + assert len(cfg.DATASETS.TEST) == len(evaluators), "{} != {}".format( + len(cfg.DATASETS.TEST), len(evaluators) + ) + + results = OrderedDict() + for idx, dataset_name in enumerate(cfg.DATASETS.TEST): + data_loader = cls.build_test_loader(cfg, dataset_name) + # When evaluators are passed in as arguments, + # implicitly assume that evaluators can be created before data_loader. + if evaluators is not None: + evaluator = evaluators[idx] + else: + try: + embedder = cls.extract_embedder_from_model(model) + evaluator = cls.build_evaluator(cfg, dataset_name, embedder=embedder) + except NotImplementedError: + logger.warn( + "No evaluator found. Use `DefaultTrainer.test(evaluators=)`, " + "or implement its `build_evaluator` method." + ) + results[dataset_name] = {} + continue + if cfg.DENSEPOSE_EVALUATION.DISTRIBUTED_INFERENCE or comm.is_main_process(): + results_i = inference_on_dataset(model, data_loader, evaluator) + else: + results_i = {} + results[dataset_name] = results_i + if comm.is_main_process(): + assert isinstance( + results_i, dict + ), "Evaluator must return a dict on the main process. Got {} instead.".format( + results_i + ) + logger.info("Evaluation results for {} in csv format:".format(dataset_name)) + print_csv_format(results_i) + + if len(results) == 1: + results = list(results.values())[0] + return results + + @classmethod + def build_evaluator( + cls, + cfg: CfgNode, + dataset_name: str, + output_folder: Optional[str] = None, + embedder: Optional[Embedder] = None, + ) -> DatasetEvaluators: + if output_folder is None: + output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") + evaluators = [] + distributed = cfg.DENSEPOSE_EVALUATION.DISTRIBUTED_INFERENCE + # Note: we currently use COCO evaluator for both COCO and LVIS datasets + # to have compatible metrics. LVIS bbox evaluator could also be used + # with an adapter to properly handle filtered / mapped categories + # evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type + # if evaluator_type == "coco": + # evaluators.append(COCOEvaluator(dataset_name, output_dir=output_folder)) + # elif evaluator_type == "lvis": + # evaluators.append(LVISEvaluator(dataset_name, output_dir=output_folder)) + evaluators.append( + Detectron2COCOEvaluatorAdapter( + dataset_name, output_dir=output_folder, distributed=distributed + ) + ) + if cfg.MODEL.DENSEPOSE_ON: + storage = build_densepose_evaluator_storage(cfg, output_folder) + evaluators.append( + DensePoseCOCOEvaluator( + dataset_name, + distributed, + output_folder, + evaluator_type=cfg.DENSEPOSE_EVALUATION.TYPE, + min_iou_threshold=cfg.DENSEPOSE_EVALUATION.MIN_IOU_THRESHOLD, + storage=storage, + embedder=embedder, + should_evaluate_mesh_alignment=cfg.DENSEPOSE_EVALUATION.EVALUATE_MESH_ALIGNMENT, + mesh_alignment_mesh_names=cfg.DENSEPOSE_EVALUATION.MESH_ALIGNMENT_MESH_NAMES, + ) + ) + return DatasetEvaluators(evaluators) + + @classmethod + def build_optimizer(cls, cfg: CfgNode, model: nn.Module): + params = get_default_optimizer_params( + model, + base_lr=cfg.SOLVER.BASE_LR, + weight_decay_norm=cfg.SOLVER.WEIGHT_DECAY_NORM, + bias_lr_factor=cfg.SOLVER.BIAS_LR_FACTOR, + weight_decay_bias=cfg.SOLVER.WEIGHT_DECAY_BIAS, + overrides={ + "features": { + "lr": cfg.SOLVER.BASE_LR * cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.FEATURES_LR_FACTOR, + }, + "embeddings": { + "lr": cfg.SOLVER.BASE_LR * cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBEDDING_LR_FACTOR, + }, + }, + ) + optimizer = torch.optim.SGD( + params, + cfg.SOLVER.BASE_LR, + momentum=cfg.SOLVER.MOMENTUM, + nesterov=cfg.SOLVER.NESTEROV, + weight_decay=cfg.SOLVER.WEIGHT_DECAY, + ) + # pyre-fixme[6]: For 2nd param expected `Type[Optimizer]` but got `SGD`. + return maybe_add_gradient_clipping(cfg, optimizer) + + @classmethod + def build_test_loader(cls, cfg: CfgNode, dataset_name): + return build_detection_test_loader(cfg, dataset_name, mapper=DatasetMapper(cfg, False)) + + @classmethod + def build_train_loader(cls, cfg: CfgNode): + data_loader = build_detection_train_loader(cfg, mapper=DatasetMapper(cfg, True)) + if not has_inference_based_loaders(cfg): + return data_loader + model = cls.build_model(cfg) + model.to(cfg.BOOTSTRAP_MODEL.DEVICE) + DetectionCheckpointer(model).resume_or_load(cfg.BOOTSTRAP_MODEL.WEIGHTS, resume=False) + inference_based_loaders, ratios = build_inference_based_loaders(cfg, model) + loaders = [data_loader] + inference_based_loaders + ratios = [1.0] + ratios + combined_data_loader = build_combined_loader(cfg, loaders, ratios) + sample_counting_loader = SampleCountingLoader(combined_data_loader) + return sample_counting_loader + + def build_writers(self): + writers = super().build_writers() + writers.append(SampleCountMetricPrinter()) + return writers + + @classmethod + def test_with_TTA(cls, cfg: CfgNode, model): + logger = logging.getLogger("detectron2.trainer") + # In the end of training, run an evaluation with TTA + # Only support some R-CNN models. + logger.info("Running inference with test-time augmentation ...") + transform_data = load_from_cfg(cfg) + model = DensePoseGeneralizedRCNNWithTTA( + cfg, model, transform_data, DensePoseDatasetMapperTTA(cfg) + ) + evaluators = [ + cls.build_evaluator( + cfg, name, output_folder=os.path.join(cfg.OUTPUT_DIR, "inference_TTA") + ) + for name in cfg.DATASETS.TEST + ] + res = cls.test(cfg, model, evaluators) # pyre-ignore[6] + res = OrderedDict({k + "_TTA": v for k, v in res.items()}) + return res diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/__init__.py b/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e5ae1f20cdc822ebf3c870f1289a0ad210c57ae7 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from .evaluator import DensePoseCOCOEvaluator diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/d2_evaluator_adapter.py b/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/d2_evaluator_adapter.py new file mode 100644 index 0000000000000000000000000000000000000000..1fbc526059a191f9414231c1b21ed3e8b7b58580 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/d2_evaluator_adapter.py @@ -0,0 +1,50 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from detectron2.data.catalog import Metadata +from detectron2.evaluation import COCOEvaluator + +from densepose.data.datasets.coco import ( + get_contiguous_id_to_category_id_map, + maybe_filter_categories_cocoapi, +) + + +def _maybe_add_iscrowd_annotations(cocoapi) -> None: + for ann in cocoapi.dataset["annotations"]: + if "iscrowd" not in ann: + ann["iscrowd"] = 0 + + +class Detectron2COCOEvaluatorAdapter(COCOEvaluator): + def __init__( + self, + dataset_name, + output_dir=None, + distributed=True, + ): + super().__init__(dataset_name, output_dir=output_dir, distributed=distributed) + maybe_filter_categories_cocoapi(dataset_name, self._coco_api) + _maybe_add_iscrowd_annotations(self._coco_api) + # substitute category metadata to account for categories + # that are mapped to the same contiguous id + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"): + self._maybe_substitute_metadata() + + def _maybe_substitute_metadata(self): + cont_id_2_cat_id = get_contiguous_id_to_category_id_map(self._metadata) + cat_id_2_cont_id = self._metadata.thing_dataset_id_to_contiguous_id + if len(cont_id_2_cat_id) == len(cat_id_2_cont_id): + return + + cat_id_2_cont_id_injective = {} + for cat_id, cont_id in cat_id_2_cont_id.items(): + if (cont_id in cont_id_2_cat_id) and (cont_id_2_cat_id[cont_id] == cat_id): + cat_id_2_cont_id_injective[cat_id] = cont_id + + metadata_new = Metadata(name=self._metadata.name) + for key, value in self._metadata.__dict__.items(): + if key == "thing_dataset_id_to_contiguous_id": + setattr(metadata_new, key, cat_id_2_cont_id_injective) + else: + setattr(metadata_new, key, value) + self._metadata = metadata_new diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/densepose_coco_evaluation.py b/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/densepose_coco_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..06965f34c4b4446e99c3df515dc39b5af0f404e0 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/densepose_coco_evaluation.py @@ -0,0 +1,1303 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# This is a modified version of cocoeval.py where we also have the densepose evaluation. + +__author__ = "tsungyi" + +import copy +import datetime +import logging +import numpy as np +import pickle +import time +from collections import defaultdict +from enum import Enum +from typing import Any, Dict, Tuple +import scipy.spatial.distance as ssd +import torch +import torch.nn.functional as F +from pycocotools import mask as maskUtils +from scipy.io import loadmat +from scipy.ndimage import zoom as spzoom + +from detectron2.utils.file_io import PathManager + +from densepose.converters.chart_output_to_chart_result import resample_uv_tensors_to_bbox +from densepose.converters.segm_to_mask import ( + resample_coarse_segm_tensor_to_bbox, + resample_fine_and_coarse_segm_tensors_to_bbox, +) +from densepose.modeling.cse.utils import squared_euclidean_distance_matrix +from densepose.structures import DensePoseDataRelative +from densepose.structures.mesh import create_mesh + +logger = logging.getLogger(__name__) + + +class DensePoseEvalMode(str, Enum): + # use both masks and geodesic distances (GPS * IOU) to compute scores + GPSM = "gpsm" + # use only geodesic distances (GPS) to compute scores + GPS = "gps" + # use only masks (IOU) to compute scores + IOU = "iou" + + +class DensePoseDataMode(str, Enum): + # use estimated IUV data (default mode) + IUV_DT = "iuvdt" + # use ground truth IUV data + IUV_GT = "iuvgt" + # use ground truth labels I and set UV to 0 + I_GT_UV_0 = "igtuv0" + # use ground truth labels I and estimated UV coordinates + I_GT_UV_DT = "igtuvdt" + # use estimated labels I and set UV to 0 + I_DT_UV_0 = "idtuv0" + + +class DensePoseCocoEval(object): + # Interface for evaluating detection on the Microsoft COCO dataset. + # + # The usage for CocoEval is as follows: + # cocoGt=..., cocoDt=... # load dataset and results + # E = CocoEval(cocoGt,cocoDt); # initialize CocoEval object + # E.params.recThrs = ...; # set parameters as desired + # E.evaluate(); # run per image evaluation + # E.accumulate(); # accumulate per image results + # E.summarize(); # display summary metrics of results + # For example usage see evalDemo.m and http://mscoco.org/. + # + # The evaluation parameters are as follows (defaults in brackets): + # imgIds - [all] N img ids to use for evaluation + # catIds - [all] K cat ids to use for evaluation + # iouThrs - [.5:.05:.95] T=10 IoU thresholds for evaluation + # recThrs - [0:.01:1] R=101 recall thresholds for evaluation + # areaRng - [...] A=4 object area ranges for evaluation + # maxDets - [1 10 100] M=3 thresholds on max detections per image + # iouType - ['segm'] set iouType to 'segm', 'bbox', 'keypoints' or 'densepose' + # iouType replaced the now DEPRECATED useSegm parameter. + # useCats - [1] if true use category labels for evaluation + # Note: if useCats=0 category labels are ignored as in proposal scoring. + # Note: multiple areaRngs [Ax2] and maxDets [Mx1] can be specified. + # + # evaluate(): evaluates detections on every image and every category and + # concats the results into the "evalImgs" with fields: + # dtIds - [1xD] id for each of the D detections (dt) + # gtIds - [1xG] id for each of the G ground truths (gt) + # dtMatches - [TxD] matching gt id at each IoU or 0 + # gtMatches - [TxG] matching dt id at each IoU or 0 + # dtScores - [1xD] confidence of each dt + # gtIgnore - [1xG] ignore flag for each gt + # dtIgnore - [TxD] ignore flag for each dt at each IoU + # + # accumulate(): accumulates the per-image, per-category evaluation + # results in "evalImgs" into the dictionary "eval" with fields: + # params - parameters used for evaluation + # date - date evaluation was performed + # counts - [T,R,K,A,M] parameter dimensions (see above) + # precision - [TxRxKxAxM] precision for every evaluation setting + # recall - [TxKxAxM] max recall for every evaluation setting + # Note: precision and recall==-1 for settings with no gt objects. + # + # See also coco, mask, pycocoDemo, pycocoEvalDemo + # + # Microsoft COCO Toolbox. version 2.0 + # Data, paper, and tutorials available at: http://mscoco.org/ + # Code written by Piotr Dollar and Tsung-Yi Lin, 2015. + # Licensed under the Simplified BSD License [see coco/license.txt] + def __init__( + self, + cocoGt=None, + cocoDt=None, + iouType: str = "densepose", + multi_storage=None, + embedder=None, + dpEvalMode: DensePoseEvalMode = DensePoseEvalMode.GPS, + dpDataMode: DensePoseDataMode = DensePoseDataMode.IUV_DT, + ): + """ + Initialize CocoEval using coco APIs for gt and dt + :param cocoGt: coco object with ground truth annotations + :param cocoDt: coco object with detection results + :return: None + """ + self.cocoGt = cocoGt # ground truth COCO API + self.cocoDt = cocoDt # detections COCO API + self.multi_storage = multi_storage + self.embedder = embedder + self._dpEvalMode = dpEvalMode + self._dpDataMode = dpDataMode + self.evalImgs = defaultdict(list) # per-image per-category eval results [KxAxI] + self.eval = {} # accumulated evaluation results + self._gts = defaultdict(list) # gt for evaluation + self._dts = defaultdict(list) # dt for evaluation + self.params = Params(iouType=iouType) # parameters + self._paramsEval = {} # parameters for evaluation + self.stats = [] # result summarization + self.ious = {} # ious between all gts and dts + if cocoGt is not None: + self.params.imgIds = sorted(cocoGt.getImgIds()) + self.params.catIds = sorted(cocoGt.getCatIds()) + self.ignoreThrBB = 0.7 + self.ignoreThrUV = 0.9 + + def _loadGEval(self): + smpl_subdiv_fpath = PathManager.get_local_path( + "https://dl.fbaipublicfiles.com/densepose/data/SMPL_subdiv.mat" + ) + pdist_transform_fpath = PathManager.get_local_path( + "https://dl.fbaipublicfiles.com/densepose/data/SMPL_SUBDIV_TRANSFORM.mat" + ) + pdist_matrix_fpath = PathManager.get_local_path( + "https://dl.fbaipublicfiles.com/densepose/data/Pdist_matrix.pkl", timeout_sec=120 + ) + SMPL_subdiv = loadmat(smpl_subdiv_fpath) + self.PDIST_transform = loadmat(pdist_transform_fpath) + self.PDIST_transform = self.PDIST_transform["index"].squeeze() + UV = np.array([SMPL_subdiv["U_subdiv"], SMPL_subdiv["V_subdiv"]]).squeeze() + ClosestVertInds = np.arange(UV.shape[1]) + 1 + self.Part_UVs = [] + self.Part_ClosestVertInds = [] + for i in np.arange(24): + self.Part_UVs.append(UV[:, SMPL_subdiv["Part_ID_subdiv"].squeeze() == (i + 1)]) + self.Part_ClosestVertInds.append( + ClosestVertInds[SMPL_subdiv["Part_ID_subdiv"].squeeze() == (i + 1)] + ) + + with open(pdist_matrix_fpath, "rb") as hFile: + arrays = pickle.load(hFile, encoding="latin1") + self.Pdist_matrix = arrays["Pdist_matrix"] + self.Part_ids = np.array(SMPL_subdiv["Part_ID_subdiv"].squeeze()) + # Mean geodesic distances for parts. + self.Mean_Distances = np.array([0, 0.351, 0.107, 0.126, 0.237, 0.173, 0.142, 0.128, 0.150]) + # Coarse Part labels. + self.CoarseParts = np.array( + [0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8] + ) + + def _prepare(self): + """ + Prepare ._gts and ._dts for evaluation based on params + :return: None + """ + + def _toMask(anns, coco): + # modify ann['segmentation'] by reference + for ann in anns: + # safeguard for invalid segmentation annotation; + # annotations containing empty lists exist in the posetrack + # dataset. This is not a correct segmentation annotation + # in terms of COCO format; we need to deal with it somehow + segm = ann["segmentation"] + if type(segm) == list and len(segm) == 0: + ann["segmentation"] = None + continue + rle = coco.annToRLE(ann) + ann["segmentation"] = rle + + def _getIgnoreRegion(iid, coco): + img = coco.imgs[iid] + + if "ignore_regions_x" not in img.keys(): + return None + + if len(img["ignore_regions_x"]) == 0: + return None + + rgns_merged = [ + [v for xy in zip(region_x, region_y) for v in xy] + for region_x, region_y in zip(img["ignore_regions_x"], img["ignore_regions_y"]) + ] + rles = maskUtils.frPyObjects(rgns_merged, img["height"], img["width"]) + rle = maskUtils.merge(rles) + return maskUtils.decode(rle) + + def _checkIgnore(dt, iregion): + if iregion is None: + return True + + bb = np.array(dt["bbox"]).astype(np.int) + x1, y1, x2, y2 = bb[0], bb[1], bb[0] + bb[2], bb[1] + bb[3] + x2 = min([x2, iregion.shape[1]]) + y2 = min([y2, iregion.shape[0]]) + + if bb[2] * bb[3] == 0: + return False + + crop_iregion = iregion[y1:y2, x1:x2] + + if crop_iregion.sum() == 0: + return True + + if "densepose" not in dt.keys(): # filtering boxes + return crop_iregion.sum() / bb[2] / bb[3] < self.ignoreThrBB + + # filtering UVs + ignoremask = np.require(crop_iregion, requirements=["F"]) + mask = self._extract_mask(dt) + uvmask = np.require(np.asarray(mask > 0), dtype=np.uint8, requirements=["F"]) + uvmask_ = maskUtils.encode(uvmask) + ignoremask_ = maskUtils.encode(ignoremask) + uviou = maskUtils.iou([uvmask_], [ignoremask_], [1])[0] + return uviou < self.ignoreThrUV + + p = self.params + + if p.useCats: + gts = self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds)) + dts = self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds)) + else: + gts = self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds)) + dts = self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds)) + + imns = self.cocoGt.loadImgs(p.imgIds) + self.size_mapping = {} + for im in imns: + self.size_mapping[im["id"]] = [im["height"], im["width"]] + + # if iouType == 'uv', add point gt annotations + if p.iouType == "densepose": + self._loadGEval() + + # convert ground truth to mask if iouType == 'segm' + if p.iouType == "segm": + _toMask(gts, self.cocoGt) + _toMask(dts, self.cocoDt) + + # set ignore flag + for gt in gts: + gt["ignore"] = gt["ignore"] if "ignore" in gt else 0 + gt["ignore"] = "iscrowd" in gt and gt["iscrowd"] + if p.iouType == "keypoints": + gt["ignore"] = (gt["num_keypoints"] == 0) or gt["ignore"] + if p.iouType == "densepose": + gt["ignore"] = ("dp_x" in gt) == 0 + if p.iouType == "segm": + gt["ignore"] = gt["segmentation"] is None + + self._gts = defaultdict(list) # gt for evaluation + self._dts = defaultdict(list) # dt for evaluation + self._igrgns = defaultdict(list) + + for gt in gts: + iid = gt["image_id"] + if iid not in self._igrgns.keys(): + self._igrgns[iid] = _getIgnoreRegion(iid, self.cocoGt) + if _checkIgnore(gt, self._igrgns[iid]): + self._gts[iid, gt["category_id"]].append(gt) + for dt in dts: + iid = dt["image_id"] + if (iid not in self._igrgns) or _checkIgnore(dt, self._igrgns[iid]): + self._dts[iid, dt["category_id"]].append(dt) + + self.evalImgs = defaultdict(list) # per-image per-category evaluation results + self.eval = {} # accumulated evaluation results + + def evaluate(self): + """ + Run per image evaluation on given images and store results (a list of dict) in self.evalImgs + :return: None + """ + tic = time.time() + logger.info("Running per image DensePose evaluation... {}".format(self.params.iouType)) + p = self.params + # add backward compatibility if useSegm is specified in params + if p.useSegm is not None: + p.iouType = "segm" if p.useSegm == 1 else "bbox" + logger.info("useSegm (deprecated) is not None. Running DensePose evaluation") + p.imgIds = list(np.unique(p.imgIds)) + if p.useCats: + p.catIds = list(np.unique(p.catIds)) + p.maxDets = sorted(p.maxDets) + self.params = p + + self._prepare() + # loop through images, area range, max detection number + catIds = p.catIds if p.useCats else [-1] + + if p.iouType in ["segm", "bbox"]: + computeIoU = self.computeIoU + elif p.iouType == "keypoints": + computeIoU = self.computeOks + elif p.iouType == "densepose": + computeIoU = self.computeOgps + if self._dpEvalMode in {DensePoseEvalMode.GPSM, DensePoseEvalMode.IOU}: + self.real_ious = { + (imgId, catId): self.computeDPIoU(imgId, catId) + for imgId in p.imgIds + for catId in catIds + } + + self.ious = { + (imgId, catId): computeIoU(imgId, catId) for imgId in p.imgIds for catId in catIds + } + + evaluateImg = self.evaluateImg + maxDet = p.maxDets[-1] + self.evalImgs = [ + evaluateImg(imgId, catId, areaRng, maxDet) + for catId in catIds + for areaRng in p.areaRng + for imgId in p.imgIds + ] + self._paramsEval = copy.deepcopy(self.params) + toc = time.time() + logger.info("DensePose evaluation DONE (t={:0.2f}s).".format(toc - tic)) + + def getDensePoseMask(self, polys): + maskGen = np.zeros([256, 256]) + stop = min(len(polys) + 1, 15) + for i in range(1, stop): + if polys[i - 1]: + currentMask = maskUtils.decode(polys[i - 1]) + maskGen[currentMask > 0] = i + return maskGen + + def _generate_rlemask_on_image(self, mask, imgId, data): + bbox_xywh = np.array(data["bbox"]) + x, y, w, h = bbox_xywh + im_h, im_w = self.size_mapping[imgId] + im_mask = np.zeros((im_h, im_w), dtype=np.uint8) + if mask is not None: + x0 = max(int(x), 0) + x1 = min(int(x + w), im_w, int(x) + mask.shape[1]) + y0 = max(int(y), 0) + y1 = min(int(y + h), im_h, int(y) + mask.shape[0]) + y = int(y) + x = int(x) + im_mask[y0:y1, x0:x1] = mask[y0 - y : y1 - y, x0 - x : x1 - x] + im_mask = np.require(np.asarray(im_mask > 0), dtype=np.uint8, requirements=["F"]) + rle_mask = maskUtils.encode(np.array(im_mask[:, :, np.newaxis], order="F"))[0] + return rle_mask + + def computeDPIoU(self, imgId, catId): + p = self.params + if p.useCats: + gt = self._gts[imgId, catId] + dt = self._dts[imgId, catId] + else: + gt = [_ for cId in p.catIds for _ in self._gts[imgId, cId]] + dt = [_ for cId in p.catIds for _ in self._dts[imgId, cId]] + if len(gt) == 0 and len(dt) == 0: + return [] + inds = np.argsort([-d["score"] for d in dt], kind="mergesort") + dt = [dt[i] for i in inds] + if len(dt) > p.maxDets[-1]: + dt = dt[0 : p.maxDets[-1]] + + gtmasks = [] + for g in gt: + if DensePoseDataRelative.S_KEY in g: + # convert DensePose mask to a binary mask + mask = np.minimum(self.getDensePoseMask(g[DensePoseDataRelative.S_KEY]), 1.0) + _, _, w, h = g["bbox"] + scale_x = float(max(w, 1)) / mask.shape[1] + scale_y = float(max(h, 1)) / mask.shape[0] + mask = spzoom(mask, (scale_y, scale_x), order=1, prefilter=False) + mask = np.array(mask > 0.5, dtype=np.uint8) + rle_mask = self._generate_rlemask_on_image(mask, imgId, g) + elif "segmentation" in g: + segmentation = g["segmentation"] + if isinstance(segmentation, list) and segmentation: + # polygons + im_h, im_w = self.size_mapping[imgId] + rles = maskUtils.frPyObjects(segmentation, im_h, im_w) + rle_mask = maskUtils.merge(rles) + elif isinstance(segmentation, dict): + if isinstance(segmentation["counts"], list): + # uncompressed RLE + im_h, im_w = self.size_mapping[imgId] + rle_mask = maskUtils.frPyObjects(segmentation, im_h, im_w) + else: + # compressed RLE + rle_mask = segmentation + else: + rle_mask = self._generate_rlemask_on_image(None, imgId, g) + else: + rle_mask = self._generate_rlemask_on_image(None, imgId, g) + gtmasks.append(rle_mask) + + dtmasks = [] + for d in dt: + mask = self._extract_mask(d) + mask = np.require(np.asarray(mask > 0), dtype=np.uint8, requirements=["F"]) + rle_mask = self._generate_rlemask_on_image(mask, imgId, d) + dtmasks.append(rle_mask) + + # compute iou between each dt and gt region + iscrowd = [int(o.get("iscrowd", 0)) for o in gt] + iousDP = maskUtils.iou(dtmasks, gtmasks, iscrowd) + return iousDP + + def computeIoU(self, imgId, catId): + p = self.params + if p.useCats: + gt = self._gts[imgId, catId] + dt = self._dts[imgId, catId] + else: + gt = [_ for cId in p.catIds for _ in self._gts[imgId, cId]] + dt = [_ for cId in p.catIds for _ in self._dts[imgId, cId]] + if len(gt) == 0 and len(dt) == 0: + return [] + inds = np.argsort([-d["score"] for d in dt], kind="mergesort") + dt = [dt[i] for i in inds] + if len(dt) > p.maxDets[-1]: + dt = dt[0 : p.maxDets[-1]] + + if p.iouType == "segm": + g = [g["segmentation"] for g in gt if g["segmentation"] is not None] + d = [d["segmentation"] for d in dt if d["segmentation"] is not None] + elif p.iouType == "bbox": + g = [g["bbox"] for g in gt] + d = [d["bbox"] for d in dt] + else: + raise Exception("unknown iouType for iou computation") + + # compute iou between each dt and gt region + iscrowd = [int(o.get("iscrowd", 0)) for o in gt] + ious = maskUtils.iou(d, g, iscrowd) + return ious + + def computeOks(self, imgId, catId): + p = self.params + # dimension here should be Nxm + gts = self._gts[imgId, catId] + dts = self._dts[imgId, catId] + inds = np.argsort([-d["score"] for d in dts], kind="mergesort") + dts = [dts[i] for i in inds] + if len(dts) > p.maxDets[-1]: + dts = dts[0 : p.maxDets[-1]] + # if len(gts) == 0 and len(dts) == 0: + if len(gts) == 0 or len(dts) == 0: + return [] + ious = np.zeros((len(dts), len(gts))) + sigmas = ( + np.array( + [ + 0.26, + 0.25, + 0.25, + 0.35, + 0.35, + 0.79, + 0.79, + 0.72, + 0.72, + 0.62, + 0.62, + 1.07, + 1.07, + 0.87, + 0.87, + 0.89, + 0.89, + ] + ) + / 10.0 + ) + vars = (sigmas * 2) ** 2 + k = len(sigmas) + # compute oks between each detection and ground truth object + for j, gt in enumerate(gts): + # create bounds for ignore regions(double the gt bbox) + g = np.array(gt["keypoints"]) + xg = g[0::3] + yg = g[1::3] + vg = g[2::3] + k1 = np.count_nonzero(vg > 0) + bb = gt["bbox"] + x0 = bb[0] - bb[2] + x1 = bb[0] + bb[2] * 2 + y0 = bb[1] - bb[3] + y1 = bb[1] + bb[3] * 2 + for i, dt in enumerate(dts): + d = np.array(dt["keypoints"]) + xd = d[0::3] + yd = d[1::3] + if k1 > 0: + # measure the per-keypoint distance if keypoints visible + dx = xd - xg + dy = yd - yg + else: + # measure minimum distance to keypoints in (x0,y0) & (x1,y1) + z = np.zeros(k) + dx = np.max((z, x0 - xd), axis=0) + np.max((z, xd - x1), axis=0) + dy = np.max((z, y0 - yd), axis=0) + np.max((z, yd - y1), axis=0) + e = (dx**2 + dy**2) / vars / (gt["area"] + np.spacing(1)) / 2 + if k1 > 0: + e = e[vg > 0] + ious[i, j] = np.sum(np.exp(-e)) / e.shape[0] + return ious + + def _extract_mask(self, dt: Dict[str, Any]) -> np.ndarray: + if "densepose" in dt: + densepose_results_quantized = dt["densepose"] + return densepose_results_quantized.labels_uv_uint8[0].numpy() + elif "cse_mask" in dt: + return dt["cse_mask"] + elif "coarse_segm" in dt: + dy = max(int(dt["bbox"][3]), 1) + dx = max(int(dt["bbox"][2]), 1) + return ( + F.interpolate( + dt["coarse_segm"].unsqueeze(0), + (dy, dx), + mode="bilinear", + align_corners=False, + ) + .squeeze(0) + .argmax(0) + .numpy() + .astype(np.uint8) + ) + elif "record_id" in dt: + assert ( + self.multi_storage is not None + ), f"Storage record id encountered in a detection {dt}, but no storage provided!" + record = self.multi_storage.get(dt["rank"], dt["record_id"]) + coarse_segm = record["coarse_segm"] + dy = max(int(dt["bbox"][3]), 1) + dx = max(int(dt["bbox"][2]), 1) + return ( + F.interpolate( + coarse_segm.unsqueeze(0), + (dy, dx), + mode="bilinear", + align_corners=False, + ) + .squeeze(0) + .argmax(0) + .numpy() + .astype(np.uint8) + ) + else: + raise Exception(f"No mask data in the detection: {dt}") + raise ValueError('The prediction dict needs to contain either "densepose" or "cse_mask"') + + def _extract_iuv( + self, densepose_data: np.ndarray, py: np.ndarray, px: np.ndarray, gt: Dict[str, Any] + ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """ + Extract arrays of I, U and V values at given points as numpy arrays + given the data mode stored in self._dpDataMode + """ + if self._dpDataMode == DensePoseDataMode.IUV_DT: + # estimated labels and UV (default) + ipoints = densepose_data[0, py, px] + upoints = densepose_data[1, py, px] / 255.0 # convert from uint8 by /255. + vpoints = densepose_data[2, py, px] / 255.0 + elif self._dpDataMode == DensePoseDataMode.IUV_GT: + # ground truth + ipoints = np.array(gt["dp_I"]) + upoints = np.array(gt["dp_U"]) + vpoints = np.array(gt["dp_V"]) + elif self._dpDataMode == DensePoseDataMode.I_GT_UV_0: + # ground truth labels, UV = 0 + ipoints = np.array(gt["dp_I"]) + upoints = upoints * 0.0 + vpoints = vpoints * 0.0 + elif self._dpDataMode == DensePoseDataMode.I_GT_UV_DT: + # ground truth labels, estimated UV + ipoints = np.array(gt["dp_I"]) + upoints = densepose_data[1, py, px] / 255.0 # convert from uint8 by /255. + vpoints = densepose_data[2, py, px] / 255.0 + elif self._dpDataMode == DensePoseDataMode.I_DT_UV_0: + # estimated labels, UV = 0 + ipoints = densepose_data[0, py, px] + upoints = upoints * 0.0 + vpoints = vpoints * 0.0 + else: + raise ValueError(f"Unknown data mode: {self._dpDataMode}") + return ipoints, upoints, vpoints + + def computeOgps_single_pair(self, dt, gt, py, px, pt_mask): + if "densepose" in dt: + ipoints, upoints, vpoints = self.extract_iuv_from_quantized(dt, gt, py, px, pt_mask) + return self.computeOgps_single_pair_iuv(dt, gt, ipoints, upoints, vpoints) + elif "u" in dt: + ipoints, upoints, vpoints = self.extract_iuv_from_raw(dt, gt, py, px, pt_mask) + return self.computeOgps_single_pair_iuv(dt, gt, ipoints, upoints, vpoints) + elif "record_id" in dt: + assert ( + self.multi_storage is not None + ), f"Storage record id encountered in detection {dt}, but no storage provided!" + record = self.multi_storage.get(dt["rank"], dt["record_id"]) + record["bbox"] = dt["bbox"] + if "u" in record: + ipoints, upoints, vpoints = self.extract_iuv_from_raw(record, gt, py, px, pt_mask) + return self.computeOgps_single_pair_iuv(dt, gt, ipoints, upoints, vpoints) + elif "embedding" in record: + return self.computeOgps_single_pair_cse( + dt, + gt, + py, + px, + pt_mask, + record["coarse_segm"], + record["embedding"], + record["bbox"], + ) + else: + raise Exception(f"Unknown record format: {record}") + elif "embedding" in dt: + return self.computeOgps_single_pair_cse( + dt, gt, py, px, pt_mask, dt["coarse_segm"], dt["embedding"], dt["bbox"] + ) + raise Exception(f"Unknown detection format: {dt}") + + def extract_iuv_from_quantized(self, dt, gt, py, px, pt_mask): + densepose_results_quantized = dt["densepose"] + ipoints, upoints, vpoints = self._extract_iuv( + densepose_results_quantized.labels_uv_uint8.numpy(), py, px, gt + ) + ipoints[pt_mask == -1] = 0 + return ipoints, upoints, vpoints + + def extract_iuv_from_raw(self, dt, gt, py, px, pt_mask): + labels_dt = resample_fine_and_coarse_segm_tensors_to_bbox( + dt["fine_segm"].unsqueeze(0), + dt["coarse_segm"].unsqueeze(0), + dt["bbox"], + ) + uv = resample_uv_tensors_to_bbox( + dt["u"].unsqueeze(0), dt["v"].unsqueeze(0), labels_dt.squeeze(0), dt["bbox"] + ) + labels_uv_uint8 = torch.cat((labels_dt.byte(), (uv * 255).clamp(0, 255).byte())) + ipoints, upoints, vpoints = self._extract_iuv(labels_uv_uint8.numpy(), py, px, gt) + ipoints[pt_mask == -1] = 0 + return ipoints, upoints, vpoints + + def computeOgps_single_pair_iuv(self, dt, gt, ipoints, upoints, vpoints): + cVertsGT, ClosestVertsGTTransformed = self.findAllClosestVertsGT(gt) + cVerts = self.findAllClosestVertsUV(upoints, vpoints, ipoints) + # Get pairwise geodesic distances between gt and estimated mesh points. + dist = self.getDistancesUV(ClosestVertsGTTransformed, cVerts) + # Compute the Ogps measure. + # Find the mean geodesic normalization distance for + # each GT point, based on which part it is on. + Current_Mean_Distances = self.Mean_Distances[ + self.CoarseParts[self.Part_ids[cVertsGT[cVertsGT > 0].astype(int) - 1]] + ] + return dist, Current_Mean_Distances + + def computeOgps_single_pair_cse( + self, dt, gt, py, px, pt_mask, coarse_segm, embedding, bbox_xywh_abs + ): + # 0-based mesh vertex indices + cVertsGT = torch.as_tensor(gt["dp_vertex"], dtype=torch.int64) + # label for each pixel of the bbox, [H, W] tensor of long + labels_dt = resample_coarse_segm_tensor_to_bbox( + coarse_segm.unsqueeze(0), bbox_xywh_abs + ).squeeze(0) + x, y, w, h = bbox_xywh_abs + # embedding for each pixel of the bbox, [D, H, W] tensor of float32 + embedding = F.interpolate( + embedding.unsqueeze(0), (int(h), int(w)), mode="bilinear", align_corners=False + ).squeeze(0) + # valid locations py, px + py_pt = torch.from_numpy(py[pt_mask > -1]) + px_pt = torch.from_numpy(px[pt_mask > -1]) + cVerts = torch.ones_like(cVertsGT) * -1 + cVerts[pt_mask > -1] = self.findClosestVertsCse( + embedding, py_pt, px_pt, labels_dt, gt["ref_model"] + ) + # Get pairwise geodesic distances between gt and estimated mesh points. + dist = self.getDistancesCse(cVertsGT, cVerts, gt["ref_model"]) + # normalize distances + if (gt["ref_model"] == "smpl_27554") and ("dp_I" in gt): + Current_Mean_Distances = self.Mean_Distances[ + self.CoarseParts[np.array(gt["dp_I"], dtype=int)] + ] + else: + Current_Mean_Distances = 0.255 + return dist, Current_Mean_Distances + + def computeOgps(self, imgId, catId): + p = self.params + # dimension here should be Nxm + g = self._gts[imgId, catId] + d = self._dts[imgId, catId] + inds = np.argsort([-d_["score"] for d_ in d], kind="mergesort") + d = [d[i] for i in inds] + if len(d) > p.maxDets[-1]: + d = d[0 : p.maxDets[-1]] + # if len(gts) == 0 and len(dts) == 0: + if len(g) == 0 or len(d) == 0: + return [] + ious = np.zeros((len(d), len(g))) + # compute opgs between each detection and ground truth object + # sigma = self.sigma #0.255 # dist = 0.3m corresponds to ogps = 0.5 + # 1 # dist = 0.3m corresponds to ogps = 0.96 + # 1.45 # dist = 1.7m (person height) corresponds to ogps = 0.5) + for j, gt in enumerate(g): + if not gt["ignore"]: + g_ = gt["bbox"] + for i, dt in enumerate(d): + # + dy = int(dt["bbox"][3]) + dx = int(dt["bbox"][2]) + dp_x = np.array(gt["dp_x"]) * g_[2] / 255.0 + dp_y = np.array(gt["dp_y"]) * g_[3] / 255.0 + py = (dp_y + g_[1] - dt["bbox"][1]).astype(np.int) + px = (dp_x + g_[0] - dt["bbox"][0]).astype(np.int) + # + pts = np.zeros(len(px)) + pts[px >= dx] = -1 + pts[py >= dy] = -1 + pts[px < 0] = -1 + pts[py < 0] = -1 + if len(pts) < 1: + ogps = 0.0 + elif np.max(pts) == -1: + ogps = 0.0 + else: + px[pts == -1] = 0 + py[pts == -1] = 0 + dists_between_matches, dist_norm_coeffs = self.computeOgps_single_pair( + dt, gt, py, px, pts + ) + # Compute gps + ogps_values = np.exp( + -(dists_between_matches**2) / (2 * (dist_norm_coeffs**2)) + ) + # + ogps = np.mean(ogps_values) if len(ogps_values) > 0 else 0.0 + ious[i, j] = ogps + + gbb = [gt["bbox"] for gt in g] + dbb = [dt["bbox"] for dt in d] + + # compute iou between each dt and gt region + iscrowd = [int(o.get("iscrowd", 0)) for o in g] + ious_bb = maskUtils.iou(dbb, gbb, iscrowd) + return ious, ious_bb + + def evaluateImg(self, imgId, catId, aRng, maxDet): + """ + perform evaluation for single category and image + :return: dict (single image results) + """ + + p = self.params + if p.useCats: + gt = self._gts[imgId, catId] + dt = self._dts[imgId, catId] + else: + gt = [_ for cId in p.catIds for _ in self._gts[imgId, cId]] + dt = [_ for cId in p.catIds for _ in self._dts[imgId, cId]] + if len(gt) == 0 and len(dt) == 0: + return None + + for g in gt: + # g['_ignore'] = g['ignore'] + if g["ignore"] or (g["area"] < aRng[0] or g["area"] > aRng[1]): + g["_ignore"] = True + else: + g["_ignore"] = False + + # sort dt highest score first, sort gt ignore last + gtind = np.argsort([g["_ignore"] for g in gt], kind="mergesort") + gt = [gt[i] for i in gtind] + dtind = np.argsort([-d["score"] for d in dt], kind="mergesort") + dt = [dt[i] for i in dtind[0:maxDet]] + iscrowd = [int(o.get("iscrowd", 0)) for o in gt] + # load computed ious + if p.iouType == "densepose": + # print('Checking the length', len(self.ious[imgId, catId])) + # if len(self.ious[imgId, catId]) == 0: + # print(self.ious[imgId, catId]) + ious = ( + self.ious[imgId, catId][0][:, gtind] + if len(self.ious[imgId, catId]) > 0 + else self.ious[imgId, catId] + ) + ioubs = ( + self.ious[imgId, catId][1][:, gtind] + if len(self.ious[imgId, catId]) > 0 + else self.ious[imgId, catId] + ) + if self._dpEvalMode in {DensePoseEvalMode.GPSM, DensePoseEvalMode.IOU}: + iousM = ( + self.real_ious[imgId, catId][:, gtind] + if len(self.real_ious[imgId, catId]) > 0 + else self.real_ious[imgId, catId] + ) + else: + ious = ( + self.ious[imgId, catId][:, gtind] + if len(self.ious[imgId, catId]) > 0 + else self.ious[imgId, catId] + ) + + T = len(p.iouThrs) + G = len(gt) + D = len(dt) + gtm = np.zeros((T, G)) + dtm = np.zeros((T, D)) + gtIg = np.array([g["_ignore"] for g in gt]) + dtIg = np.zeros((T, D)) + if np.all(gtIg) and p.iouType == "densepose": + dtIg = np.logical_or(dtIg, True) + + if len(ious) > 0: # and not p.iouType == 'densepose': + for tind, t in enumerate(p.iouThrs): + for dind, d in enumerate(dt): + # information about best match so far (m=-1 -> unmatched) + iou = min([t, 1 - 1e-10]) + m = -1 + for gind, _g in enumerate(gt): + # if this gt already matched, and not a crowd, continue + if gtm[tind, gind] > 0 and not iscrowd[gind]: + continue + # if dt matched to reg gt, and on ignore gt, stop + if m > -1 and gtIg[m] == 0 and gtIg[gind] == 1: + break + if p.iouType == "densepose": + if self._dpEvalMode == DensePoseEvalMode.GPSM: + new_iou = np.sqrt(iousM[dind, gind] * ious[dind, gind]) + elif self._dpEvalMode == DensePoseEvalMode.IOU: + new_iou = iousM[dind, gind] + elif self._dpEvalMode == DensePoseEvalMode.GPS: + new_iou = ious[dind, gind] + else: + new_iou = ious[dind, gind] + if new_iou < iou: + continue + if new_iou == 0.0: + continue + # if match successful and best so far, store appropriately + iou = new_iou + m = gind + # if match made store id of match for both dt and gt + if m == -1: + continue + dtIg[tind, dind] = gtIg[m] + dtm[tind, dind] = gt[m]["id"] + gtm[tind, m] = d["id"] + + if p.iouType == "densepose": + if not len(ioubs) == 0: + for dind, d in enumerate(dt): + # information about best match so far (m=-1 -> unmatched) + if dtm[tind, dind] == 0: + ioub = 0.8 + m = -1 + for gind, _g in enumerate(gt): + # if this gt already matched, and not a crowd, continue + if gtm[tind, gind] > 0 and not iscrowd[gind]: + continue + # continue to next gt unless better match made + if ioubs[dind, gind] < ioub: + continue + # if match successful and best so far, store appropriately + ioub = ioubs[dind, gind] + m = gind + # if match made store id of match for both dt and gt + if m > -1: + dtIg[:, dind] = gtIg[m] + if gtIg[m]: + dtm[tind, dind] = gt[m]["id"] + gtm[tind, m] = d["id"] + # set unmatched detections outside of area range to ignore + a = np.array([d["area"] < aRng[0] or d["area"] > aRng[1] for d in dt]).reshape((1, len(dt))) + dtIg = np.logical_or(dtIg, np.logical_and(dtm == 0, np.repeat(a, T, 0))) + # store results for given image and category + # print('Done with the function', len(self.ious[imgId, catId])) + return { + "image_id": imgId, + "category_id": catId, + "aRng": aRng, + "maxDet": maxDet, + "dtIds": [d["id"] for d in dt], + "gtIds": [g["id"] for g in gt], + "dtMatches": dtm, + "gtMatches": gtm, + "dtScores": [d["score"] for d in dt], + "gtIgnore": gtIg, + "dtIgnore": dtIg, + } + + def accumulate(self, p=None): + """ + Accumulate per image evaluation results and store the result in self.eval + :param p: input params for evaluation + :return: None + """ + logger.info("Accumulating evaluation results...") + tic = time.time() + if not self.evalImgs: + logger.info("Please run evaluate() first") + # allows input customized parameters + if p is None: + p = self.params + p.catIds = p.catIds if p.useCats == 1 else [-1] + T = len(p.iouThrs) + R = len(p.recThrs) + K = len(p.catIds) if p.useCats else 1 + A = len(p.areaRng) + M = len(p.maxDets) + precision = -(np.ones((T, R, K, A, M))) # -1 for the precision of absent categories + recall = -(np.ones((T, K, A, M))) + + # create dictionary for future indexing + logger.info("Categories: {}".format(p.catIds)) + _pe = self._paramsEval + catIds = _pe.catIds if _pe.useCats else [-1] + setK = set(catIds) + setA = set(map(tuple, _pe.areaRng)) + setM = set(_pe.maxDets) + setI = set(_pe.imgIds) + # get inds to evaluate + k_list = [n for n, k in enumerate(p.catIds) if k in setK] + m_list = [m for n, m in enumerate(p.maxDets) if m in setM] + a_list = [n for n, a in enumerate(map(lambda x: tuple(x), p.areaRng)) if a in setA] + i_list = [n for n, i in enumerate(p.imgIds) if i in setI] + I0 = len(_pe.imgIds) + A0 = len(_pe.areaRng) + # retrieve E at each category, area range, and max number of detections + for k, k0 in enumerate(k_list): + Nk = k0 * A0 * I0 + for a, a0 in enumerate(a_list): + Na = a0 * I0 + for m, maxDet in enumerate(m_list): + E = [self.evalImgs[Nk + Na + i] for i in i_list] + E = [e for e in E if e is not None] + if len(E) == 0: + continue + dtScores = np.concatenate([e["dtScores"][0:maxDet] for e in E]) + + # different sorting method generates slightly different results. + # mergesort is used to be consistent as Matlab implementation. + inds = np.argsort(-dtScores, kind="mergesort") + + dtm = np.concatenate([e["dtMatches"][:, 0:maxDet] for e in E], axis=1)[:, inds] + dtIg = np.concatenate([e["dtIgnore"][:, 0:maxDet] for e in E], axis=1)[:, inds] + gtIg = np.concatenate([e["gtIgnore"] for e in E]) + npig = np.count_nonzero(gtIg == 0) + if npig == 0: + continue + tps = np.logical_and(dtm, np.logical_not(dtIg)) + fps = np.logical_and(np.logical_not(dtm), np.logical_not(dtIg)) + tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float) + fp_sum = np.cumsum(fps, axis=1).astype(dtype=np.float) + for t, (tp, fp) in enumerate(zip(tp_sum, fp_sum)): + tp = np.array(tp) + fp = np.array(fp) + nd = len(tp) + rc = tp / npig + pr = tp / (fp + tp + np.spacing(1)) + q = np.zeros((R,)) + + if nd: + recall[t, k, a, m] = rc[-1] + else: + recall[t, k, a, m] = 0 + + # numpy is slow without cython optimization for accessing elements + # use python array gets significant speed improvement + pr = pr.tolist() + q = q.tolist() + + for i in range(nd - 1, 0, -1): + if pr[i] > pr[i - 1]: + pr[i - 1] = pr[i] + + inds = np.searchsorted(rc, p.recThrs, side="left") + try: + for ri, pi in enumerate(inds): + q[ri] = pr[pi] + except Exception: + pass + precision[t, :, k, a, m] = np.array(q) + logger.info( + "Final: max precision {}, min precision {}".format(np.max(precision), np.min(precision)) + ) + self.eval = { + "params": p, + "counts": [T, R, K, A, M], + "date": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "precision": precision, + "recall": recall, + } + toc = time.time() + logger.info("DONE (t={:0.2f}s).".format(toc - tic)) + + def summarize(self): + """ + Compute and display summary metrics for evaluation results. + Note this function can *only* be applied on the default parameter setting + """ + + def _summarize(ap=1, iouThr=None, areaRng="all", maxDets=100): + p = self.params + iStr = " {:<18} {} @[ {}={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}" + titleStr = "Average Precision" if ap == 1 else "Average Recall" + typeStr = "(AP)" if ap == 1 else "(AR)" + measure = "IoU" + if self.params.iouType == "keypoints": + measure = "OKS" + elif self.params.iouType == "densepose": + measure = "OGPS" + iouStr = ( + "{:0.2f}:{:0.2f}".format(p.iouThrs[0], p.iouThrs[-1]) + if iouThr is None + else "{:0.2f}".format(iouThr) + ) + + aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng] + mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets] + if ap == 1: + # dimension of precision: [TxRxKxAxM] + s = self.eval["precision"] + # IoU + if iouThr is not None: + t = np.where(np.abs(iouThr - p.iouThrs) < 0.001)[0] + s = s[t] + s = s[:, :, :, aind, mind] + else: + # dimension of recall: [TxKxAxM] + s = self.eval["recall"] + if iouThr is not None: + t = np.where(np.abs(iouThr - p.iouThrs) < 0.001)[0] + s = s[t] + s = s[:, :, aind, mind] + if len(s[s > -1]) == 0: + mean_s = -1 + else: + mean_s = np.mean(s[s > -1]) + logger.info(iStr.format(titleStr, typeStr, measure, iouStr, areaRng, maxDets, mean_s)) + return mean_s + + def _summarizeDets(): + stats = np.zeros((12,)) + stats[0] = _summarize(1) + stats[1] = _summarize(1, iouThr=0.5, maxDets=self.params.maxDets[2]) + stats[2] = _summarize(1, iouThr=0.75, maxDets=self.params.maxDets[2]) + stats[3] = _summarize(1, areaRng="small", maxDets=self.params.maxDets[2]) + stats[4] = _summarize(1, areaRng="medium", maxDets=self.params.maxDets[2]) + stats[5] = _summarize(1, areaRng="large", maxDets=self.params.maxDets[2]) + stats[6] = _summarize(0, maxDets=self.params.maxDets[0]) + stats[7] = _summarize(0, maxDets=self.params.maxDets[1]) + stats[8] = _summarize(0, maxDets=self.params.maxDets[2]) + stats[9] = _summarize(0, areaRng="small", maxDets=self.params.maxDets[2]) + stats[10] = _summarize(0, areaRng="medium", maxDets=self.params.maxDets[2]) + stats[11] = _summarize(0, areaRng="large", maxDets=self.params.maxDets[2]) + return stats + + def _summarizeKps(): + stats = np.zeros((10,)) + stats[0] = _summarize(1, maxDets=20) + stats[1] = _summarize(1, maxDets=20, iouThr=0.5) + stats[2] = _summarize(1, maxDets=20, iouThr=0.75) + stats[3] = _summarize(1, maxDets=20, areaRng="medium") + stats[4] = _summarize(1, maxDets=20, areaRng="large") + stats[5] = _summarize(0, maxDets=20) + stats[6] = _summarize(0, maxDets=20, iouThr=0.5) + stats[7] = _summarize(0, maxDets=20, iouThr=0.75) + stats[8] = _summarize(0, maxDets=20, areaRng="medium") + stats[9] = _summarize(0, maxDets=20, areaRng="large") + return stats + + def _summarizeUvs(): + stats = [_summarize(1, maxDets=self.params.maxDets[0])] + min_threshold = self.params.iouThrs.min() + if min_threshold <= 0.201: + stats += [_summarize(1, maxDets=self.params.maxDets[0], iouThr=0.2)] + if min_threshold <= 0.301: + stats += [_summarize(1, maxDets=self.params.maxDets[0], iouThr=0.3)] + if min_threshold <= 0.401: + stats += [_summarize(1, maxDets=self.params.maxDets[0], iouThr=0.4)] + stats += [ + _summarize(1, maxDets=self.params.maxDets[0], iouThr=0.5), + _summarize(1, maxDets=self.params.maxDets[0], iouThr=0.75), + _summarize(1, maxDets=self.params.maxDets[0], areaRng="medium"), + _summarize(1, maxDets=self.params.maxDets[0], areaRng="large"), + _summarize(0, maxDets=self.params.maxDets[0]), + _summarize(0, maxDets=self.params.maxDets[0], iouThr=0.5), + _summarize(0, maxDets=self.params.maxDets[0], iouThr=0.75), + _summarize(0, maxDets=self.params.maxDets[0], areaRng="medium"), + _summarize(0, maxDets=self.params.maxDets[0], areaRng="large"), + ] + return np.array(stats) + + def _summarizeUvsOld(): + stats = np.zeros((18,)) + stats[0] = _summarize(1, maxDets=self.params.maxDets[0]) + stats[1] = _summarize(1, maxDets=self.params.maxDets[0], iouThr=0.5) + stats[2] = _summarize(1, maxDets=self.params.maxDets[0], iouThr=0.55) + stats[3] = _summarize(1, maxDets=self.params.maxDets[0], iouThr=0.60) + stats[4] = _summarize(1, maxDets=self.params.maxDets[0], iouThr=0.65) + stats[5] = _summarize(1, maxDets=self.params.maxDets[0], iouThr=0.70) + stats[6] = _summarize(1, maxDets=self.params.maxDets[0], iouThr=0.75) + stats[7] = _summarize(1, maxDets=self.params.maxDets[0], iouThr=0.80) + stats[8] = _summarize(1, maxDets=self.params.maxDets[0], iouThr=0.85) + stats[9] = _summarize(1, maxDets=self.params.maxDets[0], iouThr=0.90) + stats[10] = _summarize(1, maxDets=self.params.maxDets[0], iouThr=0.95) + stats[11] = _summarize(1, maxDets=self.params.maxDets[0], areaRng="medium") + stats[12] = _summarize(1, maxDets=self.params.maxDets[0], areaRng="large") + stats[13] = _summarize(0, maxDets=self.params.maxDets[0]) + stats[14] = _summarize(0, maxDets=self.params.maxDets[0], iouThr=0.5) + stats[15] = _summarize(0, maxDets=self.params.maxDets[0], iouThr=0.75) + stats[16] = _summarize(0, maxDets=self.params.maxDets[0], areaRng="medium") + stats[17] = _summarize(0, maxDets=self.params.maxDets[0], areaRng="large") + return stats + + if not self.eval: + raise Exception("Please run accumulate() first") + iouType = self.params.iouType + if iouType in ["segm", "bbox"]: + summarize = _summarizeDets + elif iouType in ["keypoints"]: + summarize = _summarizeKps + elif iouType in ["densepose"]: + summarize = _summarizeUvs + self.stats = summarize() + + def __str__(self): + self.summarize() + + # ================ functions for dense pose ============================== + def findAllClosestVertsUV(self, U_points, V_points, Index_points): + ClosestVerts = np.ones(Index_points.shape) * -1 + for i in np.arange(24): + # + if (i + 1) in Index_points: + UVs = np.array( + [U_points[Index_points == (i + 1)], V_points[Index_points == (i + 1)]] + ) + Current_Part_UVs = self.Part_UVs[i] + Current_Part_ClosestVertInds = self.Part_ClosestVertInds[i] + D = ssd.cdist(Current_Part_UVs.transpose(), UVs.transpose()).squeeze() + ClosestVerts[Index_points == (i + 1)] = Current_Part_ClosestVertInds[ + np.argmin(D, axis=0) + ] + ClosestVertsTransformed = self.PDIST_transform[ClosestVerts.astype(int) - 1] + ClosestVertsTransformed[ClosestVerts < 0] = 0 + return ClosestVertsTransformed + + def findClosestVertsCse(self, embedding, py, px, mask, mesh_name): + mesh_vertex_embeddings = self.embedder(mesh_name) + pixel_embeddings = embedding[:, py, px].t().to(device="cuda") + mask_vals = mask[py, px] + edm = squared_euclidean_distance_matrix(pixel_embeddings, mesh_vertex_embeddings) + vertex_indices = edm.argmin(dim=1).cpu() + vertex_indices[mask_vals <= 0] = -1 + return vertex_indices + + def findAllClosestVertsGT(self, gt): + # + I_gt = np.array(gt["dp_I"]) + U_gt = np.array(gt["dp_U"]) + V_gt = np.array(gt["dp_V"]) + # + # print(I_gt) + # + ClosestVertsGT = np.ones(I_gt.shape) * -1 + for i in np.arange(24): + if (i + 1) in I_gt: + UVs = np.array([U_gt[I_gt == (i + 1)], V_gt[I_gt == (i + 1)]]) + Current_Part_UVs = self.Part_UVs[i] + Current_Part_ClosestVertInds = self.Part_ClosestVertInds[i] + D = ssd.cdist(Current_Part_UVs.transpose(), UVs.transpose()).squeeze() + ClosestVertsGT[I_gt == (i + 1)] = Current_Part_ClosestVertInds[np.argmin(D, axis=0)] + # + ClosestVertsGTTransformed = self.PDIST_transform[ClosestVertsGT.astype(int) - 1] + ClosestVertsGTTransformed[ClosestVertsGT < 0] = 0 + return ClosestVertsGT, ClosestVertsGTTransformed + + def getDistancesCse(self, cVertsGT, cVerts, mesh_name): + geodists_vertices = torch.ones_like(cVertsGT) * float("inf") + selected = (cVertsGT >= 0) * (cVerts >= 0) + mesh = create_mesh(mesh_name, "cpu") + geodists_vertices[selected] = mesh.geodists[cVertsGT[selected], cVerts[selected]] + return geodists_vertices.numpy() + + def getDistancesUV(self, cVertsGT, cVerts): + # + n = 27554 + dists = [] + for d in range(len(cVertsGT)): + if cVertsGT[d] > 0: + if cVerts[d] > 0: + i = cVertsGT[d] - 1 + j = cVerts[d] - 1 + if j == i: + dists.append(0) + elif j > i: + ccc = i + i = j + j = ccc + i = n - i - 1 + j = n - j - 1 + k = (n * (n - 1) / 2) - (n - i) * ((n - i) - 1) / 2 + j - i - 1 + k = (n * n - n) / 2 - k - 1 + dists.append(self.Pdist_matrix[int(k)][0]) + else: + i = n - i - 1 + j = n - j - 1 + k = (n * (n - 1) / 2) - (n - i) * ((n - i) - 1) / 2 + j - i - 1 + k = (n * n - n) / 2 - k - 1 + dists.append(self.Pdist_matrix[int(k)][0]) + else: + dists.append(np.inf) + return np.atleast_1d(np.array(dists).squeeze()) + + +class Params: + """ + Params for coco evaluation api + """ + + def setDetParams(self): + self.imgIds = [] + self.catIds = [] + # np.arange causes trouble. the data point on arange is slightly larger than the true value + self.iouThrs = np.linspace(0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True) + self.recThrs = np.linspace(0.0, 1.00, int(np.round((1.00 - 0.0) / 0.01)) + 1, endpoint=True) + self.maxDets = [1, 10, 100] + self.areaRng = [ + [0**2, 1e5**2], + [0**2, 32**2], + [32**2, 96**2], + [96**2, 1e5**2], + ] + self.areaRngLbl = ["all", "small", "medium", "large"] + self.useCats = 1 + + def setKpParams(self): + self.imgIds = [] + self.catIds = [] + # np.arange causes trouble. the data point on arange is slightly larger than the true value + self.iouThrs = np.linspace(0.5, 0.95, np.round((0.95 - 0.5) / 0.05) + 1, endpoint=True) + self.recThrs = np.linspace(0.0, 1.00, np.round((1.00 - 0.0) / 0.01) + 1, endpoint=True) + self.maxDets = [20] + self.areaRng = [[0**2, 1e5**2], [32**2, 96**2], [96**2, 1e5**2]] + self.areaRngLbl = ["all", "medium", "large"] + self.useCats = 1 + + def setUvParams(self): + self.imgIds = [] + self.catIds = [] + self.iouThrs = np.linspace(0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True) + self.recThrs = np.linspace(0.0, 1.00, int(np.round((1.00 - 0.0) / 0.01)) + 1, endpoint=True) + self.maxDets = [20] + self.areaRng = [[0**2, 1e5**2], [32**2, 96**2], [96**2, 1e5**2]] + self.areaRngLbl = ["all", "medium", "large"] + self.useCats = 1 + + def __init__(self, iouType="segm"): + if iouType == "segm" or iouType == "bbox": + self.setDetParams() + elif iouType == "keypoints": + self.setKpParams() + elif iouType == "densepose": + self.setUvParams() + else: + raise Exception("iouType not supported") + self.iouType = iouType + # useSegm is deprecated + self.useSegm = None diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/evaluator.py b/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..d5d1d789bbe4b8791aa8529518ba1b964d31daca --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/evaluator.py @@ -0,0 +1,421 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import contextlib +import copy +import io +import itertools +import logging +import numpy as np +import os +from collections import OrderedDict +from typing import Dict, Iterable, List, Optional +import pycocotools.mask as mask_utils +import torch +from pycocotools.coco import COCO +from tabulate import tabulate + +from detectron2.config import CfgNode +from detectron2.data import MetadataCatalog +from detectron2.evaluation import DatasetEvaluator +from detectron2.structures import BoxMode +from detectron2.utils.comm import gather, get_rank, is_main_process, synchronize +from detectron2.utils.file_io import PathManager +from detectron2.utils.logger import create_small_table + +from densepose.converters import ToChartResultConverter, ToMaskConverter +from densepose.data.datasets.coco import maybe_filter_and_map_categories_cocoapi +from densepose.structures import ( + DensePoseChartPredictorOutput, + DensePoseEmbeddingPredictorOutput, + quantize_densepose_chart_result, +) + +from .densepose_coco_evaluation import DensePoseCocoEval, DensePoseEvalMode +from .mesh_alignment_evaluator import MeshAlignmentEvaluator +from .tensor_storage import ( + SingleProcessFileTensorStorage, + SingleProcessRamTensorStorage, + SingleProcessTensorStorage, + SizeData, + storage_gather, +) + + +class DensePoseCOCOEvaluator(DatasetEvaluator): + def __init__( + self, + dataset_name, + distributed, + output_dir=None, + evaluator_type: str = "iuv", + min_iou_threshold: float = 0.5, + storage: Optional[SingleProcessTensorStorage] = None, + embedder=None, + should_evaluate_mesh_alignment: bool = False, + mesh_alignment_mesh_names: Optional[List[str]] = None, + ): + self._embedder = embedder + self._distributed = distributed + self._output_dir = output_dir + self._evaluator_type = evaluator_type + self._storage = storage + self._should_evaluate_mesh_alignment = should_evaluate_mesh_alignment + + assert not ( + should_evaluate_mesh_alignment and embedder is None + ), "Mesh alignment evaluation is activated, but no vertex embedder provided!" + if should_evaluate_mesh_alignment: + self._mesh_alignment_evaluator = MeshAlignmentEvaluator( + embedder, + mesh_alignment_mesh_names, + ) + + self._cpu_device = torch.device("cpu") + self._logger = logging.getLogger(__name__) + + self._metadata = MetadataCatalog.get(dataset_name) + self._min_threshold = min_iou_threshold + json_file = PathManager.get_local_path(self._metadata.json_file) + with contextlib.redirect_stdout(io.StringIO()): + self._coco_api = COCO(json_file) + maybe_filter_and_map_categories_cocoapi(dataset_name, self._coco_api) + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a COCO model (e.g., GeneralizedRCNN). + It is a list of dict. Each dict corresponds to an image and + contains keys like "height", "width", "file_name", "image_id". + outputs: the outputs of a COCO model. It is a list of dicts with key + "instances" that contains :class:`Instances`. + The :class:`Instances` object needs to have `densepose` field. + """ + for input, output in zip(inputs, outputs): + instances = output["instances"].to(self._cpu_device) + if not instances.has("pred_densepose"): + continue + prediction_list = prediction_to_dict( + instances, + input["image_id"], + self._embedder, + self._metadata.class_to_mesh_name, + self._storage is not None, + ) + if self._storage is not None: + for prediction_dict in prediction_list: + dict_to_store = {} + for field_name in self._storage.data_schema: + dict_to_store[field_name] = prediction_dict[field_name] + record_id = self._storage.put(dict_to_store) + prediction_dict["record_id"] = record_id + prediction_dict["rank"] = get_rank() + for field_name in self._storage.data_schema: + del prediction_dict[field_name] + self._predictions.extend(prediction_list) + + def evaluate(self, img_ids=None): + if self._distributed: + synchronize() + predictions = gather(self._predictions) + predictions = list(itertools.chain(*predictions)) + else: + predictions = self._predictions + + multi_storage = storage_gather(self._storage) if self._storage is not None else None + + if not is_main_process(): + return + return copy.deepcopy(self._eval_predictions(predictions, multi_storage, img_ids)) + + def _eval_predictions(self, predictions, multi_storage=None, img_ids=None): + """ + Evaluate predictions on densepose. + Return results with the metrics of the tasks. + """ + self._logger.info("Preparing results for COCO format ...") + + if self._output_dir: + PathManager.mkdirs(self._output_dir) + file_path = os.path.join(self._output_dir, "coco_densepose_predictions.pth") + with PathManager.open(file_path, "wb") as f: + torch.save(predictions, f) + + self._logger.info("Evaluating predictions ...") + res = OrderedDict() + results_gps, results_gpsm, results_segm = _evaluate_predictions_on_coco( + self._coco_api, + predictions, + multi_storage, + self._embedder, + class_names=self._metadata.get("thing_classes"), + min_threshold=self._min_threshold, + img_ids=img_ids, + ) + res["densepose_gps"] = results_gps + res["densepose_gpsm"] = results_gpsm + res["densepose_segm"] = results_segm + if self._should_evaluate_mesh_alignment: + res["densepose_mesh_alignment"] = self._evaluate_mesh_alignment() + return res + + def _evaluate_mesh_alignment(self): + self._logger.info("Mesh alignment evaluation ...") + mean_ge, mean_gps, per_mesh_metrics = self._mesh_alignment_evaluator.evaluate() + results = { + "GE": mean_ge * 100, + "GPS": mean_gps * 100, + } + mesh_names = set() + for metric_name in per_mesh_metrics: + for mesh_name, value in per_mesh_metrics[metric_name].items(): + results[f"{metric_name}-{mesh_name}"] = value * 100 + mesh_names.add(mesh_name) + self._print_mesh_alignment_results(results, mesh_names) + return results + + def _print_mesh_alignment_results(self, results: Dict[str, float], mesh_names: Iterable[str]): + self._logger.info("Evaluation results for densepose, mesh alignment:") + self._logger.info(f'| {"Mesh":13s} | {"GErr":7s} | {"GPS":7s} |') + self._logger.info("| :-----------: | :-----: | :-----: |") + for mesh_name in mesh_names: + ge_key = f"GE-{mesh_name}" + ge_str = f"{results[ge_key]:.4f}" if ge_key in results else " " + gps_key = f"GPS-{mesh_name}" + gps_str = f"{results[gps_key]:.4f}" if gps_key in results else " " + self._logger.info(f"| {mesh_name:13s} | {ge_str:7s} | {gps_str:7s} |") + self._logger.info("| :-------------------------------: |") + ge_key = "GE" + ge_str = f"{results[ge_key]:.4f}" if ge_key in results else " " + gps_key = "GPS" + gps_str = f"{results[gps_key]:.4f}" if gps_key in results else " " + self._logger.info(f'| {"MEAN":13s} | {ge_str:7s} | {gps_str:7s} |') + + +def prediction_to_dict(instances, img_id, embedder, class_to_mesh_name, use_storage): + """ + Args: + instances (Instances): the output of the model + img_id (str): the image id in COCO + + Returns: + list[dict]: the results in densepose evaluation format + """ + scores = instances.scores.tolist() + classes = instances.pred_classes.tolist() + raw_boxes_xywh = BoxMode.convert( + instances.pred_boxes.tensor.clone(), BoxMode.XYXY_ABS, BoxMode.XYWH_ABS + ) + + if isinstance(instances.pred_densepose, DensePoseEmbeddingPredictorOutput): + results_densepose = densepose_cse_predictions_to_dict( + instances, embedder, class_to_mesh_name, use_storage + ) + elif isinstance(instances.pred_densepose, DensePoseChartPredictorOutput): + if not use_storage: + results_densepose = densepose_chart_predictions_to_dict(instances) + else: + results_densepose = densepose_chart_predictions_to_storage_dict(instances) + + results = [] + for k in range(len(instances)): + result = { + "image_id": img_id, + "category_id": classes[k], + "bbox": raw_boxes_xywh[k].tolist(), + "score": scores[k], + } + results.append({**result, **results_densepose[k]}) + return results + + +def densepose_chart_predictions_to_dict(instances): + segmentations = ToMaskConverter.convert( + instances.pred_densepose, instances.pred_boxes, instances.image_size + ) + + results = [] + for k in range(len(instances)): + densepose_results_quantized = quantize_densepose_chart_result( + ToChartResultConverter.convert(instances.pred_densepose[k], instances.pred_boxes[k]) + ) + densepose_results_quantized.labels_uv_uint8 = ( + densepose_results_quantized.labels_uv_uint8.cpu() + ) + segmentation = segmentations.tensor[k] + segmentation_encoded = mask_utils.encode( + np.require(segmentation.numpy(), dtype=np.uint8, requirements=["F"]) + ) + segmentation_encoded["counts"] = segmentation_encoded["counts"].decode("utf-8") + result = { + "densepose": densepose_results_quantized, + "segmentation": segmentation_encoded, + } + results.append(result) + return results + + +def densepose_chart_predictions_to_storage_dict(instances): + results = [] + for k in range(len(instances)): + densepose_predictor_output = instances.pred_densepose[k] + result = { + "coarse_segm": densepose_predictor_output.coarse_segm.squeeze(0).cpu(), + "fine_segm": densepose_predictor_output.fine_segm.squeeze(0).cpu(), + "u": densepose_predictor_output.u.squeeze(0).cpu(), + "v": densepose_predictor_output.v.squeeze(0).cpu(), + } + results.append(result) + return results + + +def densepose_cse_predictions_to_dict(instances, embedder, class_to_mesh_name, use_storage): + results = [] + for k in range(len(instances)): + cse = instances.pred_densepose[k] + results.append( + { + "coarse_segm": cse.coarse_segm[0].cpu(), + "embedding": cse.embedding[0].cpu(), + } + ) + return results + + +def _evaluate_predictions_on_coco( + coco_gt, + coco_results, + multi_storage=None, + embedder=None, + class_names=None, + min_threshold: float = 0.5, + img_ids=None, +): + logger = logging.getLogger(__name__) + + densepose_metrics = _get_densepose_metrics(min_threshold) + if len(coco_results) == 0: # cocoapi does not handle empty results very well + logger.warn("No predictions from the model! Set scores to -1") + results_gps = {metric: -1 for metric in densepose_metrics} + results_gpsm = {metric: -1 for metric in densepose_metrics} + results_segm = {metric: -1 for metric in densepose_metrics} + return results_gps, results_gpsm, results_segm + + coco_dt = coco_gt.loadRes(coco_results) + + results = [] + for eval_mode_name in ["GPS", "GPSM", "IOU"]: + eval_mode = getattr(DensePoseEvalMode, eval_mode_name) + coco_eval = DensePoseCocoEval( + coco_gt, coco_dt, "densepose", multi_storage, embedder, dpEvalMode=eval_mode + ) + result = _derive_results_from_coco_eval( + coco_eval, eval_mode_name, densepose_metrics, class_names, min_threshold, img_ids + ) + results.append(result) + return results + + +def _get_densepose_metrics(min_threshold: float = 0.5): + metrics = ["AP"] + if min_threshold <= 0.201: + metrics += ["AP20"] + if min_threshold <= 0.301: + metrics += ["AP30"] + if min_threshold <= 0.401: + metrics += ["AP40"] + metrics.extend(["AP50", "AP75", "APm", "APl", "AR", "AR50", "AR75", "ARm", "ARl"]) + return metrics + + +def _derive_results_from_coco_eval( + coco_eval, eval_mode_name, metrics, class_names, min_threshold: float, img_ids +): + if img_ids is not None: + coco_eval.params.imgIds = img_ids + coco_eval.params.iouThrs = np.linspace( + min_threshold, 0.95, int(np.round((0.95 - min_threshold) / 0.05)) + 1, endpoint=True + ) + coco_eval.evaluate() + coco_eval.accumulate() + coco_eval.summarize() + results = {metric: float(coco_eval.stats[idx] * 100) for idx, metric in enumerate(metrics)} + logger = logging.getLogger(__name__) + logger.info( + f"Evaluation results for densepose, {eval_mode_name} metric: \n" + + create_small_table(results) + ) + if class_names is None or len(class_names) <= 1: + return results + + # Compute per-category AP, the same way as it is done in D2 + # (see detectron2/evaluation/coco_evaluation.py): + precisions = coco_eval.eval["precision"] + # precision has dims (iou, recall, cls, area range, max dets) + assert len(class_names) == precisions.shape[2] + + results_per_category = [] + for idx, name in enumerate(class_names): + # area range index 0: all area ranges + # max dets index -1: typically 100 per image + precision = precisions[:, :, idx, 0, -1] + precision = precision[precision > -1] + ap = np.mean(precision) if precision.size else float("nan") + results_per_category.append((f"{name}", float(ap * 100))) + + # tabulate it + n_cols = min(6, len(results_per_category) * 2) + results_flatten = list(itertools.chain(*results_per_category)) + results_2d = itertools.zip_longest(*[results_flatten[i::n_cols] for i in range(n_cols)]) + table = tabulate( + results_2d, + tablefmt="pipe", + floatfmt=".3f", + headers=["category", "AP"] * (n_cols // 2), + numalign="left", + ) + logger.info(f"Per-category {eval_mode_name} AP: \n" + table) + + results.update({"AP-" + name: ap for name, ap in results_per_category}) + return results + + +def build_densepose_evaluator_storage(cfg: CfgNode, output_folder: str): + storage_spec = cfg.DENSEPOSE_EVALUATION.STORAGE + if storage_spec == "none": + return None + evaluator_type = cfg.DENSEPOSE_EVALUATION.TYPE + # common output tensor sizes + hout = cfg.MODEL.ROI_DENSEPOSE_HEAD.HEATMAP_SIZE + wout = cfg.MODEL.ROI_DENSEPOSE_HEAD.HEATMAP_SIZE + n_csc = cfg.MODEL.ROI_DENSEPOSE_HEAD.NUM_COARSE_SEGM_CHANNELS + # specific output tensors + if evaluator_type == "iuv": + n_fsc = cfg.MODEL.ROI_DENSEPOSE_HEAD.NUM_PATCHES + 1 + schema = { + "coarse_segm": SizeData(dtype="float32", shape=(n_csc, hout, wout)), + "fine_segm": SizeData(dtype="float32", shape=(n_fsc, hout, wout)), + "u": SizeData(dtype="float32", shape=(n_fsc, hout, wout)), + "v": SizeData(dtype="float32", shape=(n_fsc, hout, wout)), + } + elif evaluator_type == "cse": + embed_size = cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBED_SIZE + schema = { + "coarse_segm": SizeData(dtype="float32", shape=(n_csc, hout, wout)), + "embedding": SizeData(dtype="float32", shape=(embed_size, hout, wout)), + } + else: + raise ValueError(f"Unknown evaluator type: {evaluator_type}") + # storage types + if storage_spec == "ram": + storage = SingleProcessRamTensorStorage(schema, io.BytesIO()) + elif storage_spec == "file": + fpath = os.path.join(output_folder, f"DensePoseEvaluatorStorage.{get_rank()}.bin") + PathManager.mkdirs(output_folder) + storage = SingleProcessFileTensorStorage(schema, fpath, "wb") + else: + raise ValueError(f"Unknown storage specification: {storage_spec}") + return storage diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/mesh_alignment_evaluator.py b/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/mesh_alignment_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..9d67c1a88a56332fb708c4618a34e96900926083 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/mesh_alignment_evaluator.py @@ -0,0 +1,66 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +import json +import logging +from typing import List, Optional +import torch +from torch import nn + +from detectron2.utils.file_io import PathManager + +from densepose.structures.mesh import create_mesh + + +class MeshAlignmentEvaluator: + """ + Class for evaluation of 3D mesh alignment based on the learned vertex embeddings + """ + + def __init__(self, embedder: nn.Module, mesh_names: Optional[List[str]]): + self.embedder = embedder + # use the provided mesh names if not None and not an empty list + self.mesh_names = mesh_names if mesh_names else embedder.mesh_names + self.logger = logging.getLogger(__name__) + with PathManager.open( + "https://dl.fbaipublicfiles.com/densepose/data/cse/mesh_keyvertices_v0.json", "r" + ) as f: + self.mesh_keyvertices = json.load(f) + + def evaluate(self): + ge_per_mesh = {} + gps_per_mesh = {} + for mesh_name_1 in self.mesh_names: + avg_errors = [] + avg_gps = [] + embeddings_1 = self.embedder(mesh_name_1) + keyvertices_1 = self.mesh_keyvertices[mesh_name_1] + keyvertex_names_1 = list(keyvertices_1.keys()) + keyvertex_indices_1 = [keyvertices_1[name] for name in keyvertex_names_1] + for mesh_name_2 in self.mesh_names: + if mesh_name_1 == mesh_name_2: + continue + embeddings_2 = self.embedder(mesh_name_2) + keyvertices_2 = self.mesh_keyvertices[mesh_name_2] + sim_matrix_12 = embeddings_1[keyvertex_indices_1].mm(embeddings_2.T) + vertices_2_matching_keyvertices_1 = sim_matrix_12.argmax(axis=1) + mesh_2 = create_mesh(mesh_name_2, embeddings_2.device) + geodists = mesh_2.geodists[ + vertices_2_matching_keyvertices_1, + [keyvertices_2[name] for name in keyvertex_names_1], + ] + Current_Mean_Distances = 0.255 + gps = (-(geodists**2) / (2 * (Current_Mean_Distances**2))).exp() + avg_errors.append(geodists.mean().item()) + avg_gps.append(gps.mean().item()) + + ge_mean = torch.as_tensor(avg_errors).mean().item() + gps_mean = torch.as_tensor(avg_gps).mean().item() + ge_per_mesh[mesh_name_1] = ge_mean + gps_per_mesh[mesh_name_1] = gps_mean + ge_mean_global = torch.as_tensor(list(ge_per_mesh.values())).mean().item() + gps_mean_global = torch.as_tensor(list(gps_per_mesh.values())).mean().item() + per_mesh_metrics = { + "GE": ge_per_mesh, + "GPS": gps_per_mesh, + } + return ge_mean_global, gps_mean_global, per_mesh_metrics diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/tensor_storage.py b/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/tensor_storage.py new file mode 100644 index 0000000000000000000000000000000000000000..72e3cb64caf91c684607a5fd7cb696b267c21e16 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/evaluation/tensor_storage.py @@ -0,0 +1,238 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import io +import numpy as np +import os +from dataclasses import dataclass +from functools import reduce +from operator import mul +from typing import BinaryIO, Dict, Optional, Tuple +import torch + +from detectron2.utils.comm import gather, get_rank +from detectron2.utils.file_io import PathManager + + +@dataclass +class SizeData: + dtype: str + shape: Tuple[int] + + +def _calculate_record_field_size_b(data_schema: Dict[str, SizeData], field_name: str) -> int: + schema = data_schema[field_name] + element_size_b = np.dtype(schema.dtype).itemsize + record_field_size_b = reduce(mul, schema.shape) * element_size_b + return record_field_size_b + + +def _calculate_record_size_b(data_schema: Dict[str, SizeData]) -> int: + record_size_b = 0 + for field_name in data_schema: + record_field_size_b = _calculate_record_field_size_b(data_schema, field_name) + record_size_b += record_field_size_b + return record_size_b + + +def _calculate_record_field_sizes_b(data_schema: Dict[str, SizeData]) -> Dict[str, int]: + field_sizes_b = {} + for field_name in data_schema: + field_sizes_b[field_name] = _calculate_record_field_size_b(data_schema, field_name) + return field_sizes_b + + +class SingleProcessTensorStorage: + """ + Compact tensor storage to keep tensor data of predefined size and type. + """ + + def __init__(self, data_schema: Dict[str, SizeData], storage_impl: BinaryIO): + """ + Construct tensor storage based on information on data shape and size. + Internally uses numpy to interpret the type specification. + The storage must support operations `seek(offset, whence=os.SEEK_SET)` and + `read(size)` to be able to perform the `get` operation. + The storage must support operation `write(bytes)` to be able to perform + the `put` operation. + + Args: + data_schema (dict: str -> SizeData): dictionary which maps tensor name + to its size data (shape and data type), e.g. + ``` + { + "coarse_segm": SizeData(dtype="float32", shape=(112, 112)), + "embedding": SizeData(dtype="float32", shape=(16, 112, 112)), + } + ``` + storage_impl (BinaryIO): io instance that handles file-like seek, read + and write operations, e.g. a file handle or a memory buffer like io.BytesIO + """ + self.data_schema = data_schema + self.record_size_b = _calculate_record_size_b(data_schema) + self.record_field_sizes_b = _calculate_record_field_sizes_b(data_schema) + self.storage_impl = storage_impl + self.next_record_id = 0 + + def get(self, record_id: int) -> Dict[str, torch.Tensor]: + """ + Load tensors from the storage by record ID + + Args: + record_id (int): Record ID, for which to load the data + + Return: + dict: str -> tensor: tensor name mapped to tensor data, recorded under the provided ID + """ + self.storage_impl.seek(record_id * self.record_size_b, os.SEEK_SET) + data_bytes = self.storage_impl.read(self.record_size_b) + assert len(data_bytes) == self.record_size_b, ( + f"Expected data size {self.record_size_b} B could not be read: " + f"got {len(data_bytes)} B" + ) + record = {} + cur_idx = 0 + # it's important to read and write in the same order + for field_name in sorted(self.data_schema): + schema = self.data_schema[field_name] + field_size_b = self.record_field_sizes_b[field_name] + chunk = data_bytes[cur_idx : cur_idx + field_size_b] + data_np = np.frombuffer( + chunk, dtype=schema.dtype, count=reduce(mul, schema.shape) + ).reshape(schema.shape) + record[field_name] = torch.from_numpy(data_np) + cur_idx += field_size_b + return record + + def put(self, data: Dict[str, torch.Tensor]) -> int: + """ + Store tensors in the storage + + Args: + data (dict: str -> tensor): data to store, a dictionary which maps + tensor names into tensors; tensor shapes must match those specified + in data schema. + Return: + int: record ID, under which the data is stored + """ + # it's important to read and write in the same order + for field_name in sorted(self.data_schema): + assert ( + field_name in data + ), f"Field '{field_name}' not present in data: data keys are {data.keys()}" + value = data[field_name] + assert value.shape == self.data_schema[field_name].shape, ( + f"Mismatched tensor shapes for field '{field_name}': " + f"expected {self.data_schema[field_name].shape}, got {value.shape}" + ) + data_bytes = value.cpu().numpy().tobytes() + assert len(data_bytes) == self.record_field_sizes_b[field_name], ( + f"Expected field {field_name} to be of size " + f"{self.record_field_sizes_b[field_name]} B, got {len(data_bytes)} B" + ) + self.storage_impl.write(data_bytes) + record_id = self.next_record_id + self.next_record_id += 1 + return record_id + + +class SingleProcessFileTensorStorage(SingleProcessTensorStorage): + """ + Implementation of a single process tensor storage which stores data in a file + """ + + def __init__(self, data_schema: Dict[str, SizeData], fpath: str, mode: str): + self.fpath = fpath + assert "b" in mode, f"Tensor storage should be opened in binary mode, got '{mode}'" + if "w" in mode: + file_h = PathManager.open(fpath, mode) + elif "r" in mode: + local_fpath = PathManager.get_local_path(fpath) + file_h = open(local_fpath, mode) + else: + raise ValueError(f"Unsupported file mode {mode}, supported modes: rb, wb") + super().__init__(data_schema, file_h) # pyre-ignore[6] + + +class SingleProcessRamTensorStorage(SingleProcessTensorStorage): + """ + Implementation of a single process tensor storage which stores data in RAM + """ + + def __init__(self, data_schema: Dict[str, SizeData], buf: io.BytesIO): + super().__init__(data_schema, buf) + + +class MultiProcessTensorStorage: + """ + Representation of a set of tensor storages created by individual processes, + allows to access those storages from a single owner process. The storages + should either be shared or broadcasted to the owner process. + The processes are identified by their rank, data is uniquely defined by + the rank of the process and the record ID. + """ + + def __init__(self, rank_to_storage: Dict[int, SingleProcessTensorStorage]): + self.rank_to_storage = rank_to_storage + + def get(self, rank: int, record_id: int) -> Dict[str, torch.Tensor]: + storage = self.rank_to_storage[rank] + return storage.get(record_id) + + def put(self, rank: int, data: Dict[str, torch.Tensor]) -> int: + storage = self.rank_to_storage[rank] + return storage.put(data) + + +class MultiProcessFileTensorStorage(MultiProcessTensorStorage): + def __init__(self, data_schema: Dict[str, SizeData], rank_to_fpath: Dict[int, str], mode: str): + rank_to_storage = { + rank: SingleProcessFileTensorStorage(data_schema, fpath, mode) + for rank, fpath in rank_to_fpath.items() + } + super().__init__(rank_to_storage) # pyre-ignore[6] + + +class MultiProcessRamTensorStorage(MultiProcessTensorStorage): + def __init__(self, data_schema: Dict[str, SizeData], rank_to_buffer: Dict[int, io.BytesIO]): + rank_to_storage = { + rank: SingleProcessRamTensorStorage(data_schema, buf) + for rank, buf in rank_to_buffer.items() + } + super().__init__(rank_to_storage) # pyre-ignore[6] + + +def _ram_storage_gather( + storage: SingleProcessRamTensorStorage, dst_rank: int = 0 +) -> Optional[MultiProcessRamTensorStorage]: + storage.storage_impl.seek(0, os.SEEK_SET) + # TODO: overhead, pickling a bytes object, can just pass bytes in a tensor directly + # see detectron2/utils.comm.py + data_list = gather(storage.storage_impl.read(), dst=dst_rank) + if get_rank() != dst_rank: + return None + rank_to_buffer = {i: io.BytesIO(data_list[i]) for i in range(len(data_list))} + multiprocess_storage = MultiProcessRamTensorStorage(storage.data_schema, rank_to_buffer) + return multiprocess_storage + + +def _file_storage_gather( + storage: SingleProcessFileTensorStorage, + dst_rank: int = 0, + mode: str = "rb", +) -> Optional[MultiProcessFileTensorStorage]: + storage.storage_impl.close() + fpath_list = gather(storage.fpath, dst=dst_rank) + if get_rank() != dst_rank: + return None + rank_to_fpath = {i: fpath_list[i] for i in range(len(fpath_list))} + return MultiProcessFileTensorStorage(storage.data_schema, rank_to_fpath, mode) + + +def storage_gather( + storage: SingleProcessTensorStorage, dst_rank: int = 0 +) -> Optional[MultiProcessTensorStorage]: + if isinstance(storage, SingleProcessRamTensorStorage): + return _ram_storage_gather(storage, dst_rank) + elif isinstance(storage, SingleProcessFileTensorStorage): + return _file_storage_gather(storage, dst_rank) + raise Exception(f"Unsupported storage for gather operation: {storage}") diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/__init__.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4c49f6da0d182cc97f5fe6b21d77c8f8330d3c3d --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from .confidence import DensePoseConfidenceModelConfig, DensePoseUVConfidenceType +from .filter import DensePoseDataFilter +from .inference import densepose_inference +from .utils import initialize_module_params +from .build import ( + build_densepose_data_filter, + build_densepose_embedder, + build_densepose_head, + build_densepose_losses, + build_densepose_predictor, +) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/build.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/build.py new file mode 100644 index 0000000000000000000000000000000000000000..bb7f54b4a1044bc518d66d89432dd52c79fdf293 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/build.py @@ -0,0 +1,87 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from typing import Optional +from torch import nn + +from detectron2.config import CfgNode + +from .cse.embedder import Embedder +from .filter import DensePoseDataFilter + + +def build_densepose_predictor(cfg: CfgNode, input_channels: int): + """ + Create an instance of DensePose predictor based on configuration options. + + Args: + cfg (CfgNode): configuration options + input_channels (int): input tensor size along the channel dimension + Return: + An instance of DensePose predictor + """ + from .predictors import DENSEPOSE_PREDICTOR_REGISTRY + + predictor_name = cfg.MODEL.ROI_DENSEPOSE_HEAD.PREDICTOR_NAME + return DENSEPOSE_PREDICTOR_REGISTRY.get(predictor_name)(cfg, input_channels) + + +def build_densepose_data_filter(cfg: CfgNode): + """ + Build DensePose data filter which selects data for training + + Args: + cfg (CfgNode): configuration options + + Return: + Callable: list(Tensor), list(Instances) -> list(Tensor), list(Instances) + An instance of DensePose filter, which takes feature tensors and proposals + as an input and returns filtered features and proposals + """ + dp_filter = DensePoseDataFilter(cfg) + return dp_filter + + +def build_densepose_head(cfg: CfgNode, input_channels: int): + """ + Build DensePose head based on configurations options + + Args: + cfg (CfgNode): configuration options + input_channels (int): input tensor size along the channel dimension + Return: + An instance of DensePose head + """ + from .roi_heads.registry import ROI_DENSEPOSE_HEAD_REGISTRY + + head_name = cfg.MODEL.ROI_DENSEPOSE_HEAD.NAME + return ROI_DENSEPOSE_HEAD_REGISTRY.get(head_name)(cfg, input_channels) + + +def build_densepose_losses(cfg: CfgNode): + """ + Build DensePose loss based on configurations options + + Args: + cfg (CfgNode): configuration options + Return: + An instance of DensePose loss + """ + from .losses import DENSEPOSE_LOSS_REGISTRY + + loss_name = cfg.MODEL.ROI_DENSEPOSE_HEAD.LOSS_NAME + return DENSEPOSE_LOSS_REGISTRY.get(loss_name)(cfg) + + +def build_densepose_embedder(cfg: CfgNode) -> Optional[nn.Module]: + """ + Build embedder used to embed mesh vertices into an embedding space. + Embedder contains sub-embedders, one for each mesh ID. + + Args: + cfg (cfgNode): configuration options + Return: + Embedding module + """ + if cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBEDDERS: + return Embedder(cfg) + return None diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/confidence.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/confidence.py new file mode 100644 index 0000000000000000000000000000000000000000..6f4a72efec06e055036ba70bc75b2624d20e1e0e --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/confidence.py @@ -0,0 +1,73 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from dataclasses import dataclass +from enum import Enum + +from detectron2.config import CfgNode + + +class DensePoseUVConfidenceType(Enum): + """ + Statistical model type for confidence learning, possible values: + - "iid_iso": statistically independent identically distributed residuals + with anisotropic covariance + - "indep_aniso": statistically independent residuals with anisotropic + covariances + For details, see: + N. Neverova, D. Novotny, A. Vedaldi "Correlated Uncertainty for Learning + Dense Correspondences from Noisy Labels", p. 918--926, in Proc. NIPS 2019 + """ + + # fmt: off + IID_ISO = "iid_iso" + INDEP_ANISO = "indep_aniso" + # fmt: on + + +@dataclass +class DensePoseUVConfidenceConfig: + """ + Configuration options for confidence on UV data + """ + + enabled: bool = False + # lower bound on UV confidences + epsilon: float = 0.01 + type: DensePoseUVConfidenceType = DensePoseUVConfidenceType.IID_ISO + + +@dataclass +class DensePoseSegmConfidenceConfig: + """ + Configuration options for confidence on segmentation + """ + + enabled: bool = False + # lower bound on confidence values + epsilon: float = 0.01 + + +@dataclass +class DensePoseConfidenceModelConfig: + """ + Configuration options for confidence models + """ + + # confidence for U and V values + uv_confidence: DensePoseUVConfidenceConfig + # segmentation confidence + segm_confidence: DensePoseSegmConfidenceConfig + + @staticmethod + def from_cfg(cfg: CfgNode) -> "DensePoseConfidenceModelConfig": + return DensePoseConfidenceModelConfig( + uv_confidence=DensePoseUVConfidenceConfig( + enabled=cfg.MODEL.ROI_DENSEPOSE_HEAD.UV_CONFIDENCE.ENABLED, + epsilon=cfg.MODEL.ROI_DENSEPOSE_HEAD.UV_CONFIDENCE.EPSILON, + type=DensePoseUVConfidenceType(cfg.MODEL.ROI_DENSEPOSE_HEAD.UV_CONFIDENCE.TYPE), + ), + segm_confidence=DensePoseSegmConfidenceConfig( + enabled=cfg.MODEL.ROI_DENSEPOSE_HEAD.SEGM_CONFIDENCE.ENABLED, + epsilon=cfg.MODEL.ROI_DENSEPOSE_HEAD.SEGM_CONFIDENCE.EPSILON, + ), + ) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/cse/__init__.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/cse/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a2273609cc54fb96d002a49dcd58788060945059 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/cse/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +from .vertex_direct_embedder import VertexDirectEmbedder +from .vertex_feature_embedder import VertexFeatureEmbedder +from .embedder import Embedder diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/cse/embedder.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/cse/embedder.py new file mode 100644 index 0000000000000000000000000000000000000000..7f52b06032ed97b2d652931646f0855ef342ada9 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/cse/embedder.py @@ -0,0 +1,130 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +import logging +import numpy as np +import pickle +from enum import Enum +from typing import Optional +import torch +from torch import nn + +from detectron2.config import CfgNode +from detectron2.utils.file_io import PathManager + +from .vertex_direct_embedder import VertexDirectEmbedder +from .vertex_feature_embedder import VertexFeatureEmbedder + + +class EmbedderType(Enum): + """ + Embedder type which defines how vertices are mapped into the embedding space: + - "vertex_direct": direct vertex embedding + - "vertex_feature": embedding vertex features + """ + + VERTEX_DIRECT = "vertex_direct" + VERTEX_FEATURE = "vertex_feature" + + +def create_embedder(embedder_spec: CfgNode, embedder_dim: int) -> nn.Module: + """ + Create an embedder based on the provided configuration + + Args: + embedder_spec (CfgNode): embedder configuration + embedder_dim (int): embedding space dimensionality + Return: + An embedder instance for the specified configuration + Raises ValueError, in case of unexpected embedder type + """ + embedder_type = EmbedderType(embedder_spec.TYPE) + if embedder_type == EmbedderType.VERTEX_DIRECT: + embedder = VertexDirectEmbedder( + num_vertices=embedder_spec.NUM_VERTICES, + embed_dim=embedder_dim, + ) + if embedder_spec.INIT_FILE != "": + embedder.load(embedder_spec.INIT_FILE) + elif embedder_type == EmbedderType.VERTEX_FEATURE: + embedder = VertexFeatureEmbedder( + num_vertices=embedder_spec.NUM_VERTICES, + feature_dim=embedder_spec.FEATURE_DIM, + embed_dim=embedder_dim, + train_features=embedder_spec.FEATURES_TRAINABLE, + ) + if embedder_spec.INIT_FILE != "": + embedder.load(embedder_spec.INIT_FILE) + else: + raise ValueError(f"Unexpected embedder type {embedder_type}") + + if not embedder_spec.IS_TRAINABLE: + embedder.requires_grad_(False) + + return embedder + + +class Embedder(nn.Module): + """ + Embedder module that serves as a container for embedders to use with different + meshes. Extends Module to automatically save / load state dict. + """ + + DEFAULT_MODEL_CHECKPOINT_PREFIX = "roi_heads.embedder." + + def __init__(self, cfg: CfgNode): + """ + Initialize mesh embedders. An embedder for mesh `i` is stored in a submodule + "embedder_{i}". + + Args: + cfg (CfgNode): configuration options + """ + super(Embedder, self).__init__() + self.mesh_names = set() + embedder_dim = cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBED_SIZE + logger = logging.getLogger(__name__) + for mesh_name, embedder_spec in cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBEDDERS.items(): + logger.info(f"Adding embedder embedder_{mesh_name} with spec {embedder_spec}") + self.add_module(f"embedder_{mesh_name}", create_embedder(embedder_spec, embedder_dim)) + self.mesh_names.add(mesh_name) + if cfg.MODEL.WEIGHTS != "": + self.load_from_model_checkpoint(cfg.MODEL.WEIGHTS) + + def load_from_model_checkpoint(self, fpath: str, prefix: Optional[str] = None): + if prefix is None: + prefix = Embedder.DEFAULT_MODEL_CHECKPOINT_PREFIX + state_dict = None + if fpath.endswith(".pkl"): + with PathManager.open(fpath, "rb") as hFile: + state_dict = pickle.load(hFile, encoding="latin1") # pyre-ignore[6] + else: + with PathManager.open(fpath, "rb") as hFile: + # pyre-fixme[6]: For 1st param expected `Union[PathLike[typing.Any], + # IO[bytes], str, BinaryIO]` but got `Union[IO[bytes], IO[str]]`. + state_dict = torch.load(hFile, map_location=torch.device("cpu")) + if state_dict is not None and "model" in state_dict: + state_dict_local = {} + for key in state_dict["model"]: + if key.startswith(prefix): + v_key = state_dict["model"][key] + if isinstance(v_key, np.ndarray): + v_key = torch.from_numpy(v_key) + state_dict_local[key[len(prefix) :]] = v_key + # non-strict loading to finetune on different meshes + self.load_state_dict(state_dict_local, strict=False) + + def forward(self, mesh_name: str) -> torch.Tensor: + """ + Produce vertex embeddings for the specific mesh; vertex embeddings are + a tensor of shape [N, D] where: + N = number of vertices + D = number of dimensions in the embedding space + Args: + mesh_name (str): name of a mesh for which to obtain vertex embeddings + Return: + Vertex embeddings, a tensor of shape [N, D] + """ + return getattr(self, f"embedder_{mesh_name}")() + + def has_embeddings(self, mesh_name: str) -> bool: + return hasattr(self, f"embedder_{mesh_name}") diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/cse/utils.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/cse/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6e70d25df7c8e2c1c408866cf7a6f0156b64114a --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/cse/utils.py @@ -0,0 +1,81 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +import torch +from torch.nn import functional as F + + +def squared_euclidean_distance_matrix(pts1: torch.Tensor, pts2: torch.Tensor) -> torch.Tensor: + """ + Get squared Euclidean Distance Matrix + Computes pairwise squared Euclidean distances between points + + Args: + pts1: Tensor [M x D], M is the number of points, D is feature dimensionality + pts2: Tensor [N x D], N is the number of points, D is feature dimensionality + + Return: + Tensor [M, N]: matrix of squared Euclidean distances; at index (m, n) + it contains || pts1[m] - pts2[n] ||^2 + """ + edm = torch.mm(-2 * pts1, pts2.t()) + edm += (pts1 * pts1).sum(1, keepdim=True) + (pts2 * pts2).sum(1, keepdim=True).t() + return edm.contiguous() + + +def normalize_embeddings(embeddings: torch.Tensor, epsilon: float = 1e-6) -> torch.Tensor: + """ + Normalize N D-dimensional embedding vectors arranged in a tensor [N, D] + + Args: + embeddings (tensor [N, D]): N D-dimensional embedding vectors + epsilon (float): minimum value for a vector norm + Return: + Normalized embeddings (tensor [N, D]), such that L2 vector norms are all equal to 1. + """ + return embeddings / torch.clamp(embeddings.norm(p=None, dim=1, keepdim=True), min=epsilon) + + +def get_closest_vertices_mask_from_ES( + E: torch.Tensor, + S: torch.Tensor, + h: int, + w: int, + mesh_vertex_embeddings: torch.Tensor, + device: torch.device, +): + """ + Interpolate Embeddings and Segmentations to the size of a given bounding box, + and compute closest vertices and the segmentation mask + + Args: + E (tensor [1, D, H, W]): D-dimensional embedding vectors for every point of the + default-sized box + S (tensor [1, 2, H, W]): 2-dimensional segmentation mask for every point of the + default-sized box + h (int): height of the target bounding box + w (int): width of the target bounding box + mesh_vertex_embeddings (tensor [N, D]): vertex embeddings for a chosen mesh + N is the number of vertices in the mesh, D is feature dimensionality + device (torch.device): device to move the tensors to + Return: + Closest Vertices (tensor [h, w]), int, for every point of the resulting box + Segmentation mask (tensor [h, w]), boolean, for every point of the resulting box + """ + embedding_resized = F.interpolate(E, size=(h, w), mode="bilinear")[0].to(device) + coarse_segm_resized = F.interpolate(S, size=(h, w), mode="bilinear")[0].to(device) + mask = coarse_segm_resized.argmax(0) > 0 + closest_vertices = torch.zeros(mask.shape, dtype=torch.long, device=device) + all_embeddings = embedding_resized[:, mask].t() + size_chunk = 10_000 # Chunking to avoid possible OOM + edm = [] + if len(all_embeddings) == 0: + return closest_vertices, mask + for chunk in range((len(all_embeddings) - 1) // size_chunk + 1): + chunk_embeddings = all_embeddings[size_chunk * chunk : size_chunk * (chunk + 1)] + edm.append( + torch.argmin( + squared_euclidean_distance_matrix(chunk_embeddings, mesh_vertex_embeddings), dim=1 + ) + ) + closest_vertices[mask] = torch.cat(edm) + return closest_vertices, mask diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/cse/vertex_direct_embedder.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/cse/vertex_direct_embedder.py new file mode 100644 index 0000000000000000000000000000000000000000..60fba277bf4c5bcb98cbd170dad168c4308bc0b4 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/cse/vertex_direct_embedder.py @@ -0,0 +1,64 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +import pickle +import torch +from torch import nn + +from detectron2.utils.file_io import PathManager + +from .utils import normalize_embeddings + + +class VertexDirectEmbedder(nn.Module): + """ + Class responsible for embedding vertices. Vertex embeddings take + the form of a tensor of size [N, D], where + N = number of vertices + D = number of dimensions in the embedding space + """ + + def __init__(self, num_vertices: int, embed_dim: int): + """ + Initialize embedder, set random embeddings + + Args: + num_vertices (int): number of vertices to embed + embed_dim (int): number of dimensions in the embedding space + """ + super(VertexDirectEmbedder, self).__init__() + self.embeddings = nn.Parameter(torch.Tensor(num_vertices, embed_dim)) + self.reset_parameters() + + @torch.no_grad() + def reset_parameters(self): + """ + Reset embeddings to random values + """ + self.embeddings.zero_() + + def forward(self) -> torch.Tensor: + """ + Produce vertex embeddings, a tensor of shape [N, D] where: + N = number of vertices + D = number of dimensions in the embedding space + + Return: + Full vertex embeddings, a tensor of shape [N, D] + """ + return normalize_embeddings(self.embeddings) + + @torch.no_grad() + def load(self, fpath: str): + """ + Load data from a file + + Args: + fpath (str): file path to load data from + """ + with PathManager.open(fpath, "rb") as hFile: + data = pickle.load(hFile) # pyre-ignore[6] + for name in ["embeddings"]: + if name in data: + getattr(self, name).copy_( + torch.tensor(data[name]).float().to(device=getattr(self, name).device) + ) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/cse/vertex_feature_embedder.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/cse/vertex_feature_embedder.py new file mode 100644 index 0000000000000000000000000000000000000000..dcb2f2039cf40b834235dc81143d0c94a7c33936 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/cse/vertex_feature_embedder.py @@ -0,0 +1,75 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +import pickle +import torch +from torch import nn + +from detectron2.utils.file_io import PathManager + +from .utils import normalize_embeddings + + +class VertexFeatureEmbedder(nn.Module): + """ + Class responsible for embedding vertex features. Mapping from + feature space to the embedding space is a tensor of size [K, D], where + K = number of dimensions in the feature space + D = number of dimensions in the embedding space + Vertex features is a tensor of size [N, K], where + N = number of vertices + K = number of dimensions in the feature space + Vertex embeddings are computed as F * E = tensor of size [N, D] + """ + + def __init__( + self, num_vertices: int, feature_dim: int, embed_dim: int, train_features: bool = False + ): + """ + Initialize embedder, set random embeddings + + Args: + num_vertices (int): number of vertices to embed + feature_dim (int): number of dimensions in the feature space + embed_dim (int): number of dimensions in the embedding space + train_features (bool): determines whether vertex features should + be trained (default: False) + """ + super(VertexFeatureEmbedder, self).__init__() + if train_features: + self.features = nn.Parameter(torch.Tensor(num_vertices, feature_dim)) + else: + self.register_buffer("features", torch.Tensor(num_vertices, feature_dim)) + self.embeddings = nn.Parameter(torch.Tensor(feature_dim, embed_dim)) + self.reset_parameters() + + @torch.no_grad() + def reset_parameters(self): + self.features.zero_() + self.embeddings.zero_() + + def forward(self) -> torch.Tensor: + """ + Produce vertex embeddings, a tensor of shape [N, D] where: + N = number of vertices + D = number of dimensions in the embedding space + + Return: + Full vertex embeddings, a tensor of shape [N, D] + """ + return normalize_embeddings(torch.mm(self.features, self.embeddings)) + + @torch.no_grad() + def load(self, fpath: str): + """ + Load data from a file + + Args: + fpath (str): file path to load data from + """ + with PathManager.open(fpath, "rb") as hFile: + data = pickle.load(hFile) # pyre-ignore[6] + for name in ["features", "embeddings"]: + if name in data: + getattr(self, name).copy_( + torch.tensor(data[name]).float().to(device=getattr(self, name).device) + ) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/densepose_checkpoint.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/densepose_checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..8c2b4f2e2cc9c6c798cf1bdb9c38dedc84058bd5 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/densepose_checkpoint.py @@ -0,0 +1,35 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from collections import OrderedDict + +from detectron2.checkpoint import DetectionCheckpointer + + +def _rename_HRNet_weights(weights): + # We detect and rename HRNet weights for DensePose. 1956 and 1716 are values that are + # common to all HRNet pretrained weights, and should be enough to accurately identify them + if ( + len(weights["model"].keys()) == 1956 + and len([k for k in weights["model"].keys() if k.startswith("stage")]) == 1716 + ): + hrnet_weights = OrderedDict() + for k in weights["model"].keys(): + hrnet_weights["backbone.bottom_up." + str(k)] = weights["model"][k] + return {"model": hrnet_weights} + else: + return weights + + +class DensePoseCheckpointer(DetectionCheckpointer): + """ + Same as :class:`DetectionCheckpointer`, but is able to handle HRNet weights + """ + + def __init__(self, model, save_dir="", *, save_to_disk=None, **checkpointables): + super().__init__(model, save_dir, save_to_disk=save_to_disk, **checkpointables) + + def _load_file(self, filename: str) -> object: + """ + Adding hrnet support + """ + weights = super()._load_file(filename) + return _rename_HRNet_weights(weights) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/filter.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/filter.py new file mode 100644 index 0000000000000000000000000000000000000000..18a856789e390e0a54484db97488e2e869c27ac8 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/filter.py @@ -0,0 +1,94 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from typing import List +import torch + +from detectron2.config import CfgNode +from detectron2.structures import Instances +from detectron2.structures.boxes import matched_pairwise_iou + + +class DensePoseDataFilter(object): + def __init__(self, cfg: CfgNode): + self.iou_threshold = cfg.MODEL.ROI_DENSEPOSE_HEAD.FG_IOU_THRESHOLD + self.keep_masks = cfg.MODEL.ROI_DENSEPOSE_HEAD.COARSE_SEGM_TRAINED_BY_MASKS + + @torch.no_grad() + def __call__(self, features: List[torch.Tensor], proposals_with_targets: List[Instances]): + """ + Filters proposals with targets to keep only the ones relevant for + DensePose training + + Args: + features (list[Tensor]): input data as a list of features, + each feature is a tensor. Axis 0 represents the number of + images `N` in the input data; axes 1-3 are channels, + height, and width, which may vary between features + (e.g., if a feature pyramid is used). + proposals_with_targets (list[Instances]): length `N` list of + `Instances`. The i-th `Instances` contains instances + (proposals, GT) for the i-th input image, + Returns: + list[Tensor]: filtered features + list[Instances]: filtered proposals + """ + proposals_filtered = [] + # TODO: the commented out code was supposed to correctly deal with situations + # where no valid DensePose GT is available for certain images. The corresponding + # image features were sliced and proposals were filtered. This led to performance + # deterioration, both in terms of runtime and in terms of evaluation results. + # + # feature_mask = torch.ones( + # len(proposals_with_targets), + # dtype=torch.bool, + # device=features[0].device if len(features) > 0 else torch.device("cpu"), + # ) + for i, proposals_per_image in enumerate(proposals_with_targets): + if not proposals_per_image.has("gt_densepose") and ( + not proposals_per_image.has("gt_masks") or not self.keep_masks + ): + # feature_mask[i] = 0 + continue + gt_boxes = proposals_per_image.gt_boxes + est_boxes = proposals_per_image.proposal_boxes + # apply match threshold for densepose head + iou = matched_pairwise_iou(gt_boxes, est_boxes) + iou_select = iou > self.iou_threshold + proposals_per_image = proposals_per_image[iou_select] # pyre-ignore[6] + + N_gt_boxes = len(proposals_per_image.gt_boxes) + assert N_gt_boxes == len(proposals_per_image.proposal_boxes), ( + f"The number of GT boxes {N_gt_boxes} is different from the " + f"number of proposal boxes {len(proposals_per_image.proposal_boxes)}" + ) + # filter out any target without suitable annotation + if self.keep_masks: + gt_masks = ( + proposals_per_image.gt_masks + if hasattr(proposals_per_image, "gt_masks") + else [None] * N_gt_boxes + ) + else: + gt_masks = [None] * N_gt_boxes + gt_densepose = ( + proposals_per_image.gt_densepose + if hasattr(proposals_per_image, "gt_densepose") + else [None] * N_gt_boxes + ) + assert len(gt_masks) == N_gt_boxes + assert len(gt_densepose) == N_gt_boxes + selected_indices = [ + i + for i, (dp_target, mask_target) in enumerate(zip(gt_densepose, gt_masks)) + if (dp_target is not None) or (mask_target is not None) + ] + # if not len(selected_indices): + # feature_mask[i] = 0 + # continue + if len(selected_indices) != N_gt_boxes: + proposals_per_image = proposals_per_image[selected_indices] # pyre-ignore[6] + assert len(proposals_per_image.gt_boxes) == len(proposals_per_image.proposal_boxes) + proposals_filtered.append(proposals_per_image) + # features_filtered = [feature[feature_mask] for feature in features] + # return features_filtered, proposals_filtered + return features, proposals_filtered diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/hrfpn.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/hrfpn.py new file mode 100644 index 0000000000000000000000000000000000000000..08ec420fa24e1e8f5074baf2e9ae737aff2ab12e --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/hrfpn.py @@ -0,0 +1,182 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +""" +MIT License +Copyright (c) 2019 Microsoft +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from detectron2.layers import ShapeSpec +from detectron2.modeling.backbone import BACKBONE_REGISTRY +from detectron2.modeling.backbone.backbone import Backbone + +from .hrnet import build_pose_hrnet_backbone + + +class HRFPN(Backbone): + """HRFPN (High Resolution Feature Pyramids) + Transforms outputs of HRNet backbone so they are suitable for the ROI_heads + arXiv: https://arxiv.org/abs/1904.04514 + Adapted from https://github.com/open-mmlab/mmdetection/blob/master/mmdet/models/necks/hrfpn.py + Args: + bottom_up: (list) output of HRNet + in_features (list): names of the input features (output of HRNet) + in_channels (list): number of channels for each branch + out_channels (int): output channels of feature pyramids + n_out_features (int): number of output stages + pooling (str): pooling for generating feature pyramids (from {MAX, AVG}) + share_conv (bool): Have one conv per output, or share one with all the outputs + """ + + def __init__( + self, + bottom_up, + in_features, + n_out_features, + in_channels, + out_channels, + pooling="AVG", + share_conv=False, + ): + super(HRFPN, self).__init__() + assert isinstance(in_channels, list) + self.bottom_up = bottom_up + self.in_features = in_features + self.n_out_features = n_out_features + self.in_channels = in_channels + self.out_channels = out_channels + self.num_ins = len(in_channels) + self.share_conv = share_conv + + if self.share_conv: + self.fpn_conv = nn.Conv2d( + in_channels=out_channels, out_channels=out_channels, kernel_size=3, padding=1 + ) + else: + self.fpn_conv = nn.ModuleList() + for _ in range(self.n_out_features): + self.fpn_conv.append( + nn.Conv2d( + in_channels=out_channels, + out_channels=out_channels, + kernel_size=3, + padding=1, + ) + ) + + # Custom change: Replaces a simple bilinear interpolation + self.interp_conv = nn.ModuleList() + for i in range(len(self.in_features)): + self.interp_conv.append( + nn.Sequential( + nn.ConvTranspose2d( + in_channels=in_channels[i], + out_channels=in_channels[i], + kernel_size=4, + stride=2**i, + padding=0, + output_padding=0, + bias=False, + ), + nn.BatchNorm2d(in_channels[i], momentum=0.1), + nn.ReLU(inplace=True), + ) + ) + + # Custom change: Replaces a couple (reduction conv + pooling) by one conv + self.reduction_pooling_conv = nn.ModuleList() + for i in range(self.n_out_features): + self.reduction_pooling_conv.append( + nn.Sequential( + nn.Conv2d(sum(in_channels), out_channels, kernel_size=2**i, stride=2**i), + nn.BatchNorm2d(out_channels, momentum=0.1), + nn.ReLU(inplace=True), + ) + ) + + if pooling == "MAX": + self.pooling = F.max_pool2d + else: + self.pooling = F.avg_pool2d + + self._out_features = [] + self._out_feature_channels = {} + self._out_feature_strides = {} + + for i in range(self.n_out_features): + self._out_features.append("p%d" % (i + 1)) + self._out_feature_channels.update({self._out_features[-1]: self.out_channels}) + self._out_feature_strides.update({self._out_features[-1]: 2 ** (i + 2)}) + + # default init_weights for conv(msra) and norm in ConvModule + def init_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, a=1) + nn.init.constant_(m.bias, 0) + + def forward(self, inputs): + bottom_up_features = self.bottom_up(inputs) + assert len(bottom_up_features) == len(self.in_features) + inputs = [bottom_up_features[f] for f in self.in_features] + + outs = [] + for i in range(len(inputs)): + outs.append(self.interp_conv[i](inputs[i])) + shape_2 = min(o.shape[2] for o in outs) + shape_3 = min(o.shape[3] for o in outs) + out = torch.cat([o[:, :, :shape_2, :shape_3] for o in outs], dim=1) + outs = [] + for i in range(self.n_out_features): + outs.append(self.reduction_pooling_conv[i](out)) + for i in range(len(outs)): # Make shapes consistent + outs[-1 - i] = outs[-1 - i][ + :, :, : outs[-1].shape[2] * 2**i, : outs[-1].shape[3] * 2**i + ] + outputs = [] + for i in range(len(outs)): + if self.share_conv: + outputs.append(self.fpn_conv(outs[i])) + else: + outputs.append(self.fpn_conv[i](outs[i])) + + assert len(self._out_features) == len(outputs) + return dict(zip(self._out_features, outputs)) + + +@BACKBONE_REGISTRY.register() +def build_hrfpn_backbone(cfg, input_shape: ShapeSpec) -> HRFPN: + + in_channels = cfg.MODEL.HRNET.STAGE4.NUM_CHANNELS + in_features = ["p%d" % (i + 1) for i in range(cfg.MODEL.HRNET.STAGE4.NUM_BRANCHES)] + n_out_features = len(cfg.MODEL.ROI_HEADS.IN_FEATURES) + out_channels = cfg.MODEL.HRNET.HRFPN.OUT_CHANNELS + hrnet = build_pose_hrnet_backbone(cfg, input_shape) + hrfpn = HRFPN( + hrnet, + in_features, + n_out_features, + in_channels, + out_channels, + pooling="AVG", + share_conv=False, + ) + + return hrfpn diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/hrnet.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/hrnet.py new file mode 100644 index 0000000000000000000000000000000000000000..ca2467107e8e5a50167de38ef6827fac646d1245 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/hrnet.py @@ -0,0 +1,474 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# ------------------------------------------------------------------------------ +# Copyright (c) Microsoft +# Licensed under the MIT License. +# Written by Bin Xiao (leoxiaobin@gmail.com) +# Modified by Bowen Cheng (bcheng9@illinois.edu) +# Adapted from https://github.com/HRNet/Higher-HRNet-Human-Pose-Estimation/blob/master/lib/models/pose_higher_hrnet.py # noqa +# ------------------------------------------------------------------------------ + +from __future__ import absolute_import, division, print_function +import logging +import torch.nn as nn + +from detectron2.layers import ShapeSpec +from detectron2.modeling.backbone import BACKBONE_REGISTRY +from detectron2.modeling.backbone.backbone import Backbone + +BN_MOMENTUM = 0.1 +logger = logging.getLogger(__name__) + +__all__ = ["build_pose_hrnet_backbone", "PoseHigherResolutionNet"] + + +def conv3x3(in_planes, out_planes, stride=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes, momentum=BN_MOMENTUM) + self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion, momentum=BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class HighResolutionModule(nn.Module): + """HighResolutionModule + Building block of the PoseHigherResolutionNet (see lower) + arXiv: https://arxiv.org/abs/1908.10357 + Args: + num_branches (int): number of branches of the modyle + blocks (str): type of block of the module + num_blocks (int): number of blocks of the module + num_inchannels (int): number of input channels of the module + num_channels (list): number of channels of each branch + multi_scale_output (bool): only used by the last module of PoseHigherResolutionNet + """ + + def __init__( + self, + num_branches, + blocks, + num_blocks, + num_inchannels, + num_channels, + multi_scale_output=True, + ): + super(HighResolutionModule, self).__init__() + self._check_branches(num_branches, blocks, num_blocks, num_inchannels, num_channels) + + self.num_inchannels = num_inchannels + self.num_branches = num_branches + + self.multi_scale_output = multi_scale_output + + self.branches = self._make_branches(num_branches, blocks, num_blocks, num_channels) + self.fuse_layers = self._make_fuse_layers() + self.relu = nn.ReLU(True) + + def _check_branches(self, num_branches, blocks, num_blocks, num_inchannels, num_channels): + if num_branches != len(num_blocks): + error_msg = "NUM_BRANCHES({}) <> NUM_BLOCKS({})".format(num_branches, len(num_blocks)) + logger.error(error_msg) + raise ValueError(error_msg) + + if num_branches != len(num_channels): + error_msg = "NUM_BRANCHES({}) <> NUM_CHANNELS({})".format( + num_branches, len(num_channels) + ) + logger.error(error_msg) + raise ValueError(error_msg) + + if num_branches != len(num_inchannels): + error_msg = "NUM_BRANCHES({}) <> NUM_INCHANNELS({})".format( + num_branches, len(num_inchannels) + ) + logger.error(error_msg) + raise ValueError(error_msg) + + def _make_one_branch(self, branch_index, block, num_blocks, num_channels, stride=1): + downsample = None + if ( + stride != 1 + or self.num_inchannels[branch_index] != num_channels[branch_index] * block.expansion + ): + downsample = nn.Sequential( + nn.Conv2d( + self.num_inchannels[branch_index], + num_channels[branch_index] * block.expansion, + kernel_size=1, + stride=stride, + bias=False, + ), + nn.BatchNorm2d(num_channels[branch_index] * block.expansion, momentum=BN_MOMENTUM), + ) + + layers = [] + layers.append( + block(self.num_inchannels[branch_index], num_channels[branch_index], stride, downsample) + ) + self.num_inchannels[branch_index] = num_channels[branch_index] * block.expansion + for _ in range(1, num_blocks[branch_index]): + layers.append(block(self.num_inchannels[branch_index], num_channels[branch_index])) + + return nn.Sequential(*layers) + + def _make_branches(self, num_branches, block, num_blocks, num_channels): + branches = [] + + for i in range(num_branches): + branches.append(self._make_one_branch(i, block, num_blocks, num_channels)) + + return nn.ModuleList(branches) + + def _make_fuse_layers(self): + if self.num_branches == 1: + return None + + num_branches = self.num_branches + num_inchannels = self.num_inchannels + fuse_layers = [] + for i in range(num_branches if self.multi_scale_output else 1): + fuse_layer = [] + for j in range(num_branches): + if j > i: + fuse_layer.append( + nn.Sequential( + nn.Conv2d(num_inchannels[j], num_inchannels[i], 1, 1, 0, bias=False), + nn.BatchNorm2d(num_inchannels[i]), + nn.Upsample(scale_factor=2 ** (j - i), mode="nearest"), + ) + ) + elif j == i: + fuse_layer.append(None) + else: + conv3x3s = [] + for k in range(i - j): + if k == i - j - 1: + num_outchannels_conv3x3 = num_inchannels[i] + conv3x3s.append( + nn.Sequential( + nn.Conv2d( + num_inchannels[j], + num_outchannels_conv3x3, + 3, + 2, + 1, + bias=False, + ), + nn.BatchNorm2d(num_outchannels_conv3x3), + ) + ) + else: + num_outchannels_conv3x3 = num_inchannels[j] + conv3x3s.append( + nn.Sequential( + nn.Conv2d( + num_inchannels[j], + num_outchannels_conv3x3, + 3, + 2, + 1, + bias=False, + ), + nn.BatchNorm2d(num_outchannels_conv3x3), + nn.ReLU(True), + ) + ) + fuse_layer.append(nn.Sequential(*conv3x3s)) + fuse_layers.append(nn.ModuleList(fuse_layer)) + + return nn.ModuleList(fuse_layers) + + def get_num_inchannels(self): + return self.num_inchannels + + def forward(self, x): + if self.num_branches == 1: + return [self.branches[0](x[0])] + + for i in range(self.num_branches): + x[i] = self.branches[i](x[i]) + + x_fuse = [] + + for i in range(len(self.fuse_layers)): + y = x[0] if i == 0 else self.fuse_layers[i][0](x[0]) + for j in range(1, self.num_branches): + if i == j: + y = y + x[j] + else: + z = self.fuse_layers[i][j](x[j])[:, :, : y.shape[2], : y.shape[3]] + y = y + z + x_fuse.append(self.relu(y)) + + return x_fuse + + +blocks_dict = {"BASIC": BasicBlock, "BOTTLENECK": Bottleneck} + + +class PoseHigherResolutionNet(Backbone): + """PoseHigherResolutionNet + Composed of several HighResolutionModule tied together with ConvNets + Adapted from the GitHub version to fit with HRFPN and the Detectron2 infrastructure + arXiv: https://arxiv.org/abs/1908.10357 + """ + + def __init__(self, cfg, **kwargs): + self.inplanes = cfg.MODEL.HRNET.STEM_INPLANES + super(PoseHigherResolutionNet, self).__init__() + + # stem net + self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64, momentum=BN_MOMENTUM) + self.conv2 = nn.Conv2d(64, 64, kernel_size=3, stride=2, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(64, momentum=BN_MOMENTUM) + self.relu = nn.ReLU(inplace=True) + self.layer1 = self._make_layer(Bottleneck, 64, 4) + + self.stage2_cfg = cfg.MODEL.HRNET.STAGE2 + num_channels = self.stage2_cfg.NUM_CHANNELS + block = blocks_dict[self.stage2_cfg.BLOCK] + num_channels = [num_channels[i] * block.expansion for i in range(len(num_channels))] + self.transition1 = self._make_transition_layer([256], num_channels) + self.stage2, pre_stage_channels = self._make_stage(self.stage2_cfg, num_channels) + + self.stage3_cfg = cfg.MODEL.HRNET.STAGE3 + num_channels = self.stage3_cfg.NUM_CHANNELS + block = blocks_dict[self.stage3_cfg.BLOCK] + num_channels = [num_channels[i] * block.expansion for i in range(len(num_channels))] + self.transition2 = self._make_transition_layer(pre_stage_channels, num_channels) + self.stage3, pre_stage_channels = self._make_stage(self.stage3_cfg, num_channels) + + self.stage4_cfg = cfg.MODEL.HRNET.STAGE4 + num_channels = self.stage4_cfg.NUM_CHANNELS + block = blocks_dict[self.stage4_cfg.BLOCK] + num_channels = [num_channels[i] * block.expansion for i in range(len(num_channels))] + self.transition3 = self._make_transition_layer(pre_stage_channels, num_channels) + self.stage4, pre_stage_channels = self._make_stage( + self.stage4_cfg, num_channels, multi_scale_output=True + ) + + self._out_features = [] + self._out_feature_channels = {} + self._out_feature_strides = {} + + for i in range(cfg.MODEL.HRNET.STAGE4.NUM_BRANCHES): + self._out_features.append("p%d" % (i + 1)) + self._out_feature_channels.update( + {self._out_features[-1]: cfg.MODEL.HRNET.STAGE4.NUM_CHANNELS[i]} + ) + self._out_feature_strides.update({self._out_features[-1]: 1}) + + def _get_deconv_cfg(self, deconv_kernel): + if deconv_kernel == 4: + padding = 1 + output_padding = 0 + elif deconv_kernel == 3: + padding = 1 + output_padding = 1 + elif deconv_kernel == 2: + padding = 0 + output_padding = 0 + + return deconv_kernel, padding, output_padding + + def _make_transition_layer(self, num_channels_pre_layer, num_channels_cur_layer): + num_branches_cur = len(num_channels_cur_layer) + num_branches_pre = len(num_channels_pre_layer) + + transition_layers = [] + for i in range(num_branches_cur): + if i < num_branches_pre: + if num_channels_cur_layer[i] != num_channels_pre_layer[i]: + transition_layers.append( + nn.Sequential( + nn.Conv2d( + num_channels_pre_layer[i], + num_channels_cur_layer[i], + 3, + 1, + 1, + bias=False, + ), + nn.BatchNorm2d(num_channels_cur_layer[i]), + nn.ReLU(inplace=True), + ) + ) + else: + transition_layers.append(None) + else: + conv3x3s = [] + for j in range(i + 1 - num_branches_pre): + inchannels = num_channels_pre_layer[-1] + outchannels = ( + num_channels_cur_layer[i] if j == i - num_branches_pre else inchannels + ) + conv3x3s.append( + nn.Sequential( + nn.Conv2d(inchannels, outchannels, 3, 2, 1, bias=False), + nn.BatchNorm2d(outchannels), + nn.ReLU(inplace=True), + ) + ) + transition_layers.append(nn.Sequential(*conv3x3s)) + + return nn.ModuleList(transition_layers) + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False, + ), + nn.BatchNorm2d(planes * block.expansion, momentum=BN_MOMENTUM), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def _make_stage(self, layer_config, num_inchannels, multi_scale_output=True): + num_modules = layer_config["NUM_MODULES"] + num_branches = layer_config["NUM_BRANCHES"] + num_blocks = layer_config["NUM_BLOCKS"] + num_channels = layer_config["NUM_CHANNELS"] + block = blocks_dict[layer_config["BLOCK"]] + + modules = [] + for i in range(num_modules): + # multi_scale_output is only used last module + if not multi_scale_output and i == num_modules - 1: + reset_multi_scale_output = False + else: + reset_multi_scale_output = True + + modules.append( + HighResolutionModule( + num_branches, + block, + num_blocks, + num_inchannels, + num_channels, + reset_multi_scale_output, + ) + ) + num_inchannels = modules[-1].get_num_inchannels() + + return nn.Sequential(*modules), num_inchannels + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.conv2(x) + x = self.bn2(x) + x = self.relu(x) + x = self.layer1(x) + + x_list = [] + for i in range(self.stage2_cfg.NUM_BRANCHES): + if self.transition1[i] is not None: + x_list.append(self.transition1[i](x)) + else: + x_list.append(x) + y_list = self.stage2(x_list) + + x_list = [] + for i in range(self.stage3_cfg.NUM_BRANCHES): + if self.transition2[i] is not None: + x_list.append(self.transition2[i](y_list[-1])) + else: + x_list.append(y_list[i]) + y_list = self.stage3(x_list) + + x_list = [] + for i in range(self.stage4_cfg.NUM_BRANCHES): + if self.transition3[i] is not None: + x_list.append(self.transition3[i](y_list[-1])) + else: + x_list.append(y_list[i]) + y_list = self.stage4(x_list) + + assert len(self._out_features) == len(y_list) + return dict(zip(self._out_features, y_list)) # final_outputs + + +@BACKBONE_REGISTRY.register() +def build_pose_hrnet_backbone(cfg, input_shape: ShapeSpec): + model = PoseHigherResolutionNet(cfg) + return model diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/inference.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..81049649edddb23aeebeac4085514da838f1463b --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/inference.py @@ -0,0 +1,44 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from dataclasses import fields +from typing import Any, List +import torch + +from detectron2.structures import Instances + + +def densepose_inference(densepose_predictor_output: Any, detections: List[Instances]) -> None: + """ + Splits DensePose predictor outputs into chunks, each chunk corresponds to + detections on one image. Predictor output chunks are stored in `pred_densepose` + attribute of the corresponding `Instances` object. + + Args: + densepose_predictor_output: a dataclass instance (can be of different types, + depending on predictor used for inference). Each field can be `None` + (if the corresponding output was not inferred) or a tensor of size + [N, ...], where N = N_1 + N_2 + .. + N_k is a total number of + detections on all images, N_1 is the number of detections on image 1, + N_2 is the number of detections on image 2, etc. + detections: a list of objects of type `Instance`, k-th object corresponds + to detections on k-th image. + """ + k = 0 + for detection_i in detections: + if densepose_predictor_output is None: + # don't add `pred_densepose` attribute + continue + n_i = detection_i.__len__() + + PredictorOutput = type(densepose_predictor_output) + output_i_dict = {} + # we assume here that `densepose_predictor_output` is a dataclass object + for field in fields(densepose_predictor_output): + field_value = getattr(densepose_predictor_output, field.name) + # slice tensors + if isinstance(field_value, torch.Tensor): + output_i_dict[field.name] = field_value[k : k + n_i] + # leave others as is + else: + output_i_dict[field.name] = field_value + detection_i.pred_densepose = PredictorOutput(**output_i_dict) + k += n_i diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/__init__.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e5c593700e7274ea9cbaf8f4a52e8a229ef4c5a1 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from .chart import DensePoseChartLoss +from .chart_with_confidences import DensePoseChartWithConfidenceLoss +from .cse import DensePoseCseLoss +from .registry import DENSEPOSE_LOSS_REGISTRY + + +__all__ = [ + "DensePoseChartLoss", + "DensePoseChartWithConfidenceLoss", + "DensePoseCseLoss", + "DENSEPOSE_LOSS_REGISTRY", +] diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/chart.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/chart.py new file mode 100644 index 0000000000000000000000000000000000000000..02cdae8db3a41fc197be7fcc792c7119c7a21726 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/chart.py @@ -0,0 +1,291 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from typing import Any, List +import torch +from torch.nn import functional as F + +from detectron2.config import CfgNode +from detectron2.structures import Instances + +from .mask_or_segm import MaskOrSegmentationLoss +from .registry import DENSEPOSE_LOSS_REGISTRY +from .utils import ( + BilinearInterpolationHelper, + ChartBasedAnnotationsAccumulator, + LossDict, + extract_packed_annotations_from_matches, +) + + +@DENSEPOSE_LOSS_REGISTRY.register() +class DensePoseChartLoss: + """ + DensePose loss for chart-based training. A mesh is split into charts, + each chart is given a label (I) and parametrized by 2 coordinates referred to + as U and V. Ground truth consists of a number of points annotated with + I, U and V values and coarse segmentation S defined for all pixels of the + object bounding box. In some cases (see `COARSE_SEGM_TRAINED_BY_MASKS`), + semantic segmentation annotations can be used as ground truth inputs as well. + + Estimated values are tensors: + * U coordinates, tensor of shape [N, C, S, S] + * V coordinates, tensor of shape [N, C, S, S] + * fine segmentation estimates, tensor of shape [N, C, S, S] with raw unnormalized + scores for each fine segmentation label at each location + * coarse segmentation estimates, tensor of shape [N, D, S, S] with raw unnormalized + scores for each coarse segmentation label at each location + where N is the number of detections, C is the number of fine segmentation + labels, S is the estimate size ( = width = height) and D is the number of + coarse segmentation channels. + + The losses are: + * regression (smooth L1) loss for U and V coordinates + * cross entropy loss for fine (I) and coarse (S) segmentations + Each loss has an associated weight + """ + + def __init__(self, cfg: CfgNode): + """ + Initialize chart-based loss from configuration options + + Args: + cfg (CfgNode): configuration options + """ + # fmt: off + self.heatmap_size = cfg.MODEL.ROI_DENSEPOSE_HEAD.HEATMAP_SIZE + self.w_points = cfg.MODEL.ROI_DENSEPOSE_HEAD.POINT_REGRESSION_WEIGHTS + self.w_part = cfg.MODEL.ROI_DENSEPOSE_HEAD.PART_WEIGHTS + self.w_segm = cfg.MODEL.ROI_DENSEPOSE_HEAD.INDEX_WEIGHTS + self.n_segm_chan = cfg.MODEL.ROI_DENSEPOSE_HEAD.NUM_COARSE_SEGM_CHANNELS + # fmt: on + self.segm_trained_by_masks = cfg.MODEL.ROI_DENSEPOSE_HEAD.COARSE_SEGM_TRAINED_BY_MASKS + self.segm_loss = MaskOrSegmentationLoss(cfg) + + def __call__( + self, proposals_with_gt: List[Instances], densepose_predictor_outputs: Any, **kwargs + ) -> LossDict: + """ + Produce chart-based DensePose losses + + Args: + proposals_with_gt (list of Instances): detections with associated ground truth data + densepose_predictor_outputs: an object of a dataclass that contains predictor outputs + with estimated values; assumed to have the following attributes: + * coarse_segm - coarse segmentation estimates, tensor of shape [N, D, S, S] + * fine_segm - fine segmentation estimates, tensor of shape [N, C, S, S] + * u - U coordinate estimates per fine labels, tensor of shape [N, C, S, S] + * v - V coordinate estimates per fine labels, tensor of shape [N, C, S, S] + where N is the number of detections, C is the number of fine segmentation + labels, S is the estimate size ( = width = height) and D is the number of + coarse segmentation channels. + + Return: + dict: str -> tensor: dict of losses with the following entries: + * `loss_densepose_U`: smooth L1 loss for U coordinate estimates + * `loss_densepose_V`: smooth L1 loss for V coordinate estimates + * `loss_densepose_I`: cross entropy for raw unnormalized scores for fine + segmentation estimates given ground truth labels; + * `loss_densepose_S`: cross entropy for raw unnormalized scores for coarse + segmentation estimates given ground truth labels; + """ + # densepose outputs are computed for all images and all bounding boxes; + # i.e. if a batch has 4 images with (3, 1, 2, 1) proposals respectively, + # the outputs will have size(0) == 3+1+2+1 == 7 + + if not len(proposals_with_gt): + return self.produce_fake_densepose_losses(densepose_predictor_outputs) + + accumulator = ChartBasedAnnotationsAccumulator() + packed_annotations = extract_packed_annotations_from_matches(proposals_with_gt, accumulator) + + # NOTE: we need to keep the same computation graph on all the GPUs to + # perform reduction properly. Hence even if we have no data on one + # of the GPUs, we still need to generate the computation graph. + # Add fake (zero) loss in the form Tensor.sum() * 0 + if packed_annotations is None: + return self.produce_fake_densepose_losses(densepose_predictor_outputs) + + h, w = densepose_predictor_outputs.u.shape[2:] + interpolator = BilinearInterpolationHelper.from_matches( + packed_annotations, + (h, w), + ) + + j_valid_fg = interpolator.j_valid * ( # pyre-ignore[16] + packed_annotations.fine_segm_labels_gt > 0 + ) + # pyre-fixme[6]: For 1st param expected `Tensor` but got `int`. + if not torch.any(j_valid_fg): + return self.produce_fake_densepose_losses(densepose_predictor_outputs) + + losses_uv = self.produce_densepose_losses_uv( + proposals_with_gt, + densepose_predictor_outputs, + packed_annotations, + interpolator, + j_valid_fg, # pyre-ignore[6] + ) + + losses_segm = self.produce_densepose_losses_segm( + proposals_with_gt, + densepose_predictor_outputs, + packed_annotations, + interpolator, + j_valid_fg, # pyre-ignore[6] + ) + + return {**losses_uv, **losses_segm} + + def produce_fake_densepose_losses(self, densepose_predictor_outputs: Any) -> LossDict: + """ + Fake losses for fine segmentation and U/V coordinates. These are used when + no suitable ground truth data was found in a batch. The loss has a value 0 + and is primarily used to construct the computation graph, so that + `DistributedDataParallel` has similar graphs on all GPUs and can perform + reduction properly. + + Args: + densepose_predictor_outputs: DensePose predictor outputs, an object + of a dataclass that is assumed to have the following attributes: + * fine_segm - fine segmentation estimates, tensor of shape [N, C, S, S] + * u - U coordinate estimates per fine labels, tensor of shape [N, C, S, S] + * v - V coordinate estimates per fine labels, tensor of shape [N, C, S, S] + Return: + dict: str -> tensor: dict of losses with the following entries: + * `loss_densepose_U`: has value 0 + * `loss_densepose_V`: has value 0 + * `loss_densepose_I`: has value 0 + * `loss_densepose_S`: has value 0 + """ + losses_uv = self.produce_fake_densepose_losses_uv(densepose_predictor_outputs) + losses_segm = self.produce_fake_densepose_losses_segm(densepose_predictor_outputs) + return {**losses_uv, **losses_segm} + + def produce_fake_densepose_losses_uv(self, densepose_predictor_outputs: Any) -> LossDict: + """ + Fake losses for U/V coordinates. These are used when no suitable ground + truth data was found in a batch. The loss has a value 0 + and is primarily used to construct the computation graph, so that + `DistributedDataParallel` has similar graphs on all GPUs and can perform + reduction properly. + + Args: + densepose_predictor_outputs: DensePose predictor outputs, an object + of a dataclass that is assumed to have the following attributes: + * u - U coordinate estimates per fine labels, tensor of shape [N, C, S, S] + * v - V coordinate estimates per fine labels, tensor of shape [N, C, S, S] + Return: + dict: str -> tensor: dict of losses with the following entries: + * `loss_densepose_U`: has value 0 + * `loss_densepose_V`: has value 0 + """ + return { + "loss_densepose_U": densepose_predictor_outputs.u.sum() * 0, + "loss_densepose_V": densepose_predictor_outputs.v.sum() * 0, + } + + def produce_fake_densepose_losses_segm(self, densepose_predictor_outputs: Any) -> LossDict: + """ + Fake losses for fine / coarse segmentation. These are used when + no suitable ground truth data was found in a batch. The loss has a value 0 + and is primarily used to construct the computation graph, so that + `DistributedDataParallel` has similar graphs on all GPUs and can perform + reduction properly. + + Args: + densepose_predictor_outputs: DensePose predictor outputs, an object + of a dataclass that is assumed to have the following attributes: + * fine_segm - fine segmentation estimates, tensor of shape [N, C, S, S] + * coarse_segm - coarse segmentation estimates, tensor of shape [N, D, S, S] + Return: + dict: str -> tensor: dict of losses with the following entries: + * `loss_densepose_I`: has value 0 + * `loss_densepose_S`: has value 0, added only if `segm_trained_by_masks` is False + """ + losses = { + "loss_densepose_I": densepose_predictor_outputs.fine_segm.sum() * 0, + "loss_densepose_S": self.segm_loss.fake_value(densepose_predictor_outputs), + } + return losses + + def produce_densepose_losses_uv( + self, + proposals_with_gt: List[Instances], + densepose_predictor_outputs: Any, + packed_annotations: Any, + interpolator: BilinearInterpolationHelper, + j_valid_fg: torch.Tensor, + ) -> LossDict: + """ + Compute losses for U/V coordinates: smooth L1 loss between + estimated coordinates and the ground truth. + + Args: + proposals_with_gt (list of Instances): detections with associated ground truth data + densepose_predictor_outputs: DensePose predictor outputs, an object + of a dataclass that is assumed to have the following attributes: + * u - U coordinate estimates per fine labels, tensor of shape [N, C, S, S] + * v - V coordinate estimates per fine labels, tensor of shape [N, C, S, S] + Return: + dict: str -> tensor: dict of losses with the following entries: + * `loss_densepose_U`: smooth L1 loss for U coordinate estimates + * `loss_densepose_V`: smooth L1 loss for V coordinate estimates + """ + u_gt = packed_annotations.u_gt[j_valid_fg] + u_est = interpolator.extract_at_points(densepose_predictor_outputs.u)[j_valid_fg] + v_gt = packed_annotations.v_gt[j_valid_fg] + v_est = interpolator.extract_at_points(densepose_predictor_outputs.v)[j_valid_fg] + return { + "loss_densepose_U": F.smooth_l1_loss(u_est, u_gt, reduction="sum") * self.w_points, + "loss_densepose_V": F.smooth_l1_loss(v_est, v_gt, reduction="sum") * self.w_points, + } + + def produce_densepose_losses_segm( + self, + proposals_with_gt: List[Instances], + densepose_predictor_outputs: Any, + packed_annotations: Any, + interpolator: BilinearInterpolationHelper, + j_valid_fg: torch.Tensor, + ) -> LossDict: + """ + Losses for fine / coarse segmentation: cross-entropy + for segmentation unnormalized scores given ground truth labels at + annotated points for fine segmentation and dense mask annotations + for coarse segmentation. + + Args: + proposals_with_gt (list of Instances): detections with associated ground truth data + densepose_predictor_outputs: DensePose predictor outputs, an object + of a dataclass that is assumed to have the following attributes: + * fine_segm - fine segmentation estimates, tensor of shape [N, C, S, S] + * coarse_segm - coarse segmentation estimates, tensor of shape [N, D, S, S] + Return: + dict: str -> tensor: dict of losses with the following entries: + * `loss_densepose_I`: cross entropy for raw unnormalized scores for fine + segmentation estimates given ground truth labels + * `loss_densepose_S`: cross entropy for raw unnormalized scores for coarse + segmentation estimates given ground truth labels; + may be included if coarse segmentation is only trained + using DensePose ground truth; if additional supervision through + instance segmentation data is performed (`segm_trained_by_masks` is True), + this loss is handled by `produce_mask_losses` instead + """ + fine_segm_gt = packed_annotations.fine_segm_labels_gt[ + interpolator.j_valid # pyre-ignore[16] + ] + fine_segm_est = interpolator.extract_at_points( + densepose_predictor_outputs.fine_segm, + slice_fine_segm=slice(None), + w_ylo_xlo=interpolator.w_ylo_xlo[:, None], # pyre-ignore[16] + w_ylo_xhi=interpolator.w_ylo_xhi[:, None], # pyre-ignore[16] + w_yhi_xlo=interpolator.w_yhi_xlo[:, None], # pyre-ignore[16] + w_yhi_xhi=interpolator.w_yhi_xhi[:, None], # pyre-ignore[16] + )[interpolator.j_valid, :] + return { + "loss_densepose_I": F.cross_entropy(fine_segm_est, fine_segm_gt.long()) * self.w_part, + "loss_densepose_S": self.segm_loss( + proposals_with_gt, densepose_predictor_outputs, packed_annotations + ) + * self.w_segm, + } diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/chart_with_confidences.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/chart_with_confidences.py new file mode 100644 index 0000000000000000000000000000000000000000..78ce7c6cb02fa01f6319d088349ff4f422001839 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/chart_with_confidences.py @@ -0,0 +1,209 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import math +from typing import Any, List +import torch +from torch import nn +from torch.nn import functional as F + +from detectron2.config import CfgNode +from detectron2.structures import Instances + +from .. import DensePoseConfidenceModelConfig, DensePoseUVConfidenceType +from .chart import DensePoseChartLoss +from .registry import DENSEPOSE_LOSS_REGISTRY +from .utils import BilinearInterpolationHelper, LossDict + + +@DENSEPOSE_LOSS_REGISTRY.register() +class DensePoseChartWithConfidenceLoss(DensePoseChartLoss): + """ """ + + def __init__(self, cfg: CfgNode): + super().__init__(cfg) + self.confidence_model_cfg = DensePoseConfidenceModelConfig.from_cfg(cfg) + if self.confidence_model_cfg.uv_confidence.type == DensePoseUVConfidenceType.IID_ISO: + self.uv_loss_with_confidences = IIDIsotropicGaussianUVLoss( + self.confidence_model_cfg.uv_confidence.epsilon + ) + elif self.confidence_model_cfg.uv_confidence.type == DensePoseUVConfidenceType.INDEP_ANISO: + self.uv_loss_with_confidences = IndepAnisotropicGaussianUVLoss( + self.confidence_model_cfg.uv_confidence.epsilon + ) + + def produce_fake_densepose_losses_uv(self, densepose_predictor_outputs: Any) -> LossDict: + """ + Overrides fake losses for fine segmentation and U/V coordinates to + include computation graphs for additional confidence parameters. + These are used when no suitable ground truth data was found in a batch. + The loss has a value 0 and is primarily used to construct the computation graph, + so that `DistributedDataParallel` has similar graphs on all GPUs and can + perform reduction properly. + + Args: + densepose_predictor_outputs: DensePose predictor outputs, an object + of a dataclass that is assumed to have the following attributes: + * fine_segm - fine segmentation estimates, tensor of shape [N, C, S, S] + * u - U coordinate estimates per fine labels, tensor of shape [N, C, S, S] + * v - V coordinate estimates per fine labels, tensor of shape [N, C, S, S] + Return: + dict: str -> tensor: dict of losses with the following entries: + * `loss_densepose_U`: has value 0 + * `loss_densepose_V`: has value 0 + * `loss_densepose_I`: has value 0 + """ + conf_type = self.confidence_model_cfg.uv_confidence.type + if self.confidence_model_cfg.uv_confidence.enabled: + loss_uv = ( + densepose_predictor_outputs.u.sum() + densepose_predictor_outputs.v.sum() + ) * 0 + if conf_type == DensePoseUVConfidenceType.IID_ISO: + loss_uv += densepose_predictor_outputs.sigma_2.sum() * 0 + elif conf_type == DensePoseUVConfidenceType.INDEP_ANISO: + loss_uv += ( + densepose_predictor_outputs.sigma_2.sum() + + densepose_predictor_outputs.kappa_u.sum() + + densepose_predictor_outputs.kappa_v.sum() + ) * 0 + return {"loss_densepose_UV": loss_uv} + else: + return super().produce_fake_densepose_losses_uv(densepose_predictor_outputs) + + def produce_densepose_losses_uv( + self, + proposals_with_gt: List[Instances], + densepose_predictor_outputs: Any, + packed_annotations: Any, + interpolator: BilinearInterpolationHelper, + j_valid_fg: torch.Tensor, + ) -> LossDict: + conf_type = self.confidence_model_cfg.uv_confidence.type + if self.confidence_model_cfg.uv_confidence.enabled: + u_gt = packed_annotations.u_gt[j_valid_fg] + u_est = interpolator.extract_at_points(densepose_predictor_outputs.u)[j_valid_fg] + v_gt = packed_annotations.v_gt[j_valid_fg] + v_est = interpolator.extract_at_points(densepose_predictor_outputs.v)[j_valid_fg] + sigma_2_est = interpolator.extract_at_points(densepose_predictor_outputs.sigma_2)[ + j_valid_fg + ] + if conf_type == DensePoseUVConfidenceType.IID_ISO: + return { + "loss_densepose_UV": ( + self.uv_loss_with_confidences(u_est, v_est, sigma_2_est, u_gt, v_gt) + * self.w_points + ) + } + elif conf_type in [DensePoseUVConfidenceType.INDEP_ANISO]: + kappa_u_est = interpolator.extract_at_points(densepose_predictor_outputs.kappa_u)[ + j_valid_fg + ] + kappa_v_est = interpolator.extract_at_points(densepose_predictor_outputs.kappa_v)[ + j_valid_fg + ] + return { + "loss_densepose_UV": ( + self.uv_loss_with_confidences( + u_est, v_est, sigma_2_est, kappa_u_est, kappa_v_est, u_gt, v_gt + ) + * self.w_points + ) + } + return super().produce_densepose_losses_uv( + proposals_with_gt, + densepose_predictor_outputs, + packed_annotations, + interpolator, + j_valid_fg, + ) + + +class IIDIsotropicGaussianUVLoss(nn.Module): + """ + Loss for the case of iid residuals with isotropic covariance: + $Sigma_i = sigma_i^2 I$ + The loss (negative log likelihood) is then: + $1/2 sum_{i=1}^n (log(2 pi) + 2 log sigma_i^2 + ||delta_i||^2 / sigma_i^2)$, + where $delta_i=(u - u', v - v')$ is a 2D vector containing UV coordinates + difference between estimated and ground truth UV values + For details, see: + N. Neverova, D. Novotny, A. Vedaldi "Correlated Uncertainty for Learning + Dense Correspondences from Noisy Labels", p. 918--926, in Proc. NIPS 2019 + """ + + def __init__(self, sigma_lower_bound: float): + super(IIDIsotropicGaussianUVLoss, self).__init__() + self.sigma_lower_bound = sigma_lower_bound + self.log2pi = math.log(2 * math.pi) + + def forward( + self, + u: torch.Tensor, + v: torch.Tensor, + sigma_u: torch.Tensor, + target_u: torch.Tensor, + target_v: torch.Tensor, + ): + # compute $\sigma_i^2$ + # use sigma_lower_bound to avoid degenerate solution for variance + # (sigma -> 0) + sigma2 = F.softplus(sigma_u) + self.sigma_lower_bound + # compute \|delta_i\|^2 + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and `int`. + delta_t_delta = (u - target_u) ** 2 + (v - target_v) ** 2 + # the total loss from the formula above: + loss = 0.5 * (self.log2pi + 2 * torch.log(sigma2) + delta_t_delta / sigma2) + return loss.sum() + + +class IndepAnisotropicGaussianUVLoss(nn.Module): + """ + Loss for the case of independent residuals with anisotropic covariances: + $Sigma_i = sigma_i^2 I + r_i r_i^T$ + The loss (negative log likelihood) is then: + $1/2 sum_{i=1}^n (log(2 pi) + + log sigma_i^2 (sigma_i^2 + ||r_i||^2) + + ||delta_i||^2 / sigma_i^2 + - ^2 / (sigma_i^2 * (sigma_i^2 + ||r_i||^2)))$, + where $delta_i=(u - u', v - v')$ is a 2D vector containing UV coordinates + difference between estimated and ground truth UV values + For details, see: + N. Neverova, D. Novotny, A. Vedaldi "Correlated Uncertainty for Learning + Dense Correspondences from Noisy Labels", p. 918--926, in Proc. NIPS 2019 + """ + + def __init__(self, sigma_lower_bound: float): + super(IndepAnisotropicGaussianUVLoss, self).__init__() + self.sigma_lower_bound = sigma_lower_bound + self.log2pi = math.log(2 * math.pi) + + def forward( + self, + u: torch.Tensor, + v: torch.Tensor, + sigma_u: torch.Tensor, + kappa_u_est: torch.Tensor, + kappa_v_est: torch.Tensor, + target_u: torch.Tensor, + target_v: torch.Tensor, + ): + # compute $\sigma_i^2$ + sigma2 = F.softplus(sigma_u) + self.sigma_lower_bound + # compute \|r_i\|^2 + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and `int`. + r_sqnorm2 = kappa_u_est**2 + kappa_v_est**2 + delta_u = u - target_u + delta_v = v - target_v + # compute \|delta_i\|^2 + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and `int`. + delta_sqnorm = delta_u**2 + delta_v**2 + delta_u_r_u = delta_u * kappa_u_est + delta_v_r_v = delta_v * kappa_v_est + # compute the scalar product + delta_r = delta_u_r_u + delta_v_r_v + # compute squared scalar product ^2 + # pyre-fixme[58]: `**` is not supported for operand types `Tensor` and `int`. + delta_r_sqnorm = delta_r**2 + denom2 = sigma2 * (sigma2 + r_sqnorm2) + loss = 0.5 * ( + self.log2pi + torch.log(denom2) + delta_sqnorm / sigma2 - delta_r_sqnorm / denom2 + ) + return loss.sum() diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/cse.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/cse.py new file mode 100644 index 0000000000000000000000000000000000000000..dd561ad518f42c769fd9a5c8517409ddc33edf6f --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/cse.py @@ -0,0 +1,115 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +from typing import Any, List +from torch import nn + +from detectron2.config import CfgNode +from detectron2.structures import Instances + +from .cycle_pix2shape import PixToShapeCycleLoss +from .cycle_shape2shape import ShapeToShapeCycleLoss +from .embed import EmbeddingLoss +from .embed_utils import CseAnnotationsAccumulator +from .mask_or_segm import MaskOrSegmentationLoss +from .registry import DENSEPOSE_LOSS_REGISTRY +from .soft_embed import SoftEmbeddingLoss +from .utils import BilinearInterpolationHelper, LossDict, extract_packed_annotations_from_matches + + +@DENSEPOSE_LOSS_REGISTRY.register() +class DensePoseCseLoss: + """ """ + + _EMBED_LOSS_REGISTRY = { + EmbeddingLoss.__name__: EmbeddingLoss, + SoftEmbeddingLoss.__name__: SoftEmbeddingLoss, + } + + def __init__(self, cfg: CfgNode): + """ + Initialize CSE loss from configuration options + + Args: + cfg (CfgNode): configuration options + """ + self.w_segm = cfg.MODEL.ROI_DENSEPOSE_HEAD.INDEX_WEIGHTS + self.w_embed = cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBED_LOSS_WEIGHT + self.segm_loss = MaskOrSegmentationLoss(cfg) + self.embed_loss = DensePoseCseLoss.create_embed_loss(cfg) + self.do_shape2shape = cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.SHAPE_TO_SHAPE_CYCLE_LOSS.ENABLED + if self.do_shape2shape: + self.w_shape2shape = cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.SHAPE_TO_SHAPE_CYCLE_LOSS.WEIGHT + self.shape2shape_loss = ShapeToShapeCycleLoss(cfg) + self.do_pix2shape = cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.PIX_TO_SHAPE_CYCLE_LOSS.ENABLED + if self.do_pix2shape: + self.w_pix2shape = cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.PIX_TO_SHAPE_CYCLE_LOSS.WEIGHT + self.pix2shape_loss = PixToShapeCycleLoss(cfg) + + @classmethod + def create_embed_loss(cls, cfg: CfgNode): + # registry not used here, since embedding losses are currently local + # and are not used anywhere else + return cls._EMBED_LOSS_REGISTRY[cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBED_LOSS_NAME](cfg) + + def __call__( + self, + proposals_with_gt: List[Instances], + densepose_predictor_outputs: Any, + embedder: nn.Module, + ) -> LossDict: + if not len(proposals_with_gt): + return self.produce_fake_losses(densepose_predictor_outputs, embedder) + accumulator = CseAnnotationsAccumulator() + packed_annotations = extract_packed_annotations_from_matches(proposals_with_gt, accumulator) + if packed_annotations is None: + return self.produce_fake_losses(densepose_predictor_outputs, embedder) + h, w = densepose_predictor_outputs.embedding.shape[2:] + interpolator = BilinearInterpolationHelper.from_matches( + packed_annotations, + (h, w), + ) + meshid_to_embed_losses = self.embed_loss( + proposals_with_gt, + densepose_predictor_outputs, + packed_annotations, + interpolator, + embedder, + ) + embed_loss_dict = { + f"loss_densepose_E{meshid}": self.w_embed * meshid_to_embed_losses[meshid] + for meshid in meshid_to_embed_losses + } + all_loss_dict = { + "loss_densepose_S": self.w_segm + * self.segm_loss(proposals_with_gt, densepose_predictor_outputs, packed_annotations), + **embed_loss_dict, + } + if self.do_shape2shape: + all_loss_dict["loss_shape2shape"] = self.w_shape2shape * self.shape2shape_loss(embedder) + if self.do_pix2shape: + all_loss_dict["loss_pix2shape"] = self.w_pix2shape * self.pix2shape_loss( + proposals_with_gt, densepose_predictor_outputs, packed_annotations, embedder + ) + return all_loss_dict + + def produce_fake_losses( + self, densepose_predictor_outputs: Any, embedder: nn.Module + ) -> LossDict: + meshname_to_embed_losses = self.embed_loss.fake_values( + densepose_predictor_outputs, embedder=embedder + ) + embed_loss_dict = { + f"loss_densepose_E{mesh_name}": meshname_to_embed_losses[mesh_name] + for mesh_name in meshname_to_embed_losses + } + all_loss_dict = { + "loss_densepose_S": self.segm_loss.fake_value(densepose_predictor_outputs), + **embed_loss_dict, + } + if self.do_shape2shape: + all_loss_dict["loss_shape2shape"] = self.shape2shape_loss.fake_value(embedder) + if self.do_pix2shape: + all_loss_dict["loss_pix2shape"] = self.pix2shape_loss.fake_value( + densepose_predictor_outputs, embedder + ) + return all_loss_dict diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/cycle_pix2shape.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/cycle_pix2shape.py new file mode 100644 index 0000000000000000000000000000000000000000..e1739182a2773ea07e08a77f1fa33735f6f3eb87 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/cycle_pix2shape.py @@ -0,0 +1,154 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +from typing import Any, List +import torch +from torch import nn +from torch.nn import functional as F + +from detectron2.config import CfgNode +from detectron2.structures import Instances + +from densepose.data.meshes.catalog import MeshCatalog +from densepose.modeling.cse.utils import normalize_embeddings, squared_euclidean_distance_matrix + +from .embed_utils import PackedCseAnnotations +from .mask import extract_data_for_mask_loss_from_matches + + +def _create_pixel_dist_matrix(grid_size: int) -> torch.Tensor: + rows = torch.arange(grid_size) + cols = torch.arange(grid_size) + # at index `i` contains [row, col], where + # row = i // grid_size + # col = i % grid_size + pix_coords = ( + torch.stack(torch.meshgrid(rows, cols), -1).reshape((grid_size * grid_size, 2)).float() + ) + return squared_euclidean_distance_matrix(pix_coords, pix_coords) + + +def _sample_fg_pixels_randperm(fg_mask: torch.Tensor, sample_size: int) -> torch.Tensor: + fg_mask_flattened = fg_mask.reshape((-1,)) + num_pixels = int(fg_mask_flattened.sum().item()) + fg_pixel_indices = fg_mask_flattened.nonzero(as_tuple=True)[0] + if (sample_size <= 0) or (num_pixels <= sample_size): + return fg_pixel_indices + sample_indices = torch.randperm(num_pixels, device=fg_mask.device)[:sample_size] + return fg_pixel_indices[sample_indices] + + +def _sample_fg_pixels_multinomial(fg_mask: torch.Tensor, sample_size: int) -> torch.Tensor: + fg_mask_flattened = fg_mask.reshape((-1,)) + num_pixels = int(fg_mask_flattened.sum().item()) + if (sample_size <= 0) or (num_pixels <= sample_size): + return fg_mask_flattened.nonzero(as_tuple=True)[0] + return fg_mask_flattened.float().multinomial(sample_size, replacement=False) + + +class PixToShapeCycleLoss(nn.Module): + """ + Cycle loss for pixel-vertex correspondence + """ + + def __init__(self, cfg: CfgNode): + super().__init__() + self.shape_names = list(cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBEDDERS.keys()) + self.embed_size = cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBED_SIZE + self.norm_p = cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.PIX_TO_SHAPE_CYCLE_LOSS.NORM_P + self.use_all_meshes_not_gt_only = ( + cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.PIX_TO_SHAPE_CYCLE_LOSS.USE_ALL_MESHES_NOT_GT_ONLY + ) + self.num_pixels_to_sample = ( + cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.PIX_TO_SHAPE_CYCLE_LOSS.NUM_PIXELS_TO_SAMPLE + ) + self.pix_sigma = cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.PIX_TO_SHAPE_CYCLE_LOSS.PIXEL_SIGMA + self.temperature_pix_to_vertex = ( + cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.PIX_TO_SHAPE_CYCLE_LOSS.TEMPERATURE_PIXEL_TO_VERTEX + ) + self.temperature_vertex_to_pix = ( + cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.PIX_TO_SHAPE_CYCLE_LOSS.TEMPERATURE_VERTEX_TO_PIXEL + ) + self.pixel_dists = _create_pixel_dist_matrix(cfg.MODEL.ROI_DENSEPOSE_HEAD.HEATMAP_SIZE) + + def forward( + self, + proposals_with_gt: List[Instances], + densepose_predictor_outputs: Any, + packed_annotations: PackedCseAnnotations, + embedder: nn.Module, + ): + """ + Args: + proposals_with_gt (list of Instances): detections with associated + ground truth data; each item corresponds to instances detected + on 1 image; the number of items corresponds to the number of + images in a batch + densepose_predictor_outputs: an object of a dataclass that contains predictor + outputs with estimated values; assumed to have the following attributes: + * embedding - embedding estimates, tensor of shape [N, D, S, S], where + N = number of instances (= sum N_i, where N_i is the number of + instances on image i) + D = embedding space dimensionality (MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBED_SIZE) + S = output size (width and height) + packed_annotations (PackedCseAnnotations): contains various data useful + for loss computation, each data is packed into a single tensor + embedder (nn.Module): module that computes vertex embeddings for different meshes + """ + pix_embeds = densepose_predictor_outputs.embedding + if self.pixel_dists.device != pix_embeds.device: + # should normally be done only once + self.pixel_dists = self.pixel_dists.to(device=pix_embeds.device) + with torch.no_grad(): + mask_loss_data = extract_data_for_mask_loss_from_matches( + proposals_with_gt, densepose_predictor_outputs.coarse_segm + ) + # GT masks - tensor of shape [N, S, S] of int64 + masks_gt = mask_loss_data.masks_gt.long() # pyre-ignore[16] + assert len(pix_embeds) == len(masks_gt), ( + f"Number of instances with embeddings {len(pix_embeds)} != " + f"number of instances with GT masks {len(masks_gt)}" + ) + losses = [] + mesh_names = ( + self.shape_names + if self.use_all_meshes_not_gt_only + else [ + MeshCatalog.get_mesh_name(mesh_id.item()) + for mesh_id in packed_annotations.vertex_mesh_ids_gt.unique() + ] + ) + for pixel_embeddings, mask_gt in zip(pix_embeds, masks_gt): + # pixel_embeddings [D, S, S] + # mask_gt [S, S] + for mesh_name in mesh_names: + mesh_vertex_embeddings = embedder(mesh_name) + # pixel indices [M] + pixel_indices_flattened = _sample_fg_pixels_randperm( + mask_gt, self.num_pixels_to_sample + ) + # pixel distances [M, M] + pixel_dists = self.pixel_dists.to(pixel_embeddings.device)[ + torch.meshgrid(pixel_indices_flattened, pixel_indices_flattened) + ] + # pixel embeddings [M, D] + pixel_embeddings_sampled = normalize_embeddings( + pixel_embeddings.reshape((self.embed_size, -1))[:, pixel_indices_flattened].T + ) + # pixel-vertex similarity [M, K] + sim_matrix = pixel_embeddings_sampled.mm(mesh_vertex_embeddings.T) + c_pix_vertex = F.softmax(sim_matrix / self.temperature_pix_to_vertex, dim=1) + c_vertex_pix = F.softmax(sim_matrix.T / self.temperature_vertex_to_pix, dim=1) + c_cycle = c_pix_vertex.mm(c_vertex_pix) + loss_cycle = torch.norm(pixel_dists * c_cycle, p=self.norm_p) + losses.append(loss_cycle) + + if len(losses) == 0: + return pix_embeds.sum() * 0 + return torch.stack(losses, dim=0).mean() + + def fake_value(self, densepose_predictor_outputs: Any, embedder: nn.Module): + losses = [ + embedder(mesh_name).sum() * 0 for mesh_name in embedder.mesh_names # pyre-ignore[29] + ] + losses.append(densepose_predictor_outputs.embedding.sum() * 0) + return torch.mean(torch.stack(losses)) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/cycle_shape2shape.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/cycle_shape2shape.py new file mode 100644 index 0000000000000000000000000000000000000000..2447e8f75aed3110b6880400517aff4ae242dfa5 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/cycle_shape2shape.py @@ -0,0 +1,117 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +import random +from typing import Tuple +import torch +from torch import nn +from torch.nn import functional as F + +from detectron2.config import CfgNode + +from densepose.structures.mesh import create_mesh + +from .utils import sample_random_indices + + +class ShapeToShapeCycleLoss(nn.Module): + """ + Cycle Loss for Shapes. + Inspired by: + "Mapping in a Cycle: Sinkhorn Regularized Unsupervised Learning for Point Cloud Shapes". + """ + + def __init__(self, cfg: CfgNode): + super().__init__() + self.shape_names = list(cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBEDDERS.keys()) + self.all_shape_pairs = [ + (x, y) for i, x in enumerate(self.shape_names) for y in self.shape_names[i + 1 :] + ] + random.shuffle(self.all_shape_pairs) + self.cur_pos = 0 + self.norm_p = cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.SHAPE_TO_SHAPE_CYCLE_LOSS.NORM_P + self.temperature = cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.SHAPE_TO_SHAPE_CYCLE_LOSS.TEMPERATURE + self.max_num_vertices = ( + cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.SHAPE_TO_SHAPE_CYCLE_LOSS.MAX_NUM_VERTICES + ) + + def _sample_random_pair(self) -> Tuple[str, str]: + """ + Produce a random pair of different mesh names + + Return: + tuple(str, str): a pair of different mesh names + """ + if self.cur_pos >= len(self.all_shape_pairs): + random.shuffle(self.all_shape_pairs) + self.cur_pos = 0 + shape_pair = self.all_shape_pairs[self.cur_pos] + self.cur_pos += 1 + return shape_pair + + def forward(self, embedder: nn.Module): + """ + Do a forward pass with a random pair (src, dst) pair of shapes + Args: + embedder (nn.Module): module that computes vertex embeddings for different meshes + """ + src_mesh_name, dst_mesh_name = self._sample_random_pair() + return self._forward_one_pair(embedder, src_mesh_name, dst_mesh_name) + + def fake_value(self, embedder: nn.Module): + losses = [] + for mesh_name in embedder.mesh_names: # pyre-ignore[29] + losses.append(embedder(mesh_name).sum() * 0) + return torch.mean(torch.stack(losses)) + + def _get_embeddings_and_geodists_for_mesh( + self, embedder: nn.Module, mesh_name: str + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Produces embeddings and geodesic distance tensors for a given mesh. May subsample + the mesh, if it contains too many vertices (controlled by + SHAPE_CYCLE_LOSS_MAX_NUM_VERTICES parameter). + Args: + embedder (nn.Module): module that computes embeddings for mesh vertices + mesh_name (str): mesh name + Return: + embeddings (torch.Tensor of size [N, D]): embeddings for selected mesh + vertices (N = number of selected vertices, D = embedding space dim) + geodists (torch.Tensor of size [N, N]): geodesic distances for the selected + mesh vertices (N = number of selected vertices) + """ + embeddings = embedder(mesh_name) + indices = sample_random_indices( + embeddings.shape[0], self.max_num_vertices, embeddings.device + ) + mesh = create_mesh(mesh_name, embeddings.device) + geodists = mesh.geodists + if indices is not None: + embeddings = embeddings[indices] + geodists = geodists[torch.meshgrid(indices, indices)] + return embeddings, geodists + + def _forward_one_pair( + self, embedder: nn.Module, mesh_name_1: str, mesh_name_2: str + ) -> torch.Tensor: + """ + Do a forward pass with a selected pair of meshes + Args: + embedder (nn.Module): module that computes vertex embeddings for different meshes + mesh_name_1 (str): first mesh name + mesh_name_2 (str): second mesh name + Return: + Tensor containing the loss value + """ + embeddings_1, geodists_1 = self._get_embeddings_and_geodists_for_mesh(embedder, mesh_name_1) + embeddings_2, geodists_2 = self._get_embeddings_and_geodists_for_mesh(embedder, mesh_name_2) + sim_matrix_12 = embeddings_1.mm(embeddings_2.T) + + c_12 = F.softmax(sim_matrix_12 / self.temperature, dim=1) + c_21 = F.softmax(sim_matrix_12.T / self.temperature, dim=1) + c_11 = c_12.mm(c_21) + c_22 = c_21.mm(c_12) + + loss_cycle_11 = torch.norm(geodists_1 * c_11, p=self.norm_p) + loss_cycle_22 = torch.norm(geodists_2 * c_22, p=self.norm_p) + + return loss_cycle_11 + loss_cycle_22 diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/embed.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/embed.py new file mode 100644 index 0000000000000000000000000000000000000000..163eebe9a663f4d46adbbd66af0546a16f32b200 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/embed.py @@ -0,0 +1,127 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +from typing import Any, Dict, List +import torch +from torch import nn +from torch.nn import functional as F + +from detectron2.config import CfgNode +from detectron2.structures import Instances + +from densepose.data.meshes.catalog import MeshCatalog +from densepose.modeling.cse.utils import normalize_embeddings, squared_euclidean_distance_matrix + +from .embed_utils import PackedCseAnnotations +from .utils import BilinearInterpolationHelper + + +class EmbeddingLoss: + """ + Computes losses for estimated embeddings given annotated vertices. + Instances in a minibatch that correspond to the same mesh are grouped + together. For each group, loss is computed as cross-entropy for + unnormalized scores given ground truth mesh vertex ids. + Scores are based on squared distances between estimated vertex embeddings + and mesh vertex embeddings. + """ + + def __init__(self, cfg: CfgNode): + """ + Initialize embedding loss from config + """ + self.embdist_gauss_sigma = cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBEDDING_DIST_GAUSS_SIGMA + + def __call__( + self, + proposals_with_gt: List[Instances], + densepose_predictor_outputs: Any, + packed_annotations: PackedCseAnnotations, + interpolator: BilinearInterpolationHelper, + embedder: nn.Module, + ) -> Dict[int, torch.Tensor]: + """ + Produces losses for estimated embeddings given annotated vertices. + Embeddings for all the vertices of a mesh are computed by the embedder. + Embeddings for observed pixels are estimated by a predictor. + Losses are computed as cross-entropy for squared distances between + observed vertex embeddings and all mesh vertex embeddings given + ground truth vertex IDs. + + Args: + proposals_with_gt (list of Instances): detections with associated + ground truth data; each item corresponds to instances detected + on 1 image; the number of items corresponds to the number of + images in a batch + densepose_predictor_outputs: an object of a dataclass that contains predictor + outputs with estimated values; assumed to have the following attributes: + * embedding - embedding estimates, tensor of shape [N, D, S, S], where + N = number of instances (= sum N_i, where N_i is the number of + instances on image i) + D = embedding space dimensionality (MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBED_SIZE) + S = output size (width and height) + packed_annotations (PackedCseAnnotations): contains various data useful + for loss computation, each data is packed into a single tensor + interpolator (BilinearInterpolationHelper): bilinear interpolation helper + embedder (nn.Module): module that computes vertex embeddings for different meshes + Return: + dict(int -> tensor): losses for different mesh IDs + """ + losses = {} + for mesh_id_tensor in packed_annotations.vertex_mesh_ids_gt.unique(): + mesh_id = mesh_id_tensor.item() + mesh_name = MeshCatalog.get_mesh_name(mesh_id) + # valid points are those that fall into estimated bbox + # and correspond to the current mesh + j_valid = interpolator.j_valid * ( # pyre-ignore[16] + packed_annotations.vertex_mesh_ids_gt == mesh_id + ) + if not torch.any(j_valid): + continue + # extract estimated embeddings for valid points + # -> tensor [J, D] + vertex_embeddings_i = normalize_embeddings( + interpolator.extract_at_points( + densepose_predictor_outputs.embedding, + slice_fine_segm=slice(None), + w_ylo_xlo=interpolator.w_ylo_xlo[:, None], # pyre-ignore[16] + w_ylo_xhi=interpolator.w_ylo_xhi[:, None], # pyre-ignore[16] + w_yhi_xlo=interpolator.w_yhi_xlo[:, None], # pyre-ignore[16] + w_yhi_xhi=interpolator.w_yhi_xhi[:, None], # pyre-ignore[16] + )[j_valid, :] + ) + # extract vertex ids for valid points + # -> tensor [J] + vertex_indices_i = packed_annotations.vertex_ids_gt[j_valid] + # embeddings for all mesh vertices + # -> tensor [K, D] + mesh_vertex_embeddings = embedder(mesh_name) + # unnormalized scores for valid points + # -> tensor [J, K] + scores = squared_euclidean_distance_matrix( + vertex_embeddings_i, mesh_vertex_embeddings + ) / (-self.embdist_gauss_sigma) + losses[mesh_name] = F.cross_entropy(scores, vertex_indices_i, ignore_index=-1) + + # pyre-fixme[29]: + # `Union[BoundMethod[typing.Callable(torch.Tensor.__iter__)[[Named(self, + # torch.Tensor)], typing.Iterator[typing.Any]], torch.Tensor], nn.Module, + # torch.Tensor]` is not a function. + for mesh_name in embedder.mesh_names: + if mesh_name not in losses: + losses[mesh_name] = self.fake_value( + densepose_predictor_outputs, embedder, mesh_name + ) + return losses + + def fake_values(self, densepose_predictor_outputs: Any, embedder: nn.Module): + losses = {} + # pyre-fixme[29]: + # `Union[BoundMethod[typing.Callable(torch.Tensor.__iter__)[[Named(self, + # torch.Tensor)], typing.Iterator[typing.Any]], torch.Tensor], nn.Module, + # torch.Tensor]` is not a function. + for mesh_name in embedder.mesh_names: + losses[mesh_name] = self.fake_value(densepose_predictor_outputs, embedder, mesh_name) + return losses + + def fake_value(self, densepose_predictor_outputs: Any, embedder: nn.Module, mesh_name: str): + return densepose_predictor_outputs.embedding.sum() * 0 + embedder(mesh_name).sum() * 0 diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/embed_utils.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/embed_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..eb9492f440e3acf04460a73f31439b0077232ef4 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/embed_utils.py @@ -0,0 +1,135 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +from dataclasses import dataclass +from typing import Any, Optional +import torch + +from detectron2.structures import BoxMode, Instances + +from .utils import AnnotationsAccumulator + + +@dataclass +class PackedCseAnnotations: + x_gt: torch.Tensor + y_gt: torch.Tensor + coarse_segm_gt: Optional[torch.Tensor] + vertex_mesh_ids_gt: torch.Tensor + vertex_ids_gt: torch.Tensor + bbox_xywh_gt: torch.Tensor + bbox_xywh_est: torch.Tensor + point_bbox_with_dp_indices: torch.Tensor + point_bbox_indices: torch.Tensor + bbox_indices: torch.Tensor + + +class CseAnnotationsAccumulator(AnnotationsAccumulator): + """ + Accumulates annotations by batches that correspond to objects detected on + individual images. Can pack them together into single tensors. + """ + + def __init__(self): + self.x_gt = [] + self.y_gt = [] + self.s_gt = [] + self.vertex_mesh_ids_gt = [] + self.vertex_ids_gt = [] + self.bbox_xywh_gt = [] + self.bbox_xywh_est = [] + self.point_bbox_with_dp_indices = [] + self.point_bbox_indices = [] + self.bbox_indices = [] + self.nxt_bbox_with_dp_index = 0 + self.nxt_bbox_index = 0 + + def accumulate(self, instances_one_image: Instances): + """ + Accumulate instances data for one image + + Args: + instances_one_image (Instances): instances data to accumulate + """ + boxes_xywh_est = BoxMode.convert( + instances_one_image.proposal_boxes.tensor.clone(), BoxMode.XYXY_ABS, BoxMode.XYWH_ABS + ) + boxes_xywh_gt = BoxMode.convert( + instances_one_image.gt_boxes.tensor.clone(), BoxMode.XYXY_ABS, BoxMode.XYWH_ABS + ) + n_matches = len(boxes_xywh_gt) + assert n_matches == len( + boxes_xywh_est + ), f"Got {len(boxes_xywh_est)} proposal boxes and {len(boxes_xywh_gt)} GT boxes" + if not n_matches: + # no detection - GT matches + return + if ( + not hasattr(instances_one_image, "gt_densepose") + or instances_one_image.gt_densepose is None + ): + # no densepose GT for the detections, just increase the bbox index + self.nxt_bbox_index += n_matches + return + for box_xywh_est, box_xywh_gt, dp_gt in zip( + boxes_xywh_est, boxes_xywh_gt, instances_one_image.gt_densepose + ): + if (dp_gt is not None) and (len(dp_gt.x) > 0): + self._do_accumulate(box_xywh_gt, box_xywh_est, dp_gt) + self.nxt_bbox_index += 1 + + def _do_accumulate(self, box_xywh_gt: torch.Tensor, box_xywh_est: torch.Tensor, dp_gt: Any): + """ + Accumulate instances data for one image, given that the data is not empty + + Args: + box_xywh_gt (tensor): GT bounding box + box_xywh_est (tensor): estimated bounding box + dp_gt: GT densepose data with the following attributes: + - x: normalized X coordinates + - y: normalized Y coordinates + - segm: tensor of size [S, S] with coarse segmentation + - + """ + self.x_gt.append(dp_gt.x) + self.y_gt.append(dp_gt.y) + if hasattr(dp_gt, "segm"): + self.s_gt.append(dp_gt.segm.unsqueeze(0)) + self.vertex_ids_gt.append(dp_gt.vertex_ids) + self.vertex_mesh_ids_gt.append(torch.full_like(dp_gt.vertex_ids, dp_gt.mesh_id)) + self.bbox_xywh_gt.append(box_xywh_gt.view(-1, 4)) + self.bbox_xywh_est.append(box_xywh_est.view(-1, 4)) + self.point_bbox_with_dp_indices.append( + torch.full_like(dp_gt.vertex_ids, self.nxt_bbox_with_dp_index) + ) + self.point_bbox_indices.append(torch.full_like(dp_gt.vertex_ids, self.nxt_bbox_index)) + self.bbox_indices.append(self.nxt_bbox_index) + self.nxt_bbox_with_dp_index += 1 + + def pack(self) -> Optional[PackedCseAnnotations]: + """ + Pack data into tensors + """ + if not len(self.x_gt): + # TODO: + # returning proper empty annotations would require + # creating empty tensors of appropriate shape and + # type on an appropriate device; + # we return None so far to indicate empty annotations + return None + return PackedCseAnnotations( + x_gt=torch.cat(self.x_gt, 0), + y_gt=torch.cat(self.y_gt, 0), + vertex_mesh_ids_gt=torch.cat(self.vertex_mesh_ids_gt, 0), + vertex_ids_gt=torch.cat(self.vertex_ids_gt, 0), + # ignore segmentation annotations, if not all the instances contain those + coarse_segm_gt=torch.cat(self.s_gt, 0) + if len(self.s_gt) == len(self.bbox_xywh_gt) + else None, + bbox_xywh_gt=torch.cat(self.bbox_xywh_gt, 0), + bbox_xywh_est=torch.cat(self.bbox_xywh_est, 0), + point_bbox_with_dp_indices=torch.cat(self.point_bbox_with_dp_indices, 0), + point_bbox_indices=torch.cat(self.point_bbox_indices, 0), + bbox_indices=torch.as_tensor( + self.bbox_indices, dtype=torch.long, device=self.x_gt[0].device + ), + ) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/mask.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/mask.py new file mode 100644 index 0000000000000000000000000000000000000000..c16b15c53de9f02dc734148e05f2bde799046aa0 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/mask.py @@ -0,0 +1,125 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +from dataclasses import dataclass +from typing import Any, Iterable, List, Optional +import torch +from torch.nn import functional as F + +from detectron2.structures import Instances + + +@dataclass +class DataForMaskLoss: + """ + Contains mask GT and estimated data for proposals from multiple images: + """ + + # tensor of size (K, H, W) containing GT labels + masks_gt: Optional[torch.Tensor] = None + # tensor of size (K, C, H, W) containing estimated scores + masks_est: Optional[torch.Tensor] = None + + +def extract_data_for_mask_loss_from_matches( + proposals_targets: Iterable[Instances], estimated_segm: torch.Tensor +) -> DataForMaskLoss: + """ + Extract data for mask loss from instances that contain matched GT and + estimated bounding boxes. + Args: + proposals_targets: Iterable[Instances] + matched GT and estimated results, each item in the iterable + corresponds to data in 1 image + estimated_segm: tensor(K, C, S, S) of float - raw unnormalized + segmentation scores, here S is the size to which GT masks are + to be resized + Return: + masks_est: tensor(K, C, S, S) of float - class scores + masks_gt: tensor(K, S, S) of int64 - labels + """ + data = DataForMaskLoss() + masks_gt = [] + offset = 0 + assert estimated_segm.shape[2] == estimated_segm.shape[3], ( + f"Expected estimated segmentation to have a square shape, " + f"but the actual shape is {estimated_segm.shape[2:]}" + ) + mask_size = estimated_segm.shape[2] + num_proposals = sum(inst.proposal_boxes.tensor.size(0) for inst in proposals_targets) + num_estimated = estimated_segm.shape[0] + assert ( + num_proposals == num_estimated + ), "The number of proposals {} must be equal to the number of estimates {}".format( + num_proposals, num_estimated + ) + + for proposals_targets_per_image in proposals_targets: + n_i = proposals_targets_per_image.proposal_boxes.tensor.size(0) + if not n_i: + continue + gt_masks_per_image = proposals_targets_per_image.gt_masks.crop_and_resize( + proposals_targets_per_image.proposal_boxes.tensor, mask_size + ).to(device=estimated_segm.device) + masks_gt.append(gt_masks_per_image) + offset += n_i + if masks_gt: + data.masks_est = estimated_segm + data.masks_gt = torch.cat(masks_gt, dim=0) + return data + + +class MaskLoss: + """ + Mask loss as cross-entropy for raw unnormalized scores given ground truth labels. + Mask ground truth labels are defined for the whole image and not only the + bounding box of interest. They are stored as objects that are assumed to implement + the `crop_and_resize` interface (e.g. BitMasks, PolygonMasks). + """ + + def __call__( + self, proposals_with_gt: List[Instances], densepose_predictor_outputs: Any + ) -> torch.Tensor: + """ + Computes segmentation loss as cross-entropy for raw unnormalized + scores given ground truth labels. + + Args: + proposals_with_gt (list of Instances): detections with associated ground truth data + densepose_predictor_outputs: an object of a dataclass that contains predictor outputs + with estimated values; assumed to have the following attribute: + * coarse_segm (tensor of shape [N, D, S, S]): coarse segmentation estimates + as raw unnormalized scores + where N is the number of detections, S is the estimate size ( = width = height) + and D is the number of coarse segmentation channels. + Return: + Cross entropy for raw unnormalized scores for coarse segmentation given + ground truth labels from masks + """ + if not len(proposals_with_gt): + return self.fake_value(densepose_predictor_outputs) + # densepose outputs are computed for all images and all bounding boxes; + # i.e. if a batch has 4 images with (3, 1, 2, 1) proposals respectively, + # the outputs will have size(0) == 3+1+2+1 == 7 + with torch.no_grad(): + mask_loss_data = extract_data_for_mask_loss_from_matches( + proposals_with_gt, densepose_predictor_outputs.coarse_segm + ) + if (mask_loss_data.masks_gt is None) or (mask_loss_data.masks_est is None): + return self.fake_value(densepose_predictor_outputs) + return F.cross_entropy(mask_loss_data.masks_est, mask_loss_data.masks_gt.long()) + + def fake_value(self, densepose_predictor_outputs: Any) -> torch.Tensor: + """ + Fake segmentation loss used when no suitable ground truth data + was found in a batch. The loss has a value 0 and is primarily used to + construct the computation graph, so that `DistributedDataParallel` + has similar graphs on all GPUs and can perform reduction properly. + + Args: + densepose_predictor_outputs: DensePose predictor outputs, an object + of a dataclass that is assumed to have `coarse_segm` + attribute + Return: + Zero value loss with proper computation graph + """ + return densepose_predictor_outputs.coarse_segm.sum() * 0 diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/mask_or_segm.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/mask_or_segm.py new file mode 100644 index 0000000000000000000000000000000000000000..98b773d99fd29a48cbdfa94c5882c9c3d94003ee --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/mask_or_segm.py @@ -0,0 +1,72 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +from typing import Any, List +import torch + +from detectron2.config import CfgNode +from detectron2.structures import Instances + +from .mask import MaskLoss +from .segm import SegmentationLoss + + +class MaskOrSegmentationLoss: + """ + Mask or segmentation loss as cross-entropy for raw unnormalized scores + given ground truth labels. Ground truth labels are either defined by coarse + segmentation annotation, or by mask annotation, depending on the config + value MODEL.ROI_DENSEPOSE_HEAD.COARSE_SEGM_TRAINED_BY_MASKS + """ + + def __init__(self, cfg: CfgNode): + """ + Initialize segmentation loss from configuration options + + Args: + cfg (CfgNode): configuration options + """ + self.segm_trained_by_masks = cfg.MODEL.ROI_DENSEPOSE_HEAD.COARSE_SEGM_TRAINED_BY_MASKS + if self.segm_trained_by_masks: + self.mask_loss = MaskLoss() + self.segm_loss = SegmentationLoss(cfg) + + def __call__( + self, + proposals_with_gt: List[Instances], + densepose_predictor_outputs: Any, + packed_annotations: Any, + ) -> torch.Tensor: + """ + Compute segmentation loss as cross-entropy between aligned unnormalized + score estimates and ground truth; with ground truth given + either by masks, or by coarse segmentation annotations. + + Args: + proposals_with_gt (list of Instances): detections with associated ground truth data + densepose_predictor_outputs: an object of a dataclass that contains predictor outputs + with estimated values; assumed to have the following attributes: + * coarse_segm - coarse segmentation estimates, tensor of shape [N, D, S, S] + packed_annotations: packed annotations for efficient loss computation + Return: + tensor: loss value as cross-entropy for raw unnormalized scores + given ground truth labels + """ + if self.segm_trained_by_masks: + return self.mask_loss(proposals_with_gt, densepose_predictor_outputs) + return self.segm_loss(proposals_with_gt, densepose_predictor_outputs, packed_annotations) + + def fake_value(self, densepose_predictor_outputs: Any) -> torch.Tensor: + """ + Fake segmentation loss used when no suitable ground truth data + was found in a batch. The loss has a value 0 and is primarily used to + construct the computation graph, so that `DistributedDataParallel` + has similar graphs on all GPUs and can perform reduction properly. + + Args: + densepose_predictor_outputs: DensePose predictor outputs, an object + of a dataclass that is assumed to have `coarse_segm` + attribute + Return: + Zero value loss with proper computation graph + """ + return densepose_predictor_outputs.coarse_segm.sum() * 0 diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/registry.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..d9c8817a743e42b2aec382818f0cc1bb39a66004 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/registry.py @@ -0,0 +1,5 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from detectron2.utils.registry import Registry + +DENSEPOSE_LOSS_REGISTRY = Registry("DENSEPOSE_LOSS") diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/segm.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/segm.py new file mode 100644 index 0000000000000000000000000000000000000000..1962b886e1946fa4896776da8a007ae0a9a4fab3 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/segm.py @@ -0,0 +1,83 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +from typing import Any, List +import torch +from torch.nn import functional as F + +from detectron2.config import CfgNode +from detectron2.structures import Instances + +from .utils import resample_data + + +class SegmentationLoss: + """ + Segmentation loss as cross-entropy for raw unnormalized scores given ground truth + labels. Segmentation ground truth labels are defined for the bounding box of + interest at some fixed resolution [S, S], where + S = MODEL.ROI_DENSEPOSE_HEAD.HEATMAP_SIZE. + """ + + def __init__(self, cfg: CfgNode): + """ + Initialize segmentation loss from configuration options + + Args: + cfg (CfgNode): configuration options + """ + self.heatmap_size = cfg.MODEL.ROI_DENSEPOSE_HEAD.HEATMAP_SIZE + self.n_segm_chan = cfg.MODEL.ROI_DENSEPOSE_HEAD.NUM_COARSE_SEGM_CHANNELS + + def __call__( + self, + proposals_with_gt: List[Instances], + densepose_predictor_outputs: Any, + packed_annotations: Any, + ) -> torch.Tensor: + """ + Compute segmentation loss as cross-entropy on aligned segmentation + ground truth and estimated scores. + + Args: + proposals_with_gt (list of Instances): detections with associated ground truth data + densepose_predictor_outputs: an object of a dataclass that contains predictor outputs + with estimated values; assumed to have the following attributes: + * coarse_segm - coarse segmentation estimates, tensor of shape [N, D, S, S] + packed_annotations: packed annotations for efficient loss computation; + the following attributes are used: + - coarse_segm_gt + - bbox_xywh_gt + - bbox_xywh_est + """ + if packed_annotations.coarse_segm_gt is None: + return self.fake_value(densepose_predictor_outputs) + coarse_segm_est = densepose_predictor_outputs.coarse_segm[packed_annotations.bbox_indices] + with torch.no_grad(): + coarse_segm_gt = resample_data( + packed_annotations.coarse_segm_gt.unsqueeze(1), + packed_annotations.bbox_xywh_gt, + packed_annotations.bbox_xywh_est, + self.heatmap_size, + self.heatmap_size, + mode="nearest", + padding_mode="zeros", + ).squeeze(1) + if self.n_segm_chan == 2: + coarse_segm_gt = coarse_segm_gt > 0 + return F.cross_entropy(coarse_segm_est, coarse_segm_gt.long()) + + def fake_value(self, densepose_predictor_outputs: Any) -> torch.Tensor: + """ + Fake segmentation loss used when no suitable ground truth data + was found in a batch. The loss has a value 0 and is primarily used to + construct the computation graph, so that `DistributedDataParallel` + has similar graphs on all GPUs and can perform reduction properly. + + Args: + densepose_predictor_outputs: DensePose predictor outputs, an object + of a dataclass that is assumed to have `coarse_segm` + attribute + Return: + Zero value loss with proper computation graph + """ + return densepose_predictor_outputs.coarse_segm.sum() * 0 diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/soft_embed.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/soft_embed.py new file mode 100644 index 0000000000000000000000000000000000000000..176d929f4adfa06164dd1ce1668b6d6743cc0983 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/soft_embed.py @@ -0,0 +1,141 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +from typing import Any, Dict, List +import torch +from torch import nn +from torch.nn import functional as F + +from detectron2.config import CfgNode +from detectron2.structures import Instances + +from densepose.data.meshes.catalog import MeshCatalog +from densepose.modeling.cse.utils import normalize_embeddings, squared_euclidean_distance_matrix +from densepose.structures.mesh import create_mesh + +from .embed_utils import PackedCseAnnotations +from .utils import BilinearInterpolationHelper + + +class SoftEmbeddingLoss: + """ + Computes losses for estimated embeddings given annotated vertices. + Instances in a minibatch that correspond to the same mesh are grouped + together. For each group, loss is computed as cross-entropy for + unnormalized scores given ground truth mesh vertex ids. + Scores are based on: + 1) squared distances between estimated vertex embeddings + and mesh vertex embeddings; + 2) geodesic distances between vertices of a mesh + """ + + def __init__(self, cfg: CfgNode): + """ + Initialize embedding loss from config + """ + self.embdist_gauss_sigma = cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBEDDING_DIST_GAUSS_SIGMA + self.geodist_gauss_sigma = cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.GEODESIC_DIST_GAUSS_SIGMA + + def __call__( + self, + proposals_with_gt: List[Instances], + densepose_predictor_outputs: Any, + packed_annotations: PackedCseAnnotations, + interpolator: BilinearInterpolationHelper, + embedder: nn.Module, + ) -> Dict[int, torch.Tensor]: + """ + Produces losses for estimated embeddings given annotated vertices. + Embeddings for all the vertices of a mesh are computed by the embedder. + Embeddings for observed pixels are estimated by a predictor. + Losses are computed as cross-entropy for unnormalized scores given + ground truth vertex IDs. + 1) squared distances between estimated vertex embeddings + and mesh vertex embeddings; + 2) geodesic distances between vertices of a mesh + + Args: + proposals_with_gt (list of Instances): detections with associated + ground truth data; each item corresponds to instances detected + on 1 image; the number of items corresponds to the number of + images in a batch + densepose_predictor_outputs: an object of a dataclass that contains predictor + outputs with estimated values; assumed to have the following attributes: + * embedding - embedding estimates, tensor of shape [N, D, S, S], where + N = number of instances (= sum N_i, where N_i is the number of + instances on image i) + D = embedding space dimensionality (MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBED_SIZE) + S = output size (width and height) + packed_annotations (PackedCseAnnotations): contains various data useful + for loss computation, each data is packed into a single tensor + interpolator (BilinearInterpolationHelper): bilinear interpolation helper + embedder (nn.Module): module that computes vertex embeddings for different meshes + Return: + dict(int -> tensor): losses for different mesh IDs + """ + losses = {} + for mesh_id_tensor in packed_annotations.vertex_mesh_ids_gt.unique(): + mesh_id = mesh_id_tensor.item() + mesh_name = MeshCatalog.get_mesh_name(mesh_id) + # valid points are those that fall into estimated bbox + # and correspond to the current mesh + j_valid = interpolator.j_valid * ( # pyre-ignore[16] + packed_annotations.vertex_mesh_ids_gt == mesh_id + ) + if not torch.any(j_valid): + continue + # extract estimated embeddings for valid points + # -> tensor [J, D] + vertex_embeddings_i = normalize_embeddings( + interpolator.extract_at_points( + densepose_predictor_outputs.embedding, + slice_fine_segm=slice(None), + w_ylo_xlo=interpolator.w_ylo_xlo[:, None], # pyre-ignore[16] + w_ylo_xhi=interpolator.w_ylo_xhi[:, None], # pyre-ignore[16] + w_yhi_xlo=interpolator.w_yhi_xlo[:, None], # pyre-ignore[16] + w_yhi_xhi=interpolator.w_yhi_xhi[:, None], # pyre-ignore[16] + )[j_valid, :] + ) + # extract vertex ids for valid points + # -> tensor [J] + vertex_indices_i = packed_annotations.vertex_ids_gt[j_valid] + # embeddings for all mesh vertices + # -> tensor [K, D] + mesh_vertex_embeddings = embedder(mesh_name) + # softmax values of geodesic distances for GT mesh vertices + # -> tensor [J, K] + mesh = create_mesh(mesh_name, mesh_vertex_embeddings.device) + geodist_softmax_values = F.softmax( + mesh.geodists[vertex_indices_i] / (-self.geodist_gauss_sigma), dim=1 + ) + # logsoftmax values for valid points + # -> tensor [J, K] + embdist_logsoftmax_values = F.log_softmax( + squared_euclidean_distance_matrix(vertex_embeddings_i, mesh_vertex_embeddings) + / (-self.embdist_gauss_sigma), + dim=1, + ) + losses[mesh_name] = (-geodist_softmax_values * embdist_logsoftmax_values).sum(1).mean() + + # pyre-fixme[29]: + # `Union[BoundMethod[typing.Callable(torch.Tensor.__iter__)[[Named(self, + # torch.Tensor)], typing.Iterator[typing.Any]], torch.Tensor], nn.Module, + # torch.Tensor]` is not a function. + for mesh_name in embedder.mesh_names: + if mesh_name not in losses: + losses[mesh_name] = self.fake_value( + densepose_predictor_outputs, embedder, mesh_name + ) + return losses + + def fake_values(self, densepose_predictor_outputs: Any, embedder: nn.Module): + losses = {} + # pyre-fixme[29]: + # `Union[BoundMethod[typing.Callable(torch.Tensor.__iter__)[[Named(self, + # torch.Tensor)], typing.Iterator[typing.Any]], torch.Tensor], nn.Module, + # torch.Tensor]` is not a function. + for mesh_name in embedder.mesh_names: + losses[mesh_name] = self.fake_value(densepose_predictor_outputs, embedder, mesh_name) + return losses + + def fake_value(self, densepose_predictor_outputs: Any, embedder: nn.Module, mesh_name: str): + return densepose_predictor_outputs.embedding.sum() * 0 + embedder(mesh_name).sum() * 0 diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/utils.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4c172aeb877df9f3e1f0765ffa804ba479861439 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/losses/utils.py @@ -0,0 +1,441 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple +import torch +from torch.nn import functional as F + +from detectron2.structures import BoxMode, Instances + +from densepose import DensePoseDataRelative + +LossDict = Dict[str, torch.Tensor] + + +def _linear_interpolation_utilities(v_norm, v0_src, size_src, v0_dst, size_dst, size_z): + """ + Computes utility values for linear interpolation at points v. + The points are given as normalized offsets in the source interval + (v0_src, v0_src + size_src), more precisely: + v = v0_src + v_norm * size_src / 256.0 + The computed utilities include lower points v_lo, upper points v_hi, + interpolation weights v_w and flags j_valid indicating whether the + points falls into the destination interval (v0_dst, v0_dst + size_dst). + + Args: + v_norm (:obj: `torch.Tensor`): tensor of size N containing + normalized point offsets + v0_src (:obj: `torch.Tensor`): tensor of size N containing + left bounds of source intervals for normalized points + size_src (:obj: `torch.Tensor`): tensor of size N containing + source interval sizes for normalized points + v0_dst (:obj: `torch.Tensor`): tensor of size N containing + left bounds of destination intervals + size_dst (:obj: `torch.Tensor`): tensor of size N containing + destination interval sizes + size_z (int): interval size for data to be interpolated + + Returns: + v_lo (:obj: `torch.Tensor`): int tensor of size N containing + indices of lower values used for interpolation, all values are + integers from [0, size_z - 1] + v_hi (:obj: `torch.Tensor`): int tensor of size N containing + indices of upper values used for interpolation, all values are + integers from [0, size_z - 1] + v_w (:obj: `torch.Tensor`): float tensor of size N containing + interpolation weights + j_valid (:obj: `torch.Tensor`): uint8 tensor of size N containing + 0 for points outside the estimation interval + (v0_est, v0_est + size_est) and 1 otherwise + """ + v = v0_src + v_norm * size_src / 256.0 + j_valid = (v - v0_dst >= 0) * (v - v0_dst < size_dst) + v_grid = (v - v0_dst) * size_z / size_dst + v_lo = v_grid.floor().long().clamp(min=0, max=size_z - 1) + v_hi = (v_lo + 1).clamp(max=size_z - 1) + v_grid = torch.min(v_hi.float(), v_grid) + v_w = v_grid - v_lo.float() + return v_lo, v_hi, v_w, j_valid + + +class BilinearInterpolationHelper: + """ + Args: + packed_annotations: object that contains packed annotations + j_valid (:obj: `torch.Tensor`): uint8 tensor of size M containing + 0 for points to be discarded and 1 for points to be selected + y_lo (:obj: `torch.Tensor`): int tensor of indices of upper values + in z_est for each point + y_hi (:obj: `torch.Tensor`): int tensor of indices of lower values + in z_est for each point + x_lo (:obj: `torch.Tensor`): int tensor of indices of left values + in z_est for each point + x_hi (:obj: `torch.Tensor`): int tensor of indices of right values + in z_est for each point + w_ylo_xlo (:obj: `torch.Tensor`): float tensor of size M; + contains upper-left value weight for each point + w_ylo_xhi (:obj: `torch.Tensor`): float tensor of size M; + contains upper-right value weight for each point + w_yhi_xlo (:obj: `torch.Tensor`): float tensor of size M; + contains lower-left value weight for each point + w_yhi_xhi (:obj: `torch.Tensor`): float tensor of size M; + contains lower-right value weight for each point + """ + + def __init__( + self, + packed_annotations: Any, + j_valid: torch.Tensor, + y_lo: torch.Tensor, + y_hi: torch.Tensor, + x_lo: torch.Tensor, + x_hi: torch.Tensor, + w_ylo_xlo: torch.Tensor, + w_ylo_xhi: torch.Tensor, + w_yhi_xlo: torch.Tensor, + w_yhi_xhi: torch.Tensor, + ): + for k, v in locals().items(): + if k != "self": + setattr(self, k, v) + + @staticmethod + def from_matches( + packed_annotations: Any, densepose_outputs_size_hw: Tuple[int, int] + ) -> "BilinearInterpolationHelper": + """ + Args: + packed_annotations: annotations packed into tensors, the following + attributes are required: + - bbox_xywh_gt + - bbox_xywh_est + - x_gt + - y_gt + - point_bbox_with_dp_indices + - point_bbox_indices + densepose_outputs_size_hw (tuple [int, int]): resolution of + DensePose predictor outputs (H, W) + Return: + An instance of `BilinearInterpolationHelper` used to perform + interpolation for the given annotation points and output resolution + """ + + zh, zw = densepose_outputs_size_hw + x0_gt, y0_gt, w_gt, h_gt = packed_annotations.bbox_xywh_gt[ + packed_annotations.point_bbox_with_dp_indices + ].unbind(dim=1) + x0_est, y0_est, w_est, h_est = packed_annotations.bbox_xywh_est[ + packed_annotations.point_bbox_with_dp_indices + ].unbind(dim=1) + x_lo, x_hi, x_w, jx_valid = _linear_interpolation_utilities( + packed_annotations.x_gt, x0_gt, w_gt, x0_est, w_est, zw + ) + y_lo, y_hi, y_w, jy_valid = _linear_interpolation_utilities( + packed_annotations.y_gt, y0_gt, h_gt, y0_est, h_est, zh + ) + j_valid = jx_valid * jy_valid + + w_ylo_xlo = (1.0 - x_w) * (1.0 - y_w) + w_ylo_xhi = x_w * (1.0 - y_w) + w_yhi_xlo = (1.0 - x_w) * y_w + w_yhi_xhi = x_w * y_w + + return BilinearInterpolationHelper( + packed_annotations, + j_valid, + y_lo, + y_hi, + x_lo, + x_hi, + w_ylo_xlo, # pyre-ignore[6] + w_ylo_xhi, + # pyre-fixme[6]: Expected `Tensor` for 9th param but got `float`. + w_yhi_xlo, + w_yhi_xhi, + ) + + def extract_at_points( + self, + z_est, + slice_fine_segm=None, + w_ylo_xlo=None, + w_ylo_xhi=None, + w_yhi_xlo=None, + w_yhi_xhi=None, + ): + """ + Extract ground truth values z_gt for valid point indices and estimated + values z_est using bilinear interpolation over top-left (y_lo, x_lo), + top-right (y_lo, x_hi), bottom-left (y_hi, x_lo) and bottom-right + (y_hi, x_hi) values in z_est with corresponding weights: + w_ylo_xlo, w_ylo_xhi, w_yhi_xlo and w_yhi_xhi. + Use slice_fine_segm to slice dim=1 in z_est + """ + slice_fine_segm = ( + self.packed_annotations.fine_segm_labels_gt + if slice_fine_segm is None + else slice_fine_segm + ) + w_ylo_xlo = self.w_ylo_xlo if w_ylo_xlo is None else w_ylo_xlo + w_ylo_xhi = self.w_ylo_xhi if w_ylo_xhi is None else w_ylo_xhi + w_yhi_xlo = self.w_yhi_xlo if w_yhi_xlo is None else w_yhi_xlo + w_yhi_xhi = self.w_yhi_xhi if w_yhi_xhi is None else w_yhi_xhi + + index_bbox = self.packed_annotations.point_bbox_indices + z_est_sampled = ( + z_est[index_bbox, slice_fine_segm, self.y_lo, self.x_lo] * w_ylo_xlo + + z_est[index_bbox, slice_fine_segm, self.y_lo, self.x_hi] * w_ylo_xhi + + z_est[index_bbox, slice_fine_segm, self.y_hi, self.x_lo] * w_yhi_xlo + + z_est[index_bbox, slice_fine_segm, self.y_hi, self.x_hi] * w_yhi_xhi + ) + return z_est_sampled + + +def resample_data( + z, bbox_xywh_src, bbox_xywh_dst, wout, hout, mode: str = "nearest", padding_mode: str = "zeros" +): + """ + Args: + z (:obj: `torch.Tensor`): tensor of size (N,C,H,W) with data to be + resampled + bbox_xywh_src (:obj: `torch.Tensor`): tensor of size (N,4) containing + source bounding boxes in format XYWH + bbox_xywh_dst (:obj: `torch.Tensor`): tensor of size (N,4) containing + destination bounding boxes in format XYWH + Return: + zresampled (:obj: `torch.Tensor`): tensor of size (N, C, Hout, Wout) + with resampled values of z, where D is the discretization size + """ + n = bbox_xywh_src.size(0) + assert n == bbox_xywh_dst.size(0), ( + "The number of " + "source ROIs for resampling ({}) should be equal to the number " + "of destination ROIs ({})".format(bbox_xywh_src.size(0), bbox_xywh_dst.size(0)) + ) + x0src, y0src, wsrc, hsrc = bbox_xywh_src.unbind(dim=1) + x0dst, y0dst, wdst, hdst = bbox_xywh_dst.unbind(dim=1) + x0dst_norm = 2 * (x0dst - x0src) / wsrc - 1 + y0dst_norm = 2 * (y0dst - y0src) / hsrc - 1 + x1dst_norm = 2 * (x0dst + wdst - x0src) / wsrc - 1 + y1dst_norm = 2 * (y0dst + hdst - y0src) / hsrc - 1 + grid_w = torch.arange(wout, device=z.device, dtype=torch.float) / wout + grid_h = torch.arange(hout, device=z.device, dtype=torch.float) / hout + grid_w_expanded = grid_w[None, None, :].expand(n, hout, wout) + grid_h_expanded = grid_h[None, :, None].expand(n, hout, wout) + dx_expanded = (x1dst_norm - x0dst_norm)[:, None, None].expand(n, hout, wout) + dy_expanded = (y1dst_norm - y0dst_norm)[:, None, None].expand(n, hout, wout) + x0_expanded = x0dst_norm[:, None, None].expand(n, hout, wout) + y0_expanded = y0dst_norm[:, None, None].expand(n, hout, wout) + grid_x = grid_w_expanded * dx_expanded + x0_expanded + grid_y = grid_h_expanded * dy_expanded + y0_expanded + grid = torch.stack((grid_x, grid_y), dim=3) + # resample Z from (N, C, H, W) into (N, C, Hout, Wout) + zresampled = F.grid_sample(z, grid, mode=mode, padding_mode=padding_mode, align_corners=True) + return zresampled + + +class AnnotationsAccumulator(ABC): + """ + Abstract class for an accumulator for annotations that can produce + dense annotations packed into tensors. + """ + + @abstractmethod + def accumulate(self, instances_one_image: Instances): + """ + Accumulate instances data for one image + + Args: + instances_one_image (Instances): instances data to accumulate + """ + pass + + @abstractmethod + def pack(self) -> Any: + """ + Pack data into tensors + """ + pass + + +@dataclass +class PackedChartBasedAnnotations: + """ + Packed annotations for chart-based model training. The following attributes + are defined: + - fine_segm_labels_gt (tensor [K] of `int64`): GT fine segmentation point labels + - x_gt (tensor [K] of `float32`): GT normalized X point coordinates + - y_gt (tensor [K] of `float32`): GT normalized Y point coordinates + - u_gt (tensor [K] of `float32`): GT point U values + - v_gt (tensor [K] of `float32`): GT point V values + - coarse_segm_gt (tensor [N, S, S] of `float32`): GT segmentation for bounding boxes + - bbox_xywh_gt (tensor [N, 4] of `float32`): selected GT bounding boxes in + XYWH format + - bbox_xywh_est (tensor [N, 4] of `float32`): selected matching estimated + bounding boxes in XYWH format + - point_bbox_with_dp_indices (tensor [K] of `int64`): indices of bounding boxes + with DensePose annotations that correspond to the point data + - point_bbox_indices (tensor [K] of `int64`): indices of bounding boxes + (not necessarily the selected ones with DensePose data) that correspond + to the point data + - bbox_indices (tensor [N] of `int64`): global indices of selected bounding + boxes with DensePose annotations; these indices could be used to access + features that are computed for all bounding boxes, not only the ones with + DensePose annotations. + Here K is the total number of points and N is the total number of instances + with DensePose annotations. + """ + + fine_segm_labels_gt: torch.Tensor + x_gt: torch.Tensor + y_gt: torch.Tensor + u_gt: torch.Tensor + v_gt: torch.Tensor + coarse_segm_gt: Optional[torch.Tensor] + bbox_xywh_gt: torch.Tensor + bbox_xywh_est: torch.Tensor + point_bbox_with_dp_indices: torch.Tensor + point_bbox_indices: torch.Tensor + bbox_indices: torch.Tensor + + +class ChartBasedAnnotationsAccumulator(AnnotationsAccumulator): + """ + Accumulates annotations by batches that correspond to objects detected on + individual images. Can pack them together into single tensors. + """ + + def __init__(self): + self.i_gt = [] + self.x_gt = [] + self.y_gt = [] + self.u_gt = [] + self.v_gt = [] + self.s_gt = [] + self.bbox_xywh_gt = [] + self.bbox_xywh_est = [] + self.point_bbox_with_dp_indices = [] + self.point_bbox_indices = [] + self.bbox_indices = [] + self.nxt_bbox_with_dp_index = 0 + self.nxt_bbox_index = 0 + + def accumulate(self, instances_one_image: Instances): + """ + Accumulate instances data for one image + + Args: + instances_one_image (Instances): instances data to accumulate + """ + boxes_xywh_est = BoxMode.convert( + instances_one_image.proposal_boxes.tensor.clone(), BoxMode.XYXY_ABS, BoxMode.XYWH_ABS + ) + boxes_xywh_gt = BoxMode.convert( + instances_one_image.gt_boxes.tensor.clone(), BoxMode.XYXY_ABS, BoxMode.XYWH_ABS + ) + n_matches = len(boxes_xywh_gt) + assert n_matches == len( + boxes_xywh_est + ), f"Got {len(boxes_xywh_est)} proposal boxes and {len(boxes_xywh_gt)} GT boxes" + if not n_matches: + # no detection - GT matches + return + if ( + not hasattr(instances_one_image, "gt_densepose") + or instances_one_image.gt_densepose is None + ): + # no densepose GT for the detections, just increase the bbox index + self.nxt_bbox_index += n_matches + return + for box_xywh_est, box_xywh_gt, dp_gt in zip( + boxes_xywh_est, boxes_xywh_gt, instances_one_image.gt_densepose + ): + if (dp_gt is not None) and (len(dp_gt.x) > 0): + self._do_accumulate(box_xywh_gt, box_xywh_est, dp_gt) + self.nxt_bbox_index += 1 + + def _do_accumulate( + self, box_xywh_gt: torch.Tensor, box_xywh_est: torch.Tensor, dp_gt: DensePoseDataRelative + ): + """ + Accumulate instances data for one image, given that the data is not empty + + Args: + box_xywh_gt (tensor): GT bounding box + box_xywh_est (tensor): estimated bounding box + dp_gt (DensePoseDataRelative): GT densepose data + """ + self.i_gt.append(dp_gt.i) + self.x_gt.append(dp_gt.x) + self.y_gt.append(dp_gt.y) + self.u_gt.append(dp_gt.u) + self.v_gt.append(dp_gt.v) + if hasattr(dp_gt, "segm"): + self.s_gt.append(dp_gt.segm.unsqueeze(0)) + self.bbox_xywh_gt.append(box_xywh_gt.view(-1, 4)) + self.bbox_xywh_est.append(box_xywh_est.view(-1, 4)) + self.point_bbox_with_dp_indices.append( + torch.full_like(dp_gt.i, self.nxt_bbox_with_dp_index) + ) + self.point_bbox_indices.append(torch.full_like(dp_gt.i, self.nxt_bbox_index)) + self.bbox_indices.append(self.nxt_bbox_index) + self.nxt_bbox_with_dp_index += 1 + + def pack(self) -> Optional[PackedChartBasedAnnotations]: + """ + Pack data into tensors + """ + if not len(self.i_gt): + # TODO: + # returning proper empty annotations would require + # creating empty tensors of appropriate shape and + # type on an appropriate device; + # we return None so far to indicate empty annotations + return None + return PackedChartBasedAnnotations( + fine_segm_labels_gt=torch.cat(self.i_gt, 0).long(), + x_gt=torch.cat(self.x_gt, 0), + y_gt=torch.cat(self.y_gt, 0), + u_gt=torch.cat(self.u_gt, 0), + v_gt=torch.cat(self.v_gt, 0), + # ignore segmentation annotations, if not all the instances contain those + coarse_segm_gt=torch.cat(self.s_gt, 0) + if len(self.s_gt) == len(self.bbox_xywh_gt) + else None, + bbox_xywh_gt=torch.cat(self.bbox_xywh_gt, 0), + bbox_xywh_est=torch.cat(self.bbox_xywh_est, 0), + point_bbox_with_dp_indices=torch.cat(self.point_bbox_with_dp_indices, 0).long(), + point_bbox_indices=torch.cat(self.point_bbox_indices, 0).long(), + bbox_indices=torch.as_tensor( + self.bbox_indices, dtype=torch.long, device=self.x_gt[0].device + ).long(), + ) + + +def extract_packed_annotations_from_matches( + proposals_with_targets: List[Instances], accumulator: AnnotationsAccumulator +) -> Any: + for proposals_targets_per_image in proposals_with_targets: + accumulator.accumulate(proposals_targets_per_image) + return accumulator.pack() + + +def sample_random_indices( + n_indices: int, n_samples: int, device: Optional[torch.device] = None +) -> Optional[torch.Tensor]: + """ + Samples `n_samples` random indices from range `[0..n_indices - 1]`. + If `n_indices` is smaller than `n_samples`, returns `None` meaning that all indices + are selected. + Args: + n_indices (int): total number of indices + n_samples (int): number of indices to sample + device (torch.device): the desired device of returned tensor + Return: + Tensor of selected vertex indices, or `None`, if all vertices are selected + """ + if (n_samples <= 0) or (n_indices <= n_samples): + return None + indices = torch.randperm(n_indices, device=device)[:n_samples] + return indices diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/__init__.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ece0757acf2a4924079c884cab44a71cea22c37 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from .chart import DensePoseChartPredictor +from .chart_confidence import DensePoseChartConfidencePredictorMixin +from .chart_with_confidence import DensePoseChartWithConfidencePredictor +from .cse import DensePoseEmbeddingPredictor +from .cse_confidence import DensePoseEmbeddingConfidencePredictorMixin +from .cse_with_confidence import DensePoseEmbeddingWithConfidencePredictor +from .registry import DENSEPOSE_PREDICTOR_REGISTRY diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/chart.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/chart.py new file mode 100644 index 0000000000000000000000000000000000000000..3bcd13f7c592e37c2751556cda1f6e9cd3400b73 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/chart.py @@ -0,0 +1,94 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import torch +from torch import nn + +from detectron2.config import CfgNode +from detectron2.layers import ConvTranspose2d, interpolate + +from ...structures import DensePoseChartPredictorOutput +from ..utils import initialize_module_params +from .registry import DENSEPOSE_PREDICTOR_REGISTRY + + +@DENSEPOSE_PREDICTOR_REGISTRY.register() +class DensePoseChartPredictor(nn.Module): + """ + Predictor (last layers of a DensePose model) that takes DensePose head outputs as an input + and produces 4 tensors which represent DensePose results for predefined body parts + (patches / charts): + * coarse segmentation, a tensor of shape [N, K, Hout, Wout] + * fine segmentation, a tensor of shape [N, C, Hout, Wout] + * U coordinates, a tensor of shape [N, C, Hout, Wout] + * V coordinates, a tensor of shape [N, C, Hout, Wout] + where + - N is the number of instances + - K is the number of coarse segmentation channels ( + 2 = foreground / background, + 15 = one of 14 body parts / background) + - C is the number of fine segmentation channels ( + 24 fine body parts / background) + - Hout and Wout are height and width of predictions + """ + + def __init__(self, cfg: CfgNode, input_channels: int): + """ + Initialize predictor using configuration options + + Args: + cfg (CfgNode): configuration options + input_channels (int): input tensor size along the channel dimension + """ + super().__init__() + dim_in = input_channels + n_segm_chan = cfg.MODEL.ROI_DENSEPOSE_HEAD.NUM_COARSE_SEGM_CHANNELS + dim_out_patches = cfg.MODEL.ROI_DENSEPOSE_HEAD.NUM_PATCHES + 1 + kernel_size = cfg.MODEL.ROI_DENSEPOSE_HEAD.DECONV_KERNEL + # coarse segmentation + self.ann_index_lowres = ConvTranspose2d( + dim_in, n_segm_chan, kernel_size, stride=2, padding=int(kernel_size / 2 - 1) + ) + # fine segmentation + self.index_uv_lowres = ConvTranspose2d( + dim_in, dim_out_patches, kernel_size, stride=2, padding=int(kernel_size / 2 - 1) + ) + # U + self.u_lowres = ConvTranspose2d( + dim_in, dim_out_patches, kernel_size, stride=2, padding=int(kernel_size / 2 - 1) + ) + # V + self.v_lowres = ConvTranspose2d( + dim_in, dim_out_patches, kernel_size, stride=2, padding=int(kernel_size / 2 - 1) + ) + self.scale_factor = cfg.MODEL.ROI_DENSEPOSE_HEAD.UP_SCALE + initialize_module_params(self) + + def interp2d(self, tensor_nchw: torch.Tensor): + """ + Bilinear interpolation method to be used for upscaling + + Args: + tensor_nchw (tensor): tensor of shape (N, C, H, W) + Return: + tensor of shape (N, C, Hout, Wout), where Hout and Wout are computed + by applying the scale factor to H and W + """ + return interpolate( + tensor_nchw, scale_factor=self.scale_factor, mode="bilinear", align_corners=False + ) + + def forward(self, head_outputs: torch.Tensor): + """ + Perform forward step on DensePose head outputs + + Args: + head_outputs (tensor): DensePose head outputs, tensor of shape [N, D, H, W] + Return: + An instance of DensePoseChartPredictorOutput + """ + return DensePoseChartPredictorOutput( + coarse_segm=self.interp2d(self.ann_index_lowres(head_outputs)), + fine_segm=self.interp2d(self.index_uv_lowres(head_outputs)), + u=self.interp2d(self.u_lowres(head_outputs)), + v=self.interp2d(self.v_lowres(head_outputs)), + ) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/chart_confidence.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/chart_confidence.py new file mode 100644 index 0000000000000000000000000000000000000000..0c0099952f3e675e42aa7d3b6d35065fdaf43dbb --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/chart_confidence.py @@ -0,0 +1,174 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from typing import Any +import torch +from torch.nn import functional as F + +from detectron2.config import CfgNode +from detectron2.layers import ConvTranspose2d + +from ...structures import decorate_predictor_output_class_with_confidences +from ..confidence import DensePoseConfidenceModelConfig, DensePoseUVConfidenceType +from ..utils import initialize_module_params + + +class DensePoseChartConfidencePredictorMixin: + """ + Predictor contains the last layers of a DensePose model that take DensePose head + outputs as an input and produce model outputs. Confidence predictor mixin is used + to generate confidences for segmentation and UV tensors estimated by some + base predictor. Several assumptions need to hold for the base predictor: + 1) the `forward` method must return SIUV tuple as the first result ( + S = coarse segmentation, I = fine segmentation, U and V are intrinsic + chart coordinates) + 2) `interp2d` method must be defined to perform bilinear interpolation; + the same method is typically used for SIUV and confidences + Confidence predictor mixin provides confidence estimates, as described in: + N. Neverova et al., Correlated Uncertainty for Learning Dense Correspondences + from Noisy Labels, NeurIPS 2019 + A. Sanakoyeu et al., Transferring Dense Pose to Proximal Animal Classes, CVPR 2020 + """ + + def __init__(self, cfg: CfgNode, input_channels: int): + """ + Initialize confidence predictor using configuration options. + + Args: + cfg (CfgNode): configuration options + input_channels (int): number of input channels + """ + # we rely on base predictor to call nn.Module.__init__ + super().__init__(cfg, input_channels) # pyre-ignore[19] + self.confidence_model_cfg = DensePoseConfidenceModelConfig.from_cfg(cfg) + self._initialize_confidence_estimation_layers(cfg, input_channels) + self._registry = {} + initialize_module_params(self) # pyre-ignore[6] + + def _initialize_confidence_estimation_layers(self, cfg: CfgNode, dim_in: int): + """ + Initialize confidence estimation layers based on configuration options + + Args: + cfg (CfgNode): configuration options + dim_in (int): number of input channels + """ + dim_out_patches = cfg.MODEL.ROI_DENSEPOSE_HEAD.NUM_PATCHES + 1 + kernel_size = cfg.MODEL.ROI_DENSEPOSE_HEAD.DECONV_KERNEL + if self.confidence_model_cfg.uv_confidence.enabled: + if self.confidence_model_cfg.uv_confidence.type == DensePoseUVConfidenceType.IID_ISO: + self.sigma_2_lowres = ConvTranspose2d( # pyre-ignore[16] + dim_in, dim_out_patches, kernel_size, stride=2, padding=int(kernel_size / 2 - 1) + ) + elif ( + self.confidence_model_cfg.uv_confidence.type + == DensePoseUVConfidenceType.INDEP_ANISO + ): + self.sigma_2_lowres = ConvTranspose2d( + dim_in, dim_out_patches, kernel_size, stride=2, padding=int(kernel_size / 2 - 1) + ) + self.kappa_u_lowres = ConvTranspose2d( # pyre-ignore[16] + dim_in, dim_out_patches, kernel_size, stride=2, padding=int(kernel_size / 2 - 1) + ) + self.kappa_v_lowres = ConvTranspose2d( # pyre-ignore[16] + dim_in, dim_out_patches, kernel_size, stride=2, padding=int(kernel_size / 2 - 1) + ) + else: + raise ValueError( + f"Unknown confidence model type: " + f"{self.confidence_model_cfg.confidence_model_type}" + ) + if self.confidence_model_cfg.segm_confidence.enabled: + self.fine_segm_confidence_lowres = ConvTranspose2d( # pyre-ignore[16] + dim_in, 1, kernel_size, stride=2, padding=int(kernel_size / 2 - 1) + ) + self.coarse_segm_confidence_lowres = ConvTranspose2d( # pyre-ignore[16] + dim_in, 1, kernel_size, stride=2, padding=int(kernel_size / 2 - 1) + ) + + def forward(self, head_outputs: torch.Tensor): + """ + Perform forward operation on head outputs used as inputs for the predictor. + Calls forward method from the base predictor and uses its outputs to compute + confidences. + + Args: + head_outputs (Tensor): head outputs used as predictor inputs + Return: + An instance of outputs with confidences, + see `decorate_predictor_output_class_with_confidences` + """ + # assuming base class returns SIUV estimates in its first result + base_predictor_outputs = super().forward(head_outputs) # pyre-ignore[16] + + # create output instance by extending base predictor outputs: + output = self._create_output_instance(base_predictor_outputs) + + if self.confidence_model_cfg.uv_confidence.enabled: + if self.confidence_model_cfg.uv_confidence.type == DensePoseUVConfidenceType.IID_ISO: + # assuming base class defines interp2d method for bilinear interpolation + output.sigma_2 = self.interp2d(self.sigma_2_lowres(head_outputs)) # pyre-ignore[16] + elif ( + self.confidence_model_cfg.uv_confidence.type + == DensePoseUVConfidenceType.INDEP_ANISO + ): + # assuming base class defines interp2d method for bilinear interpolation + output.sigma_2 = self.interp2d(self.sigma_2_lowres(head_outputs)) + output.kappa_u = self.interp2d(self.kappa_u_lowres(head_outputs)) # pyre-ignore[16] + output.kappa_v = self.interp2d(self.kappa_v_lowres(head_outputs)) # pyre-ignore[16] + else: + raise ValueError( + f"Unknown confidence model type: " + f"{self.confidence_model_cfg.confidence_model_type}" + ) + if self.confidence_model_cfg.segm_confidence.enabled: + # base predictor outputs are assumed to have `fine_segm` and `coarse_segm` attributes + # base predictor is assumed to define `interp2d` method for bilinear interpolation + output.fine_segm_confidence = ( + F.softplus( + self.interp2d(self.fine_segm_confidence_lowres(head_outputs)) # pyre-ignore[16] + ) + + self.confidence_model_cfg.segm_confidence.epsilon + ) + output.fine_segm = base_predictor_outputs.fine_segm * torch.repeat_interleave( + output.fine_segm_confidence, base_predictor_outputs.fine_segm.shape[1], dim=1 + ) + output.coarse_segm_confidence = ( + F.softplus( + self.interp2d( + self.coarse_segm_confidence_lowres(head_outputs) # pyre-ignore[16] + ) + ) + + self.confidence_model_cfg.segm_confidence.epsilon + ) + output.coarse_segm = base_predictor_outputs.coarse_segm * torch.repeat_interleave( + output.coarse_segm_confidence, base_predictor_outputs.coarse_segm.shape[1], dim=1 + ) + + return output + + def _create_output_instance(self, base_predictor_outputs: Any): + """ + Create an instance of predictor outputs by copying the outputs from the + base predictor and initializing confidence + + Args: + base_predictor_outputs: an instance of base predictor outputs + (the outputs type is assumed to be a dataclass) + Return: + An instance of outputs with confidences + """ + PredictorOutput = decorate_predictor_output_class_with_confidences( + type(base_predictor_outputs) # pyre-ignore[6] + ) + # base_predictor_outputs is assumed to be a dataclass + # reassign all the fields from base_predictor_outputs (no deep copy!), add new fields + output = PredictorOutput( + **base_predictor_outputs.__dict__, + coarse_segm_confidence=None, + fine_segm_confidence=None, + sigma_1=None, + sigma_2=None, + kappa_u=None, + kappa_v=None, + ) + return output diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/chart_with_confidence.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/chart_with_confidence.py new file mode 100644 index 0000000000000000000000000000000000000000..9c1cd6cc8fda56e831fbc02a8ffdd844866c0e4f --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/chart_with_confidence.py @@ -0,0 +1,15 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from . import DensePoseChartConfidencePredictorMixin, DensePoseChartPredictor +from .registry import DENSEPOSE_PREDICTOR_REGISTRY + + +@DENSEPOSE_PREDICTOR_REGISTRY.register() +class DensePoseChartWithConfidencePredictor( + DensePoseChartConfidencePredictorMixin, DensePoseChartPredictor +): + """ + Predictor that combines chart and chart confidence estimation + """ + + pass diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/cse.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/cse.py new file mode 100644 index 0000000000000000000000000000000000000000..466a5ecddbfa338a2b603facf06d1f4510fff6eb --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/cse.py @@ -0,0 +1,70 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +import torch +from torch import nn + +from detectron2.config import CfgNode +from detectron2.layers import ConvTranspose2d, interpolate + +from ...structures import DensePoseEmbeddingPredictorOutput +from ..utils import initialize_module_params +from .registry import DENSEPOSE_PREDICTOR_REGISTRY + + +@DENSEPOSE_PREDICTOR_REGISTRY.register() +class DensePoseEmbeddingPredictor(nn.Module): + """ + Last layers of a DensePose model that take DensePose head outputs as an input + and produce model outputs for continuous surface embeddings (CSE). + """ + + def __init__(self, cfg: CfgNode, input_channels: int): + """ + Initialize predictor using configuration options + + Args: + cfg (CfgNode): configuration options + input_channels (int): input tensor size along the channel dimension + """ + super().__init__() + dim_in = input_channels + n_segm_chan = cfg.MODEL.ROI_DENSEPOSE_HEAD.NUM_COARSE_SEGM_CHANNELS + embed_size = cfg.MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBED_SIZE + kernel_size = cfg.MODEL.ROI_DENSEPOSE_HEAD.DECONV_KERNEL + # coarse segmentation + self.coarse_segm_lowres = ConvTranspose2d( + dim_in, n_segm_chan, kernel_size, stride=2, padding=int(kernel_size / 2 - 1) + ) + # embedding + self.embed_lowres = ConvTranspose2d( + dim_in, embed_size, kernel_size, stride=2, padding=int(kernel_size / 2 - 1) + ) + self.scale_factor = cfg.MODEL.ROI_DENSEPOSE_HEAD.UP_SCALE + initialize_module_params(self) + + def interp2d(self, tensor_nchw: torch.Tensor): + """ + Bilinear interpolation method to be used for upscaling + + Args: + tensor_nchw (tensor): tensor of shape (N, C, H, W) + Return: + tensor of shape (N, C, Hout, Wout), where Hout and Wout are computed + by applying the scale factor to H and W + """ + return interpolate( + tensor_nchw, scale_factor=self.scale_factor, mode="bilinear", align_corners=False + ) + + def forward(self, head_outputs): + """ + Perform forward step on DensePose head outputs + + Args: + head_outputs (tensor): DensePose head outputs, tensor of shape [N, D, H, W] + """ + embed_lowres = self.embed_lowres(head_outputs) + coarse_segm_lowres = self.coarse_segm_lowres(head_outputs) + embed = self.interp2d(embed_lowres) + coarse_segm = self.interp2d(coarse_segm_lowres) + return DensePoseEmbeddingPredictorOutput(embedding=embed, coarse_segm=coarse_segm) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/cse_confidence.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/cse_confidence.py new file mode 100644 index 0000000000000000000000000000000000000000..8220337cea8eb87bbdf74378079551259dcc37e2 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/cse_confidence.py @@ -0,0 +1,115 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from typing import Any +import torch +from torch.nn import functional as F + +from detectron2.config import CfgNode +from detectron2.layers import ConvTranspose2d + +from densepose.modeling.confidence import DensePoseConfidenceModelConfig +from densepose.modeling.utils import initialize_module_params +from densepose.structures import decorate_cse_predictor_output_class_with_confidences + + +class DensePoseEmbeddingConfidencePredictorMixin: + """ + Predictor contains the last layers of a DensePose model that take DensePose head + outputs as an input and produce model outputs. Confidence predictor mixin is used + to generate confidences for coarse segmentation estimated by some + base predictor. Several assumptions need to hold for the base predictor: + 1) the `forward` method must return CSE DensePose head outputs, + tensor of shape [N, D, H, W] + 2) `interp2d` method must be defined to perform bilinear interpolation; + the same method is typically used for masks and confidences + Confidence predictor mixin provides confidence estimates, as described in: + N. Neverova et al., Correlated Uncertainty for Learning Dense Correspondences + from Noisy Labels, NeurIPS 2019 + A. Sanakoyeu et al., Transferring Dense Pose to Proximal Animal Classes, CVPR 2020 + """ + + def __init__(self, cfg: CfgNode, input_channels: int): + """ + Initialize confidence predictor using configuration options. + + Args: + cfg (CfgNode): configuration options + input_channels (int): number of input channels + """ + # we rely on base predictor to call nn.Module.__init__ + super().__init__(cfg, input_channels) # pyre-ignore[19] + self.confidence_model_cfg = DensePoseConfidenceModelConfig.from_cfg(cfg) + self._initialize_confidence_estimation_layers(cfg, input_channels) + self._registry = {} + initialize_module_params(self) # pyre-ignore[6] + + def _initialize_confidence_estimation_layers(self, cfg: CfgNode, dim_in: int): + """ + Initialize confidence estimation layers based on configuration options + + Args: + cfg (CfgNode): configuration options + dim_in (int): number of input channels + """ + kernel_size = cfg.MODEL.ROI_DENSEPOSE_HEAD.DECONV_KERNEL + if self.confidence_model_cfg.segm_confidence.enabled: + self.coarse_segm_confidence_lowres = ConvTranspose2d( # pyre-ignore[16] + dim_in, 1, kernel_size, stride=2, padding=int(kernel_size / 2 - 1) + ) + + def forward(self, head_outputs: torch.Tensor): + """ + Perform forward operation on head outputs used as inputs for the predictor. + Calls forward method from the base predictor and uses its outputs to compute + confidences. + + Args: + head_outputs (Tensor): head outputs used as predictor inputs + Return: + An instance of outputs with confidences, + see `decorate_cse_predictor_output_class_with_confidences` + """ + # assuming base class returns SIUV estimates in its first result + base_predictor_outputs = super().forward(head_outputs) # pyre-ignore[16] + + # create output instance by extending base predictor outputs: + output = self._create_output_instance(base_predictor_outputs) + + if self.confidence_model_cfg.segm_confidence.enabled: + # base predictor outputs are assumed to have `coarse_segm` attribute + # base predictor is assumed to define `interp2d` method for bilinear interpolation + output.coarse_segm_confidence = ( + F.softplus( + self.interp2d( # pyre-ignore[16] + self.coarse_segm_confidence_lowres(head_outputs) # pyre-ignore[16] + ) + ) + + self.confidence_model_cfg.segm_confidence.epsilon + ) + output.coarse_segm = base_predictor_outputs.coarse_segm * torch.repeat_interleave( + output.coarse_segm_confidence, base_predictor_outputs.coarse_segm.shape[1], dim=1 + ) + + return output + + def _create_output_instance(self, base_predictor_outputs: Any): + """ + Create an instance of predictor outputs by copying the outputs from the + base predictor and initializing confidence + + Args: + base_predictor_outputs: an instance of base predictor outputs + (the outputs type is assumed to be a dataclass) + Return: + An instance of outputs with confidences + """ + PredictorOutput = decorate_cse_predictor_output_class_with_confidences( + type(base_predictor_outputs) # pyre-ignore[6] + ) + # base_predictor_outputs is assumed to be a dataclass + # reassign all the fields from base_predictor_outputs (no deep copy!), add new fields + output = PredictorOutput( + **base_predictor_outputs.__dict__, + coarse_segm_confidence=None, + ) + return output diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/cse_with_confidence.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/cse_with_confidence.py new file mode 100644 index 0000000000000000000000000000000000000000..17ecef67ffb67cd0e64de73632eaede1d8f3c701 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/cse_with_confidence.py @@ -0,0 +1,15 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from . import DensePoseEmbeddingConfidencePredictorMixin, DensePoseEmbeddingPredictor +from .registry import DENSEPOSE_PREDICTOR_REGISTRY + + +@DENSEPOSE_PREDICTOR_REGISTRY.register() +class DensePoseEmbeddingWithConfidencePredictor( + DensePoseEmbeddingConfidencePredictorMixin, DensePoseEmbeddingPredictor +): + """ + Predictor that combines CSE and CSE confidence estimation + """ + + pass diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/registry.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..f96901d3242fa8f3d35d053ed0bdd7649a045b88 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/predictors/registry.py @@ -0,0 +1,5 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from detectron2.utils.registry import Registry + +DENSEPOSE_PREDICTOR_REGISTRY = Registry("DENSEPOSE_PREDICTOR") diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/roi_heads/__init__.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/roi_heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8403589f23ec2ffa8afafcd566ca0b0b7b2671a7 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/roi_heads/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from .v1convx import DensePoseV1ConvXHead +from .deeplab import DensePoseDeepLabHead +from .registry import ROI_DENSEPOSE_HEAD_REGISTRY +from .roi_head import Decoder, DensePoseROIHeads diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/roi_heads/deeplab.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/roi_heads/deeplab.py new file mode 100644 index 0000000000000000000000000000000000000000..4e5cb483037b302ff1fb2c305275a65e4ba4e941 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/roi_heads/deeplab.py @@ -0,0 +1,263 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import fvcore.nn.weight_init as weight_init +import torch +from torch import nn +from torch.nn import functional as F + +from detectron2.config import CfgNode +from detectron2.layers import Conv2d + +from .registry import ROI_DENSEPOSE_HEAD_REGISTRY + + +@ROI_DENSEPOSE_HEAD_REGISTRY.register() +class DensePoseDeepLabHead(nn.Module): + """ + DensePose head using DeepLabV3 model from + "Rethinking Atrous Convolution for Semantic Image Segmentation" + . + """ + + def __init__(self, cfg: CfgNode, input_channels: int): + super(DensePoseDeepLabHead, self).__init__() + # fmt: off + hidden_dim = cfg.MODEL.ROI_DENSEPOSE_HEAD.CONV_HEAD_DIM + kernel_size = cfg.MODEL.ROI_DENSEPOSE_HEAD.CONV_HEAD_KERNEL + norm = cfg.MODEL.ROI_DENSEPOSE_HEAD.DEEPLAB.NORM + self.n_stacked_convs = cfg.MODEL.ROI_DENSEPOSE_HEAD.NUM_STACKED_CONVS + self.use_nonlocal = cfg.MODEL.ROI_DENSEPOSE_HEAD.DEEPLAB.NONLOCAL_ON + # fmt: on + pad_size = kernel_size // 2 + n_channels = input_channels + + self.ASPP = ASPP(input_channels, [6, 12, 56], n_channels) # 6, 12, 56 + self.add_module("ASPP", self.ASPP) + + if self.use_nonlocal: + self.NLBlock = NONLocalBlock2D(input_channels, bn_layer=True) + self.add_module("NLBlock", self.NLBlock) + # weight_init.c2_msra_fill(self.ASPP) + + for i in range(self.n_stacked_convs): + norm_module = nn.GroupNorm(32, hidden_dim) if norm == "GN" else None + layer = Conv2d( + n_channels, + hidden_dim, + kernel_size, + stride=1, + padding=pad_size, + bias=not norm, + norm=norm_module, + ) + weight_init.c2_msra_fill(layer) + n_channels = hidden_dim + layer_name = self._get_layer_name(i) + self.add_module(layer_name, layer) + self.n_out_channels = hidden_dim + # initialize_module_params(self) + + def forward(self, features): + x0 = features + x = self.ASPP(x0) + if self.use_nonlocal: + x = self.NLBlock(x) + output = x + for i in range(self.n_stacked_convs): + layer_name = self._get_layer_name(i) + x = getattr(self, layer_name)(x) + x = F.relu(x) + output = x + return output + + def _get_layer_name(self, i: int): + layer_name = "body_conv_fcn{}".format(i + 1) + return layer_name + + +# Copied from +# https://github.com/pytorch/vision/blob/master/torchvision/models/segmentation/deeplabv3.py +# See https://arxiv.org/pdf/1706.05587.pdf for details +class ASPPConv(nn.Sequential): + def __init__(self, in_channels, out_channels, dilation): + modules = [ + nn.Conv2d( + in_channels, out_channels, 3, padding=dilation, dilation=dilation, bias=False + ), + nn.GroupNorm(32, out_channels), + nn.ReLU(), + ] + super(ASPPConv, self).__init__(*modules) + + +class ASPPPooling(nn.Sequential): + def __init__(self, in_channels, out_channels): + super(ASPPPooling, self).__init__( + nn.AdaptiveAvgPool2d(1), + nn.Conv2d(in_channels, out_channels, 1, bias=False), + nn.GroupNorm(32, out_channels), + nn.ReLU(), + ) + + def forward(self, x): + size = x.shape[-2:] + x = super(ASPPPooling, self).forward(x) + return F.interpolate(x, size=size, mode="bilinear", align_corners=False) + + +class ASPP(nn.Module): + def __init__(self, in_channels, atrous_rates, out_channels): + super(ASPP, self).__init__() + modules = [] + modules.append( + nn.Sequential( + nn.Conv2d(in_channels, out_channels, 1, bias=False), + nn.GroupNorm(32, out_channels), + nn.ReLU(), + ) + ) + + rate1, rate2, rate3 = tuple(atrous_rates) + modules.append(ASPPConv(in_channels, out_channels, rate1)) + modules.append(ASPPConv(in_channels, out_channels, rate2)) + modules.append(ASPPConv(in_channels, out_channels, rate3)) + modules.append(ASPPPooling(in_channels, out_channels)) + + self.convs = nn.ModuleList(modules) + + self.project = nn.Sequential( + nn.Conv2d(5 * out_channels, out_channels, 1, bias=False), + # nn.BatchNorm2d(out_channels), + nn.ReLU() + # nn.Dropout(0.5) + ) + + def forward(self, x): + res = [] + for conv in self.convs: + res.append(conv(x)) + res = torch.cat(res, dim=1) + return self.project(res) + + +# copied from +# https://github.com/AlexHex7/Non-local_pytorch/blob/master/lib/non_local_embedded_gaussian.py +# See https://arxiv.org/abs/1711.07971 for details +class _NonLocalBlockND(nn.Module): + def __init__( + self, in_channels, inter_channels=None, dimension=3, sub_sample=True, bn_layer=True + ): + super(_NonLocalBlockND, self).__init__() + + assert dimension in [1, 2, 3] + + self.dimension = dimension + self.sub_sample = sub_sample + + self.in_channels = in_channels + self.inter_channels = inter_channels + + if self.inter_channels is None: + self.inter_channels = in_channels // 2 + if self.inter_channels == 0: + self.inter_channels = 1 + + if dimension == 3: + conv_nd = nn.Conv3d + max_pool_layer = nn.MaxPool3d(kernel_size=(1, 2, 2)) + bn = nn.GroupNorm # (32, hidden_dim) #nn.BatchNorm3d + elif dimension == 2: + conv_nd = nn.Conv2d + max_pool_layer = nn.MaxPool2d(kernel_size=(2, 2)) + bn = nn.GroupNorm # (32, hidden_dim)nn.BatchNorm2d + else: + conv_nd = nn.Conv1d + max_pool_layer = nn.MaxPool1d(kernel_size=2) + bn = nn.GroupNorm # (32, hidden_dim)nn.BatchNorm1d + + self.g = conv_nd( + in_channels=self.in_channels, + out_channels=self.inter_channels, + kernel_size=1, + stride=1, + padding=0, + ) + + if bn_layer: + self.W = nn.Sequential( + conv_nd( + in_channels=self.inter_channels, + out_channels=self.in_channels, + kernel_size=1, + stride=1, + padding=0, + ), + bn(32, self.in_channels), + ) + nn.init.constant_(self.W[1].weight, 0) + nn.init.constant_(self.W[1].bias, 0) + else: + self.W = conv_nd( + in_channels=self.inter_channels, + out_channels=self.in_channels, + kernel_size=1, + stride=1, + padding=0, + ) + nn.init.constant_(self.W.weight, 0) + nn.init.constant_(self.W.bias, 0) + + self.theta = conv_nd( + in_channels=self.in_channels, + out_channels=self.inter_channels, + kernel_size=1, + stride=1, + padding=0, + ) + self.phi = conv_nd( + in_channels=self.in_channels, + out_channels=self.inter_channels, + kernel_size=1, + stride=1, + padding=0, + ) + + if sub_sample: + self.g = nn.Sequential(self.g, max_pool_layer) + self.phi = nn.Sequential(self.phi, max_pool_layer) + + def forward(self, x): + """ + :param x: (b, c, t, h, w) + :return: + """ + + batch_size = x.size(0) + + g_x = self.g(x).view(batch_size, self.inter_channels, -1) + g_x = g_x.permute(0, 2, 1) + + theta_x = self.theta(x).view(batch_size, self.inter_channels, -1) + theta_x = theta_x.permute(0, 2, 1) + phi_x = self.phi(x).view(batch_size, self.inter_channels, -1) + f = torch.matmul(theta_x, phi_x) + f_div_C = F.softmax(f, dim=-1) + + y = torch.matmul(f_div_C, g_x) + y = y.permute(0, 2, 1).contiguous() + y = y.view(batch_size, self.inter_channels, *x.size()[2:]) + W_y = self.W(y) + z = W_y + x + + return z + + +class NONLocalBlock2D(_NonLocalBlockND): + def __init__(self, in_channels, inter_channels=None, sub_sample=True, bn_layer=True): + super(NONLocalBlock2D, self).__init__( + in_channels, + inter_channels=inter_channels, + dimension=2, + sub_sample=sub_sample, + bn_layer=bn_layer, + ) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/roi_heads/registry.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/roi_heads/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..e1cea432f1fda3861266fa636d002667b3fb46a0 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/roi_heads/registry.py @@ -0,0 +1,5 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from detectron2.utils.registry import Registry + +ROI_DENSEPOSE_HEAD_REGISTRY = Registry("ROI_DENSEPOSE_HEAD") diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/roi_heads/roi_head.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/roi_heads/roi_head.py new file mode 100644 index 0000000000000000000000000000000000000000..8f9d9a612645b06c04648c2be4d556e3467204a9 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/roi_heads/roi_head.py @@ -0,0 +1,221 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import numpy as np +from typing import Dict, List, Optional +import fvcore.nn.weight_init as weight_init +import torch +import torch.nn as nn +from torch.nn import functional as F + +from detectron2.layers import Conv2d, ShapeSpec, get_norm +from detectron2.modeling import ROI_HEADS_REGISTRY, StandardROIHeads +from detectron2.modeling.poolers import ROIPooler +from detectron2.modeling.roi_heads import select_foreground_proposals +from detectron2.structures import ImageList, Instances + +from .. import ( + build_densepose_data_filter, + build_densepose_embedder, + build_densepose_head, + build_densepose_losses, + build_densepose_predictor, + densepose_inference, +) + + +class Decoder(nn.Module): + """ + A semantic segmentation head described in detail in the Panoptic Feature Pyramid Networks paper + (https://arxiv.org/abs/1901.02446). It takes FPN features as input and merges information from + all levels of the FPN into single output. + """ + + def __init__(self, cfg, input_shape: Dict[str, ShapeSpec], in_features): + super(Decoder, self).__init__() + + # fmt: off + self.in_features = in_features + feature_strides = {k: v.stride for k, v in input_shape.items()} + feature_channels = {k: v.channels for k, v in input_shape.items()} + num_classes = cfg.MODEL.ROI_DENSEPOSE_HEAD.DECODER_NUM_CLASSES + conv_dims = cfg.MODEL.ROI_DENSEPOSE_HEAD.DECODER_CONV_DIMS + self.common_stride = cfg.MODEL.ROI_DENSEPOSE_HEAD.DECODER_COMMON_STRIDE + norm = cfg.MODEL.ROI_DENSEPOSE_HEAD.DECODER_NORM + # fmt: on + + self.scale_heads = [] + for in_feature in self.in_features: + head_ops = [] + head_length = max( + 1, int(np.log2(feature_strides[in_feature]) - np.log2(self.common_stride)) + ) + for k in range(head_length): + conv = Conv2d( + feature_channels[in_feature] if k == 0 else conv_dims, + conv_dims, + kernel_size=3, + stride=1, + padding=1, + bias=not norm, + norm=get_norm(norm, conv_dims), + activation=F.relu, + ) + weight_init.c2_msra_fill(conv) + head_ops.append(conv) + if feature_strides[in_feature] != self.common_stride: + head_ops.append( + nn.Upsample(scale_factor=2, mode="bilinear", align_corners=False) + ) + self.scale_heads.append(nn.Sequential(*head_ops)) + self.add_module(in_feature, self.scale_heads[-1]) + self.predictor = Conv2d(conv_dims, num_classes, kernel_size=1, stride=1, padding=0) + weight_init.c2_msra_fill(self.predictor) + + def forward(self, features: List[torch.Tensor]): + for i, _ in enumerate(self.in_features): + if i == 0: + x = self.scale_heads[i](features[i]) + else: + x = x + self.scale_heads[i](features[i]) + x = self.predictor(x) + return x + + +@ROI_HEADS_REGISTRY.register() +class DensePoseROIHeads(StandardROIHeads): + """ + A Standard ROIHeads which contains an addition of DensePose head. + """ + + def __init__(self, cfg, input_shape): + super().__init__(cfg, input_shape) + self._init_densepose_head(cfg, input_shape) + + def _init_densepose_head(self, cfg, input_shape): + # fmt: off + self.densepose_on = cfg.MODEL.DENSEPOSE_ON + if not self.densepose_on: + return + self.densepose_data_filter = build_densepose_data_filter(cfg) + dp_pooler_resolution = cfg.MODEL.ROI_DENSEPOSE_HEAD.POOLER_RESOLUTION + dp_pooler_sampling_ratio = cfg.MODEL.ROI_DENSEPOSE_HEAD.POOLER_SAMPLING_RATIO + dp_pooler_type = cfg.MODEL.ROI_DENSEPOSE_HEAD.POOLER_TYPE + self.use_decoder = cfg.MODEL.ROI_DENSEPOSE_HEAD.DECODER_ON + # fmt: on + if self.use_decoder: + dp_pooler_scales = (1.0 / input_shape[self.in_features[0]].stride,) + else: + dp_pooler_scales = tuple(1.0 / input_shape[k].stride for k in self.in_features) + in_channels = [input_shape[f].channels for f in self.in_features][0] + + if self.use_decoder: + self.decoder = Decoder(cfg, input_shape, self.in_features) + + self.densepose_pooler = ROIPooler( + output_size=dp_pooler_resolution, + scales=dp_pooler_scales, + sampling_ratio=dp_pooler_sampling_ratio, + pooler_type=dp_pooler_type, + ) + self.densepose_head = build_densepose_head(cfg, in_channels) + self.densepose_predictor = build_densepose_predictor( + cfg, self.densepose_head.n_out_channels + ) + self.densepose_losses = build_densepose_losses(cfg) + self.embedder = build_densepose_embedder(cfg) + + def _forward_densepose(self, features: Dict[str, torch.Tensor], instances: List[Instances]): + """ + Forward logic of the densepose prediction branch. + + Args: + features (dict[str, Tensor]): input data as a mapping from feature + map name to tensor. Axis 0 represents the number of images `N` in + the input data; axes 1-3 are channels, height, and width, which may + vary between feature maps (e.g., if a feature pyramid is used). + instances (list[Instances]): length `N` list of `Instances`. The i-th + `Instances` contains instances for the i-th input image, + In training, they can be the proposals. + In inference, they can be the predicted boxes. + + Returns: + In training, a dict of losses. + In inference, update `instances` with new fields "densepose" and return it. + """ + if not self.densepose_on: + return {} if self.training else instances + + features_list = [features[f] for f in self.in_features] + if self.training: + proposals, _ = select_foreground_proposals(instances, self.num_classes) + features_list, proposals = self.densepose_data_filter(features_list, proposals) + if len(proposals) > 0: + proposal_boxes = [x.proposal_boxes for x in proposals] + + if self.use_decoder: + # pyre-fixme[29]: `Union[nn.Module, torch.Tensor]` is not a + # function. + features_list = [self.decoder(features_list)] + + features_dp = self.densepose_pooler(features_list, proposal_boxes) + densepose_head_outputs = self.densepose_head(features_dp) + densepose_predictor_outputs = self.densepose_predictor(densepose_head_outputs) + densepose_loss_dict = self.densepose_losses( + proposals, densepose_predictor_outputs, embedder=self.embedder + ) + return densepose_loss_dict + else: + pred_boxes = [x.pred_boxes for x in instances] + + if self.use_decoder: + # pyre-fixme[29]: `Union[nn.Module, torch.Tensor]` is not a function. + features_list = [self.decoder(features_list)] + + features_dp = self.densepose_pooler(features_list, pred_boxes) + if len(features_dp) > 0: + densepose_head_outputs = self.densepose_head(features_dp) + densepose_predictor_outputs = self.densepose_predictor(densepose_head_outputs) + else: + densepose_predictor_outputs = None + + densepose_inference(densepose_predictor_outputs, instances) + return instances + + def forward( + self, + images: ImageList, + features: Dict[str, torch.Tensor], + proposals: List[Instances], + targets: Optional[List[Instances]] = None, + ): + instances, losses = super().forward(images, features, proposals, targets) + del targets, images + + if self.training: + losses.update(self._forward_densepose(features, instances)) + return instances, losses + + def forward_with_given_boxes( + self, features: Dict[str, torch.Tensor], instances: List[Instances] + ): + """ + Use the given boxes in `instances` to produce other (non-box) per-ROI outputs. + + This is useful for downstream tasks where a box is known, but need to obtain + other attributes (outputs of other heads). + Test-time augmentation also uses this. + + Args: + features: same as in `forward()` + instances (list[Instances]): instances to predict other outputs. Expect the keys + "pred_boxes" and "pred_classes" to exist. + + Returns: + instances (list[Instances]): + the same `Instances` objects, with extra + fields such as `pred_masks` or `pred_keypoints`. + """ + + instances = super().forward_with_given_boxes(features, instances) + instances = self._forward_densepose(features, instances) + return instances diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/roi_heads/v1convx.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/roi_heads/v1convx.py new file mode 100644 index 0000000000000000000000000000000000000000..df79f658d8f7149e44aa1a31072adc4dadd89a48 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/roi_heads/v1convx.py @@ -0,0 +1,64 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import torch +from torch import nn +from torch.nn import functional as F + +from detectron2.config import CfgNode +from detectron2.layers import Conv2d + +from ..utils import initialize_module_params +from .registry import ROI_DENSEPOSE_HEAD_REGISTRY + + +@ROI_DENSEPOSE_HEAD_REGISTRY.register() +class DensePoseV1ConvXHead(nn.Module): + """ + Fully convolutional DensePose head. + """ + + def __init__(self, cfg: CfgNode, input_channels: int): + """ + Initialize DensePose fully convolutional head + + Args: + cfg (CfgNode): configuration options + input_channels (int): number of input channels + """ + super(DensePoseV1ConvXHead, self).__init__() + # fmt: off + hidden_dim = cfg.MODEL.ROI_DENSEPOSE_HEAD.CONV_HEAD_DIM + kernel_size = cfg.MODEL.ROI_DENSEPOSE_HEAD.CONV_HEAD_KERNEL + self.n_stacked_convs = cfg.MODEL.ROI_DENSEPOSE_HEAD.NUM_STACKED_CONVS + # fmt: on + pad_size = kernel_size // 2 + n_channels = input_channels + for i in range(self.n_stacked_convs): + layer = Conv2d(n_channels, hidden_dim, kernel_size, stride=1, padding=pad_size) + layer_name = self._get_layer_name(i) + self.add_module(layer_name, layer) + n_channels = hidden_dim + self.n_out_channels = n_channels + initialize_module_params(self) + + def forward(self, features: torch.Tensor): + """ + Apply DensePose fully convolutional head to the input features + + Args: + features (tensor): input features + Result: + A tensor of DensePose head outputs + """ + x = features + output = x + for i in range(self.n_stacked_convs): + layer_name = self._get_layer_name(i) + x = getattr(self, layer_name)(x) + x = F.relu(x) + output = x + return output + + def _get_layer_name(self, i: int): + layer_name = "body_conv_fcn{}".format(i + 1) + return layer_name diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/test_time_augmentation.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/test_time_augmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..ec2022ed16727f538993d2c7db60a60a1183b90d --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/test_time_augmentation.py @@ -0,0 +1,207 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import numpy as np +import torch +from fvcore.transforms import HFlipTransform, TransformList +from torch.nn import functional as F + +from detectron2.data.transforms import RandomRotation, RotationTransform, apply_transform_gens +from detectron2.modeling.postprocessing import detector_postprocess +from detectron2.modeling.test_time_augmentation import DatasetMapperTTA, GeneralizedRCNNWithTTA + +from ..converters import HFlipConverter + + +class DensePoseDatasetMapperTTA(DatasetMapperTTA): + def __init__(self, cfg): + super().__init__(cfg=cfg) + self.angles = cfg.TEST.AUG.ROTATION_ANGLES + + def __call__(self, dataset_dict): + ret = super().__call__(dataset_dict=dataset_dict) + numpy_image = dataset_dict["image"].permute(1, 2, 0).numpy() + for angle in self.angles: + rotate = RandomRotation(angle=angle, expand=True) + new_numpy_image, tfms = apply_transform_gens([rotate], np.copy(numpy_image)) + torch_image = torch.from_numpy(np.ascontiguousarray(new_numpy_image.transpose(2, 0, 1))) + dic = copy.deepcopy(dataset_dict) + # In DatasetMapperTTA, there is a pre_tfm transform (resize or no-op) that is + # added at the beginning of each TransformList. That's '.transforms[0]'. + dic["transforms"] = TransformList( + [ret[-1]["transforms"].transforms[0]] + tfms.transforms + ) + dic["image"] = torch_image + ret.append(dic) + return ret + + +class DensePoseGeneralizedRCNNWithTTA(GeneralizedRCNNWithTTA): + def __init__(self, cfg, model, transform_data, tta_mapper=None, batch_size=1): + """ + Args: + cfg (CfgNode): + model (GeneralizedRCNN): a GeneralizedRCNN to apply TTA on. + transform_data (DensePoseTransformData): contains symmetry label + transforms used for horizontal flip + tta_mapper (callable): takes a dataset dict and returns a list of + augmented versions of the dataset dict. Defaults to + `DatasetMapperTTA(cfg)`. + batch_size (int): batch the augmented images into this batch size for inference. + """ + self._transform_data = transform_data.to(model.device) + super().__init__(cfg=cfg, model=model, tta_mapper=tta_mapper, batch_size=batch_size) + + # the implementation follows closely the one from detectron2/modeling + def _inference_one_image(self, input): + """ + Args: + input (dict): one dataset dict with "image" field being a CHW tensor + + Returns: + dict: one output dict + """ + orig_shape = (input["height"], input["width"]) + # For some reason, resize with uint8 slightly increases box AP but decreases densepose AP + input["image"] = input["image"].to(torch.uint8) + augmented_inputs, tfms = self._get_augmented_inputs(input) + # Detect boxes from all augmented versions + with self._turn_off_roi_heads(["mask_on", "keypoint_on", "densepose_on"]): + # temporarily disable roi heads + all_boxes, all_scores, all_classes = self._get_augmented_boxes(augmented_inputs, tfms) + merged_instances = self._merge_detections(all_boxes, all_scores, all_classes, orig_shape) + + if self.cfg.MODEL.MASK_ON or self.cfg.MODEL.DENSEPOSE_ON: + # Use the detected boxes to obtain new fields + augmented_instances = self._rescale_detected_boxes( + augmented_inputs, merged_instances, tfms + ) + # run forward on the detected boxes + outputs = self._batch_inference(augmented_inputs, augmented_instances) + # Delete now useless variables to avoid being out of memory + del augmented_inputs, augmented_instances + # average the predictions + if self.cfg.MODEL.MASK_ON: + merged_instances.pred_masks = self._reduce_pred_masks(outputs, tfms) + if self.cfg.MODEL.DENSEPOSE_ON: + merged_instances.pred_densepose = self._reduce_pred_densepose(outputs, tfms) + # postprocess + merged_instances = detector_postprocess(merged_instances, *orig_shape) + return {"instances": merged_instances} + else: + return {"instances": merged_instances} + + def _get_augmented_boxes(self, augmented_inputs, tfms): + # Heavily based on detectron2/modeling/test_time_augmentation.py + # Only difference is that RotationTransform is excluded from bbox computation + # 1: forward with all augmented images + outputs = self._batch_inference(augmented_inputs) + # 2: union the results + all_boxes = [] + all_scores = [] + all_classes = [] + for output, tfm in zip(outputs, tfms): + # Need to inverse the transforms on boxes, to obtain results on original image + if not any(isinstance(t, RotationTransform) for t in tfm.transforms): + # Some transforms can't compute bbox correctly + pred_boxes = output.pred_boxes.tensor + original_pred_boxes = tfm.inverse().apply_box(pred_boxes.cpu().numpy()) + all_boxes.append(torch.from_numpy(original_pred_boxes).to(pred_boxes.device)) + all_scores.extend(output.scores) + all_classes.extend(output.pred_classes) + all_boxes = torch.cat(all_boxes, dim=0) + return all_boxes, all_scores, all_classes + + def _reduce_pred_densepose(self, outputs, tfms): + # Should apply inverse transforms on densepose preds. + # We assume only rotation, resize & flip are used. pred_masks is a scale-invariant + # representation, so we handle the other ones specially + for idx, (output, tfm) in enumerate(zip(outputs, tfms)): + for t in tfm.transforms: + for attr in ["coarse_segm", "fine_segm", "u", "v"]: + setattr( + output.pred_densepose, + attr, + _inverse_rotation( + getattr(output.pred_densepose, attr), output.pred_boxes.tensor, t + ), + ) + if any(isinstance(t, HFlipTransform) for t in tfm.transforms): + output.pred_densepose = HFlipConverter.convert( + output.pred_densepose, self._transform_data + ) + self._incremental_avg_dp(outputs[0].pred_densepose, output.pred_densepose, idx) + return outputs[0].pred_densepose + + # incrementally computed average: u_(n + 1) = u_n + (x_(n+1) - u_n) / (n + 1). + def _incremental_avg_dp(self, avg, new_el, idx): + for attr in ["coarse_segm", "fine_segm", "u", "v"]: + setattr(avg, attr, (getattr(avg, attr) * idx + getattr(new_el, attr)) / (idx + 1)) + if idx: + # Deletion of the > 0 index intermediary values to prevent GPU OOM + setattr(new_el, attr, None) + return avg + + +def _inverse_rotation(densepose_attrs, boxes, transform): + # resample outputs to image size and rotate back the densepose preds + # on the rotated images to the space of the original image + if len(boxes) == 0 or not isinstance(transform, RotationTransform): + return densepose_attrs + boxes = boxes.int().cpu().numpy() + wh_boxes = boxes[:, 2:] - boxes[:, :2] # bboxes in the rotated space + inv_boxes = rotate_box_inverse(transform, boxes).astype(int) # bboxes in original image + wh_diff = (inv_boxes[:, 2:] - inv_boxes[:, :2] - wh_boxes) // 2 # diff between new/old bboxes + rotation_matrix = torch.tensor([transform.rm_image]).to(device=densepose_attrs.device).float() + rotation_matrix[:, :, -1] = 0 + # To apply grid_sample for rotation, we need to have enough space to fit the original and + # rotated bboxes. l_bds and r_bds are the left/right bounds that will be used to + # crop the difference once the rotation is done + l_bds = np.maximum(0, -wh_diff) + for i in range(len(densepose_attrs)): + if min(wh_boxes[i]) <= 0: + continue + densepose_attr = densepose_attrs[[i]].clone() + # 1. Interpolate densepose attribute to size of the rotated bbox + densepose_attr = F.interpolate(densepose_attr, wh_boxes[i].tolist()[::-1], mode="bilinear") + # 2. Pad the interpolated attribute so it has room for the original + rotated bbox + densepose_attr = F.pad(densepose_attr, tuple(np.repeat(np.maximum(0, wh_diff[i]), 2))) + # 3. Compute rotation grid and transform + grid = F.affine_grid(rotation_matrix, size=densepose_attr.shape) + densepose_attr = F.grid_sample(densepose_attr, grid) + # 4. Compute right bounds and crop the densepose_attr to the size of the original bbox + r_bds = densepose_attr.shape[2:][::-1] - l_bds[i] + densepose_attr = densepose_attr[:, :, l_bds[i][1] : r_bds[1], l_bds[i][0] : r_bds[0]] + if min(densepose_attr.shape) > 0: + # Interpolate back to the original size of the densepose attribute + densepose_attr = F.interpolate( + densepose_attr, densepose_attrs.shape[-2:], mode="bilinear" + ) + # Adding a very small probability to the background class to fill padded zones + densepose_attr[:, 0] += 1e-10 + densepose_attrs[i] = densepose_attr + return densepose_attrs + + +def rotate_box_inverse(rot_tfm, rotated_box): + """ + rotated_box is a N * 4 array of [x0, y0, x1, y1] boxes + When a bbox is rotated, it gets bigger, because we need to surround the tilted bbox + So when a bbox is rotated then inverse-rotated, it is much bigger than the original + This function aims to invert the rotation on the box, but also resize it to its original size + """ + # 1. Compute the inverse rotation of the rotated bboxes (bigger than it ) + invrot_box = rot_tfm.inverse().apply_box(rotated_box) + h, w = rotated_box[:, 3] - rotated_box[:, 1], rotated_box[:, 2] - rotated_box[:, 0] + ih, iw = invrot_box[:, 3] - invrot_box[:, 1], invrot_box[:, 2] - invrot_box[:, 0] + assert 2 * rot_tfm.abs_sin**2 != 1, "45 degrees angle can't be inverted" + # 2. Inverse the corresponding computation in the rotation transform + # to get the original height/width of the rotated boxes + orig_h = (h * rot_tfm.abs_cos - w * rot_tfm.abs_sin) / (1 - 2 * rot_tfm.abs_sin**2) + orig_w = (w * rot_tfm.abs_cos - h * rot_tfm.abs_sin) / (1 - 2 * rot_tfm.abs_sin**2) + # 3. Resize the inverse-rotated bboxes to their original size + invrot_box[:, 0] += (iw - orig_w) / 2 + invrot_box[:, 1] += (ih - orig_h) / 2 + invrot_box[:, 2] -= (iw - orig_w) / 2 + invrot_box[:, 3] -= (ih - orig_h) / 2 + + return invrot_box diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/modeling/utils.py b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2e76eb9535a68dcb4ccb065556c55289294e42c8 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/modeling/utils.py @@ -0,0 +1,11 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from torch import nn + + +def initialize_module_params(module: nn.Module) -> None: + for name, param in module.named_parameters(): + if "bias" in name: + nn.init.constant_(param, 0) + elif "weight" in name: + nn.init.kaiming_normal_(param, mode="fan_out", nonlinearity="relu") diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/structures/__init__.py b/approach/ovod/detectron2/projects/DensePose/densepose/structures/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ed32c5e9d6c4c1599ba960681d9e86889e2cdbd8 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/structures/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from .chart import DensePoseChartPredictorOutput +from .chart_confidence import decorate_predictor_output_class_with_confidences +from .cse_confidence import decorate_cse_predictor_output_class_with_confidences +from .chart_result import ( + DensePoseChartResult, + DensePoseChartResultWithConfidences, + quantize_densepose_chart_result, + compress_quantized_densepose_chart_result, + decompress_compressed_densepose_chart_result, +) +from .cse import DensePoseEmbeddingPredictorOutput +from .data_relative import DensePoseDataRelative +from .list import DensePoseList +from .mesh import Mesh, create_mesh +from .transform_data import DensePoseTransformData, normalized_coords_transform diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/structures/chart.py b/approach/ovod/detectron2/projects/DensePose/densepose/structures/chart.py new file mode 100644 index 0000000000000000000000000000000000000000..115cc084e98115c537382494af9eb0e246cd375b --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/structures/chart.py @@ -0,0 +1,70 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from dataclasses import dataclass +from typing import Union +import torch + + +@dataclass +class DensePoseChartPredictorOutput: + """ + Predictor output that contains segmentation and inner coordinates predictions for predefined + body parts: + * coarse segmentation, a tensor of shape [N, K, Hout, Wout] + * fine segmentation, a tensor of shape [N, C, Hout, Wout] + * U coordinates, a tensor of shape [N, C, Hout, Wout] + * V coordinates, a tensor of shape [N, C, Hout, Wout] + where + - N is the number of instances + - K is the number of coarse segmentation channels ( + 2 = foreground / background, + 15 = one of 14 body parts / background) + - C is the number of fine segmentation channels ( + 24 fine body parts / background) + - Hout and Wout are height and width of predictions + """ + + coarse_segm: torch.Tensor + fine_segm: torch.Tensor + u: torch.Tensor + v: torch.Tensor + + def __len__(self): + """ + Number of instances (N) in the output + """ + return self.coarse_segm.size(0) + + def __getitem__( + self, item: Union[int, slice, torch.BoolTensor] + ) -> "DensePoseChartPredictorOutput": + """ + Get outputs for the selected instance(s) + + Args: + item (int or slice or tensor): selected items + """ + if isinstance(item, int): + return DensePoseChartPredictorOutput( + coarse_segm=self.coarse_segm[item].unsqueeze(0), + fine_segm=self.fine_segm[item].unsqueeze(0), + u=self.u[item].unsqueeze(0), + v=self.v[item].unsqueeze(0), + ) + else: + return DensePoseChartPredictorOutput( + coarse_segm=self.coarse_segm[item], + fine_segm=self.fine_segm[item], + u=self.u[item], + v=self.v[item], + ) + + def to(self, device: torch.device): + """ + Transfers all tensors to the given device + """ + coarse_segm = self.coarse_segm.to(device) + fine_segm = self.fine_segm.to(device) + u = self.u.to(device) + v = self.v.to(device) + return DensePoseChartPredictorOutput(coarse_segm=coarse_segm, fine_segm=fine_segm, u=u, v=v) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/structures/chart_confidence.py b/approach/ovod/detectron2/projects/DensePose/densepose/structures/chart_confidence.py new file mode 100644 index 0000000000000000000000000000000000000000..57c63257a7c176af1522e2f143ed594c26906c76 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/structures/chart_confidence.py @@ -0,0 +1,98 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from dataclasses import make_dataclass +from functools import lru_cache +from typing import Any, Optional +import torch + + +@lru_cache(maxsize=None) +def decorate_predictor_output_class_with_confidences(BasePredictorOutput: type) -> type: + """ + Create a new output class from an existing one by adding new attributes + related to confidence estimation: + - sigma_1 (tensor) + - sigma_2 (tensor) + - kappa_u (tensor) + - kappa_v (tensor) + - fine_segm_confidence (tensor) + - coarse_segm_confidence (tensor) + + Details on confidence estimation parameters can be found in: + N. Neverova, D. Novotny, A. Vedaldi "Correlated Uncertainty for Learning + Dense Correspondences from Noisy Labels", p. 918--926, in Proc. NIPS 2019 + A. Sanakoyeu et al., Transferring Dense Pose to Proximal Animal Classes, CVPR 2020 + + The new class inherits the provided `BasePredictorOutput` class, + it's name is composed of the name of the provided class and + "WithConfidences" suffix. + + Args: + BasePredictorOutput (type): output type to which confidence data + is to be added, assumed to be a dataclass + Return: + New dataclass derived from the provided one that has attributes + for confidence estimation + """ + + PredictorOutput = make_dataclass( + BasePredictorOutput.__name__ + "WithConfidences", + fields=[ + ("sigma_1", Optional[torch.Tensor], None), + ("sigma_2", Optional[torch.Tensor], None), + ("kappa_u", Optional[torch.Tensor], None), + ("kappa_v", Optional[torch.Tensor], None), + ("fine_segm_confidence", Optional[torch.Tensor], None), + ("coarse_segm_confidence", Optional[torch.Tensor], None), + ], + bases=(BasePredictorOutput,), + ) + + # add possibility to index PredictorOutput + + def slice_if_not_none(data, item): + if data is None: + return None + if isinstance(item, int): + return data[item].unsqueeze(0) + return data[item] + + def PredictorOutput_getitem(self, item): + PredictorOutput = type(self) + base_predictor_output_sliced = super(PredictorOutput, self).__getitem__(item) + return PredictorOutput( + **base_predictor_output_sliced.__dict__, + coarse_segm_confidence=slice_if_not_none(self.coarse_segm_confidence, item), + fine_segm_confidence=slice_if_not_none(self.fine_segm_confidence, item), + sigma_1=slice_if_not_none(self.sigma_1, item), + sigma_2=slice_if_not_none(self.sigma_2, item), + kappa_u=slice_if_not_none(self.kappa_u, item), + kappa_v=slice_if_not_none(self.kappa_v, item), + ) + + PredictorOutput.__getitem__ = PredictorOutput_getitem + + def PredictorOutput_to(self, device: torch.device): + """ + Transfers all tensors to the given device + """ + PredictorOutput = type(self) + base_predictor_output_to = super(PredictorOutput, self).to(device) # pyre-ignore[16] + + def to_device_if_tensor(var: Any): + if isinstance(var, torch.Tensor): + return var.to(device) + return var + + return PredictorOutput( + **base_predictor_output_to.__dict__, + sigma_1=to_device_if_tensor(self.sigma_1), + sigma_2=to_device_if_tensor(self.sigma_2), + kappa_u=to_device_if_tensor(self.kappa_u), + kappa_v=to_device_if_tensor(self.kappa_v), + fine_segm_confidence=to_device_if_tensor(self.fine_segm_confidence), + coarse_segm_confidence=to_device_if_tensor(self.coarse_segm_confidence), + ) + + PredictorOutput.to = PredictorOutput_to + return PredictorOutput diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/structures/chart_result.py b/approach/ovod/detectron2/projects/DensePose/densepose/structures/chart_result.py new file mode 100644 index 0000000000000000000000000000000000000000..003933d03d153d045c0bf551c465bc7a224d90cb --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/structures/chart_result.py @@ -0,0 +1,183 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from dataclasses import dataclass +from typing import Any, Optional, Tuple +import torch + + +@dataclass +class DensePoseChartResult: + """ + DensePose results for chart-based methods represented by labels and inner + coordinates (U, V) of individual charts. Each chart is a 2D manifold + that has an associated label and is parameterized by two coordinates U and V. + Both U and V take values in [0, 1]. + Thus the results are represented by two tensors: + - labels (tensor [H, W] of long): contains estimated label for each pixel of + the detection bounding box of size (H, W) + - uv (tensor [2, H, W] of float): contains estimated U and V coordinates + for each pixel of the detection bounding box of size (H, W) + """ + + labels: torch.Tensor + uv: torch.Tensor + + def to(self, device: torch.device): + """ + Transfers all tensors to the given device + """ + labels = self.labels.to(device) + uv = self.uv.to(device) + return DensePoseChartResult(labels=labels, uv=uv) + + +@dataclass +class DensePoseChartResultWithConfidences: + """ + We add confidence values to DensePoseChartResult + Thus the results are represented by two tensors: + - labels (tensor [H, W] of long): contains estimated label for each pixel of + the detection bounding box of size (H, W) + - uv (tensor [2, H, W] of float): contains estimated U and V coordinates + for each pixel of the detection bounding box of size (H, W) + Plus one [H, W] tensor of float for each confidence type + """ + + labels: torch.Tensor + uv: torch.Tensor + sigma_1: Optional[torch.Tensor] = None + sigma_2: Optional[torch.Tensor] = None + kappa_u: Optional[torch.Tensor] = None + kappa_v: Optional[torch.Tensor] = None + fine_segm_confidence: Optional[torch.Tensor] = None + coarse_segm_confidence: Optional[torch.Tensor] = None + + def to(self, device: torch.device): + """ + Transfers all tensors to the given device, except if their value is None + """ + + def to_device_if_tensor(var: Any): + if isinstance(var, torch.Tensor): + return var.to(device) + return var + + return DensePoseChartResultWithConfidences( + labels=self.labels.to(device), + uv=self.uv.to(device), + sigma_1=to_device_if_tensor(self.sigma_1), + sigma_2=to_device_if_tensor(self.sigma_2), + kappa_u=to_device_if_tensor(self.kappa_u), + kappa_v=to_device_if_tensor(self.kappa_v), + fine_segm_confidence=to_device_if_tensor(self.fine_segm_confidence), + coarse_segm_confidence=to_device_if_tensor(self.coarse_segm_confidence), + ) + + +@dataclass +class DensePoseChartResultQuantized: + """ + DensePose results for chart-based methods represented by labels and quantized + inner coordinates (U, V) of individual charts. Each chart is a 2D manifold + that has an associated label and is parameterized by two coordinates U and V. + Both U and V take values in [0, 1]. + Quantized coordinates Uq and Vq have uint8 values which are obtained as: + Uq = U * 255 (hence 0 <= Uq <= 255) + Vq = V * 255 (hence 0 <= Vq <= 255) + Thus the results are represented by one tensor: + - labels_uv_uint8 (tensor [3, H, W] of uint8): contains estimated label + and quantized coordinates Uq and Vq for each pixel of the detection + bounding box of size (H, W) + """ + + labels_uv_uint8: torch.Tensor + + def to(self, device: torch.device): + """ + Transfers all tensors to the given device + """ + labels_uv_uint8 = self.labels_uv_uint8.to(device) + return DensePoseChartResultQuantized(labels_uv_uint8=labels_uv_uint8) + + +@dataclass +class DensePoseChartResultCompressed: + """ + DensePose results for chart-based methods represented by a PNG-encoded string. + The tensor of quantized DensePose results of size [3, H, W] is considered + as an image with 3 color channels. PNG compression is applied and the result + is stored as a Base64-encoded string. The following attributes are defined: + - shape_chw (tuple of 3 int): contains shape of the result tensor + (number of channels, height, width) + - labels_uv_str (str): contains Base64-encoded results tensor of size + [3, H, W] compressed with PNG compression methods + """ + + shape_chw: Tuple[int, int, int] + labels_uv_str: str + + +def quantize_densepose_chart_result(result: DensePoseChartResult) -> DensePoseChartResultQuantized: + """ + Applies quantization to DensePose chart-based result. + + Args: + result (DensePoseChartResult): DensePose chart-based result + Return: + Quantized DensePose chart-based result (DensePoseChartResultQuantized) + """ + h, w = result.labels.shape + labels_uv_uint8 = torch.zeros([3, h, w], dtype=torch.uint8, device=result.labels.device) + labels_uv_uint8[0] = result.labels + labels_uv_uint8[1:] = (result.uv * 255).clamp(0, 255).byte() + return DensePoseChartResultQuantized(labels_uv_uint8=labels_uv_uint8) + + +def compress_quantized_densepose_chart_result( + result: DensePoseChartResultQuantized, +) -> DensePoseChartResultCompressed: + """ + Compresses quantized DensePose chart-based result + + Args: + result (DensePoseChartResultQuantized): quantized DensePose chart-based result + Return: + Compressed DensePose chart-based result (DensePoseChartResultCompressed) + """ + import base64 + import numpy as np + from io import BytesIO + from PIL import Image + + labels_uv_uint8_np_chw = result.labels_uv_uint8.cpu().numpy() + labels_uv_uint8_np_hwc = np.moveaxis(labels_uv_uint8_np_chw, 0, -1) + im = Image.fromarray(labels_uv_uint8_np_hwc) + fstream = BytesIO() + im.save(fstream, format="png", optimize=True) + labels_uv_str = base64.encodebytes(fstream.getvalue()).decode() + shape_chw = labels_uv_uint8_np_chw.shape + return DensePoseChartResultCompressed(labels_uv_str=labels_uv_str, shape_chw=shape_chw) + + +def decompress_compressed_densepose_chart_result( + result: DensePoseChartResultCompressed, +) -> DensePoseChartResultQuantized: + """ + Decompresses DensePose chart-based result encoded into a base64 string + + Args: + result (DensePoseChartResultCompressed): compressed DensePose chart result + Return: + Quantized DensePose chart-based result (DensePoseChartResultQuantized) + """ + import base64 + import numpy as np + from io import BytesIO + from PIL import Image + + fstream = BytesIO(base64.decodebytes(result.labels_uv_str.encode())) + im = Image.open(fstream) + labels_uv_uint8_np_chw = np.moveaxis(np.array(im, dtype=np.uint8), -1, 0) + return DensePoseChartResultQuantized( + labels_uv_uint8=torch.from_numpy(labels_uv_uint8_np_chw.reshape(result.shape_chw)) + ) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/structures/cse.py b/approach/ovod/detectron2/projects/DensePose/densepose/structures/cse.py new file mode 100644 index 0000000000000000000000000000000000000000..9cd65da96c04613053e21494bc2dcc04f37fe1fd --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/structures/cse.py @@ -0,0 +1,52 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +from dataclasses import dataclass +from typing import Union +import torch + + +@dataclass +class DensePoseEmbeddingPredictorOutput: + """ + Predictor output that contains embedding and coarse segmentation data: + * embedding: float tensor of size [N, D, H, W], contains estimated embeddings + * coarse_segm: float tensor of size [N, K, H, W] + Here D = MODEL.ROI_DENSEPOSE_HEAD.CSE.EMBED_SIZE + K = MODEL.ROI_DENSEPOSE_HEAD.NUM_COARSE_SEGM_CHANNELS + """ + + embedding: torch.Tensor + coarse_segm: torch.Tensor + + def __len__(self): + """ + Number of instances (N) in the output + """ + return self.coarse_segm.size(0) + + def __getitem__( + self, item: Union[int, slice, torch.BoolTensor] + ) -> "DensePoseEmbeddingPredictorOutput": + """ + Get outputs for the selected instance(s) + + Args: + item (int or slice or tensor): selected items + """ + if isinstance(item, int): + return DensePoseEmbeddingPredictorOutput( + coarse_segm=self.coarse_segm[item].unsqueeze(0), + embedding=self.embedding[item].unsqueeze(0), + ) + else: + return DensePoseEmbeddingPredictorOutput( + coarse_segm=self.coarse_segm[item], embedding=self.embedding[item] + ) + + def to(self, device: torch.device): + """ + Transfers all tensors to the given device + """ + coarse_segm = self.coarse_segm.to(device) + embedding = self.embedding.to(device) + return DensePoseEmbeddingPredictorOutput(coarse_segm=coarse_segm, embedding=embedding) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/structures/cse_confidence.py b/approach/ovod/detectron2/projects/DensePose/densepose/structures/cse_confidence.py new file mode 100644 index 0000000000000000000000000000000000000000..ee5166f82d45ecb4ea829ec2ecab248161c19421 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/structures/cse_confidence.py @@ -0,0 +1,78 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from dataclasses import make_dataclass +from functools import lru_cache +from typing import Any, Optional +import torch + + +@lru_cache(maxsize=None) +def decorate_cse_predictor_output_class_with_confidences(BasePredictorOutput: type) -> type: + """ + Create a new output class from an existing one by adding new attributes + related to confidence estimation: + - coarse_segm_confidence (tensor) + + Details on confidence estimation parameters can be found in: + N. Neverova, D. Novotny, A. Vedaldi "Correlated Uncertainty for Learning + Dense Correspondences from Noisy Labels", p. 918--926, in Proc. NIPS 2019 + A. Sanakoyeu et al., Transferring Dense Pose to Proximal Animal Classes, CVPR 2020 + + The new class inherits the provided `BasePredictorOutput` class, + it's name is composed of the name of the provided class and + "WithConfidences" suffix. + + Args: + BasePredictorOutput (type): output type to which confidence data + is to be added, assumed to be a dataclass + Return: + New dataclass derived from the provided one that has attributes + for confidence estimation + """ + + PredictorOutput = make_dataclass( + BasePredictorOutput.__name__ + "WithConfidences", + fields=[ + ("coarse_segm_confidence", Optional[torch.Tensor], None), + ], + bases=(BasePredictorOutput,), + ) + + # add possibility to index PredictorOutput + + def slice_if_not_none(data, item): + if data is None: + return None + if isinstance(item, int): + return data[item].unsqueeze(0) + return data[item] + + def PredictorOutput_getitem(self, item): + PredictorOutput = type(self) + base_predictor_output_sliced = super(PredictorOutput, self).__getitem__(item) + return PredictorOutput( + **base_predictor_output_sliced.__dict__, + coarse_segm_confidence=slice_if_not_none(self.coarse_segm_confidence, item), + ) + + PredictorOutput.__getitem__ = PredictorOutput_getitem + + def PredictorOutput_to(self, device: torch.device): + """ + Transfers all tensors to the given device + """ + PredictorOutput = type(self) + base_predictor_output_to = super(PredictorOutput, self).to(device) # pyre-ignore[16] + + def to_device_if_tensor(var: Any): + if isinstance(var, torch.Tensor): + return var.to(device) + return var + + return PredictorOutput( + **base_predictor_output_to.__dict__, + coarse_segm_confidence=to_device_if_tensor(self.coarse_segm_confidence), + ) + + PredictorOutput.to = PredictorOutput_to + return PredictorOutput diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/structures/data_relative.py b/approach/ovod/detectron2/projects/DensePose/densepose/structures/data_relative.py new file mode 100644 index 0000000000000000000000000000000000000000..a148fa75dcf33eb610ef2a2758969c0277bc0906 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/structures/data_relative.py @@ -0,0 +1,243 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import numpy as np +import torch +from torch.nn import functional as F + +from densepose.data.meshes.catalog import MeshCatalog +from densepose.structures.mesh import load_mesh_symmetry +from densepose.structures.transform_data import DensePoseTransformData + + +class DensePoseDataRelative(object): + """ + Dense pose relative annotations that can be applied to any bounding box: + x - normalized X coordinates [0, 255] of annotated points + y - normalized Y coordinates [0, 255] of annotated points + i - body part labels 0,...,24 for annotated points + u - body part U coordinates [0, 1] for annotated points + v - body part V coordinates [0, 1] for annotated points + segm - 256x256 segmentation mask with values 0,...,14 + To obtain absolute x and y data wrt some bounding box one needs to first + divide the data by 256, multiply by the respective bounding box size + and add bounding box offset: + x_img = x0 + x_norm * w / 256.0 + y_img = y0 + y_norm * h / 256.0 + Segmentation masks are typically sampled to get image-based masks. + """ + + # Key for normalized X coordinates in annotation dict + X_KEY = "dp_x" + # Key for normalized Y coordinates in annotation dict + Y_KEY = "dp_y" + # Key for U part coordinates in annotation dict (used in chart-based annotations) + U_KEY = "dp_U" + # Key for V part coordinates in annotation dict (used in chart-based annotations) + V_KEY = "dp_V" + # Key for I point labels in annotation dict (used in chart-based annotations) + I_KEY = "dp_I" + # Key for segmentation mask in annotation dict + S_KEY = "dp_masks" + # Key for vertex ids (used in continuous surface embeddings annotations) + VERTEX_IDS_KEY = "dp_vertex" + # Key for mesh id (used in continuous surface embeddings annotations) + MESH_NAME_KEY = "ref_model" + # Number of body parts in segmentation masks + N_BODY_PARTS = 14 + # Number of parts in point labels + N_PART_LABELS = 24 + MASK_SIZE = 256 + + def __init__(self, annotation, cleanup=False): + self.x = torch.as_tensor(annotation[DensePoseDataRelative.X_KEY]) + self.y = torch.as_tensor(annotation[DensePoseDataRelative.Y_KEY]) + if ( + DensePoseDataRelative.I_KEY in annotation + and DensePoseDataRelative.U_KEY in annotation + and DensePoseDataRelative.V_KEY in annotation + ): + self.i = torch.as_tensor(annotation[DensePoseDataRelative.I_KEY]) + self.u = torch.as_tensor(annotation[DensePoseDataRelative.U_KEY]) + self.v = torch.as_tensor(annotation[DensePoseDataRelative.V_KEY]) + if ( + DensePoseDataRelative.VERTEX_IDS_KEY in annotation + and DensePoseDataRelative.MESH_NAME_KEY in annotation + ): + self.vertex_ids = torch.as_tensor( + annotation[DensePoseDataRelative.VERTEX_IDS_KEY], dtype=torch.long + ) + self.mesh_id = MeshCatalog.get_mesh_id(annotation[DensePoseDataRelative.MESH_NAME_KEY]) + if DensePoseDataRelative.S_KEY in annotation: + self.segm = DensePoseDataRelative.extract_segmentation_mask(annotation) + self.device = torch.device("cpu") + if cleanup: + DensePoseDataRelative.cleanup_annotation(annotation) + + def to(self, device): + if self.device == device: + return self + new_data = DensePoseDataRelative.__new__(DensePoseDataRelative) + new_data.x = self.x.to(device) + new_data.y = self.y.to(device) + for attr in ["i", "u", "v", "vertex_ids", "segm"]: + if hasattr(self, attr): + setattr(new_data, attr, getattr(self, attr).to(device)) + if hasattr(self, "mesh_id"): + new_data.mesh_id = self.mesh_id + new_data.device = device + return new_data + + @staticmethod + def extract_segmentation_mask(annotation): + import pycocotools.mask as mask_utils + + # TODO: annotation instance is accepted if it contains either + # DensePose segmentation or instance segmentation. However, here we + # only rely on DensePose segmentation + poly_specs = annotation[DensePoseDataRelative.S_KEY] + if isinstance(poly_specs, torch.Tensor): + # data is already given as mask tensors, no need to decode + return poly_specs + segm = torch.zeros((DensePoseDataRelative.MASK_SIZE,) * 2, dtype=torch.float32) + if isinstance(poly_specs, dict): + if poly_specs: + mask = mask_utils.decode(poly_specs) + segm[mask > 0] = 1 + else: + for i in range(len(poly_specs)): + poly_i = poly_specs[i] + if poly_i: + mask_i = mask_utils.decode(poly_i) + segm[mask_i > 0] = i + 1 + return segm + + @staticmethod + def validate_annotation(annotation): + for key in [ + DensePoseDataRelative.X_KEY, + DensePoseDataRelative.Y_KEY, + ]: + if key not in annotation: + return False, "no {key} data in the annotation".format(key=key) + valid_for_iuv_setting = all( + key in annotation + for key in [ + DensePoseDataRelative.I_KEY, + DensePoseDataRelative.U_KEY, + DensePoseDataRelative.V_KEY, + ] + ) + valid_for_cse_setting = all( + key in annotation + for key in [ + DensePoseDataRelative.VERTEX_IDS_KEY, + DensePoseDataRelative.MESH_NAME_KEY, + ] + ) + if not valid_for_iuv_setting and not valid_for_cse_setting: + return ( + False, + "expected either {} (IUV setting) or {} (CSE setting) annotations".format( + ", ".join( + [ + DensePoseDataRelative.I_KEY, + DensePoseDataRelative.U_KEY, + DensePoseDataRelative.V_KEY, + ] + ), + ", ".join( + [ + DensePoseDataRelative.VERTEX_IDS_KEY, + DensePoseDataRelative.MESH_NAME_KEY, + ] + ), + ), + ) + return True, None + + @staticmethod + def cleanup_annotation(annotation): + for key in [ + DensePoseDataRelative.X_KEY, + DensePoseDataRelative.Y_KEY, + DensePoseDataRelative.I_KEY, + DensePoseDataRelative.U_KEY, + DensePoseDataRelative.V_KEY, + DensePoseDataRelative.S_KEY, + DensePoseDataRelative.VERTEX_IDS_KEY, + DensePoseDataRelative.MESH_NAME_KEY, + ]: + if key in annotation: + del annotation[key] + + def apply_transform(self, transforms, densepose_transform_data): + self._transform_pts(transforms, densepose_transform_data) + if hasattr(self, "segm"): + self._transform_segm(transforms, densepose_transform_data) + + def _transform_pts(self, transforms, dp_transform_data): + import detectron2.data.transforms as T + + # NOTE: This assumes that HorizFlipTransform is the only one that does flip + do_hflip = sum(isinstance(t, T.HFlipTransform) for t in transforms.transforms) % 2 == 1 + if do_hflip: + self.x = self.MASK_SIZE - self.x + if hasattr(self, "i"): + self._flip_iuv_semantics(dp_transform_data) + if hasattr(self, "vertex_ids"): + self._flip_vertices() + + for t in transforms.transforms: + if isinstance(t, T.RotationTransform): + xy_scale = np.array((t.w, t.h)) / DensePoseDataRelative.MASK_SIZE + xy = t.apply_coords(np.stack((self.x, self.y), axis=1) * xy_scale) + self.x, self.y = torch.tensor(xy / xy_scale, dtype=self.x.dtype).T + + def _flip_iuv_semantics(self, dp_transform_data: DensePoseTransformData) -> None: + i_old = self.i.clone() + uv_symmetries = dp_transform_data.uv_symmetries + pt_label_symmetries = dp_transform_data.point_label_symmetries + for i in range(self.N_PART_LABELS): + if i + 1 in i_old: + annot_indices_i = i_old == i + 1 + if pt_label_symmetries[i + 1] != i + 1: + self.i[annot_indices_i] = pt_label_symmetries[i + 1] + u_loc = (self.u[annot_indices_i] * 255).long() + v_loc = (self.v[annot_indices_i] * 255).long() + self.u[annot_indices_i] = uv_symmetries["U_transforms"][i][v_loc, u_loc].to( + device=self.u.device + ) + self.v[annot_indices_i] = uv_symmetries["V_transforms"][i][v_loc, u_loc].to( + device=self.v.device + ) + + def _flip_vertices(self): + mesh_info = MeshCatalog[MeshCatalog.get_mesh_name(self.mesh_id)] + mesh_symmetry = ( + load_mesh_symmetry(mesh_info.symmetry) if mesh_info.symmetry is not None else None + ) + self.vertex_ids = mesh_symmetry["vertex_transforms"][self.vertex_ids] + + def _transform_segm(self, transforms, dp_transform_data): + import detectron2.data.transforms as T + + # NOTE: This assumes that HorizFlipTransform is the only one that does flip + do_hflip = sum(isinstance(t, T.HFlipTransform) for t in transforms.transforms) % 2 == 1 + if do_hflip: + self.segm = torch.flip(self.segm, [1]) + self._flip_segm_semantics(dp_transform_data) + + for t in transforms.transforms: + if isinstance(t, T.RotationTransform): + self._transform_segm_rotation(t) + + def _flip_segm_semantics(self, dp_transform_data): + old_segm = self.segm.clone() + mask_label_symmetries = dp_transform_data.mask_label_symmetries + for i in range(self.N_BODY_PARTS): + if mask_label_symmetries[i + 1] != i + 1: + self.segm[old_segm == i + 1] = mask_label_symmetries[i + 1] + + def _transform_segm_rotation(self, rotation): + self.segm = F.interpolate(self.segm[None, None, :], (rotation.h, rotation.w)).numpy() + self.segm = torch.tensor(rotation.apply_segmentation(self.segm[0, 0]))[None, None, :] + self.segm = F.interpolate(self.segm, [DensePoseDataRelative.MASK_SIZE] * 2)[0, 0] diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/structures/list.py b/approach/ovod/detectron2/projects/DensePose/densepose/structures/list.py new file mode 100644 index 0000000000000000000000000000000000000000..3dc40b0a7c04c7144c8e33c826a7354bf5d59819 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/structures/list.py @@ -0,0 +1,70 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import torch + +from densepose.structures.data_relative import DensePoseDataRelative + + +class DensePoseList(object): + + _TORCH_DEVICE_CPU = torch.device("cpu") + + def __init__(self, densepose_datas, boxes_xyxy_abs, image_size_hw, device=_TORCH_DEVICE_CPU): + assert len(densepose_datas) == len( + boxes_xyxy_abs + ), "Attempt to initialize DensePoseList with {} DensePose datas " "and {} boxes".format( + len(densepose_datas), len(boxes_xyxy_abs) + ) + self.densepose_datas = [] + for densepose_data in densepose_datas: + assert isinstance(densepose_data, DensePoseDataRelative) or densepose_data is None, ( + "Attempt to initialize DensePoseList with DensePose datas " + "of type {}, expected DensePoseDataRelative".format(type(densepose_data)) + ) + densepose_data_ondevice = ( + densepose_data.to(device) if densepose_data is not None else None + ) + self.densepose_datas.append(densepose_data_ondevice) + self.boxes_xyxy_abs = boxes_xyxy_abs.to(device) + self.image_size_hw = image_size_hw + self.device = device + + def to(self, device): + if self.device == device: + return self + return DensePoseList(self.densepose_datas, self.boxes_xyxy_abs, self.image_size_hw, device) + + def __iter__(self): + return iter(self.densepose_datas) + + def __len__(self): + return len(self.densepose_datas) + + def __repr__(self): + s = self.__class__.__name__ + "(" + s += "num_instances={}, ".format(len(self.densepose_datas)) + s += "image_width={}, ".format(self.image_size_hw[1]) + s += "image_height={})".format(self.image_size_hw[0]) + return s + + def __getitem__(self, item): + if isinstance(item, int): + densepose_data_rel = self.densepose_datas[item] + return densepose_data_rel + elif isinstance(item, slice): + densepose_datas_rel = self.densepose_datas[item] + boxes_xyxy_abs = self.boxes_xyxy_abs[item] + return DensePoseList( + densepose_datas_rel, boxes_xyxy_abs, self.image_size_hw, self.device + ) + elif isinstance(item, torch.Tensor) and (item.dtype == torch.bool): + densepose_datas_rel = [self.densepose_datas[i] for i, x in enumerate(item) if x > 0] + boxes_xyxy_abs = self.boxes_xyxy_abs[item] + return DensePoseList( + densepose_datas_rel, boxes_xyxy_abs, self.image_size_hw, self.device + ) + else: + densepose_datas_rel = [self.densepose_datas[i] for i in item] + boxes_xyxy_abs = self.boxes_xyxy_abs[item] + return DensePoseList( + densepose_datas_rel, boxes_xyxy_abs, self.image_size_hw, self.device + ) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/structures/mesh.py b/approach/ovod/detectron2/projects/DensePose/densepose/structures/mesh.py new file mode 100644 index 0000000000000000000000000000000000000000..589515d2c4dfc6f94fdd3973e874c0a01fddb5eb --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/structures/mesh.py @@ -0,0 +1,172 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +import pickle +from functools import lru_cache +from typing import Dict, Optional, Tuple +import torch + +from detectron2.utils.file_io import PathManager + +from densepose.data.meshes.catalog import MeshCatalog, MeshInfo + + +def _maybe_copy_to_device( + attribute: Optional[torch.Tensor], device: torch.device +) -> Optional[torch.Tensor]: + if attribute is None: + return None + return attribute.to(device) + + +class Mesh: + def __init__( + self, + vertices: Optional[torch.Tensor] = None, + faces: Optional[torch.Tensor] = None, + geodists: Optional[torch.Tensor] = None, + symmetry: Optional[Dict[str, torch.Tensor]] = None, + texcoords: Optional[torch.Tensor] = None, + mesh_info: Optional[MeshInfo] = None, + device: Optional[torch.device] = None, + ): + """ + Args: + vertices (tensor [N, 3] of float32): vertex coordinates in 3D + faces (tensor [M, 3] of long): triangular face represented as 3 + vertex indices + geodists (tensor [N, N] of float32): geodesic distances from + vertex `i` to vertex `j` (optional, default: None) + symmetry (dict: str -> tensor): various mesh symmetry data: + - "vertex_transforms": vertex mapping under horizontal flip, + tensor of size [N] of type long; vertex `i` is mapped to + vertex `tensor[i]` (optional, default: None) + texcoords (tensor [N, 2] of float32): texture coordinates, i.e. global + and normalized mesh UVs (optional, default: None) + mesh_info (MeshInfo type): necessary to load the attributes on-the-go, + can be used instead of passing all the variables one by one + device (torch.device): device of the Mesh. If not provided, will use + the device of the vertices + """ + self._vertices = vertices + self._faces = faces + self._geodists = geodists + self._symmetry = symmetry + self._texcoords = texcoords + self.mesh_info = mesh_info + self.device = device + + assert self._vertices is not None or self.mesh_info is not None + + all_fields = [self._vertices, self._faces, self._geodists, self._texcoords] + + if self.device is None: + for field in all_fields: + if field is not None: + self.device = field.device + break + if self.device is None and symmetry is not None: + for key in symmetry: + self.device = symmetry[key].device + break + self.device = torch.device("cpu") if self.device is None else self.device + + assert all([var.device == self.device for var in all_fields if var is not None]) + if symmetry: + assert all(symmetry[key].device == self.device for key in symmetry) + if texcoords and vertices: + assert len(vertices) == len(texcoords) + + def to(self, device: torch.device): + device_symmetry = self._symmetry + if device_symmetry: + device_symmetry = {key: value.to(device) for key, value in device_symmetry.items()} + return Mesh( + _maybe_copy_to_device(self._vertices, device), + _maybe_copy_to_device(self._faces, device), + _maybe_copy_to_device(self._geodists, device), + device_symmetry, + _maybe_copy_to_device(self._texcoords, device), + self.mesh_info, + device, + ) + + @property + def vertices(self): + if self._vertices is None and self.mesh_info is not None: + self._vertices = load_mesh_data(self.mesh_info.data, "vertices", self.device) + return self._vertices + + @property + def faces(self): + if self._faces is None and self.mesh_info is not None: + self._faces = load_mesh_data(self.mesh_info.data, "faces", self.device) + return self._faces + + @property + def geodists(self): + if self._geodists is None and self.mesh_info is not None: + self._geodists = load_mesh_auxiliary_data(self.mesh_info.geodists, self.device) + return self._geodists + + @property + def symmetry(self): + if self._symmetry is None and self.mesh_info is not None: + self._symmetry = load_mesh_symmetry(self.mesh_info.symmetry, self.device) + return self._symmetry + + @property + def texcoords(self): + if self._texcoords is None and self.mesh_info is not None: + self._texcoords = load_mesh_auxiliary_data(self.mesh_info.texcoords, self.device) + return self._texcoords + + def get_geodists(self): + if self.geodists is None: + self.geodists = self._compute_geodists() + return self.geodists + + def _compute_geodists(self): + # TODO: compute using Laplace-Beltrami + geodists = None + return geodists + + +def load_mesh_data( + mesh_fpath: str, field: str, device: Optional[torch.device] = None +) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]: + with PathManager.open(mesh_fpath, "rb") as hFile: + # pyre-fixme[7]: Expected `Tuple[Optional[Tensor], Optional[Tensor]]` but + # got `Tensor`. + return torch.as_tensor(pickle.load(hFile)[field], dtype=torch.float).to( # pyre-ignore[6] + device + ) + return None + + +def load_mesh_auxiliary_data( + fpath: str, device: Optional[torch.device] = None +) -> Optional[torch.Tensor]: + fpath_local = PathManager.get_local_path(fpath) + with PathManager.open(fpath_local, "rb") as hFile: + return torch.as_tensor(pickle.load(hFile), dtype=torch.float).to(device) # pyre-ignore[6] + return None + + +@lru_cache() +def load_mesh_symmetry( + symmetry_fpath: str, device: Optional[torch.device] = None +) -> Optional[Dict[str, torch.Tensor]]: + with PathManager.open(symmetry_fpath, "rb") as hFile: + symmetry_loaded = pickle.load(hFile) # pyre-ignore[6] + symmetry = { + "vertex_transforms": torch.as_tensor( + symmetry_loaded["vertex_transforms"], dtype=torch.long + ).to(device), + } + return symmetry + return None + + +@lru_cache() +def create_mesh(mesh_name: str, device: Optional[torch.device] = None) -> Mesh: + return Mesh(mesh_info=MeshCatalog[mesh_name], device=device) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/structures/transform_data.py b/approach/ovod/detectron2/projects/DensePose/densepose/structures/transform_data.py new file mode 100644 index 0000000000000000000000000000000000000000..7cac1bb7663b985165000b2b351d6ff630d2ba3f --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/structures/transform_data.py @@ -0,0 +1,71 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import BinaryIO, Dict, Union +import torch + + +def normalized_coords_transform(x0, y0, w, h): + """ + Coordinates transform that maps top left corner to (-1, -1) and bottom + right corner to (1, 1). Used for torch.grid_sample to initialize the + grid + """ + + def f(p): + return (2 * (p[0] - x0) / w - 1, 2 * (p[1] - y0) / h - 1) + + return f + + +class DensePoseTransformData(object): + + # Horizontal symmetry label transforms used for horizontal flip + MASK_LABEL_SYMMETRIES = [0, 1, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 14] + # fmt: off + POINT_LABEL_SYMMETRIES = [ 0, 1, 2, 4, 3, 6, 5, 8, 7, 10, 9, 12, 11, 14, 13, 16, 15, 18, 17, 20, 19, 22, 21, 24, 23] # noqa + # fmt: on + + def __init__(self, uv_symmetries: Dict[str, torch.Tensor], device: torch.device): + self.mask_label_symmetries = DensePoseTransformData.MASK_LABEL_SYMMETRIES + self.point_label_symmetries = DensePoseTransformData.POINT_LABEL_SYMMETRIES + self.uv_symmetries = uv_symmetries + self.device = torch.device("cpu") + + def to(self, device: torch.device, copy: bool = False) -> "DensePoseTransformData": + """ + Convert transform data to the specified device + + Args: + device (torch.device): device to convert the data to + copy (bool): flag that specifies whether to copy or to reference the data + in case the device is the same + Return: + An instance of `DensePoseTransformData` with data stored on the specified device + """ + if self.device == device and not copy: + return self + uv_symmetry_map = {} + for key in self.uv_symmetries: + uv_symmetry_map[key] = self.uv_symmetries[key].to(device=device, copy=copy) + return DensePoseTransformData(uv_symmetry_map, device) + + @staticmethod + def load(io: Union[str, BinaryIO]): + """ + Args: + io: (str or binary file-like object): input file to load data from + Returns: + An instance of `DensePoseTransformData` with transforms loaded from the file + """ + import scipy.io + + uv_symmetry_map = scipy.io.loadmat(io) + uv_symmetry_map_torch = {} + for key in ["U_transforms", "V_transforms"]: + uv_symmetry_map_torch[key] = [] + map_src = uv_symmetry_map[key] + map_dst = uv_symmetry_map_torch[key] + for i in range(map_src.shape[1]): + map_dst.append(torch.from_numpy(map_src[0, i]).to(dtype=torch.float)) + uv_symmetry_map_torch[key] = torch.stack(map_dst, dim=0) + transform_data = DensePoseTransformData(uv_symmetry_map_torch, device=torch.device("cpu")) + return transform_data diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/utils/__init__.py b/approach/ovod/detectron2/projects/DensePose/densepose/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/utils/dbhelper.py b/approach/ovod/detectron2/projects/DensePose/densepose/utils/dbhelper.py new file mode 100644 index 0000000000000000000000000000000000000000..65b615739a2b1df8b90002995dbd45098858e048 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/utils/dbhelper.py @@ -0,0 +1,147 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import Any, Dict, Optional, Tuple + + +class EntrySelector(object): + """ + Base class for entry selectors + """ + + @staticmethod + def from_string(spec: str) -> "EntrySelector": + if spec == "*": + return AllEntrySelector() + return FieldEntrySelector(spec) + + +class AllEntrySelector(EntrySelector): + """ + Selector that accepts all entries + """ + + SPECIFIER = "*" + + def __call__(self, entry): + return True + + +class FieldEntrySelector(EntrySelector): + """ + Selector that accepts only entries that match provided field + specifier(s). Only a limited set of specifiers is supported for now: + ::=[] + ::=[] + is a valid identifier + ::= "int" | "str" + ::= "=" + ::= "," + ::= ":" + ::= | + ::= + ::= "-" + is a string without spaces and special symbols + (e.g. , , , ) + """ + + _SPEC_DELIM = "," + _TYPE_DELIM = ":" + _RANGE_DELIM = "-" + _EQUAL = "=" + _ERROR_PREFIX = "Invalid field selector specifier" + + class _FieldEntryValuePredicate(object): + """ + Predicate that checks strict equality for the specified entry field + """ + + def __init__(self, name: str, typespec: Optional[str], value: str): + import builtins + + self.name = name + self.type = getattr(builtins, typespec) if typespec is not None else str + self.value = value + + def __call__(self, entry): + return entry[self.name] == self.type(self.value) + + class _FieldEntryRangePredicate(object): + """ + Predicate that checks whether an entry field falls into the specified range + """ + + def __init__(self, name: str, typespec: Optional[str], vmin: str, vmax: str): + import builtins + + self.name = name + self.type = getattr(builtins, typespec) if typespec is not None else str + self.vmin = vmin + self.vmax = vmax + + def __call__(self, entry): + return (entry[self.name] >= self.type(self.vmin)) and ( + entry[self.name] <= self.type(self.vmax) + ) + + def __init__(self, spec: str): + self._predicates = self._parse_specifier_into_predicates(spec) + + def __call__(self, entry: Dict[str, Any]): + for predicate in self._predicates: + if not predicate(entry): + return False + return True + + def _parse_specifier_into_predicates(self, spec: str): + predicates = [] + specs = spec.split(self._SPEC_DELIM) + for subspec in specs: + eq_idx = subspec.find(self._EQUAL) + if eq_idx > 0: + field_name_with_type = subspec[:eq_idx] + field_name, field_type = self._parse_field_name_type(field_name_with_type) + field_value_or_range = subspec[eq_idx + 1 :] + if self._is_range_spec(field_value_or_range): + vmin, vmax = self._get_range_spec(field_value_or_range) + predicate = FieldEntrySelector._FieldEntryRangePredicate( + field_name, field_type, vmin, vmax + ) + else: + predicate = FieldEntrySelector._FieldEntryValuePredicate( + field_name, field_type, field_value_or_range + ) + predicates.append(predicate) + elif eq_idx == 0: + self._parse_error(f'"{subspec}", field name is empty!') + else: + self._parse_error(f'"{subspec}", should have format ' "=!") + return predicates + + def _parse_field_name_type(self, field_name_with_type: str) -> Tuple[str, Optional[str]]: + type_delim_idx = field_name_with_type.find(self._TYPE_DELIM) + if type_delim_idx > 0: + field_name = field_name_with_type[:type_delim_idx] + field_type = field_name_with_type[type_delim_idx + 1 :] + elif type_delim_idx == 0: + self._parse_error(f'"{field_name_with_type}", field name is empty!') + else: + field_name = field_name_with_type + field_type = None + # pyre-fixme[61]: `field_name` may not be initialized here. + # pyre-fixme[61]: `field_type` may not be initialized here. + return field_name, field_type + + def _is_range_spec(self, field_value_or_range): + delim_idx = field_value_or_range.find(self._RANGE_DELIM) + return delim_idx > 0 + + def _get_range_spec(self, field_value_or_range): + if self._is_range_spec(field_value_or_range): + delim_idx = field_value_or_range.find(self._RANGE_DELIM) + vmin = field_value_or_range[:delim_idx] + vmax = field_value_or_range[delim_idx + 1 :] + return vmin, vmax + else: + self._parse_error('"field_value_or_range", range of values expected!') + + def _parse_error(self, msg): + raise ValueError(f"{self._ERROR_PREFIX}: {msg}") diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/utils/logger.py b/approach/ovod/detectron2/projects/DensePose/densepose/utils/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..70cd3cb0eb0fc7495b1a4b50a05725a0e5b1baba --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/utils/logger.py @@ -0,0 +1,13 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging + + +def verbosity_to_level(verbosity) -> int: + if verbosity is not None: + if verbosity == 0: + return logging.WARNING + elif verbosity == 1: + return logging.INFO + elif verbosity >= 2: + return logging.DEBUG + return logging.WARNING diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/utils/transform.py b/approach/ovod/detectron2/projects/DensePose/densepose/utils/transform.py new file mode 100644 index 0000000000000000000000000000000000000000..8dc4ae7be878302ec39b7f235e3ae1b7a3ca29ee --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/utils/transform.py @@ -0,0 +1,15 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from detectron2.data import MetadataCatalog +from detectron2.utils.file_io import PathManager + +from densepose import DensePoseTransformData + + +def load_for_dataset(dataset_name): + path = MetadataCatalog.get(dataset_name).densepose_transform_src + densepose_transform_data_fpath = PathManager.get_local_path(path) + return DensePoseTransformData.load(densepose_transform_data_fpath) + + +def load_from_cfg(cfg): + return load_for_dataset(cfg.DATASETS.TEST[0]) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/vis/__init__.py b/approach/ovod/detectron2/projects/DensePose/densepose/vis/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/vis/base.py b/approach/ovod/detectron2/projects/DensePose/densepose/vis/base.py new file mode 100644 index 0000000000000000000000000000000000000000..7b35397b18e62c195dc15771aa79a1d42b321e7f --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/vis/base.py @@ -0,0 +1,191 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import numpy as np +import cv2 +import torch + +Image = np.ndarray +Boxes = torch.Tensor + + +class MatrixVisualizer(object): + """ + Base visualizer for matrix data + """ + + def __init__( + self, + inplace=True, + cmap=cv2.COLORMAP_PARULA, + val_scale=1.0, + alpha=0.7, + interp_method_matrix=cv2.INTER_LINEAR, + interp_method_mask=cv2.INTER_NEAREST, + ): + self.inplace = inplace + self.cmap = cmap + self.val_scale = val_scale + self.alpha = alpha + self.interp_method_matrix = interp_method_matrix + self.interp_method_mask = interp_method_mask + + def visualize(self, image_bgr, mask, matrix, bbox_xywh): + self._check_image(image_bgr) + self._check_mask_matrix(mask, matrix) + if self.inplace: + image_target_bgr = image_bgr + else: + image_target_bgr = image_bgr * 0 + x, y, w, h = [int(v) for v in bbox_xywh] + if w <= 0 or h <= 0: + return image_bgr + mask, matrix = self._resize(mask, matrix, w, h) + mask_bg = np.tile((mask == 0)[:, :, np.newaxis], [1, 1, 3]) + matrix_scaled = matrix.astype(np.float32) * self.val_scale + _EPSILON = 1e-6 + if np.any(matrix_scaled > 255 + _EPSILON): + logger = logging.getLogger(__name__) + logger.warning( + f"Matrix has values > {255 + _EPSILON} after " f"scaling, clipping to [0..255]" + ) + matrix_scaled_8u = matrix_scaled.clip(0, 255).astype(np.uint8) + matrix_vis = cv2.applyColorMap(matrix_scaled_8u, self.cmap) + matrix_vis[mask_bg] = image_target_bgr[y : y + h, x : x + w, :][mask_bg] + image_target_bgr[y : y + h, x : x + w, :] = ( + image_target_bgr[y : y + h, x : x + w, :] * (1.0 - self.alpha) + matrix_vis * self.alpha + ) + return image_target_bgr.astype(np.uint8) + + def _resize(self, mask, matrix, w, h): + if (w != mask.shape[1]) or (h != mask.shape[0]): + mask = cv2.resize(mask, (w, h), self.interp_method_mask) + if (w != matrix.shape[1]) or (h != matrix.shape[0]): + matrix = cv2.resize(matrix, (w, h), self.interp_method_matrix) + return mask, matrix + + def _check_image(self, image_rgb): + assert len(image_rgb.shape) == 3 + assert image_rgb.shape[2] == 3 + assert image_rgb.dtype == np.uint8 + + def _check_mask_matrix(self, mask, matrix): + assert len(matrix.shape) == 2 + assert len(mask.shape) == 2 + assert mask.dtype == np.uint8 + + +class RectangleVisualizer(object): + + _COLOR_GREEN = (18, 127, 15) + + def __init__(self, color=_COLOR_GREEN, thickness=1): + self.color = color + self.thickness = thickness + + def visualize(self, image_bgr, bbox_xywh, color=None, thickness=None): + x, y, w, h = bbox_xywh + color = color or self.color + thickness = thickness or self.thickness + cv2.rectangle(image_bgr, (int(x), int(y)), (int(x + w), int(y + h)), color, thickness) + return image_bgr + + +class PointsVisualizer(object): + + _COLOR_GREEN = (18, 127, 15) + + def __init__(self, color_bgr=_COLOR_GREEN, r=5): + self.color_bgr = color_bgr + self.r = r + + def visualize(self, image_bgr, pts_xy, colors_bgr=None, rs=None): + for j, pt_xy in enumerate(pts_xy): + x, y = pt_xy + color_bgr = colors_bgr[j] if colors_bgr is not None else self.color_bgr + r = rs[j] if rs is not None else self.r + cv2.circle(image_bgr, (x, y), r, color_bgr, -1) + return image_bgr + + +class TextVisualizer(object): + + _COLOR_GRAY = (218, 227, 218) + _COLOR_WHITE = (255, 255, 255) + + def __init__( + self, + font_face=cv2.FONT_HERSHEY_SIMPLEX, + font_color_bgr=_COLOR_GRAY, + font_scale=0.35, + font_line_type=cv2.LINE_AA, + font_line_thickness=1, + fill_color_bgr=_COLOR_WHITE, + fill_color_transparency=1.0, + frame_color_bgr=_COLOR_WHITE, + frame_color_transparency=1.0, + frame_thickness=1, + ): + self.font_face = font_face + self.font_color_bgr = font_color_bgr + self.font_scale = font_scale + self.font_line_type = font_line_type + self.font_line_thickness = font_line_thickness + self.fill_color_bgr = fill_color_bgr + self.fill_color_transparency = fill_color_transparency + self.frame_color_bgr = frame_color_bgr + self.frame_color_transparency = frame_color_transparency + self.frame_thickness = frame_thickness + + def visualize(self, image_bgr, txt, topleft_xy): + txt_w, txt_h = self.get_text_size_wh(txt) + topleft_xy = tuple(map(int, topleft_xy)) + x, y = topleft_xy + if self.frame_color_transparency < 1.0: + t = self.frame_thickness + image_bgr[y - t : y + txt_h + t, x - t : x + txt_w + t, :] = ( + image_bgr[y - t : y + txt_h + t, x - t : x + txt_w + t, :] + * self.frame_color_transparency + + np.array(self.frame_color_bgr) * (1.0 - self.frame_color_transparency) + ).astype(np.float) + if self.fill_color_transparency < 1.0: + image_bgr[y : y + txt_h, x : x + txt_w, :] = ( + image_bgr[y : y + txt_h, x : x + txt_w, :] * self.fill_color_transparency + + np.array(self.fill_color_bgr) * (1.0 - self.fill_color_transparency) + ).astype(np.float) + cv2.putText( + image_bgr, + txt, + topleft_xy, + self.font_face, + self.font_scale, + self.font_color_bgr, + self.font_line_thickness, + self.font_line_type, + ) + return image_bgr + + def get_text_size_wh(self, txt): + ((txt_w, txt_h), _) = cv2.getTextSize( + txt, self.font_face, self.font_scale, self.font_line_thickness + ) + return txt_w, txt_h + + +class CompoundVisualizer(object): + def __init__(self, visualizers): + self.visualizers = visualizers + + def visualize(self, image_bgr, data): + assert len(data) == len( + self.visualizers + ), "The number of datas {} should match the number of visualizers" " {}".format( + len(data), len(self.visualizers) + ) + image = image_bgr + for i, visualizer in enumerate(self.visualizers): + image = visualizer.visualize(image, data[i]) + return image + + def __str__(self): + visualizer_str = ", ".join([str(v) for v in self.visualizers]) + return "Compound Visualizer [{}]".format(visualizer_str) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/vis/bounding_box.py b/approach/ovod/detectron2/projects/DensePose/densepose/vis/bounding_box.py new file mode 100644 index 0000000000000000000000000000000000000000..4f83957221f4503e707f2270a20e8d3829a299af --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/vis/bounding_box.py @@ -0,0 +1,37 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .base import RectangleVisualizer, TextVisualizer + + +class BoundingBoxVisualizer(object): + def __init__(self): + self.rectangle_visualizer = RectangleVisualizer() + + def visualize(self, image_bgr, boxes_xywh): + for bbox_xywh in boxes_xywh: + image_bgr = self.rectangle_visualizer.visualize(image_bgr, bbox_xywh) + return image_bgr + + +class ScoredBoundingBoxVisualizer(object): + def __init__(self, bbox_visualizer_params=None, score_visualizer_params=None, **kwargs): + if bbox_visualizer_params is None: + bbox_visualizer_params = {} + if score_visualizer_params is None: + score_visualizer_params = {} + self.visualizer_bbox = RectangleVisualizer(**bbox_visualizer_params) + self.visualizer_score = TextVisualizer(**score_visualizer_params) + + def visualize(self, image_bgr, scored_bboxes): + boxes_xywh, box_scores = scored_bboxes + assert len(boxes_xywh) == len( + box_scores + ), "Number of bounding boxes {} should be equal to the number of scores {}".format( + len(boxes_xywh), len(box_scores) + ) + for i, box_xywh in enumerate(boxes_xywh): + score_i = box_scores[i] + image_bgr = self.visualizer_bbox.visualize(image_bgr, box_xywh) + score_txt = "{0:6.4f}".format(score_i) + topleft_xy = box_xywh[0], box_xywh[1] + image_bgr = self.visualizer_score.visualize(image_bgr, score_txt, topleft_xy) + return image_bgr diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/vis/densepose_data_points.py b/approach/ovod/detectron2/projects/DensePose/densepose/vis/densepose_data_points.py new file mode 100644 index 0000000000000000000000000000000000000000..b6839a984a4df5691f31314efd4a815d75121b3b --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/vis/densepose_data_points.py @@ -0,0 +1,106 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import numpy as np +from typing import Iterable, Optional, Tuple +import cv2 + +from densepose.structures import DensePoseDataRelative + +from .base import Boxes, Image, MatrixVisualizer, PointsVisualizer + + +class DensePoseDataCoarseSegmentationVisualizer(object): + """ + Visualizer for ground truth segmentation + """ + + def __init__(self, inplace=True, cmap=cv2.COLORMAP_PARULA, alpha=0.7, **kwargs): + self.mask_visualizer = MatrixVisualizer( + inplace=inplace, + cmap=cmap, + val_scale=255.0 / DensePoseDataRelative.N_BODY_PARTS, + alpha=alpha, + ) + + def visualize( + self, + image_bgr: Image, + bbox_densepose_datas: Optional[Tuple[Iterable[Boxes], Iterable[DensePoseDataRelative]]], + ) -> Image: + if bbox_densepose_datas is None: + return image_bgr + for bbox_xywh, densepose_data in zip(*bbox_densepose_datas): + matrix = densepose_data.segm.numpy() + mask = np.zeros(matrix.shape, dtype=np.uint8) + mask[matrix > 0] = 1 + image_bgr = self.mask_visualizer.visualize(image_bgr, mask, matrix, bbox_xywh.numpy()) + return image_bgr + + +class DensePoseDataPointsVisualizer(object): + def __init__(self, densepose_data_to_value_fn=None, cmap=cv2.COLORMAP_PARULA, **kwargs): + self.points_visualizer = PointsVisualizer() + self.densepose_data_to_value_fn = densepose_data_to_value_fn + self.cmap = cmap + + def visualize( + self, + image_bgr: Image, + bbox_densepose_datas: Optional[Tuple[Iterable[Boxes], Iterable[DensePoseDataRelative]]], + ) -> Image: + if bbox_densepose_datas is None: + return image_bgr + for bbox_xywh, densepose_data in zip(*bbox_densepose_datas): + x0, y0, w, h = bbox_xywh.numpy() + x = densepose_data.x.numpy() * w / 255.0 + x0 + y = densepose_data.y.numpy() * h / 255.0 + y0 + pts_xy = zip(x, y) + if self.densepose_data_to_value_fn is None: + image_bgr = self.points_visualizer.visualize(image_bgr, pts_xy) + else: + v = self.densepose_data_to_value_fn(densepose_data) + img_colors_bgr = cv2.applyColorMap(v, self.cmap) + colors_bgr = [ + [int(v) for v in img_color_bgr.ravel()] for img_color_bgr in img_colors_bgr + ] + image_bgr = self.points_visualizer.visualize(image_bgr, pts_xy, colors_bgr) + return image_bgr + + +def _densepose_data_u_for_cmap(densepose_data): + u = np.clip(densepose_data.u.numpy(), 0, 1) * 255.0 + return u.astype(np.uint8) + + +def _densepose_data_v_for_cmap(densepose_data): + v = np.clip(densepose_data.v.numpy(), 0, 1) * 255.0 + return v.astype(np.uint8) + + +def _densepose_data_i_for_cmap(densepose_data): + i = ( + np.clip(densepose_data.i.numpy(), 0.0, DensePoseDataRelative.N_PART_LABELS) + * 255.0 + / DensePoseDataRelative.N_PART_LABELS + ) + return i.astype(np.uint8) + + +class DensePoseDataPointsUVisualizer(DensePoseDataPointsVisualizer): + def __init__(self, **kwargs): + super(DensePoseDataPointsUVisualizer, self).__init__( + densepose_data_to_value_fn=_densepose_data_u_for_cmap, **kwargs + ) + + +class DensePoseDataPointsVVisualizer(DensePoseDataPointsVisualizer): + def __init__(self, **kwargs): + super(DensePoseDataPointsVVisualizer, self).__init__( + densepose_data_to_value_fn=_densepose_data_v_for_cmap, **kwargs + ) + + +class DensePoseDataPointsIVisualizer(DensePoseDataPointsVisualizer): + def __init__(self, **kwargs): + super(DensePoseDataPointsIVisualizer, self).__init__( + densepose_data_to_value_fn=_densepose_data_i_for_cmap, **kwargs + ) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/vis/densepose_outputs_iuv.py b/approach/ovod/detectron2/projects/DensePose/densepose/vis/densepose_outputs_iuv.py new file mode 100644 index 0000000000000000000000000000000000000000..a32a418b33e0f54988e4ebc2b8725021fe6f19dc --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/vis/densepose_outputs_iuv.py @@ -0,0 +1,101 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import numpy as np +from typing import Optional, Tuple +import cv2 + +from densepose.structures import DensePoseDataRelative + +from ..structures import DensePoseChartPredictorOutput +from .base import Boxes, Image, MatrixVisualizer + + +class DensePoseOutputsVisualizer(object): + def __init__( + self, inplace=True, cmap=cv2.COLORMAP_PARULA, alpha=0.7, to_visualize=None, **kwargs + ): + assert to_visualize in "IUV", "can only visualize IUV" + self.to_visualize = to_visualize + + if self.to_visualize == "I": + val_scale = 255.0 / DensePoseDataRelative.N_PART_LABELS + else: + val_scale = 1.0 + self.mask_visualizer = MatrixVisualizer( + inplace=inplace, cmap=cmap, val_scale=val_scale, alpha=alpha + ) + + def visualize( + self, + image_bgr: Image, + dp_output_with_bboxes: Tuple[Optional[DensePoseChartPredictorOutput], Optional[Boxes]], + ) -> Image: + densepose_output, bboxes_xywh = dp_output_with_bboxes + if densepose_output is None or bboxes_xywh is None: + return image_bgr + + assert isinstance( + densepose_output, DensePoseChartPredictorOutput + ), "DensePoseChartPredictorOutput expected, {} encountered".format(type(densepose_output)) + + S = densepose_output.coarse_segm + I = densepose_output.fine_segm # noqa + U = densepose_output.u + V = densepose_output.v + N = S.size(0) + assert N == I.size( + 0 + ), "densepose outputs S {} and I {}" " should have equal first dim size".format( + S.size(), I.size() + ) + assert N == U.size( + 0 + ), "densepose outputs S {} and U {}" " should have equal first dim size".format( + S.size(), U.size() + ) + assert N == V.size( + 0 + ), "densepose outputs S {} and V {}" " should have equal first dim size".format( + S.size(), V.size() + ) + assert N == len( + bboxes_xywh + ), "number of bounding boxes {}" " should be equal to first dim size of outputs {}".format( + len(bboxes_xywh), N + ) + for n in range(N): + Sn = S[n].argmax(dim=0) + In = I[n].argmax(dim=0) * (Sn > 0).long() + segmentation = In.cpu().numpy().astype(np.uint8) + mask = np.zeros(segmentation.shape, dtype=np.uint8) + mask[segmentation > 0] = 1 + bbox_xywh = bboxes_xywh[n] + + if self.to_visualize == "I": + vis = segmentation + elif self.to_visualize in "UV": + U_or_Vn = {"U": U, "V": V}[self.to_visualize][n].cpu().numpy().astype(np.float32) + vis = np.zeros(segmentation.shape, dtype=np.float32) + for partId in range(U_or_Vn.shape[0]): + vis[segmentation == partId] = ( + U_or_Vn[partId][segmentation == partId].clip(0, 1) * 255 + ) + + # pyre-fixme[61]: `vis` may not be initialized here. + image_bgr = self.mask_visualizer.visualize(image_bgr, mask, vis, bbox_xywh) + + return image_bgr + + +class DensePoseOutputsUVisualizer(DensePoseOutputsVisualizer): + def __init__(self, inplace=True, cmap=cv2.COLORMAP_PARULA, alpha=0.7, **kwargs): + super().__init__(inplace=inplace, cmap=cmap, alpha=alpha, to_visualize="U", **kwargs) + + +class DensePoseOutputsVVisualizer(DensePoseOutputsVisualizer): + def __init__(self, inplace=True, cmap=cv2.COLORMAP_PARULA, alpha=0.7, **kwargs): + super().__init__(inplace=inplace, cmap=cmap, alpha=alpha, to_visualize="V", **kwargs) + + +class DensePoseOutputsFineSegmentationVisualizer(DensePoseOutputsVisualizer): + def __init__(self, inplace=True, cmap=cv2.COLORMAP_PARULA, alpha=0.7, **kwargs): + super().__init__(inplace=inplace, cmap=cmap, alpha=alpha, to_visualize="I", **kwargs) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/vis/densepose_outputs_vertex.py b/approach/ovod/detectron2/projects/DensePose/densepose/vis/densepose_outputs_vertex.py new file mode 100644 index 0000000000000000000000000000000000000000..71e5323c2bd3a29bc90e66d7d59d524033c120bf --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/vis/densepose_outputs_vertex.py @@ -0,0 +1,229 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import json +import numpy as np +from functools import lru_cache +from typing import Dict, List, Optional, Tuple +import cv2 +import torch + +from detectron2.utils.file_io import PathManager + +from densepose.modeling import build_densepose_embedder +from densepose.modeling.cse.utils import get_closest_vertices_mask_from_ES + +from ..data.utils import get_class_to_mesh_name_mapping +from ..structures import DensePoseEmbeddingPredictorOutput +from ..structures.mesh import create_mesh +from .base import Boxes, Image, MatrixVisualizer +from .densepose_results_textures import get_texture_atlas + + +@lru_cache() +def get_xyz_vertex_embedding(mesh_name: str, device: torch.device): + if mesh_name == "smpl_27554": + embed_path = PathManager.get_local_path( + "https://dl.fbaipublicfiles.com/densepose/data/cse/mds_d=256.npy" + ) + embed_map, _ = np.load(embed_path, allow_pickle=True) + embed_map = torch.tensor(embed_map).float()[:, 0] + embed_map -= embed_map.min() + embed_map /= embed_map.max() + else: + mesh = create_mesh(mesh_name, device) + embed_map = mesh.vertices.sum(dim=1) + embed_map -= embed_map.min() + embed_map /= embed_map.max() + embed_map = embed_map**2 + return embed_map + + +class DensePoseOutputsVertexVisualizer(object): + def __init__( + self, + cfg, + inplace=True, + cmap=cv2.COLORMAP_JET, + alpha=0.7, + device="cuda", + default_class=0, + **kwargs, + ): + self.mask_visualizer = MatrixVisualizer( + inplace=inplace, cmap=cmap, val_scale=1.0, alpha=alpha + ) + self.class_to_mesh_name = get_class_to_mesh_name_mapping(cfg) + self.embedder = build_densepose_embedder(cfg) + self.device = torch.device(device) + self.default_class = default_class + + self.mesh_vertex_embeddings = { + mesh_name: self.embedder(mesh_name).to(self.device) + for mesh_name in self.class_to_mesh_name.values() + if self.embedder.has_embeddings(mesh_name) + } + + def visualize( + self, + image_bgr: Image, + outputs_boxes_xywh_classes: Tuple[ + Optional[DensePoseEmbeddingPredictorOutput], Optional[Boxes], Optional[List[int]] + ], + ) -> Image: + if outputs_boxes_xywh_classes[0] is None: + return image_bgr + + S, E, N, bboxes_xywh, pred_classes = self.extract_and_check_outputs_and_boxes( + outputs_boxes_xywh_classes + ) + + for n in range(N): + x, y, w, h = bboxes_xywh[n].int().tolist() + mesh_name = self.class_to_mesh_name[pred_classes[n]] + closest_vertices, mask = get_closest_vertices_mask_from_ES( + E[[n]], + S[[n]], + h, + w, + self.mesh_vertex_embeddings[mesh_name], + self.device, + ) + embed_map = get_xyz_vertex_embedding(mesh_name, self.device) + vis = (embed_map[closest_vertices].clip(0, 1) * 255.0).cpu().numpy() + mask_numpy = mask.cpu().numpy().astype(dtype=np.uint8) + image_bgr = self.mask_visualizer.visualize(image_bgr, mask_numpy, vis, [x, y, w, h]) + + return image_bgr + + def extract_and_check_outputs_and_boxes(self, outputs_boxes_xywh_classes): + + densepose_output, bboxes_xywh, pred_classes = outputs_boxes_xywh_classes + + if pred_classes is None: + pred_classes = [self.default_class] * len(bboxes_xywh) + + assert isinstance( + densepose_output, DensePoseEmbeddingPredictorOutput + ), "DensePoseEmbeddingPredictorOutput expected, {} encountered".format( + type(densepose_output) + ) + + S = densepose_output.coarse_segm + E = densepose_output.embedding + N = S.size(0) + assert N == E.size( + 0 + ), "CSE coarse_segm {} and embeddings {}" " should have equal first dim size".format( + S.size(), E.size() + ) + assert N == len( + bboxes_xywh + ), "number of bounding boxes {}" " should be equal to first dim size of outputs {}".format( + len(bboxes_xywh), N + ) + assert N == len(pred_classes), ( + "number of predicted classes {}" + " should be equal to first dim size of outputs {}".format(len(bboxes_xywh), N) + ) + + return S, E, N, bboxes_xywh, pred_classes + + +def get_texture_atlases(json_str: Optional[str]) -> Optional[Dict[str, Optional[np.ndarray]]]: + """ + json_str is a JSON string representing a mesh_name -> texture_atlas_path dictionary + """ + if json_str is None: + return None + + paths = json.loads(json_str) + return {mesh_name: get_texture_atlas(path) for mesh_name, path in paths.items()} + + +class DensePoseOutputsTextureVisualizer(DensePoseOutputsVertexVisualizer): + def __init__( + self, + cfg, + texture_atlases_dict, + device="cuda", + default_class=0, + **kwargs, + ): + self.embedder = build_densepose_embedder(cfg) + + self.texture_image_dict = {} + self.alpha_dict = {} + + for mesh_name in texture_atlases_dict.keys(): + if texture_atlases_dict[mesh_name].shape[-1] == 4: # Image with alpha channel + self.alpha_dict[mesh_name] = texture_atlases_dict[mesh_name][:, :, -1] / 255.0 + self.texture_image_dict[mesh_name] = texture_atlases_dict[mesh_name][:, :, :3] + else: + self.alpha_dict[mesh_name] = texture_atlases_dict[mesh_name].sum(axis=-1) > 0 + self.texture_image_dict[mesh_name] = texture_atlases_dict[mesh_name] + + self.device = torch.device(device) + self.class_to_mesh_name = get_class_to_mesh_name_mapping(cfg) + self.default_class = default_class + + self.mesh_vertex_embeddings = { + mesh_name: self.embedder(mesh_name).to(self.device) + for mesh_name in self.class_to_mesh_name.values() + } + + def visualize( + self, + image_bgr: Image, + outputs_boxes_xywh_classes: Tuple[ + Optional[DensePoseEmbeddingPredictorOutput], Optional[Boxes], Optional[List[int]] + ], + ) -> Image: + image_target_bgr = image_bgr.copy() + if outputs_boxes_xywh_classes[0] is None: + return image_target_bgr + + S, E, N, bboxes_xywh, pred_classes = self.extract_and_check_outputs_and_boxes( + outputs_boxes_xywh_classes + ) + + meshes = { + p: create_mesh(self.class_to_mesh_name[p], self.device) for p in np.unique(pred_classes) + } + + for n in range(N): + x, y, w, h = bboxes_xywh[n].int().cpu().numpy() + mesh_name = self.class_to_mesh_name[pred_classes[n]] + closest_vertices, mask = get_closest_vertices_mask_from_ES( + E[[n]], + S[[n]], + h, + w, + self.mesh_vertex_embeddings[mesh_name], + self.device, + ) + uv_array = meshes[pred_classes[n]].texcoords[closest_vertices].permute((2, 0, 1)) + uv_array = uv_array.cpu().numpy().clip(0, 1) + textured_image = self.generate_image_with_texture( + image_target_bgr[y : y + h, x : x + w], + uv_array, + mask.cpu().numpy(), + self.class_to_mesh_name[pred_classes[n]], + ) + if textured_image is None: + continue + image_target_bgr[y : y + h, x : x + w] = textured_image + + return image_target_bgr + + def generate_image_with_texture(self, bbox_image_bgr, uv_array, mask, mesh_name): + alpha = self.alpha_dict.get(mesh_name) + texture_image = self.texture_image_dict.get(mesh_name) + if alpha is None or texture_image is None: + return None + U, V = uv_array + x_index = (U * texture_image.shape[1]).astype(int) + y_index = (V * texture_image.shape[0]).astype(int) + local_texture = texture_image[y_index, x_index][mask] + local_alpha = np.expand_dims(alpha[y_index, x_index][mask], -1) + output_image = bbox_image_bgr.copy() + output_image[mask] = output_image[mask] * (1 - local_alpha) + local_texture * local_alpha + return output_image.astype(np.uint8) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/vis/densepose_results.py b/approach/ovod/detectron2/projects/DensePose/densepose/vis/densepose_results.py new file mode 100644 index 0000000000000000000000000000000000000000..ce8a7c0e207f5b3b6e755c759a59f5bed9965cef --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/vis/densepose_results.py @@ -0,0 +1,355 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import numpy as np +from typing import List, Optional, Tuple +import cv2 +import torch + +from densepose.structures import DensePoseDataRelative + +from ..structures import DensePoseChartResult +from .base import Boxes, Image, MatrixVisualizer + + +class DensePoseResultsVisualizer(object): + def visualize( + self, + image_bgr: Image, + results_and_boxes_xywh: Tuple[Optional[List[DensePoseChartResult]], Optional[Boxes]], + ) -> Image: + densepose_result, boxes_xywh = results_and_boxes_xywh + if densepose_result is None or boxes_xywh is None: + return image_bgr + + boxes_xywh = boxes_xywh.cpu().numpy() + context = self.create_visualization_context(image_bgr) + for i, result in enumerate(densepose_result): + iuv_array = torch.cat( + (result.labels[None].type(torch.float32), result.uv * 255.0) + ).type(torch.uint8) + self.visualize_iuv_arr(context, iuv_array.cpu().numpy(), boxes_xywh[i]) + image_bgr = self.context_to_image_bgr(context) + return image_bgr + + def create_visualization_context(self, image_bgr: Image): + return image_bgr + + def visualize_iuv_arr(self, context, iuv_arr: np.ndarray, bbox_xywh) -> None: + pass + + def context_to_image_bgr(self, context): + return context + + def get_image_bgr_from_context(self, context): + return context + + +class DensePoseMaskedColormapResultsVisualizer(DensePoseResultsVisualizer): + def __init__( + self, + data_extractor, + segm_extractor, + inplace=True, + cmap=cv2.COLORMAP_PARULA, + alpha=0.7, + val_scale=1.0, + **kwargs, + ): + self.mask_visualizer = MatrixVisualizer( + inplace=inplace, cmap=cmap, val_scale=val_scale, alpha=alpha + ) + self.data_extractor = data_extractor + self.segm_extractor = segm_extractor + + def context_to_image_bgr(self, context): + return context + + def visualize_iuv_arr(self, context, iuv_arr: np.ndarray, bbox_xywh) -> None: + image_bgr = self.get_image_bgr_from_context(context) + matrix = self.data_extractor(iuv_arr) + segm = self.segm_extractor(iuv_arr) + mask = np.zeros(matrix.shape, dtype=np.uint8) + mask[segm > 0] = 1 + image_bgr = self.mask_visualizer.visualize(image_bgr, mask, matrix, bbox_xywh) + + +def _extract_i_from_iuvarr(iuv_arr): + return iuv_arr[0, :, :] + + +def _extract_u_from_iuvarr(iuv_arr): + return iuv_arr[1, :, :] + + +def _extract_v_from_iuvarr(iuv_arr): + return iuv_arr[2, :, :] + + +class DensePoseResultsMplContourVisualizer(DensePoseResultsVisualizer): + def __init__(self, levels=10, **kwargs): + self.levels = levels + self.plot_args = kwargs + + def create_visualization_context(self, image_bgr: Image): + import matplotlib.pyplot as plt + from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas + + context = {} + context["image_bgr"] = image_bgr + dpi = 100 + height_inches = float(image_bgr.shape[0]) / dpi + width_inches = float(image_bgr.shape[1]) / dpi + fig = plt.figure(figsize=(width_inches, height_inches), dpi=dpi) + plt.axes([0, 0, 1, 1]) + plt.axis("off") + context["fig"] = fig + canvas = FigureCanvas(fig) + context["canvas"] = canvas + extent = (0, image_bgr.shape[1], image_bgr.shape[0], 0) + plt.imshow(image_bgr[:, :, ::-1], extent=extent) + return context + + def context_to_image_bgr(self, context): + fig = context["fig"] + w, h = map(int, fig.get_size_inches() * fig.get_dpi()) + canvas = context["canvas"] + canvas.draw() + image_1d = np.fromstring(canvas.tostring_rgb(), dtype="uint8") + image_rgb = image_1d.reshape(h, w, 3) + image_bgr = image_rgb[:, :, ::-1].copy() + return image_bgr + + def visualize_iuv_arr(self, context, iuv_arr: np.ndarray, bbox_xywh: Boxes) -> None: + import matplotlib.pyplot as plt + + u = _extract_u_from_iuvarr(iuv_arr).astype(float) / 255.0 + v = _extract_v_from_iuvarr(iuv_arr).astype(float) / 255.0 + extent = ( + bbox_xywh[0], + bbox_xywh[0] + bbox_xywh[2], + bbox_xywh[1], + bbox_xywh[1] + bbox_xywh[3], + ) + plt.contour(u, self.levels, extent=extent, **self.plot_args) + plt.contour(v, self.levels, extent=extent, **self.plot_args) + + +class DensePoseResultsCustomContourVisualizer(DensePoseResultsVisualizer): + """ + Contour visualization using marching squares + """ + + def __init__(self, levels=10, **kwargs): + # TODO: colormap is hardcoded + cmap = cv2.COLORMAP_PARULA + if isinstance(levels, int): + self.levels = np.linspace(0, 1, levels) + else: + self.levels = levels + if "linewidths" in kwargs: + self.linewidths = kwargs["linewidths"] + else: + self.linewidths = [1] * len(self.levels) + self.plot_args = kwargs + img_colors_bgr = cv2.applyColorMap((self.levels * 255).astype(np.uint8), cmap) + self.level_colors_bgr = [ + [int(v) for v in img_color_bgr.ravel()] for img_color_bgr in img_colors_bgr + ] + + def visualize_iuv_arr(self, context, iuv_arr: np.ndarray, bbox_xywh: Boxes) -> None: + image_bgr = self.get_image_bgr_from_context(context) + segm = _extract_i_from_iuvarr(iuv_arr) + u = _extract_u_from_iuvarr(iuv_arr).astype(float) / 255.0 + v = _extract_v_from_iuvarr(iuv_arr).astype(float) / 255.0 + self._contours(image_bgr, u, segm, bbox_xywh) + self._contours(image_bgr, v, segm, bbox_xywh) + + def _contours(self, image_bgr, arr, segm, bbox_xywh): + for part_idx in range(1, DensePoseDataRelative.N_PART_LABELS + 1): + mask = segm == part_idx + if not np.any(mask): + continue + arr_min = np.amin(arr[mask]) + arr_max = np.amax(arr[mask]) + I, J = np.nonzero(mask) + i0 = np.amin(I) + i1 = np.amax(I) + 1 + j0 = np.amin(J) + j1 = np.amax(J) + 1 + if (j1 == j0 + 1) or (i1 == i0 + 1): + continue + Nw = arr.shape[1] - 1 + Nh = arr.shape[0] - 1 + for level_idx, level in enumerate(self.levels): + if (level < arr_min) or (level > arr_max): + continue + vp = arr[i0:i1, j0:j1] >= level + bin_codes = vp[:-1, :-1] + vp[1:, :-1] * 2 + vp[1:, 1:] * 4 + vp[:-1, 1:] * 8 + mp = mask[i0:i1, j0:j1] + bin_mask_codes = mp[:-1, :-1] + mp[1:, :-1] * 2 + mp[1:, 1:] * 4 + mp[:-1, 1:] * 8 + it = np.nditer(bin_codes, flags=["multi_index"]) + color_bgr = self.level_colors_bgr[level_idx] + linewidth = self.linewidths[level_idx] + while not it.finished: + if (it[0] != 0) and (it[0] != 15): + i, j = it.multi_index + if bin_mask_codes[i, j] != 0: + self._draw_line( + image_bgr, + arr, + mask, + level, + color_bgr, + linewidth, + it[0], + it.multi_index, + bbox_xywh, + Nw, + Nh, + (i0, j0), + ) + it.iternext() + + def _draw_line( + self, + image_bgr, + arr, + mask, + v, + color_bgr, + linewidth, + bin_code, + multi_idx, + bbox_xywh, + Nw, + Nh, + offset, + ): + lines = self._bin_code_2_lines(arr, v, bin_code, multi_idx, Nw, Nh, offset) + x0, y0, w, h = bbox_xywh + x1 = x0 + w + y1 = y0 + h + for line in lines: + x0r, y0r = line[0] + x1r, y1r = line[1] + pt0 = (int(x0 + x0r * (x1 - x0)), int(y0 + y0r * (y1 - y0))) + pt1 = (int(x0 + x1r * (x1 - x0)), int(y0 + y1r * (y1 - y0))) + cv2.line(image_bgr, pt0, pt1, color_bgr, linewidth) + + def _bin_code_2_lines(self, arr, v, bin_code, multi_idx, Nw, Nh, offset): + i0, j0 = offset + i, j = multi_idx + i += i0 + j += j0 + v0, v1, v2, v3 = arr[i, j], arr[i + 1, j], arr[i + 1, j + 1], arr[i, j + 1] + x0i = float(j) / Nw + y0j = float(i) / Nh + He = 1.0 / Nh + We = 1.0 / Nw + if (bin_code == 1) or (bin_code == 14): + a = (v - v0) / (v1 - v0) + b = (v - v0) / (v3 - v0) + pt1 = (x0i, y0j + a * He) + pt2 = (x0i + b * We, y0j) + return [(pt1, pt2)] + elif (bin_code == 2) or (bin_code == 13): + a = (v - v0) / (v1 - v0) + b = (v - v1) / (v2 - v1) + pt1 = (x0i, y0j + a * He) + pt2 = (x0i + b * We, y0j + He) + return [(pt1, pt2)] + elif (bin_code == 3) or (bin_code == 12): + a = (v - v0) / (v3 - v0) + b = (v - v1) / (v2 - v1) + pt1 = (x0i + a * We, y0j) + pt2 = (x0i + b * We, y0j + He) + return [(pt1, pt2)] + elif (bin_code == 4) or (bin_code == 11): + a = (v - v1) / (v2 - v1) + b = (v - v3) / (v2 - v3) + pt1 = (x0i + a * We, y0j + He) + pt2 = (x0i + We, y0j + b * He) + return [(pt1, pt2)] + elif (bin_code == 6) or (bin_code == 9): + a = (v - v0) / (v1 - v0) + b = (v - v3) / (v2 - v3) + pt1 = (x0i, y0j + a * He) + pt2 = (x0i + We, y0j + b * He) + return [(pt1, pt2)] + elif (bin_code == 7) or (bin_code == 8): + a = (v - v0) / (v3 - v0) + b = (v - v3) / (v2 - v3) + pt1 = (x0i + a * We, y0j) + pt2 = (x0i + We, y0j + b * He) + return [(pt1, pt2)] + elif bin_code == 5: + a1 = (v - v0) / (v1 - v0) + b1 = (v - v1) / (v2 - v1) + pt11 = (x0i, y0j + a1 * He) + pt12 = (x0i + b1 * We, y0j + He) + a2 = (v - v0) / (v3 - v0) + b2 = (v - v3) / (v2 - v3) + pt21 = (x0i + a2 * We, y0j) + pt22 = (x0i + We, y0j + b2 * He) + return [(pt11, pt12), (pt21, pt22)] + elif bin_code == 10: + a1 = (v - v0) / (v3 - v0) + b1 = (v - v0) / (v1 - v0) + pt11 = (x0i + a1 * We, y0j) + pt12 = (x0i, y0j + b1 * He) + a2 = (v - v1) / (v2 - v1) + b2 = (v - v3) / (v2 - v3) + pt21 = (x0i + a2 * We, y0j + He) + pt22 = (x0i + We, y0j + b2 * He) + return [(pt11, pt12), (pt21, pt22)] + return [] + + +try: + import matplotlib + + matplotlib.use("Agg") + DensePoseResultsContourVisualizer = DensePoseResultsMplContourVisualizer +except ModuleNotFoundError: + logger = logging.getLogger(__name__) + logger.warning("Could not import matplotlib, using custom contour visualizer") + DensePoseResultsContourVisualizer = DensePoseResultsCustomContourVisualizer + + +class DensePoseResultsFineSegmentationVisualizer(DensePoseMaskedColormapResultsVisualizer): + def __init__(self, inplace=True, cmap=cv2.COLORMAP_PARULA, alpha=0.7, **kwargs): + super(DensePoseResultsFineSegmentationVisualizer, self).__init__( + _extract_i_from_iuvarr, + _extract_i_from_iuvarr, + inplace, + cmap, + alpha, + val_scale=255.0 / DensePoseDataRelative.N_PART_LABELS, + **kwargs, + ) + + +class DensePoseResultsUVisualizer(DensePoseMaskedColormapResultsVisualizer): + def __init__(self, inplace=True, cmap=cv2.COLORMAP_PARULA, alpha=0.7, **kwargs): + super(DensePoseResultsUVisualizer, self).__init__( + _extract_u_from_iuvarr, + _extract_i_from_iuvarr, + inplace, + cmap, + alpha, + val_scale=1.0, + **kwargs, + ) + + +class DensePoseResultsVVisualizer(DensePoseMaskedColormapResultsVisualizer): + def __init__(self, inplace=True, cmap=cv2.COLORMAP_PARULA, alpha=0.7, **kwargs): + super(DensePoseResultsVVisualizer, self).__init__( + _extract_v_from_iuvarr, + _extract_i_from_iuvarr, + inplace, + cmap, + alpha, + val_scale=1.0, + **kwargs, + ) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/vis/densepose_results_textures.py b/approach/ovod/detectron2/projects/DensePose/densepose/vis/densepose_results_textures.py new file mode 100644 index 0000000000000000000000000000000000000000..8b02f2bdbaa8bb1b70bc0f690a568ac4f8f1c91a --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/vis/densepose_results_textures.py @@ -0,0 +1,91 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import numpy as np +from typing import List, Optional, Tuple +import torch + +from detectron2.data.detection_utils import read_image + +from ..structures import DensePoseChartResult +from .base import Boxes, Image +from .densepose_results import DensePoseResultsVisualizer + + +def get_texture_atlas(path: Optional[str]) -> Optional[np.ndarray]: + if path is None: + return None + + # Reading images like that downsamples 16-bit images to 8-bit + # If 16-bit images are needed, we can replace that by cv2.imread with the + # cv2.IMREAD_UNCHANGED flag (with cv2 we also need it to keep alpha channels) + # The rest of the pipeline would need to be adapted to 16-bit images too + bgr_image = read_image(path) + rgb_image = np.copy(bgr_image) # Convert BGR -> RGB + rgb_image[:, :, :3] = rgb_image[:, :, 2::-1] # Works with alpha channel + return rgb_image + + +class DensePoseResultsVisualizerWithTexture(DensePoseResultsVisualizer): + """ + texture_atlas: An image, size 6N * 4N, with N * N squares for each of the 24 body parts. + It must follow the grid found at https://github.com/facebookresearch/DensePose/blob/master/DensePoseData/demo_data/texture_atlas_200.png # noqa + For each body part, U is proportional to the x coordinate, and (1 - V) to y + """ + + def __init__(self, texture_atlas, **kwargs): + self.texture_atlas = texture_atlas + self.body_part_size = texture_atlas.shape[0] // 6 + assert self.body_part_size == texture_atlas.shape[1] // 4 + + def visualize( + self, + image_bgr: Image, + results_and_boxes_xywh: Tuple[Optional[List[DensePoseChartResult]], Optional[Boxes]], + ) -> Image: + densepose_result, boxes_xywh = results_and_boxes_xywh + if densepose_result is None or boxes_xywh is None: + return image_bgr + + boxes_xywh = boxes_xywh.int().cpu().numpy() + texture_image, alpha = self.get_texture() + for i, result in enumerate(densepose_result): + iuv_array = torch.cat((result.labels[None], result.uv.clamp(0, 1))) + x, y, w, h = boxes_xywh[i] + bbox_image = image_bgr[y : y + h, x : x + w] + image_bgr[y : y + h, x : x + w] = self.generate_image_with_texture( + texture_image, alpha, bbox_image, iuv_array.cpu().numpy() + ) + return image_bgr + + def get_texture(self): + N = self.body_part_size + texture_image = np.zeros([24, N, N, self.texture_atlas.shape[-1]]) + for i in range(4): + for j in range(6): + texture_image[(6 * i + j), :, :, :] = self.texture_atlas[ + N * j : N * (j + 1), N * i : N * (i + 1), : + ] + + if texture_image.shape[-1] == 4: # Image with alpha channel + alpha = texture_image[:, :, :, -1] / 255.0 + texture_image = texture_image[:, :, :, :3] + else: + alpha = texture_image.sum(axis=-1) > 0 + + return texture_image, alpha + + def generate_image_with_texture(self, texture_image, alpha, bbox_image_bgr, iuv_array): + + I, U, V = iuv_array + generated_image_bgr = bbox_image_bgr.copy() + + for PartInd in range(1, 25): + x, y = np.where(I == PartInd) + x_index = (U[x, y] * (self.body_part_size - 1)).astype(int) + y_index = ((1 - V[x, y]) * (self.body_part_size - 1)).astype(int) + part_alpha = np.expand_dims(alpha[PartInd - 1, y_index, x_index], -1) + generated_image_bgr[I == PartInd] = ( + generated_image_bgr[I == PartInd] * (1 - part_alpha) + + texture_image[PartInd - 1, y_index, x_index] * part_alpha + ) + + return generated_image_bgr.astype(np.uint8) diff --git a/approach/ovod/detectron2/projects/DensePose/densepose/vis/extractor.py b/approach/ovod/detectron2/projects/DensePose/densepose/vis/extractor.py new file mode 100644 index 0000000000000000000000000000000000000000..bfb2bdf693254a954e54a74b8766e5f574f6cf3a --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/densepose/vis/extractor.py @@ -0,0 +1,199 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +from typing import List, Optional, Sequence, Tuple +import torch + +from detectron2.layers.nms import batched_nms +from detectron2.structures.instances import Instances + +from densepose.converters import ToChartResultConverterWithConfidences +from densepose.structures import ( + DensePoseChartResultWithConfidences, + DensePoseEmbeddingPredictorOutput, +) +from densepose.vis.bounding_box import BoundingBoxVisualizer, ScoredBoundingBoxVisualizer +from densepose.vis.densepose_outputs_vertex import DensePoseOutputsVertexVisualizer +from densepose.vis.densepose_results import DensePoseResultsVisualizer + +from .base import CompoundVisualizer + +Scores = Sequence[float] +DensePoseChartResultsWithConfidences = List[DensePoseChartResultWithConfidences] + + +def extract_scores_from_instances(instances: Instances, select=None): + if instances.has("scores"): + return instances.scores if select is None else instances.scores[select] + return None + + +def extract_boxes_xywh_from_instances(instances: Instances, select=None): + if instances.has("pred_boxes"): + boxes_xywh = instances.pred_boxes.tensor.clone() + boxes_xywh[:, 2] -= boxes_xywh[:, 0] + boxes_xywh[:, 3] -= boxes_xywh[:, 1] + return boxes_xywh if select is None else boxes_xywh[select] + return None + + +def create_extractor(visualizer: object): + """ + Create an extractor for the provided visualizer + """ + if isinstance(visualizer, CompoundVisualizer): + extractors = [create_extractor(v) for v in visualizer.visualizers] + return CompoundExtractor(extractors) + elif isinstance(visualizer, DensePoseResultsVisualizer): + return DensePoseResultExtractor() + elif isinstance(visualizer, ScoredBoundingBoxVisualizer): + return CompoundExtractor([extract_boxes_xywh_from_instances, extract_scores_from_instances]) + elif isinstance(visualizer, BoundingBoxVisualizer): + return extract_boxes_xywh_from_instances + elif isinstance(visualizer, DensePoseOutputsVertexVisualizer): + return DensePoseOutputsExtractor() + else: + logger = logging.getLogger(__name__) + logger.error(f"Could not create extractor for {visualizer}") + return None + + +class BoundingBoxExtractor(object): + """ + Extracts bounding boxes from instances + """ + + def __call__(self, instances: Instances): + boxes_xywh = extract_boxes_xywh_from_instances(instances) + return boxes_xywh + + +class ScoredBoundingBoxExtractor(object): + """ + Extracts bounding boxes from instances + """ + + def __call__(self, instances: Instances, select=None): + scores = extract_scores_from_instances(instances) + boxes_xywh = extract_boxes_xywh_from_instances(instances) + if (scores is None) or (boxes_xywh is None): + return (boxes_xywh, scores) + if select is not None: + scores = scores[select] + boxes_xywh = boxes_xywh[select] + return (boxes_xywh, scores) + + +class DensePoseResultExtractor(object): + """ + Extracts DensePose chart result with confidences from instances + """ + + def __call__( + self, instances: Instances, select=None + ) -> Tuple[Optional[DensePoseChartResultsWithConfidences], Optional[torch.Tensor]]: + if instances.has("pred_densepose") and instances.has("pred_boxes"): + dpout = instances.pred_densepose + boxes_xyxy = instances.pred_boxes + boxes_xywh = extract_boxes_xywh_from_instances(instances) + if select is not None: + dpout = dpout[select] + boxes_xyxy = boxes_xyxy[select] + converter = ToChartResultConverterWithConfidences() + results = [converter.convert(dpout[i], boxes_xyxy[[i]]) for i in range(len(dpout))] + return results, boxes_xywh + else: + return None, None + + +class DensePoseOutputsExtractor(object): + """ + Extracts DensePose result from instances + """ + + def __call__( + self, + instances: Instances, + select=None, + ) -> Tuple[ + Optional[DensePoseEmbeddingPredictorOutput], Optional[torch.Tensor], Optional[List[int]] + ]: + if not (instances.has("pred_densepose") and instances.has("pred_boxes")): + return None, None, None + + dpout = instances.pred_densepose + boxes_xyxy = instances.pred_boxes + boxes_xywh = extract_boxes_xywh_from_instances(instances) + + if instances.has("pred_classes"): + classes = instances.pred_classes.tolist() + else: + classes = None + + if select is not None: + dpout = dpout[select] + boxes_xyxy = boxes_xyxy[select] + if classes is not None: + classes = classes[select] + + return dpout, boxes_xywh, classes + + +class CompoundExtractor(object): + """ + Extracts data for CompoundVisualizer + """ + + def __init__(self, extractors): + self.extractors = extractors + + def __call__(self, instances: Instances, select=None): + datas = [] + for extractor in self.extractors: + data = extractor(instances, select) + datas.append(data) + return datas + + +class NmsFilteredExtractor(object): + """ + Extracts data in the format accepted by NmsFilteredVisualizer + """ + + def __init__(self, extractor, iou_threshold): + self.extractor = extractor + self.iou_threshold = iou_threshold + + def __call__(self, instances: Instances, select=None): + scores = extract_scores_from_instances(instances) + boxes_xywh = extract_boxes_xywh_from_instances(instances) + if boxes_xywh is None: + return None + select_local_idx = batched_nms( + boxes_xywh, + scores, + torch.zeros(len(scores), dtype=torch.int32), + iou_threshold=self.iou_threshold, + ).squeeze() + select_local = torch.zeros(len(boxes_xywh), dtype=torch.bool, device=boxes_xywh.device) + select_local[select_local_idx] = True + select = select_local if select is None else (select & select_local) + return self.extractor(instances, select=select) + + +class ScoreThresholdedExtractor(object): + """ + Extracts data in the format accepted by ScoreThresholdedVisualizer + """ + + def __init__(self, extractor, min_score): + self.extractor = extractor + self.min_score = min_score + + def __call__(self, instances: Instances, select=None): + scores = extract_scores_from_instances(instances) + if scores is None: + return None + select_local = scores > self.min_score + select = select_local if select is None else (select & select_local) + data = self.extractor(instances, select=select) + return data diff --git a/approach/ovod/detectron2/projects/DensePose/dev/README.md b/approach/ovod/detectron2/projects/DensePose/dev/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e3a94b67ed4b4d0c2934f074802cd00f3660f9a9 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/dev/README.md @@ -0,0 +1,7 @@ + +## Some scripts for developers to use, include: + +- `run_instant_tests.sh`: run training for a few iterations. +- `run_inference_tests.sh`: run inference on a small dataset. +- `../../dev/linter.sh`: lint the codebase before commit +- `../../dev/parse_results.sh`: parse results from log file. diff --git a/approach/ovod/detectron2/projects/DensePose/dev/run_inference_tests.sh b/approach/ovod/detectron2/projects/DensePose/dev/run_inference_tests.sh new file mode 100644 index 0000000000000000000000000000000000000000..46556b80a3ee793bdf6a79f5de2ec88cac902189 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/dev/run_inference_tests.sh @@ -0,0 +1,33 @@ +#!/bin/bash -e +# Copyright (c) Facebook, Inc. and its affiliates. + +BIN="python train_net.py" +OUTPUT="inference_test_output" +NUM_GPUS=2 +IMS_PER_GPU=2 +IMS_PER_BATCH=$(( NUM_GPUS * IMS_PER_GPU )) + +CFG_LIST=( "${@:1}" ) + +if [ ${#CFG_LIST[@]} -eq 0 ]; then + CFG_LIST=( ./configs/quick_schedules/*inference_acc_test.yaml ) +fi + +echo "========================================================================" +echo "Configs to run:" +echo "${CFG_LIST[@]}" +echo "========================================================================" + +for cfg in "${CFG_LIST[@]}"; do + echo "========================================================================" + echo "Running $cfg ..." + echo "========================================================================" + $BIN \ + --eval-only \ + --num-gpus $NUM_GPUS \ + --config-file "$cfg" \ + OUTPUT_DIR "$OUTPUT" \ + SOLVER.IMS_PER_BATCH $IMS_PER_BATCH + rm -rf $OUTPUT +done + diff --git a/approach/ovod/detectron2/projects/DensePose/dev/run_instant_tests.sh b/approach/ovod/detectron2/projects/DensePose/dev/run_instant_tests.sh new file mode 100644 index 0000000000000000000000000000000000000000..23a9c67cefe3cfca790181c90b27f2471d8a7771 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/dev/run_instant_tests.sh @@ -0,0 +1,28 @@ +#!/bin/bash -e +# Copyright (c) Facebook, Inc. and its affiliates. + +BIN="python train_net.py" +OUTPUT="instant_test_output" +NUM_GPUS=2 +SOLVER_IMS_PER_BATCH=$((NUM_GPUS * 2)) + +CFG_LIST=( "${@:1}" ) +if [ ${#CFG_LIST[@]} -eq 0 ]; then + CFG_LIST=( ./configs/quick_schedules/*instant_test.yaml ) +fi + +echo "========================================================================" +echo "Configs to run:" +echo "${CFG_LIST[@]}" +echo "========================================================================" + +for cfg in "${CFG_LIST[@]}"; do + echo "========================================================================" + echo "Running $cfg ..." + echo "========================================================================" + $BIN --num-gpus $NUM_GPUS --config-file "$cfg" \ + SOLVER.IMS_PER_BATCH $SOLVER_IMS_PER_BATCH \ + OUTPUT_DIR "$OUTPUT" + rm -rf "$OUTPUT" +done + diff --git a/approach/ovod/detectron2/projects/DensePose/doc/BOOTSTRAPPING_PIPELINE.md b/approach/ovod/detectron2/projects/DensePose/doc/BOOTSTRAPPING_PIPELINE.md new file mode 100644 index 0000000000000000000000000000000000000000..a1326862abe5479140269f5e6af50b68e7c2d0aa --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/doc/BOOTSTRAPPING_PIPELINE.md @@ -0,0 +1,197 @@ +# Bootstrapping Pipeline + +Bootstrapping pipeline for DensePose was proposed in +[Sanakoyeu et al., 2020](https://arxiv.org/pdf/2003.00080.pdf) +to extend DensePose from humans to proximal animal classes +(chimpanzees). Currently, the pipeline is only implemented for +[chart-based models](DENSEPOSE_IUV.md). +Bootstrapping proceeds in two steps. + +## Master Model Training + +Master model is trained on data from source domain (humans) +and supporting domain (animals). Instances from the source domain +contain full DensePose annotations (`S`, `I`, `U` and `V`) and +instances from the supporting domain have segmentation annotations only. +To ensure segmentation quality in the target domain, only a subset of +supporting domain classes is included into the training. This is achieved +through category filters, e.g. +(see [configs/evolution/Base-RCNN-FPN-Atop10P_CA.yaml](../configs/evolution/Base-RCNN-FPN-Atop10P_CA.yaml)): + +``` + WHITELISTED_CATEGORIES: + "base_coco_2017_train": + - 1 # person + - 16 # bird + - 17 # cat + - 18 # dog + - 19 # horse + - 20 # sheep + - 21 # cow + - 22 # elephant + - 23 # bear + - 24 # zebra + - 25 # girafe +``` +The acronym `Atop10P` in config file names indicates that categories are filtered to +only contain top 10 animals and person. + +The training is performed in a *class-agnostic* manner: all instances +are mapped into the same class (person), e.g. +(see [configs/evolution/Base-RCNN-FPN-Atop10P_CA.yaml](../configs/evolution/Base-RCNN-FPN-Atop10P_CA.yaml)): + +``` + CATEGORY_MAPS: + "base_coco_2017_train": + "16": 1 # bird -> person + "17": 1 # cat -> person + "18": 1 # dog -> person + "19": 1 # horse -> person + "20": 1 # sheep -> person + "21": 1 # cow -> person + "22": 1 # elephant -> person + "23": 1 # bear -> person + "24": 1 # zebra -> person + "25": 1 # girafe -> person +``` +The acronym `CA` in config file names indicates that the training is class-agnostic. + +## Student Model Training + +Student model is trained on data from source domain (humans), +supporting domain (animals) and target domain (chimpanzees). +Annotations in source and supporting domains are similar to the ones +used for the master model training. +Annotations in target domain are obtained by applying the master model +to images that contain instances from the target category and sampling +sparse annotations from dense results. This process is called *bootstrapping*. +Below we give details on how the bootstrapping pipeline is implemented. + +### Data Loaders + +The central components that enable bootstrapping are +[`InferenceBasedLoader`](../densepose/data/inference_based_loader.py) and +[`CombinedDataLoader`](../densepose/data/combined_loader.py). + +`InferenceBasedLoader` takes images from a data loader, applies a model +to the images, filters the model outputs based on the selected criteria and +samples the filtered outputs to produce annotations. + +`CombinedDataLoader` combines data obtained from the loaders based on specified +ratios. The standard data loader has the default ratio of 1.0, +ratios for bootstrap datasets are specified in the configuration file. +The higher the ratio the higher the probability to include samples from the +particular data loader into a batch. + +Here is an example of the bootstrapping configuration taken from +[`configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_uniform.yaml`](../configs/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_uniform.yaml): +``` +BOOTSTRAP_DATASETS: + - DATASET: "chimpnsee" + RATIO: 1.0 + IMAGE_LOADER: + TYPE: "video_keyframe" + SELECT: + STRATEGY: "random_k" + NUM_IMAGES: 4 + TRANSFORM: + TYPE: "resize" + MIN_SIZE: 800 + MAX_SIZE: 1333 + BATCH_SIZE: 8 + NUM_WORKERS: 1 + INFERENCE: + INPUT_BATCH_SIZE: 1 + OUTPUT_BATCH_SIZE: 1 + DATA_SAMPLER: + # supported types: + # densepose_uniform + # densepose_UV_confidence + # densepose_fine_segm_confidence + # densepose_coarse_segm_confidence + TYPE: "densepose_uniform" + COUNT_PER_CLASS: 8 + FILTER: + TYPE: "detection_score" + MIN_VALUE: 0.8 +BOOTSTRAP_MODEL: + WEIGHTS: https://dl.fbaipublicfiles.com/densepose/evolution/densepose_R_50_FPN_DL_WC1M_3x_Atop10P_CA/217578784/model_final_9fe1cc.pkl +``` + +The above example has one bootstrap dataset (`chimpnsee`). This dataset is registered as +a [VIDEO_LIST](../densepose/data/datasets/chimpnsee.py) dataset, which means that +it consists of a number of videos specified in a text file. For videos there can be +different strategies to sample individual images. Here we use `video_keyframe` strategy +which considers only keyframes; this ensures temporal offset between sampled images and +faster seek operations. We select at most 4 random keyframes in each video: + +``` +SELECT: + STRATEGY: "random_k" + NUM_IMAGES: 4 +``` + +The frames are then resized + +``` +TRANSFORM: + TYPE: "resize" + MIN_SIZE: 800 + MAX_SIZE: 1333 +``` + +and batched using the standard +[PyTorch DataLoader](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader): + +``` +BATCH_SIZE: 8 +NUM_WORKERS: 1 +``` + +`InferenceBasedLoader` decomposes those batches into batches of size `INPUT_BATCH_SIZE` +and applies the master model specified by `BOOTSTRAP_MODEL`. Models outputs are filtered +by detection score: + +``` +FILTER: + TYPE: "detection_score" + MIN_VALUE: 0.8 +``` + +and sampled using the specified sampling strategy: + +``` +DATA_SAMPLER: + # supported types: + # densepose_uniform + # densepose_UV_confidence + # densepose_fine_segm_confidence + # densepose_coarse_segm_confidence + TYPE: "densepose_uniform" + COUNT_PER_CLASS: 8 +``` + +The current implementation supports +[uniform sampling](../densepose/data/samplers/densepose_uniform.py) and +[confidence-based sampling](../densepose/data/samplers/densepose_confidence_based.py) +to obtain sparse annotations from dense results. For confidence-based +sampling one needs to use the master model which produces confidence estimates. +The `WC1M` master model used in the example above produces all three types of confidence +estimates. + +Finally, sampled data is grouped into batches of size `OUTPUT_BATCH_SIZE`: + +``` +INFERENCE: + INPUT_BATCH_SIZE: 1 + OUTPUT_BATCH_SIZE: 1 +``` + +The proportion of data from annotated datasets and bootstrapped dataset can be tracked +in the logs, e.g.: + +``` +[... densepose.engine.trainer]: batch/ 1.8, batch/base_coco_2017_train 6.4, batch/densepose_coco_2014_train 3.85 +``` + +which means that over the last 20 iterations, on average for 1.8 bootstrapped data samples there were 6.4 samples from `base_coco_2017_train` and 3.85 samples from `densepose_coco_2014_train`. diff --git a/approach/ovod/detectron2/projects/DensePose/doc/DENSEPOSE_CSE.md b/approach/ovod/detectron2/projects/DensePose/doc/DENSEPOSE_CSE.md new file mode 100644 index 0000000000000000000000000000000000000000..d5761ef989bdfb441a2a61f4e508cc826f93d2d1 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/doc/DENSEPOSE_CSE.md @@ -0,0 +1,336 @@ +# Continuous Surface Embeddings for Dense Pose Estimation for Humans and Animals + +## Overview + +
+ +
+ +The pipeline uses [Faster R-CNN](https://arxiv.org/abs/1506.01497) +with [Feature Pyramid Network](https://arxiv.org/abs/1612.03144) meta architecture +outlined in Figure 1. For each detected object, the model predicts +its coarse segmentation `S` (2 channels: foreground / background) +and the embedding `E` (16 channels). At the same time, the embedder produces vertex +embeddings `Ê` for the corresponding mesh. Universal positional embeddings `E` +and vertex embeddings `Ê` are matched to derive for each pixel its continuous +surface embedding. + +
+ +
+

Figure 1. DensePose continuous surface embeddings architecture based on Faster R-CNN with Feature Pyramid Network (FPN).

+ +### Datasets + +For more details on datasets used for training and validation of +continuous surface embeddings models, +please refer to the [DensePose Datasets](DENSEPOSE_DATASETS.md) page. + +## Model Zoo and Baselines + +### Human CSE Models + +Continuous surface embeddings models for humans trained using the protocols from [Neverova et al, 2020](https://arxiv.org/abs/2011.12438). + +Models trained with hard assignment loss ℒ: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Namelr
sched		train
time
(s/iter)		inference
time
(s/im)		train
mem
(GB)		box
AP		segm
AP		dp. AP
GPS		dp. AP
GPSm		model iddownload
R_50_FPN_s1xs1x0.3490.0606.361.167.164.465.7251155172model | metrics
R_101_FPN_s1xs1x0.4610.0717.462.367.264.765.8251155500model | metrics
R_50_FPN_DL_s1xs1x0.3990.0617.060.867.865.566.4251156349model | metrics
R_101_FPN_DL_s1xs1x0.5040.0748.361.568.065.666.6251156606model | metrics
+ +Models trained with soft assignment loss ℒσ: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Namelr
sched		train
time
(s/iter)		inference
time
(s/im)		train
mem
(GB)		box
AP		segm
AP		dp. AP
GPS		dp. AP
GPSm		model iddownload
R_50_FPN_soft_s1xs1x0.3570.0579.761.366.964.365.4250533982model | metrics
R_101_FPN_soft_s1xs1x0.4640.07110.562.167.364.566.0250712522model | metrics
R_50_FPN_DL_soft_s1xs1x0.4270.06211.360.868.066.166.7250713703model | metrics
R_101_FPN_DL_soft_s1xs1x0.4830.07112.261.568.266.267.1250713061model | metrics
+ +### Animal CSE Models + +Models obtained by finetuning human CSE models on animals data from `ds1_train` +(see the [DensePose LVIS](DENSEPOSE_DATASETS.md#continuous-surface-embeddings-annotations-3) +section for more details on the datasets) with soft assignment loss ℒσ: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Namelr
sched		train
time
(s/iter)		inference
time
(s/im)		train
mem
(GB)		box
AP		segm
AP		dp. AP
GPS		dp. AP
GPSm		model iddownload
R_50_FPN_soft_chimps_finetune_4k4K0.5690.0514.762.059.032.239.6253146869model | metrics
R_50_FPN_soft_animals_finetune_4k4K0.3810.0617.344.955.521.328.8253145793model | metrics
R_50_FPN_soft_animals_CA_finetune_4k4K0.4120.0597.153.459.525.433.4253498611model | metrics
+ +Acronyms: + +`CA`: class agnostic training, where all annotated instances are mapped into a single category + + +Models obtained by finetuning human CSE models on animals data from `ds2_train` dataset +with soft assignment loss ℒσ and, for some schedules, cycle losses. +Please refer to [DensePose LVIS](DENSEPOSE_DATASETS.md#continuous-surface-embeddings-annotations-3) +section for details on the dataset and to [Neverova et al, 2021]() for details on cycle losses. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Namelr
sched		train
time
(s/iter)		inference
time
(s/im)		train
mem
(GB)		box
AP		segm
AP		dp. AP
GPS		dp. AP
GPSm		GErrGPSmodel iddownload
R_50_FPN_soft_animals_I0_finetune_16k16k0.3860.0588.454.267.029.038.613.285.4270727112model | metrics
R_50_FPN_soft_animals_I0_finetune_m2m_16k16k0.5080.05612.254.167.328.638.412.587.6270982215model | metrics
R_50_FPN_soft_animals_I0_finetune_i2m_16k16k0.4830.0569.754.066.628.938.311.088.9270727461model | metrics
+ +## References + +If you use DensePose methods based on continuous surface embeddings, please take the +references from the following BibTeX entries: + +Continuous surface embeddings: +``` +@InProceedings{Neverova2020ContinuousSurfaceEmbeddings, + title = {Continuous Surface Embeddings}, + author = {Neverova, Natalia and Novotny, David and Khalidov, Vasil and Szafraniec, Marc and Labatut, Patrick and Vedaldi, Andrea}, + journal = {Advances in Neural Information Processing Systems}, + year = {2020}, +} +``` + +Cycle Losses: +``` +@InProceedings{Neverova2021UniversalCanonicalMaps, + title = {Discovering Relationships between Object Categories via Universal Canonical Maps}, + author = {Neverova, Natalia and Sanakoyeu, Artsiom and Novotny, David and Labatut, Patrick and Vedaldi, Andrea}, + journal = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, + year = {2021}, +} +``` diff --git a/approach/ovod/detectron2/projects/DensePose/doc/DENSEPOSE_DATASETS.md b/approach/ovod/detectron2/projects/DensePose/doc/DENSEPOSE_DATASETS.md new file mode 100644 index 0000000000000000000000000000000000000000..6943741e104310e7ec1837951e602e9c79061b10 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/doc/DENSEPOSE_DATASETS.md @@ -0,0 +1,513 @@ +# DensePose Datasets + +We summarize the datasets used in various DensePose training +schedules and describe different available annotation types. + +## Table of Contents + +[General Information](#general-information) + +[DensePose COCO](#densepose-coco) + +[DensePose PoseTrack](#densepose-posetrack) + +[DensePose Chimps](#densepose-chimps) + +[DensePose LVIS](#densepose-lvis) + +## General Information + +DensePose annotations are typically stored in JSON files. Their +structure follows the [COCO Data Format](https://cocodataset.org/#format-data), +the basic data structure is outlined below: + +``` +{ + "info": info, + "images": [image], + "annotations": [annotation], + "licenses": [license], +} + +info{ + "year": int, + "version": str, + "description": str, + "contributor": str, + "url": str, + "date_created": datetime, +} + +image{ + "id": int, + "width": int, + "height": int, + "file_name": str, + "license": int, + "flickr_url": str, + "coco_url": str, + "date_captured": datetime, +} + +license{ + "id": int, "name": str, "url": str, +} +``` + +DensePose annotations can be of two types: +*chart-based annotations* or *continuous surface embeddings annotations*. +We give more details on each of the two annotation types below. + +### Chart-based Annotations + +These annotations assume a single 3D model which corresponds to +all the instances in a given dataset. +3D model is assumed to be split into *charts*. Each chart has its own +2D parametrization through inner coordinates `U` and `V`, typically +taking values in `[0, 1]`. + +Chart-based annotations consist of *point-based annotations* and +*segmentation annotations*. Point-based annotations specify, for a given +image point, which model part it belongs to and what are its coordinates +in the corresponding chart. Segmentation annotations specify regions +in an image that are occupied by a given part. In some cases, charts +associated with point annotations are more detailed than the ones +associated with segmentation annotations. In this case we distinguish +*fine segmentation* (associated with points) and *coarse segmentation* +(associated with masks). + +**Point-based annotations**: + +`dp_x` and `dp_y`: image coordinates of the annotated points along +the horizontal and vertical axes respectively. The coordinates are defined +with respect to the top-left corner of the annotated bounding box and are +normalized assuming the bounding box size to be `256x256`; + +`dp_I`: for each point specifies the index of the fine segmentation chart +it belongs to; + +`dp_U` and `dp_V`: point coordinates on the corresponding chart. +Each fine segmentation part has its own parametrization in terms of chart +coordinates. + +**Segmentation annotations**: + +`dp_masks`: RLE encoded dense masks (`dict` containing keys `counts` and `size`). +The masks are typically of size `256x256`, they define segmentation within the +bounding box. + +### Continuous Surface Embeddings Annotations + +Continuous surface embeddings annotations also consist of *point-based annotations* +and *segmentation annotations*. Point-based annotations establish correspondence +between image points and 3D model vertices. Segmentation annotations specify +foreground regions for a given instane. + +**Point-based annotations**: + +`dp_x` and `dp_y` specify image point coordinates the same way as for chart-based +annotations; + +`dp_vertex` gives indices of 3D model vertices, which the annotated image points +correspond to; + +`ref_model` specifies 3D model name. + +**Segmentation annotations**: + +Segmentations can either be given by `dp_masks` field or by `segmentation` field. + +`dp_masks`: RLE encoded dense masks (`dict` containing keys `counts` and `size`). +The masks are typically of size `256x256`, they define segmentation within the +bounding box. + +`segmentation`: polygon-based masks stored as a 2D list +`[[x1 y1 x2 y2...],[x1 y1 ...],...]` of polygon vertex coordinates in a given +image. + +## DensePose COCO + +
+ +
+

+ Figure 1. Annotation examples from the DensePose COCO dataset. +

+ +DensePose COCO dataset contains about 50K annotated persons on images from the +[COCO dataset](https://cocodataset.org/#home) +The images are available for download from the +[COCO Dataset download page](https://cocodataset.org/#download): +[train2014](http://images.cocodataset.org/zips/train2014.zip), +[val2014](http://images.cocodataset.org/zips/val2014.zip). +The details on available annotations and their download links are given below. + +### Chart-based Annotations + +Chart-based DensePose COCO annotations are available for the instances of category +`person` and correspond to the model shown in Figure 2. +They include `dp_x`, `dp_y`, `dp_I`, `dp_U` and `dp_V` fields for annotated points +(~100 points per annotated instance) and `dp_masks` field, which encodes +coarse segmentation into 14 parts in the following order: +`Torso`, `Right Hand`, `Left Hand`, `Left Foot`, `Right Foot`, +`Upper Leg Right`, `Upper Leg Left`, `Lower Leg Right`, `Lower Leg Left`, +`Upper Arm Left`, `Upper Arm Right`, `Lower Arm Left`, `Lower Arm Right`, +`Head`. + +
+ +
+

+ Figure 2. Human body charts (fine segmentation) + and the associated 14 body parts depicted with rounded rectangles + (coarse segmentation). +

+ +The dataset splits used in the training schedules are +`train2014`, `valminusminival2014` and `minival2014`. +`train2014` and `valminusminival2014` are used for training, +and `minival2014` is used for validation. +The table with annotation download links, which summarizes the number of annotated +instances and images for each of the dataset splits is given below: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Name# inst# imagesfile sizedownload
densepose_train20143921026437526Mdensepose_train2014.json
densepose_valminusminival201472975984105Mdensepose_valminusminival2014.json
densepose_minival20142243150831Mdensepose_minival2014.json
+ +### Continuous Surface Embeddings Annotations + +DensePose COCO continuous surface embeddings annotations are available for the instances +of category `person`. The annotations correspond to the 3D model shown in Figure 2, +and include `dp_x`, `dp_y` and `dp_vertex` and `ref_model` fields. +All chart-based annotations were also kept for convenience. + +As with chart-based annotations, the dataset splits used in the training schedules are +`train2014`, `valminusminival2014` and `minival2014`. +`train2014` and `valminusminival2014` are used for training, +and `minival2014` is used for validation. +The table with annotation download links, which summarizes the number of annotated +instances and images for each of the dataset splits is given below: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Name# inst# imagesfile sizedownload
densepose_train2014_cse3921026437554Mdensepose_train2014_cse.json
densepose_valminusminival2014_cse72975984110Mdensepose_valminusminival2014_cse.json
densepose_minival2014_cse2243150832Mdensepose_minival2014_cse.json
+ +## DensePose PoseTrack + +
+ +
+

+ Figure 3. Annotation examples from the PoseTrack dataset. +

+ +DensePose PoseTrack dataset contains annotated image sequences. +To download the images for this dataset, please follow the instructions +from the [PoseTrack Download Page](https://posetrack.net/users/download.php). + +### Chart-based Annotations + +Chart-based DensePose PoseTrack annotations are available for the instances with category +`person` and correspond to the model shown in Figure 2. +They include `dp_x`, `dp_y`, `dp_I`, `dp_U` and `dp_V` fields for annotated points +(~100 points per annotated instance) and `dp_masks` field, which encodes +coarse segmentation into the same 14 parts as in DensePose COCO. + +The dataset splits used in the training schedules are +`posetrack_train2017` (train set) and `posetrack_val2017` (validation set). +The table with annotation download links, which summarizes the number of annotated +instances, instance tracks and images for the dataset splits is given below: + + + + + + + + + + + + + + + + + + + + + + + + + + +
Name# inst# images# tracksfile sizedownload
densepose_posetrack_train20178274168036118Mdensepose_posetrack_train2017.json
densepose_posetrack_val201747537824659Mdensepose_posetrack_val2017.json
+ +## DensePose Chimps + +
+ +
+

+ Figure 4. Example images from the DensePose Chimps dataset. +

+ +DensePose Chimps dataset contains annotated images of chimpanzees. +To download the images for this dataset, please use the URL specified in +`image_url` field in the annotations. + +### Chart-based Annotations + +Chart-based DensePose Chimps annotations correspond to the human model shown in Figure 2, +the instances are thus annotated to belong to the `person` category. +They include `dp_x`, `dp_y`, `dp_I`, `dp_U` and `dp_V` fields for annotated points +(~3 points per annotated instance) and `dp_masks` field, which encodes +foreground mask in RLE format. + +Chart-base DensePose Chimps annotations are used for validation only. +The table with annotation download link, which summarizes the number of annotated +instances and images is given below: + + + + + + + + + + + + + + + + + +
Name# inst# imagesfile sizedownload
densepose_chimps9306546Mdensepose_chimps_full_v2.json
+ +### Continuous Surface Embeddings Annotations + +Continuous surface embeddings annotations for DensePose Chimps +include `dp_x`, `dp_y` and `dp_vertex` point-based annotations +(~3 points per annotated instance), `dp_masks` field with the same +contents as for chart-based annotations and `ref_model` field +which refers to a chimpanzee 3D model `chimp_5029`. + +The dataset is split into training and validation subsets. +The table with annotation download links, which summarizes the number of annotated +instances and images for each of the dataset splits is given below: + +The table below outlines the dataset splits: + + + + + + + + + + + + + + + + + + + + + + + +
Name# inst# imagesfile sizedownload
densepose_chimps_cse_train5003503Mdensepose_chimps_cse_train.json
densepose_chimps_cse_val4303043Mdensepose_chimps_cse_val.json
+ +## DensePose LVIS + +
+ +
+

+ Figure 5. Example images from the DensePose LVIS dataset. +

+ +DensePose LVIS dataset contains segmentation and DensePose annotations for animals +on images from the [LVIS dataset](https://www.lvisdataset.org/dataset). +The images are available for download through the links: +[train2017](http://images.cocodataset.org/zips/train2017.zip), +[val2017](http://images.cocodataset.org/zips/val2017.zip). + +### Continuous Surface Embeddings Annotations + +Continuous surface embeddings (CSE) annotations for DensePose LVIS +include `dp_x`, `dp_y` and `dp_vertex` point-based annotations +(~3 points per annotated instance) and a `ref_model` field +which refers to a 3D model that corresponds to the instance. +Instances from 9 animal categories were annotated with CSE DensePose data: +bear, cow, cat, dog, elephant, giraffe, horse, sheep and zebra. + +Foreground masks are available from instance segmentation annotations +(`segmentation` field) in polygon format, they are stored as a 2D list +`[[x1 y1 x2 y2...],[x1 y1 ...],...]`. + +We used two datasets, each constising of one training (`train`) +and validation (`val`) subsets: the first one (`ds1`) +was used in [Neverova et al, 2020](https://arxiv.org/abs/2011.12438). +The second one (`ds2`), was used in [Neverova et al, 2021](). + +The summary of the available datasets is given below: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
All DataSelected Animals
(9 categories)		File
Name# cat# img# segm# img# segm# dpsizedownload
ds1_train55641412398541419472518446Mdensepose_lvis_v1_ds1_train_v1.json
ds1_val2515713281571153710365Mdensepose_lvis_v1_ds1_val_v1.json
ds2_train12039938812701411374646964189321051Mdensepose_lvis_v1_ds2_train_v1.json
ds2_val92690915526909155360424Mdensepose_lvis_v1_ds2_val_v1.json
+ +Legend: + +`#cat` - number of categories in the dataset for which annotations are available; + +`#img` - number of images with annotations in the dataset; + +`#segm` - number of segmentation annotations; + +`#dp` - number of DensePose annotations. + + +Important Notes: + +1. The reference models used for `ds1_train` and `ds1_val` are +`bear_4936`, `cow_5002`, `cat_5001`, `dog_5002`, `elephant_5002`, `giraffe_5002`, +`horse_5004`, `sheep_5004` and `zebra_5002`. The reference models used for +`ds2_train` and `ds2_val` are `bear_4936`, `cow_5002`, `cat_7466`, +`dog_7466`, `elephant_5002`, `giraffe_5002`, `horse_5004`, `sheep_5004` and `zebra_5002`. +So reference models for categories `cat` aind `dog` are different for `ds1` and `ds2`. + +2. Some annotations from `ds1_train` are reused in `ds2_train` (4538 DensePose annotations +and 21275 segmentation annotations). The ones for cat and dog categories were remapped +from `cat_5001` and `dog_5002` reference models used in `ds1` to `cat_7466` and `dog_7466` +used in `ds2`. + +3. All annotations from `ds1_val` are included into `ds2_val` after the remapping +procedure mentioned in note 2. + +4. Some annotations from `ds1_train` are part of `ds2_val` (646 DensePose annotations and +1225 segmentation annotations). Thus one should not train on `ds1_train` if evaluating on `ds2_val`. diff --git a/approach/ovod/detectron2/projects/DensePose/doc/DENSEPOSE_IUV.md b/approach/ovod/detectron2/projects/DensePose/doc/DENSEPOSE_IUV.md new file mode 100644 index 0000000000000000000000000000000000000000..de158e0eea0c287507b701376abc9307ce92c0f1 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/doc/DENSEPOSE_IUV.md @@ -0,0 +1,627 @@ +# Chart-based Dense Pose Estimation for Humans and Animals + +## Overview + +The goal of chart-based DensePose methods is to establish dense correspondences +between image pixels and 3D object mesh by splitting the latter into charts and estimating +for each pixel the corresponding chart index `I` and local chart coordinates `(U, V)`. + +
+ +
+ +The charts used for human DensePose estimation are shown in Figure 1. +The human body is split into 24 parts, each part is parametrized by `U` and `V` +coordinates, each taking values in `[0, 1]`. + +
+ +
+

Figure 1. Partitioning and parametrization of human body surface.

+ +The pipeline uses [Faster R-CNN](https://arxiv.org/abs/1506.01497) +with [Feature Pyramid Network](https://arxiv.org/abs/1612.03144) meta architecture +outlined in Figure 2. For each detected object, the model predicts +its coarse segmentation `S` (2 or 15 channels: foreground / background or +background + 14 predefined body parts), fine segmentation `I` (25 channels: +background + 24 predefined body parts) and local chart coordinates `U` and `V`. + +
+ +
+

Figure 2. DensePose chart-based architecture based on Faster R-CNN with Feature Pyramid Network (FPN).

+ +### Bootstrapping Chart-Based Models + +[Sanakoyeu et al., 2020](https://arxiv.org/pdf/2003.00080.pdf) introduced a pipeline +to transfer DensePose models trained on humans to proximal animal classes (chimpanzees), +which is summarized in Figure 3. The training proceeds in two stages: + +First, a *master* model is trained on data from source domain (humans with full +DensePose annotation `S`, `I`, `U` and `V`) +and supporting domain (animals with segmentation annotation only). +Only selected animal classes are chosen from the supporting +domain through *category filters* to guarantee the quality of target domain results. +The training is done in *class-agnostic manner*: all selected categories are mapped +to a single category (human). + +Second, a *student* model is trained on data from source and supporting domains, +as well as data from target domain obtained by applying the master model, selecting +high-confidence detections and sampling the results. + +
+ +
+

Figure 3. Domain adaptation: master model is trained on data from source and +supporting domains to produce predictions in target domain; student model combines data from source and +supporting domains, as well as sampled predictions from the master model on target domain to improve +target domain predictions quality.

+ +Examples of pretrained master and student models are available in the [Model Zoo](#ModelZooBootstrap). +For more details on the bootstrapping pipeline, please see [Bootstrapping Pipeline](BOOTSTRAPPING_PIPELINE.md). + +### Datasets + +For more details on datasets used for chart-based model training and validation, +please refer to the [DensePose Datasets](DENSEPOSE_DATASETS.md) page. + +## Model Zoo and Baselines + +### Legacy Models + +Baselines trained using schedules from [Güler et al, 2018](https://arxiv.org/pdf/1802.00434.pdf) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Namelr
sched		train
time
(s/iter)		inference
time
(s/im)		train
mem
(GB)		box
AP		segm
AP		dp. AP
GPS		dp. AP
GPSm		model iddownload
R_50_FPN_s1x_legacys1x0.3070.0513.258.158.252.154.9164832157model | metrics
R_101_FPN_s1x_legacys1x0.3900.0634.359.559.353.256.0164832182model | metrics
+ +### Improved Baselines, Original Fully Convolutional Head + +These models use an improved training schedule and Panoptic FPN head from [Kirillov et al, 2019](https://arxiv.org/abs/1901.02446). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Namelr
sched		train
time
(s/iter)		inference
time
(s/im)		train
mem
(GB)		box
AP		segm
AP		dp. AP
GPS		dp. AP
GPSm		model iddownload
R_50_FPN_s1xs1x0.3590.0664.561.267.263.765.3165712039model | metrics
R_101_FPN_s1xs1x0.4280.0795.862.367.864.566.2165712084model | metrics
+ +### Improved Baselines, DeepLabV3 Head + +These models use an improved training schedule, Panoptic FPN head from [Kirillov et al, 2019](https://arxiv.org/abs/1901.02446) and DeepLabV3 head from [Chen et al, 2017](https://arxiv.org/abs/1706.05587). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Namelr
sched		train
time
(s/iter)		inference
time
(s/im)		train
mem
(GB)		box
AP		segm
AP		dp. AP
GPS		dp. AP
GPSm		model iddownload
R_50_FPN_DL_s1xs1x0.3920.0706.761.168.365.666.7165712097model | metrics
R_101_FPN_DL_s1xs1x0.4780.0837.062.368.766.367.6165712116model | metrics
+ +### Baselines with Confidence Estimation + +These models perform additional estimation of confidence in regressed UV coodrinates, along the lines of [Neverova et al., 2019](https://papers.nips.cc/paper/8378-correlated-uncertainty-for-learning-dense-correspondences-from-noisy-labels). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Namelr
sched		train
time
(s/iter)		inference
time
(s/im)		train
mem
(GB)		box
AP		segm
AP		dp. AP
GPS		dp. AP
GPSm		model iddownload
R_50_FPN_WC1_s1xs1x0.3530.0644.660.567.064.265.4173862049model | metrics
R_50_FPN_WC2_s1xs1x0.3640.0664.860.766.964.265.7173861455model | metrics
R_50_FPN_DL_WC1_s1xs1x0.3970.0686.761.168.165.867.0173067973model | metrics
R_50_FPN_DL_WC2_s1xs1x0.4100.0706.860.867.965.666.7173859335model | metrics
R_101_FPN_WC1_s1xs1x0.4350.0765.762.567.664.966.3171402969model | metrics
R_101_FPN_WC2_s1xs1x0.4500.0785.762.367.664.866.4173860702model | metrics
R_101_FPN_DL_WC1_s1xs1x0.4790.0817.962.068.466.267.2173858525model | metrics
R_101_FPN_DL_WC2_s1xs1x0.4910.0827.661.768.365.967.2173294801model | metrics
+ +Acronyms: + +`WC1`: with confidence estimation model type 1 for `U` and `V` + +`WC2`: with confidence estimation model type 2 for `U` and `V` + +### Baselines with Mask Confidence Estimation + +Models that perform estimation of confidence in regressed UV coodrinates +as well as confidences associated with coarse and fine segmentation, +see [Sanakoyeu et al., 2020](https://arxiv.org/pdf/2003.00080.pdf) for details. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Namelr
sched		train
time
(s/iter)		inference
time
(s/im)		train
mem
(GB)		box
AP		segm
AP		dp. AP
GPS		dp. AP
GPSm		model iddownload
R_50_FPN_WC1M_s1xs1x0.3810.0664.860.666.764.065.4217144516model | metrics
R_50_FPN_WC2M_s1xs1x0.3420.0685.060.766.964.265.5216245640model | metrics
R_50_FPN_DL_WC1M_s1xs1x0.3710.0686.060.768.065.266.7216245703model | metrics
R_50_FPN_DL_WC2M_s1xs1x0.3850.0716.160.868.165.066.4216245758model | metrics
R_101_FPN_WC1M_s1xs1x0.4230.0795.962.067.364.866.0216453687model | metrics
R_101_FPN_WC2M_s1xs1x0.4360.0805.962.567.464.566.0216245682model | metrics
R_101_FPN_DL_WC1M_s1xs1x0.4530.0796.862.068.166.467.1216245771model | metrics
R_101_FPN_DL_WC2M_s1xs1x0.4640.0806.961.968.266.167.1216245790model | metrics
+ +Acronyms: + +`WC1M`: with confidence estimation model type 1 for `U` and `V` and mask confidence estimation + +`WC2M`: with confidence estimation model type 2 for `U` and `V` and mask confidence estimation + +### Bootstrapping Baselines + +Master and student models trained using the bootstrapping pipeline with chimpanzee as the target category, +see [Sanakoyeu et al., 2020](https://arxiv.org/pdf/2003.00080.pdf) +and [Bootstrapping Pipeline](BOOTSTRAPPING_PIPELINE.md) for details. +Evaluation is performed on [DensePose Chimps](DENSEPOSE_DATASETS.md#densepose-chimps) dataset. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Namelr
sched		train
time
(s/iter)		inference
time
(s/im)		train
mem
(GB)		box
AP		segm
AP		dp. APex
GPS		dp. AP
GPS		dp. AP
GPSm		model iddownload
R_50_FPN_DL_WC1M_3x_Atop10P_CA3x0.5220.0739.761.359.136.220.030.2217578784model | metrics
R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_uniform3x1.9390.07210.160.958.537.221.531.0256453729model | metrics
R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_uv3x1.9850.0729.661.458.938.322.232.1256452095model | metrics
R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_finesegm3x2.0470.07210.360.958.536.720.730.7256452819model | metrics
R_50_FPN_DL_WC1M_3x_Atop10P_CA_B_coarsesegm3x1.8300.0709.661.359.237.921.531.6256455697model | metrics
+ +Acronyms: + +`WC1M`: with confidence estimation model type 1 for `U` and `V` and mask confidence estimation + +`Atop10P`: humans and animals from the 10 best suitable categories are used for training + +`CA`: class agnostic training, where all annotated instances are mapped into a single category + +`B_<...>`: schedule with bootstrapping with the specified results sampling strategy + +Note: + +The relaxed `dp. APex GPS` metric was used in +[Sanakoyeu et al., 2020](https://arxiv.org/pdf/2003.00080.pdf) to evaluate DensePose +results. This metric considers matches at thresholds 0.2, 0.3 and 0.4 additionally +to the standard ones used in the evaluation protocol. The minimum threshold is +controlled by `DENSEPOSE_EVALUATION.MIN_IOU_THRESHOLD` config option. + +### License + +All models available for download are licensed under the +[Creative Commons Attribution-ShareAlike 3.0 license](https://creativecommons.org/licenses/by-sa/3.0/) + +## References + +If you use chart-based DensePose methods, please take the references from the following +BibTeX entries: + +DensePose bootstrapping pipeline: +``` +@InProceedings{Sanakoyeu2020TransferringDensePose, + title = {Transferring Dense Pose to Proximal Animal Classes}, + author = {Artsiom Sanakoyeu and Vasil Khalidov and Maureen S. McCarthy and Andrea Vedaldi and Natalia Neverova}, + journal = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, + year = {2020}, +} +``` + +DensePose with confidence estimation: +``` +@InProceedings{Neverova2019DensePoseConfidences, + title = {Correlated Uncertainty for Learning Dense Correspondences from Noisy Labels}, + author = {Neverova, Natalia and Novotny, David and Vedaldi, Andrea}, + journal = {Advances in Neural Information Processing Systems}, + year = {2019}, +} +``` + +Original DensePose: +``` +@InProceedings{Guler2018DensePose, + title={DensePose: Dense Human Pose Estimation In The Wild}, + author={R\{i}za Alp G\"uler, Natalia Neverova, Iasonas Kokkinos}, + journal={The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, + year={2018} +} +``` diff --git a/approach/ovod/detectron2/projects/DensePose/doc/GETTING_STARTED.md b/approach/ovod/detectron2/projects/DensePose/doc/GETTING_STARTED.md new file mode 100644 index 0000000000000000000000000000000000000000..a5c86f3ab5e66dc3dee4f7836aa79bd5d41b68f2 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/doc/GETTING_STARTED.md @@ -0,0 +1,76 @@ +# Getting Started with DensePose + +## Inference with Pre-trained Models + +1. Pick a model and its config file from [Model Zoo(IUV)](DENSEPOSE_IUV.md#ModelZoo), [Model Zoo(CSE)](DENSEPOSE_CSE.md#ModelZoo), for example [densepose_rcnn_R_50_FPN_s1x.yaml](../configs/densepose_rcnn_R_50_FPN_s1x.yaml) +2. Run the [Apply Net](TOOL_APPLY_NET.md) tool to visualize the results or save the to disk. For example, to use contour visualization for DensePose, one can run: +```bash +python apply_net.py show configs/densepose_rcnn_R_50_FPN_s1x.yaml densepose_rcnn_R_50_FPN_s1x.pkl image.jpg dp_contour,bbox --output image_densepose_contour.png +``` +Please see [Apply Net](TOOL_APPLY_NET.md) for more details on the tool. + +## Training + +First, prepare the [dataset](http://densepose.org/#dataset) into the following structure under the directory you'll run training scripts: +
+datasets/coco/
+  annotations/
+    densepose_{train,minival,valminusminival}2014.json
+    densepose_minival2014_100.json   (optional, for testing only)
+  {train,val}2014/
+    # image files that are mentioned in the corresponding json
+
+ +To train a model one can use the [train_net.py](../train_net.py) script. +This script was used to train all DensePose models in [Model Zoo(IUV)](DENSEPOSE_IUV.md#ModelZoo), [Model Zoo(CSE)](DENSEPOSE_CSE.md#ModelZoo). +For example, to launch end-to-end DensePose-RCNN training with ResNet-50 FPN backbone +on 8 GPUs following the s1x schedule, one can run +```bash +python train_net.py --config-file configs/densepose_rcnn_R_50_FPN_s1x.yaml --num-gpus 8 +``` +The configs are made for 8-GPU training. To train on 1 GPU, one can apply the +[linear learning rate scaling rule](https://arxiv.org/abs/1706.02677): +```bash +python train_net.py --config-file configs/densepose_rcnn_R_50_FPN_s1x.yaml \ + SOLVER.IMS_PER_BATCH 2 SOLVER.BASE_LR 0.0025 +``` + +## Evaluation + +Model testing can be done in the same way as training, except for an additional flag `--eval-only` and +model location specification through `MODEL.WEIGHTS model.pth` in the command line +```bash +python train_net.py --config-file configs/densepose_rcnn_R_50_FPN_s1x.yaml \ + --eval-only MODEL.WEIGHTS model.pth +``` + +## Tools + +We provide tools which allow one to: + - easily view DensePose annotated data in a dataset; + - perform DensePose inference on a set of images; + - visualize DensePose model results; + +`query_db` is a tool to print or visualize DensePose data in a dataset. +Please refer to [Query DB](TOOL_QUERY_DB.md) for more details on this tool + +`apply_net` is a tool to print or visualize DensePose results. +Please refer to [Apply Net](TOOL_APPLY_NET.md) for more details on this tool + + +## Installation as a package + +DensePose can also be installed as a Python package for integration with other software. + +The following dependencies are needed: +- Python >= 3.7 +- [PyTorch](https://pytorch.org/get-started/locally/#start-locally) >= 1.7 (to match [detectron2 requirements](https://detectron2.readthedocs.io/en/latest/tutorials/install.html#requirements)) +- [torchvision](https://pytorch.org/vision/stable/) version [compatible with your version of PyTorch](https://github.com/pytorch/vision#installation) + +DensePose can then be installed from this repository with: + +``` +pip install git+https://github.com/facebookresearch/detectron2@main#subdirectory=projects/DensePose +``` + +After installation, the package will be importable as `densepose`. diff --git a/approach/ovod/detectron2/projects/DensePose/doc/RELEASE_2020_04.md b/approach/ovod/detectron2/projects/DensePose/doc/RELEASE_2020_04.md new file mode 100644 index 0000000000000000000000000000000000000000..2fab6ae78e887c630ad94e71aa6e946115c61593 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/doc/RELEASE_2020_04.md @@ -0,0 +1,6 @@ +# DensePose Confidence Estimation and Model Zoo Improvements + +* [DensePose models with confidence estimation](doc/DENSEPOSE_IUV.md#ModelZooConfidence) +* [Panoptic FPN and DeepLabV3 head implementation](doc/DENSEPOSE_IUV.md#ModelZooDeepLabV3) +* Test time augmentations for DensePose +* New evaluation metric (GPSm) that yields more reliable scores diff --git a/approach/ovod/detectron2/projects/DensePose/doc/RELEASE_2021_03.md b/approach/ovod/detectron2/projects/DensePose/doc/RELEASE_2021_03.md new file mode 100644 index 0000000000000000000000000000000000000000..eb908a67f7e48d1d3aba51f946c0ca884cfcfe79 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/doc/RELEASE_2021_03.md @@ -0,0 +1,45 @@ +# DensePose CSE and DensePose Evolution + +* [DensePose Evolution pipeline](DENSEPOSE_IUV.md#ModelZooBootstrap), a framework to bootstrap + DensePose on unlabeled data + * [`InferenceBasedLoader`](../densepose/data/inference_based_loader.py) + with data samplers to use inference results from one model + to train another model (bootstrap); + * [`VideoKeyframeDataset`](../densepose/data/video/video_keyframe_dataset.py) + to efficiently load images from video keyframes; + * Category maps and filters to combine annotations from different categories + and train in a class-agnostic manner; + * [Pretrained models](DENSEPOSE_IUV.md#ModelZooBootstrap) for DensePose estimation on chimpanzees; + * DensePose head training from partial data (segmentation only); + * [DensePose models with mask confidence estimation](DENSEPOSE_IUV.md#ModelZooMaskConfidence); + * [DensePose Chimps]() dataset for IUV evaluation +* [DensePose Continuous Surface Embeddings](DENSEPOSE_CSE.md), a framework to extend DensePose + to various categories using 3D models + * [Hard embedding](../densepose/modeling/losses/embed.py) and + [soft embedding](../densepose/modeling/losses/soft_embed.py) + losses to train universal positional embeddings; + * [Embedder](../(densepose/modeling/cse/embedder.py) to handle + mesh vertex embeddings; + * [Storage](../densepose/evaluation/tensor_storage.py) for evaluation with high volumes of data; + * [Pretrained models](DENSEPOSE_CSE.md#ModelZoo) for DensePose CSE estimation on humans and animals; + * [DensePose Chimps](DENSEPOSE_DATASETS.md#densepose-chimps) and + [DensePose LVIS](DENSEPOSE_DATASETS.md#densepose-lvis) datasets for CSE finetuning and evaluation; + * [Vertex and texture mapping visualizers](../densepose/vis/densepose_outputs_vertex.py); +* Refactoring of all major components: losses, predictors, model outputs, model results, visualizers; + * Dedicated structures for [chart outputs](../densepose/structures/chart.py), + [chart outputs with confidences](../densepose/structures/chart_confidence.py), + [chart results](../densepose/structures/chart_result.py), + [CSE outputs](../densepose/structures/cse.py); + * Dedicated predictors for + [chart-based estimation](../densepose/modeling/predictors/chart.py), + [confidence estimation](../densepose/modeling/predictors/chart_confidence.py) + and [CSE estimation](../densepose/modeling/predictors/cse.py); + * Generic handling of various [conversions](../densepose/converters) (e.g. from outputs to results); + * Better organization of various [losses](../densepose/modeling/losses); + * Segregation of loss data accumulators for + [IUV setting](../densepose/modeling/losses/utils.py) + and [CSE setting](../densepose/modeling/losses/embed_utils.py); + * Splitting visualizers into separate modules; +* [HRNet](../densepose/modeling/hrnet.py) and [HRFPN](../densepose/modeling/hrfpn.py) backbones; +* [PoseTrack](DENSEPOSE_DATASETS.md#densepose-posetrack) dataset; +* [IUV texture visualizer](../densepose/vis/densepose_results_textures.py) diff --git a/approach/ovod/detectron2/projects/DensePose/doc/RELEASE_2021_06.md b/approach/ovod/detectron2/projects/DensePose/doc/RELEASE_2021_06.md new file mode 100644 index 0000000000000000000000000000000000000000..fb5ff4facdfaf5559d7be26c49852f4f6bc5495e --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/doc/RELEASE_2021_06.md @@ -0,0 +1,12 @@ +# DensePose CSE with Cycle Losses + +This release follows the paper [Neverova et al, 2021]() and +adds CSE datasets with more annotations, better CSE animal models +to the model zoo, losses to ensure cycle consistency for models and mesh +alignment evaluator. In particular: + +* [Pixel to shape](../densepose/modeling/losses/cycle_pix2shape.py) and [shape to shape](../densepose/modeling/losses/cycle_shape2shape.py) cycle consistency losses; +* Mesh alignment [evaluator](../densepose/evaluation/mesh_alignment_evaluator.py); +* Existing CSE datasets renamed to [ds1_train](https://dl.fbaipublicfiles.com/densepose/annotations/lvis/densepose_lvis_v1_ds1_train_v1.json) and [ds1_val](https://dl.fbaipublicfiles.com/densepose/annotations/lvis/densepose_lvis_v1_ds1_val_v1.json); +* New CSE datasets [ds2_train](https://dl.fbaipublicfiles.com/densepose/annotations/lvis/densepose_lvis_v1_ds2_train_v1.json) and [ds2_val](https://dl.fbaipublicfiles.com/densepose/annotations/lvis/densepose_lvis_v1_ds2_val_v1.json) added; +* Better CSE animal models trained with the 16k schedule added to the [model zoo](DENSEPOSE_CSE.md#animal-cse-models). diff --git a/approach/ovod/detectron2/projects/DensePose/doc/TOOL_APPLY_NET.md b/approach/ovod/detectron2/projects/DensePose/doc/TOOL_APPLY_NET.md new file mode 100644 index 0000000000000000000000000000000000000000..ca8e1ddafc7b1003ba98cce2826157ab995a2443 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/doc/TOOL_APPLY_NET.md @@ -0,0 +1,203 @@ +# Apply Net + +`apply_net` is a tool to print or visualize DensePose results on a set of images. +It has two modes: `dump` to save DensePose model results to a pickle file +and `show` to visualize them on images. + +The `image.jpg` file that is used as an example in this doc can be found [here](http://images.cocodataset.org/train2017/000000117508.jpg) + +## Dump Mode + +The general command form is: +```bash +python apply_net.py dump [-h] [-v] [--output ] +``` + +There are three mandatory arguments: + - ``, configuration file for a given model; + - ``, model file with trained parameters + - ``, input image file name, pattern or folder + +One can additionally provide `--output` argument to define the output file name, +which defaults to `output.pkl`. + + +Examples: + +1. Dump results of the [R_50_FPN_s1x](https://dl.fbaipublicfiles.com/densepose/densepose_rcnn_R_50_FPN_s1x/165712039/model_final_162be9.pkl) DensePose model for images in a folder `images` to file `dump.pkl`: +```bash +python apply_net.py dump configs/densepose_rcnn_R_50_FPN_s1x.yaml \ +https://dl.fbaipublicfiles.com/densepose/densepose_rcnn_R_50_FPN_s1x/165712039/model_final_162be9.pkl \ +images --output dump.pkl -v +``` + +2. Dump results of the [R_50_FPN_s1x](https://dl.fbaipublicfiles.com/densepose/densepose_rcnn_R_50_FPN_s1x/165712039/model_final_162be9.pkl) DensePose model for images with file name matching a pattern `image*.jpg` to file `results.pkl`: +```bash +python apply_net.py dump configs/densepose_rcnn_R_50_FPN_s1x.yaml \ +https://dl.fbaipublicfiles.com/densepose/densepose_rcnn_R_50_FPN_s1x/165712039/model_final_162be9.pkl \ +"image*.jpg" --output results.pkl -v +``` + +If you want to load the pickle file generated by the above command: +``` +# make sure DensePose is in your PYTHONPATH, or use the following line to add it: +sys.path.append("/your_detectron2_path/detectron2_repo/projects/DensePose/") + +f = open('/your_result_path/results.pkl', 'rb') +data = pickle.load(f) +``` + +The file `results.pkl` contains the list of results per image, for each image the result is a dictionary. + +**If you use a [IUV model](DENSEPOSE_IUV.md#-model-zoo-and-baselines)**, the dumped data will have the following format: + +``` +data: [{'file_name': '/your_path/image1.jpg', + 'scores': tensor([0.9884]), + 'pred_boxes_XYXY': tensor([[ 69.6114, 0.0000, 706.9797, 706.0000]]), + 'pred_densepose': [DensePoseChartResultWithConfidences(labels=tensor(...), uv=tensor(...), sigma_1=None, + sigma_2=None, kappa_u=None, kappa_v=None, fine_segm_confidence=None, coarse_segm_confidence=None), + DensePoseChartResultWithConfidences, ...] + } + {'file_name': '/your_path/image2.jpg', + 'scores': tensor([0.9999, 0.5373, 0.3991]), + 'pred_boxes_XYXY': tensor([[ 59.5734, 7.7535, 579.9311, 932.3619], + [612.9418, 686.1254, 612.9999, 704.6053], + [164.5081, 407.4034, 598.3944, 920.4266]]), + 'pred_densepose': [DensePoseChartResultWithConfidences(labels=tensor(...), uv=tensor(...), sigma_1=None, + sigma_2=None, kappa_u=None, kappa_v=None, fine_segm_confidence=None, coarse_segm_confidence=None), + DensePoseChartResultWithConfidences, ...] + }] +``` + +`DensePoseChartResultWithConfidences` contains the following fields: +- `labels` - a tensor of size `[H, W]` of type `torch.long` which contains fine segmentation labels (previously called `I`) +- `uv` - a tensor of size `[2, H, W]` of type `torch.float` which contains `U` and `V` coordinates +- various optional confidence-related fields (`sigma_1`, `sigma_2`, `kappa_u`, `kappa_v`, `fine_segm_confidence`, `coarse_segm_confidence`) + + +**If you use a [CSE model](DENSEPOSE_CSE.md#-model-zoo-and-baselines)**, the dumped data will have the following format: +``` +data: [{'file_name': '/your_path/image1.jpg', + 'scores': tensor([0.9984, 0.9961]), + 'pred_boxes_XYXY': tensor([[480.0093, 461.0796, 698.3614, 696.1011], + [78.1589, 168.6614, 307.1287, 653.8522]]), + 'pred_densepose': DensePoseEmbeddingPredictorOutput(embedding=tensor(...), coarse_segm=tensor(...))} + {'file_name': '/your_path/image2.jpg', + 'scores': tensor([0.9189, 0.9491]), + 'pred_boxes_XYXY': tensor([[734.9685, 534.2003, 287.3923, 254.8859], + [434.2853, 765.1219, 132.1029, 867.9283]]), + 'pred_densepose': DensePoseEmbeddingPredictorOutput(embedding=tensor(...), coarse_segm=tensor(...))}] +``` + +`DensePoseEmbeddingPredictorOutput` contains the following fields: +- `embedding` - a tensor of size `[N, D, sz, sz]` of type `torch.float`, which contains embeddings of size `D` of the `N` detections in the image +- `coarse_segm` - a tensor of size `[N, 2, sz, sz]` of type `torch.float` which contains segmentation scores of the `N` detections in the image; e.g. a mask can be obtained by `coarse_segm.argmax(dim=1)` + +`sz` is a fixed size for the tensors; you can resize them to the size of the bounding box, if needed + +We can use the following code, to parse the outputs of the first +detected instance on the first image (IUV model). +``` +img_id, instance_id = 0, 0 # Look at the first image and the first detected instance +bbox_xyxy = data[img_id]['pred_boxes_XYXY'][instance_id] +result = data[img_id]['pred_densepose'][instance_id] +uv = result.uv +``` +The array `bbox_xyxy` contains (x0, y0, x1, y1) of the bounding box. + + +## Visualization Mode + +The general command form is: +```bash +python apply_net.py show [-h] [-v] [--min_score ] [--nms_thresh ] [--output ] +``` + +There are four mandatory arguments: + - ``, configuration file for a given model; + - ``, model file with trained parameters + - ``, input image file name, pattern or folder + - ``, visualizations specifier; currently available visualizations are: + * `bbox` - bounding boxes of detected persons; + * `dp_segm` - segmentation masks for detected persons; + * `dp_u` - each body part is colored according to the estimated values of the + U coordinate in part parameterization; + * `dp_v` - each body part is colored according to the estimated values of the + V coordinate in part parameterization; + * `dp_contour` - plots contours with color-coded U and V coordinates; + * `dp_iuv_texture` - transfers the texture from a given texture image file to detected instances, in IUV mode; + * `dp_vertex` - plots the rainbow visualization of the closest vertices prediction for a given mesh, in CSE mode; + * `dp_cse_texture` - transfers the texture from a given list of texture image files (one from each human or animal mesh) to detected instances, in CSE mode + + +One can additionally provide the following optional arguments: + - `--min_score` to only show detections with sufficient scores that are not lower than provided value + - `--nms_thresh` to additionally apply non-maximum suppression to detections at a given threshold + - `--output` to define visualization file name template, which defaults to `output.png`. + To distinguish output file names for different images, the tool appends 1-based entry index, + e.g. output.0001.png, output.0002.png, etc... +- `--texture_atlas` to define the texture atlas image for IUV texture transfer +- `--texture_atlases_map` to define the texture atlas images map (a dictionary `{mesh name: texture atlas image}`) for CSE texture transfer + + +The following examples show how to output results of a DensePose model +with ResNet-50 FPN backbone using different visualizations for image `image.jpg`: + +1. Show bounding box and segmentation: +```bash +python apply_net.py show configs/densepose_rcnn_R_50_FPN_s1x.yaml \ +https://dl.fbaipublicfiles.com/densepose/densepose_rcnn_R_50_FPN_s1x/165712039/model_final_162be9.pkl \ +image.jpg bbox,dp_segm -v +``` +![Bounding Box + Segmentation Visualization](https://dl.fbaipublicfiles.com/densepose/web/apply_net/res_bbox_dp_segm.jpg) + +2. Show bounding box and estimated U coordinates for body parts: +```bash +python apply_net.py show configs/densepose_rcnn_R_50_FPN_s1x.yaml \ +https://dl.fbaipublicfiles.com/densepose/densepose_rcnn_R_50_FPN_s1x/165712039/model_final_162be9.pkl \ +image.jpg bbox,dp_u -v +``` +![Bounding Box + U Coordinate Visualization](https://dl.fbaipublicfiles.com/densepose/web/apply_net/res_bbox_dp_u.jpg) + +3. Show bounding box and estimated V coordinates for body parts: +```bash +python apply_net.py show configs/densepose_rcnn_R_50_FPN_s1x.yaml \ +https://dl.fbaipublicfiles.com/densepose/densepose_rcnn_R_50_FPN_s1x/165712039/model_final_162be9.pkl \ +image.jpg bbox,dp_v -v +``` +![Bounding Box + V Coordinate Visualization](https://dl.fbaipublicfiles.com/densepose/web/apply_net/res_bbox_dp_v.jpg) + +4. Show bounding box and estimated U and V coordinates via contour plots: +```bash +python apply_net.py show configs/densepose_rcnn_R_50_FPN_s1x.yaml \ +https://dl.fbaipublicfiles.com/densepose/densepose_rcnn_R_50_FPN_s1x/165712039/model_final_162be9.pkl \ +image.jpg dp_contour,bbox -v +``` +![Bounding Box + Contour Visualization](https://dl.fbaipublicfiles.com/densepose/web/apply_net/res_bbox_dp_contour.jpg) + +5. Show bounding box and texture transfer: +```bash +python apply_net.py show configs/densepose_rcnn_R_50_FPN_s1x.yaml \ +https://dl.fbaipublicfiles.com/densepose/densepose_rcnn_R_50_FPN_s1x/165712039/model_final_162be9.pkl \ +image.jpg dp_iuv_texture,bbox --texture_atlas texture_from_SURREAL.jpg -v +``` +![Bounding Box + IUV Texture Transfer Visualization](https://dl.fbaipublicfiles.com/densepose/web/apply_net/res_bbox_dp_iuv_texture.jpg) + +6. Show bounding box and CSE rainbow visualization: +```bash +python apply_net.py show configs/cse/densepose_rcnn_R_50_FPN_s1x.yaml \ +https://dl.fbaipublicfiles.com/densepose/cse/densepose_rcnn_R_50_FPN_s1x/251155172/model_final_c4ea5f.pkl \ +image.jpg dp_vertex,bbox -v +``` +![Bounding Box + CSE Rainbow Visualization](https://dl.fbaipublicfiles.com/densepose/web/apply_net/res_bbox_dp_vertex.jpg) + +7. Show bounding box and CSE texture transfer: +```bash +python apply_net.py show configs/cse/densepose_rcnn_R_50_FPN_s1x.yaml \ +https://dl.fbaipublicfiles.com/densepose/cse/densepose_rcnn_R_50_FPN_s1x/251155172/model_final_c4ea5f.pkl \ +image.jpg dp_cse_texture,bbox --texture_atlases_map '{"smpl_27554": "smpl_uvSnapshot_colors.jpg"}' -v +``` +![Bounding Box + CSE Texture Transfer Visualization](https://dl.fbaipublicfiles.com/densepose/web/apply_net/res_bbox_dp_cse_texture.jpg) + +The texture files can be found in the `doc/images` folder diff --git a/approach/ovod/detectron2/projects/DensePose/doc/TOOL_QUERY_DB.md b/approach/ovod/detectron2/projects/DensePose/doc/TOOL_QUERY_DB.md new file mode 100644 index 0000000000000000000000000000000000000000..b0a764b8740597c6af634127b80b53d28913726f --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/doc/TOOL_QUERY_DB.md @@ -0,0 +1,105 @@ + +# Query Dataset + +`query_db` is a tool to print or visualize DensePose data from a dataset. +It has two modes: `print` and `show` to output dataset entries to standard +output or to visualize them on images. + +## Print Mode + +The general command form is: +```bash +python query_db.py print [-h] [-v] [--max-entries N] +``` + +There are two mandatory arguments: + - ``, DensePose dataset specification, from which to select + the entries (e.g. `densepose_coco_2014_train`). + - ``, dataset entry selector which can be a single specification, + or a comma-separated list of specifications of the form + `field[:type]=value` for exact match with the value + or `field[:type]=min-max` for a range of values + +One can additionally limit the maximum number of entries to output +by providing `--max-entries` argument. + +Examples: + +1. Output at most 10 first entries from the `densepose_coco_2014_train` dataset: +```bash +python query_db.py print densepose_coco_2014_train \* --max-entries 10 -v +``` + +2. Output all entries with `file_name` equal to `COCO_train2014_000000000036.jpg`: +```bash +python query_db.py print densepose_coco_2014_train file_name=COCO_train2014_000000000036.jpg -v +``` + +3. Output all entries with `image_id` between 36 and 156: +```bash +python query_db.py print densepose_coco_2014_train image_id:int=36-156 -v +``` + +## Visualization Mode + +The general command form is: +```bash +python query_db.py show [-h] [-v] [--max-entries N] [--output ] +``` + +There are three mandatory arguments: + - ``, DensePose dataset specification, from which to select + the entries (e.g. `densepose_coco_2014_train`). + - ``, dataset entry selector which can be a single specification, + or a comma-separated list of specifications of the form + `field[:type]=value` for exact match with the value + or `field[:type]=min-max` for a range of values + - ``, visualizations specifier; currently available visualizations are: + * `bbox` - bounding boxes of annotated persons; + * `dp_i` - annotated points colored according to the containing part; + * `dp_pts` - annotated points in green color; + * `dp_segm` - segmentation masks for annotated persons; + * `dp_u` - annotated points colored according to their U coordinate in part parameterization; + * `dp_v` - annotated points colored according to their V coordinate in part parameterization; + +One can additionally provide one of the two optional arguments: + - `--max_entries` to limit the maximum number of entries to visualize + - `--output` to provide visualization file name template, which defaults + to `output.png`. To distinguish file names for different dataset + entries, the tool appends 1-based entry index to the output file name, + e.g. output.0001.png, output.0002.png, etc. + +The following examples show how to output different visualizations for image with `id = 322` +from `densepose_coco_2014_train` dataset: + +1. Show bounding box and segmentation: +```bash +python query_db.py show densepose_coco_2014_train image_id:int=322 bbox,dp_segm -v +``` +![Bounding Box + Segmentation Visualization](images/vis_bbox_dp_segm.jpg) + +2. Show bounding box and points colored according to the containing part: +```bash +python query_db.py show densepose_coco_2014_train image_id:int=322 bbox,dp_i -v +``` +![Bounding Box + Point Label Visualization](images/vis_bbox_dp_i.jpg) + +3. Show bounding box and annotated points in green color: +```bash +python query_db.py show densepose_coco_2014_train image_id:int=322 bbox,dp_segm -v +``` +![Bounding Box + Point Visualization](images/vis_bbox_dp_pts.jpg) + +4. Show bounding box and annotated points colored according to their U coordinate in part parameterization: +```bash +python query_db.py show densepose_coco_2014_train image_id:int=322 bbox,dp_u -v +``` +![Bounding Box + Point U Visualization](images/vis_bbox_dp_u.jpg) + +5. Show bounding box and annotated points colored according to their V coordinate in part parameterization: +```bash +python query_db.py show densepose_coco_2014_train image_id:int=322 bbox,dp_v -v +``` +![Bounding Box + Point V Visualization](images/vis_bbox_dp_v.jpg) + + diff --git a/approach/ovod/detectron2/projects/DensePose/query_db.py b/approach/ovod/detectron2/projects/DensePose/query_db.py new file mode 100644 index 0000000000000000000000000000000000000000..8b2745ebbc4206441d8af6ac0bc4f1f74faf4d20 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/query_db.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. + +import argparse +import logging +import os +import sys +from timeit import default_timer as timer +from typing import Any, ClassVar, Dict, List +import torch + +from detectron2.data.catalog import DatasetCatalog +from detectron2.utils.file_io import PathManager +from detectron2.utils.logger import setup_logger + +from densepose.structures import DensePoseDataRelative +from densepose.utils.dbhelper import EntrySelector +from densepose.utils.logger import verbosity_to_level +from densepose.vis.base import CompoundVisualizer +from densepose.vis.bounding_box import BoundingBoxVisualizer +from densepose.vis.densepose_data_points import ( + DensePoseDataCoarseSegmentationVisualizer, + DensePoseDataPointsIVisualizer, + DensePoseDataPointsUVisualizer, + DensePoseDataPointsVisualizer, + DensePoseDataPointsVVisualizer, +) + +DOC = """Query DB - a tool to print / visualize data from a database +""" + +LOGGER_NAME = "query_db" + +logger = logging.getLogger(LOGGER_NAME) + +_ACTION_REGISTRY: Dict[str, "Action"] = {} + + +class Action(object): + @classmethod + def add_arguments(cls: type, parser: argparse.ArgumentParser): + parser.add_argument( + "-v", + "--verbosity", + action="count", + help="Verbose mode. Multiple -v options increase the verbosity.", + ) + + +def register_action(cls: type): + """ + Decorator for action classes to automate action registration + """ + global _ACTION_REGISTRY + _ACTION_REGISTRY[cls.COMMAND] = cls + return cls + + +class EntrywiseAction(Action): + @classmethod + def add_arguments(cls: type, parser: argparse.ArgumentParser): + super(EntrywiseAction, cls).add_arguments(parser) + parser.add_argument( + "dataset", metavar="", help="Dataset name (e.g. densepose_coco_2014_train)" + ) + parser.add_argument( + "selector", + metavar="", + help="Dataset entry selector in the form field1[:type]=value1[," + "field2[:type]=value_min-value_max...] which selects all " + "entries from the dataset that satisfy the constraints", + ) + parser.add_argument( + "--max-entries", metavar="N", help="Maximum number of entries to process", type=int + ) + + @classmethod + def execute(cls: type, args: argparse.Namespace): + dataset = setup_dataset(args.dataset) + entry_selector = EntrySelector.from_string(args.selector) + context = cls.create_context(args) + if args.max_entries is not None: + for _, entry in zip(range(args.max_entries), dataset): + if entry_selector(entry): + cls.execute_on_entry(entry, context) + else: + for entry in dataset: + if entry_selector(entry): + cls.execute_on_entry(entry, context) + + @classmethod + def create_context(cls: type, args: argparse.Namespace) -> Dict[str, Any]: + context = {} + return context + + +@register_action +class PrintAction(EntrywiseAction): + """ + Print action that outputs selected entries to stdout + """ + + COMMAND: ClassVar[str] = "print" + + @classmethod + def add_parser(cls: type, subparsers: argparse._SubParsersAction): + parser = subparsers.add_parser(cls.COMMAND, help="Output selected entries to stdout. ") + cls.add_arguments(parser) + parser.set_defaults(func=cls.execute) + + @classmethod + def add_arguments(cls: type, parser: argparse.ArgumentParser): + super(PrintAction, cls).add_arguments(parser) + + @classmethod + def execute_on_entry(cls: type, entry: Dict[str, Any], context: Dict[str, Any]): + import pprint + + printer = pprint.PrettyPrinter(indent=2, width=200, compact=True) + printer.pprint(entry) + + +@register_action +class ShowAction(EntrywiseAction): + """ + Show action that visualizes selected entries on an image + """ + + COMMAND: ClassVar[str] = "show" + VISUALIZERS: ClassVar[Dict[str, object]] = { + "dp_segm": DensePoseDataCoarseSegmentationVisualizer(), + "dp_i": DensePoseDataPointsIVisualizer(), + "dp_u": DensePoseDataPointsUVisualizer(), + "dp_v": DensePoseDataPointsVVisualizer(), + "dp_pts": DensePoseDataPointsVisualizer(), + "bbox": BoundingBoxVisualizer(), + } + + @classmethod + def add_parser(cls: type, subparsers: argparse._SubParsersAction): + parser = subparsers.add_parser(cls.COMMAND, help="Visualize selected entries") + cls.add_arguments(parser) + parser.set_defaults(func=cls.execute) + + @classmethod + def add_arguments(cls: type, parser: argparse.ArgumentParser): + super(ShowAction, cls).add_arguments(parser) + parser.add_argument( + "visualizations", + metavar="", + help="Comma separated list of visualizations, possible values: " + "[{}]".format(",".join(sorted(cls.VISUALIZERS.keys()))), + ) + parser.add_argument( + "--output", + metavar="", + default="output.png", + help="File name to save output to", + ) + + @classmethod + def execute_on_entry(cls: type, entry: Dict[str, Any], context: Dict[str, Any]): + import cv2 + import numpy as np + + image_fpath = PathManager.get_local_path(entry["file_name"]) + image = cv2.imread(image_fpath, cv2.IMREAD_GRAYSCALE) + image = np.tile(image[:, :, np.newaxis], [1, 1, 3]) + datas = cls._extract_data_for_visualizers_from_entry(context["vis_specs"], entry) + visualizer = context["visualizer"] + image_vis = visualizer.visualize(image, datas) + entry_idx = context["entry_idx"] + 1 + out_fname = cls._get_out_fname(entry_idx, context["out_fname"]) + cv2.imwrite(out_fname, image_vis) + logger.info(f"Output saved to {out_fname}") + context["entry_idx"] += 1 + + @classmethod + def _get_out_fname(cls: type, entry_idx: int, fname_base: str): + base, ext = os.path.splitext(fname_base) + return base + ".{0:04d}".format(entry_idx) + ext + + @classmethod + def create_context(cls: type, args: argparse.Namespace) -> Dict[str, Any]: + vis_specs = args.visualizations.split(",") + visualizers = [] + for vis_spec in vis_specs: + vis = cls.VISUALIZERS[vis_spec] + visualizers.append(vis) + context = { + "vis_specs": vis_specs, + "visualizer": CompoundVisualizer(visualizers), + "out_fname": args.output, + "entry_idx": 0, + } + return context + + @classmethod + def _extract_data_for_visualizers_from_entry( + cls: type, vis_specs: List[str], entry: Dict[str, Any] + ): + dp_list = [] + bbox_list = [] + for annotation in entry["annotations"]: + is_valid, _ = DensePoseDataRelative.validate_annotation(annotation) + if not is_valid: + continue + bbox = torch.as_tensor(annotation["bbox"]) + bbox_list.append(bbox) + dp_data = DensePoseDataRelative(annotation) + dp_list.append(dp_data) + datas = [] + for vis_spec in vis_specs: + datas.append(bbox_list if "bbox" == vis_spec else (bbox_list, dp_list)) + return datas + + +def setup_dataset(dataset_name): + logger.info("Loading dataset {}".format(dataset_name)) + start = timer() + dataset = DatasetCatalog.get(dataset_name) + stop = timer() + logger.info("Loaded dataset {} in {:.3f}s".format(dataset_name, stop - start)) + return dataset + + +def create_argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=DOC, + formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=120), + ) + parser.set_defaults(func=lambda _: parser.print_help(sys.stdout)) + subparsers = parser.add_subparsers(title="Actions") + for _, action in _ACTION_REGISTRY.items(): + action.add_parser(subparsers) + return parser + + +def main(): + parser = create_argument_parser() + args = parser.parse_args() + verbosity = args.verbosity if hasattr(args, "verbosity") else None + global logger + logger = setup_logger(name=LOGGER_NAME) + logger.setLevel(verbosity_to_level(verbosity)) + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/approach/ovod/detectron2/projects/DensePose/setup.py b/approach/ovod/detectron2/projects/DensePose/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..22ad239fe320b8f9501f783afb134b975276a628 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/setup.py @@ -0,0 +1,42 @@ +import re +from pathlib import Path +from setuptools import find_packages, setup + +try: + import torch # noqa: F401 +except ImportError as e: + raise Exception( + """ +You must install PyTorch prior to installing DensePose: +pip install torch + +For more information: + https://pytorch.org/get-started/locally/ + """ + ) from e + + +def get_detectron2_current_version(): + """Version is not available for import through Python since it is + above the top level of the package. Instead, we parse it from the + file with a regex.""" + # Get version info from detectron2 __init__.py + version_source = (Path(__file__).parents[2] / "detectron2" / "__init__.py").read_text() + version_number = re.findall(r'__version__ = "([0-9\.]+)"', version_source)[0] + return version_number + + +setup( + name="detectron2-densepose", + author="FAIR", + version=get_detectron2_current_version(), + url="https://github.com/facebookresearch/detectron2/tree/main/projects/DensePose", + packages=find_packages(), + python_requires=">=3.7", + install_requires=[ + "av>=8.0.3", + "detectron2@git+https://github.com/facebookresearch/detectron2.git", + "opencv-python-headless>=4.5.3.56", + "scipy>=1.5.4", + ], +) diff --git a/approach/ovod/detectron2/projects/DensePose/tests/common.py b/approach/ovod/detectron2/projects/DensePose/tests/common.py new file mode 100644 index 0000000000000000000000000000000000000000..ff22b9ab6eceb7c9de0f769c3cbd3197ecd51222 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/tests/common.py @@ -0,0 +1,124 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import os +import torch + +from detectron2.config import get_cfg +from detectron2.engine import default_setup +from detectron2.modeling import build_model + +from densepose import add_densepose_config + +_BASE_CONFIG_DIR = "configs" +_EVOLUTION_CONFIG_SUB_DIR = "evolution" +_HRNET_CONFIG_SUB_DIR = "HRNet" +_QUICK_SCHEDULES_CONFIG_SUB_DIR = "quick_schedules" +_BASE_CONFIG_FILE_PREFIX = "Base-" +_CONFIG_FILE_EXT = ".yaml" + + +def _get_base_config_dir(): + """ + Return the base directory for configurations + """ + return os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", _BASE_CONFIG_DIR) + + +def _get_evolution_config_dir(): + """ + Return the base directory for evolution configurations + """ + return os.path.join(_get_base_config_dir(), _EVOLUTION_CONFIG_SUB_DIR) + + +def _get_hrnet_config_dir(): + """ + Return the base directory for HRNet configurations + """ + return os.path.join(_get_base_config_dir(), _HRNET_CONFIG_SUB_DIR) + + +def _get_quick_schedules_config_dir(): + """ + Return the base directory for quick schedules configurations + """ + return os.path.join(_get_base_config_dir(), _QUICK_SCHEDULES_CONFIG_SUB_DIR) + + +def _collect_config_files(config_dir): + """ + Collect all configuration files (i.e. densepose_*.yaml) directly in the specified directory + """ + start = _get_base_config_dir() + results = [] + for entry in os.listdir(config_dir): + path = os.path.join(config_dir, entry) + if not os.path.isfile(path): + continue + _, ext = os.path.splitext(entry) + if ext != _CONFIG_FILE_EXT: + continue + if entry.startswith(_BASE_CONFIG_FILE_PREFIX): + continue + config_file = os.path.relpath(path, start) + results.append(config_file) + return results + + +def get_config_files(): + """ + Get all the configuration files (relative to the base configuration directory) + """ + return _collect_config_files(_get_base_config_dir()) + + +def get_evolution_config_files(): + """ + Get all the evolution configuration files (relative to the base configuration directory) + """ + return _collect_config_files(_get_evolution_config_dir()) + + +def get_hrnet_config_files(): + """ + Get all the HRNet configuration files (relative to the base configuration directory) + """ + return _collect_config_files(_get_hrnet_config_dir()) + + +def get_quick_schedules_config_files(): + """ + Get all the quick schedules configuration files (relative to the base configuration directory) + """ + return _collect_config_files(_get_quick_schedules_config_dir()) + + +def get_model_config(config_file): + """ + Load and return the configuration from the specified file (relative to the base configuration + directory) + """ + cfg = get_cfg() + add_densepose_config(cfg) + path = os.path.join(_get_base_config_dir(), config_file) + cfg.merge_from_file(path) + if not torch.cuda.is_available(): + cfg.MODEL.DEVICE = "cpu" + return cfg + + +def get_model(config_file): + """ + Get the model from the specified file (relative to the base configuration directory) + """ + cfg = get_model_config(config_file) + return build_model(cfg) + + +def setup(config_file): + """ + Setup the configuration from the specified file (relative to the base configuration directory) + """ + cfg = get_model_config(config_file) + cfg.freeze() + default_setup(cfg, {}) diff --git a/approach/ovod/detectron2/projects/DensePose/tests/test_chart_based_annotations_accumulator.py b/approach/ovod/detectron2/projects/DensePose/tests/test_chart_based_annotations_accumulator.py new file mode 100644 index 0000000000000000000000000000000000000000..a1c4f8565a3c55b79b6ed96b03635e6c2932958d --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/tests/test_chart_based_annotations_accumulator.py @@ -0,0 +1,76 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import unittest +import torch + +from detectron2.structures import Boxes, BoxMode, Instances + +from densepose.modeling.losses.utils import ChartBasedAnnotationsAccumulator +from densepose.structures import DensePoseDataRelative, DensePoseList + +image_shape = (100, 100) +instances = Instances(image_shape) +n_instances = 3 +instances.proposal_boxes = Boxes(torch.rand(n_instances, 4)) +instances.gt_boxes = Boxes(torch.rand(n_instances, 4)) + + +# instances.gt_densepose = None cannot happen because instances attributes need a length +class TestChartBasedAnnotationsAccumulator(unittest.TestCase): + def test_chart_based_annotations_accumulator_no_gt_densepose(self): + accumulator = ChartBasedAnnotationsAccumulator() + accumulator.accumulate(instances) + expected_values = {"nxt_bbox_with_dp_index": 0, "nxt_bbox_index": n_instances} + for key in accumulator.__dict__: + self.assertEqual(getattr(accumulator, key), expected_values.get(key, [])) + + def test_chart_based_annotations_accumulator_gt_densepose_none(self): + instances.gt_densepose = [None] * n_instances + accumulator = ChartBasedAnnotationsAccumulator() + accumulator.accumulate(instances) + expected_values = {"nxt_bbox_with_dp_index": 0, "nxt_bbox_index": n_instances} + for key in accumulator.__dict__: + self.assertEqual(getattr(accumulator, key), expected_values.get(key, [])) + + def test_chart_based_annotations_accumulator_gt_densepose(self): + data_relative_keys = [ + DensePoseDataRelative.X_KEY, + DensePoseDataRelative.Y_KEY, + DensePoseDataRelative.I_KEY, + DensePoseDataRelative.U_KEY, + DensePoseDataRelative.V_KEY, + DensePoseDataRelative.S_KEY, + ] + annotations = [DensePoseDataRelative({k: [0] for k in data_relative_keys})] * n_instances + instances.gt_densepose = DensePoseList(annotations, instances.gt_boxes, image_shape) + accumulator = ChartBasedAnnotationsAccumulator() + accumulator.accumulate(instances) + bbox_xywh_est = BoxMode.convert( + instances.proposal_boxes.tensor.clone(), BoxMode.XYXY_ABS, BoxMode.XYWH_ABS + ) + bbox_xywh_gt = BoxMode.convert( + instances.gt_boxes.tensor.clone(), BoxMode.XYXY_ABS, BoxMode.XYWH_ABS + ) + expected_values = { + "s_gt": [ + torch.zeros((3, DensePoseDataRelative.MASK_SIZE, DensePoseDataRelative.MASK_SIZE)) + ] + * n_instances, + "bbox_xywh_est": bbox_xywh_est.split(1), + "bbox_xywh_gt": bbox_xywh_gt.split(1), + "point_bbox_with_dp_indices": [torch.tensor([i]) for i in range(n_instances)], + "point_bbox_indices": [torch.tensor([i]) for i in range(n_instances)], + "bbox_indices": list(range(n_instances)), + "nxt_bbox_with_dp_index": n_instances, + "nxt_bbox_index": n_instances, + } + default_value = [torch.tensor([0])] * 3 + for key in accumulator.__dict__: + to_test = getattr(accumulator, key) + gt_value = expected_values.get(key, default_value) + if key in ["nxt_bbox_with_dp_index", "nxt_bbox_index"]: + self.assertEqual(to_test, gt_value) + elif key == "bbox_indices": + self.assertListEqual(to_test, gt_value) + else: + self.assertTrue(torch.allclose(torch.stack(to_test), torch.stack(gt_value))) diff --git a/approach/ovod/detectron2/projects/DensePose/tests/test_combine_data_loader.py b/approach/ovod/detectron2/projects/DensePose/tests/test_combine_data_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..832903a8e133b124669830b378af582c3b58b3dc --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/tests/test_combine_data_loader.py @@ -0,0 +1,46 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import random +import unittest +from typing import Any, Iterable, Iterator, Tuple + +from densepose.data import CombinedDataLoader + + +def _grouper(iterable: Iterable[Any], n: int, fillvalue=None) -> Iterator[Tuple[Any]]: + """ + Group elements of an iterable by chunks of size `n`, e.g. + grouper(range(9), 4) -> + (0, 1, 2, 3), (4, 5, 6, 7), (8, None, None, None) + """ + it = iter(iterable) + while True: + values = [] + for _ in range(n): + try: + value = next(it) + except StopIteration: + values.extend([fillvalue] * (n - len(values))) + yield tuple(values) + return + values.append(value) + yield tuple(values) + + +class TestCombinedDataLoader(unittest.TestCase): + def test_combine_loaders_1(self): + loader1 = _grouper([f"1_{i}" for i in range(10)], 2) + loader2 = _grouper([f"2_{i}" for i in range(11)], 3) + batch_size = 4 + ratios = (0.1, 0.9) + random.seed(43) + combined = CombinedDataLoader((loader1, loader2), batch_size, ratios) + BATCHES_GT = [ + ["1_0", "1_1", "2_0", "2_1"], + ["2_2", "2_3", "2_4", "2_5"], + ["1_2", "1_3", "2_6", "2_7"], + ["2_8", "2_9", "2_10", None], + ] + for i, batch in enumerate(combined): + self.assertEqual(len(batch), batch_size) + self.assertEqual(batch, BATCHES_GT[i]) diff --git a/approach/ovod/detectron2/projects/DensePose/tests/test_cse_annotations_accumulator.py b/approach/ovod/detectron2/projects/DensePose/tests/test_cse_annotations_accumulator.py new file mode 100644 index 0000000000000000000000000000000000000000..a22dce9ce00532d60dc3f4edbef4cea26b006b92 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/tests/test_cse_annotations_accumulator.py @@ -0,0 +1,240 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +import unittest +import torch + +from detectron2.structures import Boxes, BoxMode, Instances + +from densepose.modeling.losses.embed_utils import CseAnnotationsAccumulator +from densepose.structures import DensePoseDataRelative, DensePoseList + + +class TestCseAnnotationsAccumulator(unittest.TestCase): + def test_cse_annotations_accumulator_nodp(self): + instances_lst = [ + self._create_instances_nodp(), + ] + self._test_template(instances_lst) + + def test_cse_annotations_accumulator_sparsedp(self): + instances_lst = [ + self._create_instances_sparsedp(), + ] + self._test_template(instances_lst) + + def test_cse_annotations_accumulator_fulldp(self): + instances_lst = [ + self._create_instances_fulldp(), + ] + self._test_template(instances_lst) + + def test_cse_annotations_accumulator_combined(self): + instances_lst = [ + self._create_instances_nodp(), + self._create_instances_sparsedp(), + self._create_instances_fulldp(), + ] + self._test_template(instances_lst) + + def _test_template(self, instances_lst): + acc = CseAnnotationsAccumulator() + for instances in instances_lst: + acc.accumulate(instances) + packed_anns = acc.pack() + self._check_correspondence(packed_anns, instances_lst) + + def _create_instances_nodp(self): + image_shape = (480, 640) + instances = Instances(image_shape) + instances.gt_boxes = Boxes( + torch.as_tensor( + [ + [40.0, 40.0, 140.0, 140.0], + [160.0, 160.0, 270.0, 270.0], + [40.0, 160.0, 160.0, 280.0], + ] + ) + ) + instances.proposal_boxes = Boxes( + torch.as_tensor( + [ + [41.0, 39.0, 142.0, 138.0], + [161.0, 159.0, 272.0, 268.0], + [41.0, 159.0, 162.0, 278.0], + ] + ) + ) + # do not add gt_densepose + return instances + + def _create_instances_sparsedp(self): + image_shape = (540, 720) + instances = Instances(image_shape) + instances.gt_boxes = Boxes( + torch.as_tensor( + [ + [50.0, 50.0, 130.0, 130.0], + [150.0, 150.0, 240.0, 240.0], + [50.0, 150.0, 230.0, 330.0], + ] + ) + ) + instances.proposal_boxes = Boxes( + torch.as_tensor( + [ + [49.0, 51.0, 131.0, 129.0], + [151.0, 149.0, 241.0, 239.0], + [51.0, 149.0, 232.0, 329.0], + ] + ) + ) + instances.gt_densepose = DensePoseList( + [ + None, + self._create_dp_data( + { + "dp_x": [81.69, 153.47, 151.00], + "dp_y": [162.24, 128.71, 113.81], + "dp_vertex": [0, 1, 2], + "ref_model": "zebra_5002", + "dp_masks": [], + }, + {"c": (166, 133), "r": 64}, + ), + None, + ], + instances.gt_boxes, + image_shape, + ) + return instances + + def _create_instances_fulldp(self): + image_shape = (680, 840) + instances = Instances(image_shape) + instances.gt_boxes = Boxes( + torch.as_tensor( + [ + [65.0, 55.0, 165.0, 155.0], + [170.0, 175.0, 275.0, 280.0], + [55.0, 165.0, 165.0, 275.0], + ] + ) + ) + instances.proposal_boxes = Boxes( + torch.as_tensor( + [ + [66.0, 54.0, 166.0, 154.0], + [171.0, 174.0, 276.0, 279.0], + [56.0, 164.0, 166.0, 274.0], + ] + ) + ) + instances.gt_densepose = DensePoseList( + [ + self._create_dp_data( + { + "dp_x": [149.99, 198.62, 157.59], + "dp_y": [170.74, 197.73, 123.12], + "dp_vertex": [3, 4, 5], + "ref_model": "cat_5001", + "dp_masks": [], + }, + {"c": (100, 100), "r": 50}, + ), + self._create_dp_data( + { + "dp_x": [234.53, 116.72, 71.66], + "dp_y": [107.53, 11.31, 142.32], + "dp_vertex": [6, 7, 8], + "ref_model": "dog_5002", + "dp_masks": [], + }, + {"c": (200, 150), "r": 40}, + ), + self._create_dp_data( + { + "dp_x": [225.54, 202.61, 135.90], + "dp_y": [167.46, 181.00, 211.47], + "dp_vertex": [9, 10, 11], + "ref_model": "elephant_5002", + "dp_masks": [], + }, + {"c": (100, 200), "r": 45}, + ), + ], + instances.gt_boxes, + image_shape, + ) + return instances + + def _create_dp_data(self, anns, blob_def=None): + dp_data = DensePoseDataRelative(anns) + if blob_def is not None: + dp_data.segm[ + blob_def["c"][0] - blob_def["r"] : blob_def["c"][0] + blob_def["r"], + blob_def["c"][1] - blob_def["r"] : blob_def["c"][1] + blob_def["r"], + ] = 1 + return dp_data + + def _check_correspondence(self, packed_anns, instances_lst): + instance_idx = 0 + data_idx = 0 + pt_offset = 0 + if packed_anns is not None: + bbox_xyxy_gt = BoxMode.convert( + packed_anns.bbox_xywh_gt.clone(), BoxMode.XYWH_ABS, BoxMode.XYXY_ABS + ) + bbox_xyxy_est = BoxMode.convert( + packed_anns.bbox_xywh_est.clone(), BoxMode.XYWH_ABS, BoxMode.XYXY_ABS + ) + for instances in instances_lst: + if not hasattr(instances, "gt_densepose"): + instance_idx += len(instances) + continue + for i, dp_data in enumerate(instances.gt_densepose): + if dp_data is None: + instance_idx += 1 + continue + n_pts = len(dp_data.x) + self.assertTrue( + torch.allclose(dp_data.x, packed_anns.x_gt[pt_offset : pt_offset + n_pts]) + ) + self.assertTrue( + torch.allclose(dp_data.y, packed_anns.y_gt[pt_offset : pt_offset + n_pts]) + ) + self.assertTrue(torch.allclose(dp_data.segm, packed_anns.coarse_segm_gt[data_idx])) + self.assertTrue( + torch.allclose( + torch.ones(n_pts, dtype=torch.long) * dp_data.mesh_id, + packed_anns.vertex_mesh_ids_gt[pt_offset : pt_offset + n_pts], + ) + ) + self.assertTrue( + torch.allclose( + dp_data.vertex_ids, packed_anns.vertex_ids_gt[pt_offset : pt_offset + n_pts] + ) + ) + self.assertTrue( + torch.allclose(instances.gt_boxes.tensor[i], bbox_xyxy_gt[data_idx]) + ) + self.assertTrue( + torch.allclose(instances.proposal_boxes.tensor[i], bbox_xyxy_est[data_idx]) + ) + self.assertTrue( + torch.allclose( + torch.ones(n_pts, dtype=torch.long) * data_idx, + packed_anns.point_bbox_with_dp_indices[pt_offset : pt_offset + n_pts], + ) + ) + self.assertTrue( + torch.allclose( + torch.ones(n_pts, dtype=torch.long) * instance_idx, + packed_anns.point_bbox_indices[pt_offset : pt_offset + n_pts], + ) + ) + self.assertEqual(instance_idx, packed_anns.bbox_indices[data_idx]) + pt_offset += n_pts + instance_idx += 1 + data_idx += 1 + if data_idx == 0: + self.assertIsNone(packed_anns) diff --git a/approach/ovod/detectron2/projects/DensePose/tests/test_dataset_loaded_annotations.py b/approach/ovod/detectron2/projects/DensePose/tests/test_dataset_loaded_annotations.py new file mode 100644 index 0000000000000000000000000000000000000000..cf8035b87c6477221a113ba9fcb794495c04af7c --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/tests/test_dataset_loaded_annotations.py @@ -0,0 +1,87 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +import unittest + +from densepose.data.datasets.builtin import COCO_DATASETS, DENSEPOSE_ANNOTATIONS_DIR, LVIS_DATASETS +from densepose.data.datasets.coco import load_coco_json +from densepose.data.datasets.lvis import load_lvis_json +from densepose.data.utils import maybe_prepend_base_path +from densepose.structures import DensePoseDataRelative + + +class TestDatasetLoadedAnnotations(unittest.TestCase): + COCO_DATASET_DATA = { + "densepose_coco_2014_train": {"n_instances": 39210}, + "densepose_coco_2014_minival": {"n_instances": 2243}, + "densepose_coco_2014_minival_100": {"n_instances": 164}, + "densepose_coco_2014_valminusminival": {"n_instances": 7297}, + "densepose_coco_2014_train_cse": {"n_instances": 39210}, + "densepose_coco_2014_minival_cse": {"n_instances": 2243}, + "densepose_coco_2014_minival_100_cse": {"n_instances": 164}, + "densepose_coco_2014_valminusminival_cse": {"n_instances": 7297}, + "densepose_chimps": {"n_instances": 930}, + "posetrack2017_train": {"n_instances": 8274}, + "posetrack2017_val": {"n_instances": 4753}, + "lvis_v05_train": {"n_instances": 5186}, + "lvis_v05_val": {"n_instances": 1037}, + } + + LVIS_DATASET_DATA = { + "densepose_lvis_v1_train1": {"n_instances": 3394}, + "densepose_lvis_v1_train2": {"n_instances": 1800}, + "densepose_lvis_v1_val": {"n_instances": 1037}, + "densepose_lvis_v1_val_animals_100": {"n_instances": 89}, + } + + def generic_coco_test(self, dataset_info): + if dataset_info.name not in self.COCO_DATASET_DATA: + return + n_inst = self.COCO_DATASET_DATA[dataset_info.name]["n_instances"] + self.generic_test(dataset_info, n_inst, load_coco_json) + + def generic_lvis_test(self, dataset_info): + if dataset_info.name not in self.LVIS_DATASET_DATA: + return + n_inst = self.LVIS_DATASET_DATA[dataset_info.name]["n_instances"] + self.generic_test(dataset_info, n_inst, load_lvis_json) + + def generic_test(self, dataset_info, n_inst, loader_fun): + datasets_root = DENSEPOSE_ANNOTATIONS_DIR + annotations_fpath = maybe_prepend_base_path(datasets_root, dataset_info.annotations_fpath) + images_root = maybe_prepend_base_path(datasets_root, dataset_info.images_root) + image_annotation_dicts = loader_fun( + annotations_json_file=annotations_fpath, + image_root=images_root, + dataset_name=dataset_info.name, + ) + num_valid = sum( + 1 + for image_annotation_dict in image_annotation_dicts + for ann in image_annotation_dict["annotations"] + if DensePoseDataRelative.validate_annotation(ann)[0] + ) + self.assertEqual(num_valid, n_inst) + + +def coco_test_fun(dataset_info): + return lambda self: self.generic_coco_test(dataset_info) + + +for dataset_info in COCO_DATASETS: + setattr( + TestDatasetLoadedAnnotations, + f"test_coco_builtin_loaded_annotations_{dataset_info.name}", + coco_test_fun(dataset_info), + ) + + +def lvis_test_fun(dataset_info): + return lambda self: self.generic_lvis_test(dataset_info) + + +for dataset_info in LVIS_DATASETS: + setattr( + TestDatasetLoadedAnnotations, + f"test_lvis_builtin_loaded_annotations_{dataset_info.name}", + lvis_test_fun(dataset_info), + ) diff --git a/approach/ovod/detectron2/projects/DensePose/tests/test_frame_selector.py b/approach/ovod/detectron2/projects/DensePose/tests/test_frame_selector.py new file mode 100644 index 0000000000000000000000000000000000000000..65f05f55c78d4ab24950e5335818b3e1f981aa0d --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/tests/test_frame_selector.py @@ -0,0 +1,60 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import random +import unittest + +from densepose.data.video import FirstKFramesSelector, LastKFramesSelector, RandomKFramesSelector + + +class TestFrameSelector(unittest.TestCase): + def test_frame_selector_random_k_1(self): + _SEED = 43 + _K = 4 + random.seed(_SEED) + selector = RandomKFramesSelector(_K) + frame_tss = list(range(0, 20, 2)) + _SELECTED_GT = [0, 8, 4, 6] + selected = selector(frame_tss) + self.assertEqual(_SELECTED_GT, selected) + + def test_frame_selector_random_k_2(self): + _SEED = 43 + _K = 10 + random.seed(_SEED) + selector = RandomKFramesSelector(_K) + frame_tss = list(range(0, 6, 2)) + _SELECTED_GT = [0, 2, 4] + selected = selector(frame_tss) + self.assertEqual(_SELECTED_GT, selected) + + def test_frame_selector_first_k_1(self): + _K = 4 + selector = FirstKFramesSelector(_K) + frame_tss = list(range(0, 20, 2)) + _SELECTED_GT = frame_tss[:_K] + selected = selector(frame_tss) + self.assertEqual(_SELECTED_GT, selected) + + def test_frame_selector_first_k_2(self): + _K = 10 + selector = FirstKFramesSelector(_K) + frame_tss = list(range(0, 6, 2)) + _SELECTED_GT = frame_tss[:_K] + selected = selector(frame_tss) + self.assertEqual(_SELECTED_GT, selected) + + def test_frame_selector_last_k_1(self): + _K = 4 + selector = LastKFramesSelector(_K) + frame_tss = list(range(0, 20, 2)) + _SELECTED_GT = frame_tss[-_K:] + selected = selector(frame_tss) + self.assertEqual(_SELECTED_GT, selected) + + def test_frame_selector_last_k_2(self): + _K = 10 + selector = LastKFramesSelector(_K) + frame_tss = list(range(0, 6, 2)) + _SELECTED_GT = frame_tss[-_K:] + selected = selector(frame_tss) + self.assertEqual(_SELECTED_GT, selected) diff --git a/approach/ovod/detectron2/projects/DensePose/tests/test_image_list_dataset.py b/approach/ovod/detectron2/projects/DensePose/tests/test_image_list_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..7932602448b49b9be4fcea9645fe7a9c4d53c00e --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/tests/test_image_list_dataset.py @@ -0,0 +1,48 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import contextlib +import os +import tempfile +import unittest +import torch +from torchvision.utils import save_image + +from densepose.data.image_list_dataset import ImageListDataset +from densepose.data.transform import ImageResizeTransform + + +@contextlib.contextmanager +def temp_image(height, width): + random_image = torch.rand(height, width) + with tempfile.NamedTemporaryFile(suffix=".jpg") as f: + f.close() + save_image(random_image, f.name) + yield f.name + os.unlink(f.name) + + +class TestImageListDataset(unittest.TestCase): + def test_image_list_dataset(self): + height, width = 720, 1280 + with temp_image(height, width) as image_fpath: + image_list = [image_fpath] + category_list = [None] + dataset = ImageListDataset(image_list, category_list) + self.assertEqual(len(dataset), 1) + data1, categories1 = dataset[0]["images"], dataset[0]["categories"] + self.assertEqual(data1.shape, torch.Size((1, 3, height, width))) + self.assertEqual(data1.dtype, torch.float32) + self.assertIsNone(categories1[0]) + + def test_image_list_dataset_with_transform(self): + height, width = 720, 1280 + with temp_image(height, width) as image_fpath: + image_list = [image_fpath] + category_list = [None] + transform = ImageResizeTransform() + dataset = ImageListDataset(image_list, category_list, transform) + self.assertEqual(len(dataset), 1) + data1, categories1 = dataset[0]["images"], dataset[0]["categories"] + self.assertEqual(data1.shape, torch.Size((1, 3, 749, 1333))) + self.assertEqual(data1.dtype, torch.float32) + self.assertIsNone(categories1[0]) diff --git a/approach/ovod/detectron2/projects/DensePose/tests/test_image_resize_transform.py b/approach/ovod/detectron2/projects/DensePose/tests/test_image_resize_transform.py new file mode 100644 index 0000000000000000000000000000000000000000..01c3373b64ee243198af682928939781a15f929a --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/tests/test_image_resize_transform.py @@ -0,0 +1,16 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import unittest +import torch + +from densepose.data.transform import ImageResizeTransform + + +class TestImageResizeTransform(unittest.TestCase): + def test_image_resize_1(self): + images_batch = torch.ones((3, 3, 100, 100), dtype=torch.uint8) * 100 + transform = ImageResizeTransform() + images_transformed = transform(images_batch) + IMAGES_GT = torch.ones((3, 3, 800, 800), dtype=torch.float) * 100 + self.assertEqual(images_transformed.size(), IMAGES_GT.size()) + self.assertAlmostEqual(torch.abs(IMAGES_GT - images_transformed).max().item(), 0.0) diff --git a/approach/ovod/detectron2/projects/DensePose/tests/test_model_e2e.py b/approach/ovod/detectron2/projects/DensePose/tests/test_model_e2e.py new file mode 100644 index 0000000000000000000000000000000000000000..055fadfd781adcdfd661795edbc621d5eca763fe --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/tests/test_model_e2e.py @@ -0,0 +1,43 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import unittest +import torch + +from detectron2.structures import BitMasks, Boxes, Instances + +from .common import get_model + + +# TODO(plabatut): Modularize detectron2 tests and re-use +def make_model_inputs(image, instances=None): + if instances is None: + return {"image": image} + + return {"image": image, "instances": instances} + + +def make_empty_instances(h, w): + instances = Instances((h, w)) + instances.gt_boxes = Boxes(torch.rand(0, 4)) + instances.gt_classes = torch.tensor([]).to(dtype=torch.int64) + instances.gt_masks = BitMasks(torch.rand(0, h, w)) + return instances + + +class ModelE2ETest(unittest.TestCase): + CONFIG_PATH = "" + + def setUp(self): + self.model = get_model(self.CONFIG_PATH) + + def _test_eval(self, sizes): + inputs = [make_model_inputs(torch.rand(3, size[0], size[1])) for size in sizes] + self.model.eval() + self.model(inputs) + + +class DensePoseRCNNE2ETest(ModelE2ETest): + CONFIG_PATH = "densepose_rcnn_R_101_FPN_s1x.yaml" + + def test_empty_data(self): + self._test_eval([(200, 250), (200, 249)]) diff --git a/approach/ovod/detectron2/projects/DensePose/tests/test_setup.py b/approach/ovod/detectron2/projects/DensePose/tests/test_setup.py new file mode 100644 index 0000000000000000000000000000000000000000..165a1b9a7b64aa8a0fbe5b862ebfb6594e77c256 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/tests/test_setup.py @@ -0,0 +1,36 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import unittest + +from .common import ( + get_config_files, + get_evolution_config_files, + get_hrnet_config_files, + get_quick_schedules_config_files, + setup, +) + + +class TestSetup(unittest.TestCase): + def _test_setup(self, config_file): + setup(config_file) + + def test_setup_configs(self): + config_files = get_config_files() + for config_file in config_files: + self._test_setup(config_file) + + def test_setup_evolution_configs(self): + config_files = get_evolution_config_files() + for config_file in config_files: + self._test_setup(config_file) + + def test_setup_hrnet_configs(self): + config_files = get_hrnet_config_files() + for config_file in config_files: + self._test_setup(config_file) + + def test_setup_quick_schedules_configs(self): + config_files = get_quick_schedules_config_files() + for config_file in config_files: + self._test_setup(config_file) diff --git a/approach/ovod/detectron2/projects/DensePose/tests/test_structures.py b/approach/ovod/detectron2/projects/DensePose/tests/test_structures.py new file mode 100644 index 0000000000000000000000000000000000000000..54082d3abf119bf2fdba7206124893f35b4b4ae1 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/tests/test_structures.py @@ -0,0 +1,25 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import unittest + +from densepose.structures import normalized_coords_transform + + +class TestStructures(unittest.TestCase): + def test_normalized_coords_transform(self): + bbox = (32, 24, 288, 216) + x0, y0, w, h = bbox + xmin, ymin, xmax, ymax = x0, y0, x0 + w, y0 + h + f = normalized_coords_transform(*bbox) + # Top-left + expected_p, actual_p = (-1, -1), f((xmin, ymin)) + self.assertEqual(expected_p, actual_p) + # Top-right + expected_p, actual_p = (1, -1), f((xmax, ymin)) + self.assertEqual(expected_p, actual_p) + # Bottom-left + expected_p, actual_p = (-1, 1), f((xmin, ymax)) + self.assertEqual(expected_p, actual_p) + # Bottom-right + expected_p, actual_p = (1, 1), f((xmax, ymax)) + self.assertEqual(expected_p, actual_p) diff --git a/approach/ovod/detectron2/projects/DensePose/tests/test_tensor_storage.py b/approach/ovod/detectron2/projects/DensePose/tests/test_tensor_storage.py new file mode 100644 index 0000000000000000000000000000000000000000..aeeeffae4675f8d607d0471250dadb2ece5361a0 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/tests/test_tensor_storage.py @@ -0,0 +1,256 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import io +import tempfile +import unittest +from contextlib import ExitStack +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from detectron2.utils import comm + +from densepose.evaluation.tensor_storage import ( + SingleProcessFileTensorStorage, + SingleProcessRamTensorStorage, + SizeData, + storage_gather, +) + + +class TestSingleProcessRamTensorStorage(unittest.TestCase): + def test_read_write_1(self): + schema = { + "tf": SizeData(dtype="float32", shape=(112, 112)), + "ti": SizeData(dtype="int32", shape=(4, 64, 64)), + } + # generate data which corresponds to the schema + data_elts = [] + torch.manual_seed(23) + for _i in range(3): + data_elt = { + "tf": torch.rand((112, 112), dtype=torch.float32), + "ti": (torch.rand(4, 64, 64) * 1000).to(dtype=torch.int32), + } + data_elts.append(data_elt) + storage = SingleProcessRamTensorStorage(schema, io.BytesIO()) + # write data to the storage + for i in range(3): + record_id = storage.put(data_elts[i]) + self.assertEqual(record_id, i) + # read data from the storage + for i in range(3): + record = storage.get(i) + self.assertEqual(len(record), len(schema)) + for field_name in schema: + self.assertTrue(field_name in record) + self.assertEqual(data_elts[i][field_name].shape, record[field_name].shape) + self.assertEqual(data_elts[i][field_name].dtype, record[field_name].dtype) + self.assertTrue(torch.allclose(data_elts[i][field_name], record[field_name])) + + +class TestSingleProcessFileTensorStorage(unittest.TestCase): + def test_read_write_1(self): + schema = { + "tf": SizeData(dtype="float32", shape=(112, 112)), + "ti": SizeData(dtype="int32", shape=(4, 64, 64)), + } + # generate data which corresponds to the schema + data_elts = [] + torch.manual_seed(23) + for _i in range(3): + data_elt = { + "tf": torch.rand((112, 112), dtype=torch.float32), + "ti": (torch.rand(4, 64, 64) * 1000).to(dtype=torch.int32), + } + data_elts.append(data_elt) + # WARNING: opens the file several times! may not work on all platforms + with tempfile.NamedTemporaryFile() as hFile: + storage = SingleProcessFileTensorStorage(schema, hFile.name, "wb") + # write data to the storage + for i in range(3): + record_id = storage.put(data_elts[i]) + self.assertEqual(record_id, i) + hFile.seek(0) + storage = SingleProcessFileTensorStorage(schema, hFile.name, "rb") + # read data from the storage + for i in range(3): + record = storage.get(i) + self.assertEqual(len(record), len(schema)) + for field_name in schema: + self.assertTrue(field_name in record) + self.assertEqual(data_elts[i][field_name].shape, record[field_name].shape) + self.assertEqual(data_elts[i][field_name].dtype, record[field_name].dtype) + self.assertTrue(torch.allclose(data_elts[i][field_name], record[field_name])) + + +def _find_free_port(): + """ + Copied from detectron2/engine/launch.py + """ + import socket + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Binding to port 0 will cause the OS to find an available port for us + sock.bind(("", 0)) + port = sock.getsockname()[1] + sock.close() + # NOTE: there is still a chance the port could be taken by other processes. + return port + + +def launch(main_func, nprocs, args=()): + port = _find_free_port() + dist_url = f"tcp://127.0.0.1:{port}" + # dist_url = "env://" + mp.spawn( + distributed_worker, nprocs=nprocs, args=(main_func, nprocs, dist_url, args), daemon=False + ) + + +def distributed_worker(local_rank, main_func, nprocs, dist_url, args): + dist.init_process_group( + backend="gloo", init_method=dist_url, world_size=nprocs, rank=local_rank + ) + comm.synchronize() + assert comm._LOCAL_PROCESS_GROUP is None + pg = dist.new_group(list(range(nprocs))) + comm._LOCAL_PROCESS_GROUP = pg + main_func(*args) + + +def ram_read_write_worker(): + schema = { + "tf": SizeData(dtype="float32", shape=(112, 112)), + "ti": SizeData(dtype="int32", shape=(4, 64, 64)), + } + storage = SingleProcessRamTensorStorage(schema, io.BytesIO()) + world_size = comm.get_world_size() + rank = comm.get_rank() + data_elts = [] + # prepare different number of tensors in different processes + for i in range(rank + 1): + data_elt = { + "tf": torch.ones((112, 112), dtype=torch.float32) * (rank + i * world_size), + "ti": torch.ones((4, 64, 64), dtype=torch.int32) * (rank + i * world_size), + } + data_elts.append(data_elt) + # write data to the single process storage + for i in range(rank + 1): + record_id = storage.put(data_elts[i]) + assert record_id == i, f"Process {rank}: record ID {record_id}, expected {i}" + comm.synchronize() + # gather all data in process rank 0 + multi_storage = storage_gather(storage) + if rank != 0: + return + # read and check data from the multiprocess storage + for j in range(world_size): + for i in range(j): + record = multi_storage.get(j, i) + record_gt = { + "tf": torch.ones((112, 112), dtype=torch.float32) * (j + i * world_size), + "ti": torch.ones((4, 64, 64), dtype=torch.int32) * (j + i * world_size), + } + assert len(record) == len(schema), ( + f"Process {rank}: multi storage record, rank {j}, id {i}: " + f"expected {len(schema)} fields in the record, got {len(record)}" + ) + for field_name in schema: + assert field_name in record, ( + f"Process {rank}: multi storage record, rank {j}, id {i}: " + f"field {field_name} not in the record" + ) + + assert record_gt[field_name].shape == record[field_name].shape, ( + f"Process {rank}: multi storage record, rank {j}, id {i}: " + f"field {field_name}, expected shape {record_gt[field_name].shape} " + f"got {record[field_name].shape}" + ) + assert record_gt[field_name].dtype == record[field_name].dtype, ( + f"Process {rank}: multi storage record, rank {j}, id {i}: " + f"field {field_name}, expected dtype {record_gt[field_name].dtype} " + f"got {record[field_name].dtype}" + ) + assert torch.allclose(record_gt[field_name], record[field_name]), ( + f"Process {rank}: multi storage record, rank {j}, id {i}: " + f"field {field_name}, tensors are not close enough:" + f"L-inf {(record_gt[field_name]-record[field_name]).abs_().max()} " + f"L1 {(record_gt[field_name]-record[field_name]).abs_().sum()} " + ) + + +def file_read_write_worker(rank_to_fpath): + schema = { + "tf": SizeData(dtype="float32", shape=(112, 112)), + "ti": SizeData(dtype="int32", shape=(4, 64, 64)), + } + world_size = comm.get_world_size() + rank = comm.get_rank() + storage = SingleProcessFileTensorStorage(schema, rank_to_fpath[rank], "wb") + data_elts = [] + # prepare different number of tensors in different processes + for i in range(rank + 1): + data_elt = { + "tf": torch.ones((112, 112), dtype=torch.float32) * (rank + i * world_size), + "ti": torch.ones((4, 64, 64), dtype=torch.int32) * (rank + i * world_size), + } + data_elts.append(data_elt) + # write data to the single process storage + for i in range(rank + 1): + record_id = storage.put(data_elts[i]) + assert record_id == i, f"Process {rank}: record ID {record_id}, expected {i}" + comm.synchronize() + # gather all data in process rank 0 + multi_storage = storage_gather(storage) + if rank != 0: + return + # read and check data from the multiprocess storage + for j in range(world_size): + for i in range(j): + record = multi_storage.get(j, i) + record_gt = { + "tf": torch.ones((112, 112), dtype=torch.float32) * (j + i * world_size), + "ti": torch.ones((4, 64, 64), dtype=torch.int32) * (j + i * world_size), + } + assert len(record) == len(schema), ( + f"Process {rank}: multi storage record, rank {j}, id {i}: " + f"expected {len(schema)} fields in the record, got {len(record)}" + ) + for field_name in schema: + assert field_name in record, ( + f"Process {rank}: multi storage record, rank {j}, id {i}: " + f"field {field_name} not in the record" + ) + + assert record_gt[field_name].shape == record[field_name].shape, ( + f"Process {rank}: multi storage record, rank {j}, id {i}: " + f"field {field_name}, expected shape {record_gt[field_name].shape} " + f"got {record[field_name].shape}" + ) + assert record_gt[field_name].dtype == record[field_name].dtype, ( + f"Process {rank}: multi storage record, rank {j}, id {i}: " + f"field {field_name}, expected dtype {record_gt[field_name].dtype} " + f"got {record[field_name].dtype}" + ) + assert torch.allclose(record_gt[field_name], record[field_name]), ( + f"Process {rank}: multi storage record, rank {j}, id {i}: " + f"field {field_name}, tensors are not close enough:" + f"L-inf {(record_gt[field_name]-record[field_name]).abs_().max()} " + f"L1 {(record_gt[field_name]-record[field_name]).abs_().sum()} " + ) + + +class TestMultiProcessRamTensorStorage(unittest.TestCase): + def test_read_write_1(self): + launch(ram_read_write_worker, 8) + + +class TestMultiProcessFileTensorStorage(unittest.TestCase): + def test_read_write_1(self): + with ExitStack() as stack: + # WARNING: opens the files several times! may not work on all platforms + rank_to_fpath = { + i: stack.enter_context(tempfile.NamedTemporaryFile()).name for i in range(8) + } + launch(file_read_write_worker, 8, (rank_to_fpath,)) diff --git a/approach/ovod/detectron2/projects/DensePose/tests/test_video_keyframe_dataset.py b/approach/ovod/detectron2/projects/DensePose/tests/test_video_keyframe_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..988e1616cdd30757157b479990050d1ca494ce7b --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/tests/test_video_keyframe_dataset.py @@ -0,0 +1,98 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import contextlib +import os +import random +import tempfile +import unittest +import torch +import torchvision.io as io + +from densepose.data.transform import ImageResizeTransform +from densepose.data.video import RandomKFramesSelector, VideoKeyframeDataset + +try: + import av +except ImportError: + av = None + + +# copied from torchvision test/test_io.py +def _create_video_frames(num_frames, height, width): + y, x = torch.meshgrid(torch.linspace(-2, 2, height), torch.linspace(-2, 2, width)) + data = [] + for i in range(num_frames): + xc = float(i) / num_frames + yc = 1 - float(i) / (2 * num_frames) + d = torch.exp(-((x - xc) ** 2 + (y - yc) ** 2) / 2) * 255 + data.append(d.unsqueeze(2).repeat(1, 1, 3).byte()) + return torch.stack(data, 0) + + +# adapted from torchvision test/test_io.py +@contextlib.contextmanager +def temp_video(num_frames, height, width, fps, lossless=False, video_codec=None, options=None): + if lossless: + if video_codec is not None: + raise ValueError("video_codec can't be specified together with lossless") + if options is not None: + raise ValueError("options can't be specified together with lossless") + video_codec = "libx264rgb" + options = {"crf": "0"} + if video_codec is None: + video_codec = "libx264" + if options is None: + options = {} + data = _create_video_frames(num_frames, height, width) + with tempfile.NamedTemporaryFile(suffix=".mp4") as f: + f.close() + io.write_video(f.name, data, fps=fps, video_codec=video_codec, options=options) + yield f.name, data + os.unlink(f.name) + + +@unittest.skipIf(av is None, "PyAV unavailable") +class TestVideoKeyframeDataset(unittest.TestCase): + def test_read_keyframes_all(self): + with temp_video(60, 300, 300, 5, video_codec="mpeg4") as (fname, data): + video_list = [fname] + category_list = [None] + dataset = VideoKeyframeDataset(video_list, category_list) + self.assertEqual(len(dataset), 1) + data1, categories1 = dataset[0]["images"], dataset[0]["categories"] + self.assertEqual(data1.shape, torch.Size((5, 3, 300, 300))) + self.assertEqual(data1.dtype, torch.float32) + self.assertIsNone(categories1[0]) + return + self.assertTrue(False) + + def test_read_keyframes_with_selector(self): + with temp_video(60, 300, 300, 5, video_codec="mpeg4") as (fname, data): + video_list = [fname] + category_list = [None] + random.seed(0) + frame_selector = RandomKFramesSelector(3) + dataset = VideoKeyframeDataset(video_list, category_list, frame_selector) + self.assertEqual(len(dataset), 1) + data1, categories1 = dataset[0]["images"], dataset[0]["categories"] + self.assertEqual(data1.shape, torch.Size((3, 3, 300, 300))) + self.assertEqual(data1.dtype, torch.float32) + self.assertIsNone(categories1[0]) + return + self.assertTrue(False) + + def test_read_keyframes_with_selector_with_transform(self): + with temp_video(60, 300, 300, 5, video_codec="mpeg4") as (fname, data): + video_list = [fname] + category_list = [None] + random.seed(0) + frame_selector = RandomKFramesSelector(1) + transform = ImageResizeTransform() + dataset = VideoKeyframeDataset(video_list, category_list, frame_selector, transform) + data1, categories1 = dataset[0]["images"], dataset[0]["categories"] + self.assertEqual(len(dataset), 1) + self.assertEqual(data1.shape, torch.Size((1, 3, 800, 800))) + self.assertEqual(data1.dtype, torch.float32) + self.assertIsNone(categories1[0]) + return + self.assertTrue(False) diff --git a/approach/ovod/detectron2/projects/DensePose/train_net.py b/approach/ovod/detectron2/projects/DensePose/train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..e8d77b9f41159bfaca1406973678a2c2c6a14f25 --- /dev/null +++ b/approach/ovod/detectron2/projects/DensePose/train_net.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. + +""" +DensePose Training Script. + +This script is similar to the training script in detectron2/tools. + +It is an example of how a user might use detectron2 for a new project. +""" + +from datetime import timedelta + +import detectron2.utils.comm as comm +from detectron2.config import get_cfg +from detectron2.engine import DEFAULT_TIMEOUT, default_argument_parser, default_setup, hooks, launch +from detectron2.evaluation import verify_results +from detectron2.utils.file_io import PathManager +from detectron2.utils.logger import setup_logger + +from densepose import add_densepose_config +from densepose.engine import Trainer +from densepose.modeling.densepose_checkpoint import DensePoseCheckpointer + + +def setup(args): + cfg = get_cfg() + add_densepose_config(cfg) + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + cfg.freeze() + default_setup(cfg, args) + # Setup logger for "densepose" module + setup_logger(output=cfg.OUTPUT_DIR, distributed_rank=comm.get_rank(), name="densepose") + return cfg + + +def main(args): + cfg = setup(args) + # disable strict kwargs checking: allow one to specify path handle + # hints through kwargs, like timeout in DP evaluation + PathManager.set_strict_kwargs_checking(False) + + if args.eval_only: + model = Trainer.build_model(cfg) + DensePoseCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( + cfg.MODEL.WEIGHTS, resume=args.resume + ) + res = Trainer.test(cfg, model) + if cfg.TEST.AUG.ENABLED: + res.update(Trainer.test_with_TTA(cfg, model)) + if comm.is_main_process(): + verify_results(cfg, res) + return res + + trainer = Trainer(cfg) + trainer.resume_or_load(resume=args.resume) + if cfg.TEST.AUG.ENABLED: + trainer.register_hooks( + [hooks.EvalHook(0, lambda: trainer.test_with_TTA(cfg, trainer.model))] + ) + return trainer.train() + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + cfg = setup(args) + timeout = ( + DEFAULT_TIMEOUT if cfg.DENSEPOSE_EVALUATION.DISTRIBUTED_INFERENCE else timedelta(hours=4) + ) + print("Command Line Args:", args) + launch( + main, + args.num_gpus, + num_machines=args.num_machines, + machine_rank=args.machine_rank, + dist_url=args.dist_url, + args=(args,), + timeout=timeout, + ) diff --git a/approach/ovod/detectron2/projects/MViTv2/README.md b/approach/ovod/detectron2/projects/MViTv2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..64afd79cac8d83de5518b57199fd618eebe83645 --- /dev/null +++ b/approach/ovod/detectron2/projects/MViTv2/README.md @@ -0,0 +1,142 @@ +# MViTv2: Improved Multiscale Vision Transformers for Classification and Detection + +Yanghao Li*, Chao-Yuan Wu*, Haoqi Fan, Karttikeya Mangalam, Bo Xiong, Jitendra Malik, Christoph Feichtenhofer* + +[[`arXiv`](https://arxiv.org/abs/2112.01526)] [[`BibTeX`](#CitingMViTv2)] + +In this repository, we provide detection configs and models for MViTv2 (CVPR 2022) in Detectron2. For image classification tasks, please refer to [MViTv2 repo](https://github.com/facebookresearch/mvit). + +## Results and Pretrained Models + +### COCO + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Namepre-trainMethodepochsbox
AP		mask
AP		#paramsFLOPSmodel iddownload
MViTV2-TIN1KMask R-CNN3648.343.844M279G307611773model
MViTV2-TIN1KCascade Mask R-CNN3652.245.076M701G308344828model
MViTV2-SIN1KCascade Mask R-CNN3653.246.087M748G308344647model
MViTV2-BIN1KCascade Mask R-CNN3654.146.7103M814G308109448model
MViTV2-BIN21KCascade Mask R-CNN3654.947.4103M814G309003202model
MViTV2-LIN21KCascade Mask R-CNN5055.848.3270M1519G308099658model
MViTV2-HIN21KCascade Mask R-CNN3656.148.5718M3084G309013744model
+ +Note that the above models were trained and measured on 8-node with 64 NVIDIA A100 GPUs in total. The ImageNet pre-trained model weights are obtained from [MViTv2 repo](https://github.com/facebookresearch/mvit). + +## Training +All configs can be trained with: + +``` +../../tools/lazyconfig_train_net.py --config-file configs/path/to/config.py +``` +By default, we use 64 GPUs with batch size as 64 for training. + +## Evaluation +Model evaluation can be done similarly: +``` +../../tools/lazyconfig_train_net.py --config-file configs/path/to/config.py --eval-only train.init_checkpoint=/path/to/model_checkpoint +``` + + + +## Citing MViTv2 + +If you use MViTv2, please use the following BibTeX entry. + +```BibTeX +@inproceedings{li2021improved, + title={MViTv2: Improved multiscale vision transformers for classification and detection}, + author={Li, Yanghao and Wu, Chao-Yuan and Fan, Haoqi and Mangalam, Karttikeya and Xiong, Bo and Malik, Jitendra and Feichtenhofer, Christoph}, + booktitle={CVPR}, + year={2022} +} +``` diff --git a/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_b_3x.py b/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_b_3x.py new file mode 100644 index 0000000000000000000000000000000000000000..61366bf11477136e8950b81dd24a1a7af9b37f8b --- /dev/null +++ b/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_b_3x.py @@ -0,0 +1,8 @@ +from .cascade_mask_rcnn_mvitv2_t_3x import model, dataloader, optimizer, lr_multiplier, train + + +model.backbone.bottom_up.depth = 24 +model.backbone.bottom_up.last_block_indexes = (1, 4, 20, 23) +model.backbone.bottom_up.drop_path_rate = 0.4 + +train.init_checkpoint = "detectron2://ImageNetPretrained/mvitv2/MViTv2_B_in1k.pyth" diff --git a/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_b_in21k_3x.py b/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_b_in21k_3x.py new file mode 100644 index 0000000000000000000000000000000000000000..7c3bdce0a2206b3afd1a33245a193292f0cd2a35 --- /dev/null +++ b/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_b_in21k_3x.py @@ -0,0 +1,3 @@ +from .cascade_mask_rcnn_mvitv2_b_3x import model, dataloader, optimizer, lr_multiplier, train + +train.init_checkpoint = "detectron2://ImageNetPretrained/mvitv2/MViTv2_B_in21k.pyth" diff --git a/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_h_in21k_lsj_3x.py b/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_h_in21k_lsj_3x.py new file mode 100644 index 0000000000000000000000000000000000000000..6fee5e99b7d5d611d27dca62a7db7d88808f87da --- /dev/null +++ b/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_h_in21k_lsj_3x.py @@ -0,0 +1,12 @@ +from .cascade_mask_rcnn_mvitv2_b_3x import model, optimizer, train, lr_multiplier +from .common.coco_loader_lsj import dataloader + + +model.backbone.bottom_up.embed_dim = 192 +model.backbone.bottom_up.depth = 80 +model.backbone.bottom_up.num_heads = 3 +model.backbone.bottom_up.last_block_indexes = (3, 11, 71, 79) +model.backbone.bottom_up.drop_path_rate = 0.6 +model.backbone.bottom_up.use_act_checkpoint = True + +train.init_checkpoint = "detectron2://ImageNetPretrained/mvitv2/MViTv2_H_in21k.pyth" diff --git a/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_l_in21k_lsj_50ep.py b/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_l_in21k_lsj_50ep.py new file mode 100644 index 0000000000000000000000000000000000000000..38da8958e0174d378555887d72a9956f4b3f8e58 --- /dev/null +++ b/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_l_in21k_lsj_50ep.py @@ -0,0 +1,31 @@ +from fvcore.common.param_scheduler import MultiStepParamScheduler + +from detectron2.config import LazyCall as L +from detectron2.solver import WarmupParamScheduler + +from .cascade_mask_rcnn_mvitv2_b_3x import model, optimizer, train +from .common.coco_loader_lsj import dataloader + + +model.backbone.bottom_up.embed_dim = 144 +model.backbone.bottom_up.depth = 48 +model.backbone.bottom_up.num_heads = 2 +model.backbone.bottom_up.last_block_indexes = (1, 7, 43, 47) +model.backbone.bottom_up.drop_path_rate = 0.5 + +train.init_checkpoint = "detectron2://ImageNetPretrained/mvitv2/MViTv2_L_in21k.pyth" + +# Schedule +# 50ep = 184375 // 2 iters * 64 images/iter / 118000 images/ep +train.max_iter = 184375 // 2 +lr_multiplier = L(WarmupParamScheduler)( + scheduler=L(MultiStepParamScheduler)( + values=[1.0, 0.1, 0.01], + milestones=[163889 // 2, 177546 // 2], + num_updates=train.max_iter, + ), + warmup_length=250 / train.max_iter, + warmup_factor=0.001, +) + +optimizer.lr = 1e-4 diff --git a/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_s_3x.py b/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_s_3x.py new file mode 100644 index 0000000000000000000000000000000000000000..ad8eeb4df25476893c5a966a669ecceaec2a6dbc --- /dev/null +++ b/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_s_3x.py @@ -0,0 +1,7 @@ +from .cascade_mask_rcnn_mvitv2_t_3x import model, dataloader, optimizer, lr_multiplier, train + + +model.backbone.bottom_up.depth = 16 +model.backbone.bottom_up.last_block_indexes = (0, 2, 13, 15) + +train.init_checkpoint = "detectron2://ImageNetPretrained/mvitv2/MViTv2_S_in1k.pyth" diff --git a/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_t_3x.py b/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_t_3x.py new file mode 100644 index 0000000000000000000000000000000000000000..51327dd9379b011c2d6cdc8299515b6df8112f4e --- /dev/null +++ b/approach/ovod/detectron2/projects/MViTv2/configs/cascade_mask_rcnn_mvitv2_t_3x.py @@ -0,0 +1,48 @@ +from detectron2.config import LazyCall as L +from detectron2.layers import ShapeSpec +from detectron2.modeling.box_regression import Box2BoxTransform +from detectron2.modeling.matcher import Matcher +from detectron2.modeling.roi_heads import FastRCNNOutputLayers, FastRCNNConvFCHead, CascadeROIHeads +from detectron2.layers.batch_norm import NaiveSyncBatchNorm + +from .mask_rcnn_mvitv2_t_3x import model, dataloader, optimizer, lr_multiplier, train + + +# arguments that don't exist for Cascade R-CNN +[model.roi_heads.pop(k) for k in ["box_head", "box_predictor", "proposal_matcher"]] + +model.roi_heads.update( + _target_=CascadeROIHeads, + box_heads=[ + L(FastRCNNConvFCHead)( + input_shape=ShapeSpec(channels=256, height=7, width=7), + conv_dims=[256, 256, 256, 256], + fc_dims=[1024], + conv_norm=lambda c: NaiveSyncBatchNorm(c, stats_mode="N"), + ) + for _ in range(3) + ], + box_predictors=[ + L(FastRCNNOutputLayers)( + input_shape=ShapeSpec(channels=1024), + test_score_thresh=0.05, + box2box_transform=L(Box2BoxTransform)(weights=(w1, w1, w2, w2)), + cls_agnostic_bbox_reg=True, + num_classes="${...num_classes}", + ) + for (w1, w2) in [(10, 5), (20, 10), (30, 15)] + ], + proposal_matchers=[ + L(Matcher)(thresholds=[th], labels=[0, 1], allow_low_quality_matches=False) + for th in [0.5, 0.6, 0.7] + ], +) + +# Using NaiveSyncBatchNorm becase heads may have empty input. That is not supported by +# torch.nn.SyncBatchNorm. We can remove this after +# https://github.com/pytorch/pytorch/issues/36530 is fixed. +model.roi_heads.mask_head.conv_norm = lambda c: NaiveSyncBatchNorm(c, stats_mode="N") + +# 2conv in RPN: +# https://github.com/tensorflow/tpu/blob/b24729de804fdb751b06467d3dce0637fa652060/models/official/detection/modeling/architecture/heads.py#L95-L97 # noqa: E501, B950 +model.proposal_generator.head.conv_dims = [-1, -1] diff --git a/approach/ovod/detectron2/projects/MViTv2/configs/common/coco_loader.py b/approach/ovod/detectron2/projects/MViTv2/configs/common/coco_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..923878b8d4cdda9292738550f1c6aa18e38d5757 --- /dev/null +++ b/approach/ovod/detectron2/projects/MViTv2/configs/common/coco_loader.py @@ -0,0 +1,59 @@ +from omegaconf import OmegaConf + +import detectron2.data.transforms as T +from detectron2.config import LazyCall as L +from detectron2.data import ( + DatasetMapper, + build_detection_test_loader, + build_detection_train_loader, + get_detection_dataset_dicts, +) +from detectron2.evaluation import COCOEvaluator + +dataloader = OmegaConf.create() + +dataloader.train = L(build_detection_train_loader)( + dataset=L(get_detection_dataset_dicts)(names="coco_2017_train"), + mapper=L(DatasetMapper)( + is_train=True, + augmentations=[ + L(T.RandomApply)( + tfm_or_aug=L(T.AugmentationList)( + augs=[ + L(T.ResizeShortestEdge)( + short_edge_length=[400, 500, 600], sample_style="choice" + ), + L(T.RandomCrop)(crop_type="absolute_range", crop_size=(384, 600)), + ] + ), + prob=0.5, + ), + L(T.ResizeShortestEdge)( + short_edge_length=(480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800), + sample_style="choice", + max_size=1333, + ), + L(T.RandomFlip)(horizontal=True), + ], + image_format="RGB", + use_instance_mask=True, + ), + total_batch_size=16, + num_workers=4, +) + +dataloader.test = L(build_detection_test_loader)( + dataset=L(get_detection_dataset_dicts)(names="coco_2017_val", filter_empty=False), + mapper=L(DatasetMapper)( + is_train=False, + augmentations=[ + L(T.ResizeShortestEdge)(short_edge_length=800, max_size=1333), + ], + image_format="${...train.mapper.image_format}", + ), + num_workers=4, +) + +dataloader.evaluator = L(COCOEvaluator)( + dataset_name="${..test.dataset.names}", +) diff --git a/approach/ovod/detectron2/projects/MViTv2/configs/common/coco_loader_lsj.py b/approach/ovod/detectron2/projects/MViTv2/configs/common/coco_loader_lsj.py new file mode 100644 index 0000000000000000000000000000000000000000..019b21fb23299542f757459da12a56df1c538e2b --- /dev/null +++ b/approach/ovod/detectron2/projects/MViTv2/configs/common/coco_loader_lsj.py @@ -0,0 +1,19 @@ +import detectron2.data.transforms as T +from detectron2 import model_zoo +from detectron2.config import LazyCall as L + +from .coco_loader import dataloader + +# Data using LSJ +image_size = 1024 +dataloader.train.mapper.augmentations = [ + L(T.RandomFlip)(horizontal=True), # flip first + L(T.ResizeScale)( + min_scale=0.1, max_scale=2.0, target_height=image_size, target_width=image_size + ), + L(T.FixedSizeCrop)(crop_size=(image_size, image_size)), +] +dataloader.train.mapper.image_format = "RGB" +dataloader.train.total_batch_size = 64 +# recompute boxes due to cropping +dataloader.train.mapper.recompute_boxes = True diff --git a/approach/ovod/detectron2/projects/MViTv2/configs/mask_rcnn_mvitv2_t_3x.py b/approach/ovod/detectron2/projects/MViTv2/configs/mask_rcnn_mvitv2_t_3x.py new file mode 100644 index 0000000000000000000000000000000000000000..ba4bdfecf2fc996f3e06480a2f02781c71b5aa44 --- /dev/null +++ b/approach/ovod/detectron2/projects/MViTv2/configs/mask_rcnn_mvitv2_t_3x.py @@ -0,0 +1,55 @@ +from functools import partial +import torch.nn as nn +from fvcore.common.param_scheduler import MultiStepParamScheduler + +from detectron2 import model_zoo +from detectron2.config import LazyCall as L +from detectron2.solver import WarmupParamScheduler +from detectron2.modeling import MViT + +from .common.coco_loader import dataloader + +model = model_zoo.get_config("common/models/mask_rcnn_fpn.py").model +constants = model_zoo.get_config("common/data/constants.py").constants +model.pixel_mean = constants.imagenet_rgb256_mean +model.pixel_std = constants.imagenet_rgb256_std +model.input_format = "RGB" +model.backbone.bottom_up = L(MViT)( + embed_dim=96, + depth=10, + num_heads=1, + last_block_indexes=(0, 2, 7, 9), + residual_pooling=True, + drop_path_rate=0.2, + norm_layer=partial(nn.LayerNorm, eps=1e-6), + out_features=("scale2", "scale3", "scale4", "scale5"), +) +model.backbone.in_features = "${.bottom_up.out_features}" + + +# Initialization and trainer settings +train = model_zoo.get_config("common/train.py").train +train.amp.enabled = True +train.ddp.fp16_compression = True +train.init_checkpoint = "detectron2://ImageNetPretrained/mvitv2/MViTv2_T_in1k.pyth" + +dataloader.train.total_batch_size = 64 + +# 36 epochs +train.max_iter = 67500 +lr_multiplier = L(WarmupParamScheduler)( + scheduler=L(MultiStepParamScheduler)( + values=[1.0, 0.1, 0.01], + milestones=[52500, 62500, 67500], + ), + warmup_length=250 / train.max_iter, + warmup_factor=0.001, +) + +optimizer = model_zoo.get_config("common/optim.py").AdamW +optimizer.params.overrides = { + "pos_embed": {"weight_decay": 0.0}, + "rel_pos_h": {"weight_decay": 0.0}, + "rel_pos_w": {"weight_decay": 0.0}, +} +optimizer.lr = 1.6e-4 diff --git a/approach/ovod/detectron2/projects/Panoptic-DeepLab/README.md b/approach/ovod/detectron2/projects/Panoptic-DeepLab/README.md new file mode 100644 index 0000000000000000000000000000000000000000..86b6d42ba059d7da602b95cfdf3fe7d37ea7d4ec --- /dev/null +++ b/approach/ovod/detectron2/projects/Panoptic-DeepLab/README.md @@ -0,0 +1,175 @@ +# Panoptic-DeepLab: A Simple, Strong, and Fast Baseline for Bottom-Up Panoptic Segmentation + +Bowen Cheng, Maxwell D. Collins, Yukun Zhu, Ting Liu, Thomas S. Huang, Hartwig Adam, Liang-Chieh Chen + +[[`arXiv`](https://arxiv.org/abs/1911.10194)] [[`BibTeX`](#CitingPanopticDeepLab)] [[`Reference implementation`](https://github.com/bowenc0221/panoptic-deeplab)] + +
+ +

+ +## Installation +Install Detectron2 following [the instructions](https://detectron2.readthedocs.io/tutorials/install.html). +To use cityscapes, prepare data follow the [tutorial](https://detectron2.readthedocs.io/tutorials/builtin_datasets.html#expected-dataset-structure-for-cityscapes). + +## Training + +To train a model with 8 GPUs run: +```bash +cd /path/to/detectron2/projects/Panoptic-DeepLab +python train_net.py --config-file configs/Cityscapes-PanopticSegmentation/panoptic_deeplab_R_52_os16_mg124_poly_90k_bs32_crop_512_1024_dsconv.yaml --num-gpus 8 +``` + +## Evaluation + +Model evaluation can be done similarly: +```bash +cd /path/to/detectron2/projects/Panoptic-DeepLab +python train_net.py --config-file configs/Cityscapes-PanopticSegmentation/panoptic_deeplab_R_52_os16_mg124_poly_90k_bs32_crop_512_1024_dsconv.yaml --eval-only MODEL.WEIGHTS /path/to/model_checkpoint +``` + +## Benchmark network speed + +If you want to benchmark the network speed without post-processing, you can run the evaluation script with `MODEL.PANOPTIC_DEEPLAB.BENCHMARK_NETWORK_SPEED True`: +```bash +cd /path/to/detectron2/projects/Panoptic-DeepLab +python train_net.py --config-file configs/Cityscapes-PanopticSegmentation/panoptic_deeplab_R_52_os16_mg124_poly_90k_bs32_crop_512_1024_dsconv.yaml --eval-only MODEL.WEIGHTS /path/to/model_checkpoint MODEL.PANOPTIC_DEEPLAB.BENCHMARK_NETWORK_SPEED True +``` + +## Cityscapes Panoptic Segmentation +Cityscapes models are trained with ImageNet pretraining. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MethodBackboneOutput
resolution		PQSQRQmIoUAPMemory (M)model iddownload
Panoptic-DeepLabR50-DC51024×2048 58.6 80.9 71.2 75.9 29.8 8668 - model | metrics
Panoptic-DeepLabR52-DC51024×2048 60.3 81.5 72.9 78.2 33.2 9682 30841561 model | metrics
Panoptic-DeepLab (DSConv)R52-DC51024×2048 60.3 81.0 73.2 78.7 32.1 10466 33148034 model | metrics
+ +Note: +- [R52](https://dl.fbaipublicfiles.com/detectron2/DeepLab/R-52.pkl): a ResNet-50 with its first 7x7 convolution replaced by 3 3x3 convolutions. This modification has been used in most semantic segmentation papers. We pre-train this backbone on ImageNet using the default recipe of [pytorch examples](https://github.com/pytorch/examples/tree/master/imagenet). +- DC5 means using dilated convolution in `res5`. +- We use a smaller training crop size (512x1024) than the original paper (1025x2049), we find using larger crop size (1024x2048) could further improve PQ by 1.5% but also degrades AP by 3%. +- The implementation with regular Conv2d in ASPP and head is much heavier head than the original paper. +- This implementation does not include optimized post-processing code needed for deployment. Post-processing the network + outputs now takes similar amount of time to the network itself. Please refer to speed in the + original paper for comparison. +- DSConv refers to using DepthwiseSeparableConv2d in ASPP and decoder. The implementation with DSConv is identical to the original paper. + +## COCO Panoptic Segmentation +COCO models are trained with ImageNet pretraining on 16 V100s. + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MethodBackboneOutput
resolution		PQSQRQBox APMask APMemory (M)model iddownload
Panoptic-DeepLab (DSConv)R52-DC5640×640 35.5 77.3 44.7 18.6 19.7 246448865 model | metrics
+ +Note: +- [R52](https://dl.fbaipublicfiles.com/detectron2/DeepLab/R-52.pkl): a ResNet-50 with its first 7x7 convolution replaced by 3 3x3 convolutions. This modification has been used in most semantic segmentation papers. We pre-train this backbone on ImageNet using the default recipe of [pytorch examples](https://github.com/pytorch/examples/tree/master/imagenet). +- DC5 means using dilated convolution in `res5`. +- This reproduced number matches the original paper (35.5 vs. 35.1 PQ). +- This implementation does not include optimized post-processing code needed for deployment. Post-processing the network + outputs now takes more time than the network itself. Please refer to speed in the original paper for comparison. +- DSConv refers to using DepthwiseSeparableConv2d in ASPP and decoder. + +## Citing Panoptic-DeepLab + +If you use Panoptic-DeepLab, please use the following BibTeX entry. + +* CVPR 2020 paper: + +``` +@inproceedings{cheng2020panoptic, + title={Panoptic-DeepLab: A Simple, Strong, and Fast Baseline for Bottom-Up Panoptic Segmentation}, + author={Cheng, Bowen and Collins, Maxwell D and Zhu, Yukun and Liu, Ting and Huang, Thomas S and Adam, Hartwig and Chen, Liang-Chieh}, + booktitle={CVPR}, + year={2020} +} +``` + +* ICCV 2019 COCO-Mapillary workshp challenge report: + +``` +@inproceedings{cheng2019panoptic, + title={Panoptic-DeepLab}, + author={Cheng, Bowen and Collins, Maxwell D and Zhu, Yukun and Liu, Ting and Huang, Thomas S and Adam, Hartwig and Chen, Liang-Chieh}, + booktitle={ICCV COCO + Mapillary Joint Recognition Challenge Workshop}, + year={2019} +} +``` diff --git a/approach/ovod/detectron2/projects/Panoptic-DeepLab/configs/COCO-PanopticSegmentation/panoptic_deeplab_R_52_os16_mg124_poly_200k_bs64_crop_640_640_coco_dsconv.yaml b/approach/ovod/detectron2/projects/Panoptic-DeepLab/configs/COCO-PanopticSegmentation/panoptic_deeplab_R_52_os16_mg124_poly_200k_bs64_crop_640_640_coco_dsconv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6944c6fdf3dcaafdc0a740188610fe604cb7d3be --- /dev/null +++ b/approach/ovod/detectron2/projects/Panoptic-DeepLab/configs/COCO-PanopticSegmentation/panoptic_deeplab_R_52_os16_mg124_poly_200k_bs64_crop_640_640_coco_dsconv.yaml @@ -0,0 +1,42 @@ +_BASE_: ../Cityscapes-PanopticSegmentation/Base-PanopticDeepLab-OS16.yaml +MODEL: + WEIGHTS: "detectron2://DeepLab/R-52.pkl" + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.120, 57.375] + BACKBONE: + NAME: "build_resnet_deeplab_backbone" + RESNETS: + DEPTH: 50 + NORM: "SyncBN" + RES5_MULTI_GRID: [1, 2, 4] + STEM_TYPE: "deeplab" + STEM_OUT_CHANNELS: 128 + STRIDE_IN_1X1: False + SEM_SEG_HEAD: + NUM_CLASSES: 133 + LOSS_TOP_K: 1.0 + USE_DEPTHWISE_SEPARABLE_CONV: True + PANOPTIC_DEEPLAB: + STUFF_AREA: 4096 + NMS_KERNEL: 41 + SIZE_DIVISIBILITY: 640 + USE_DEPTHWISE_SEPARABLE_CONV: True +DATASETS: + TRAIN: ("coco_2017_train_panoptic",) + TEST: ("coco_2017_val_panoptic",) +SOLVER: + BASE_LR: 0.0005 + MAX_ITER: 200000 + IMS_PER_BATCH: 64 +INPUT: + FORMAT: "RGB" + GAUSSIAN_SIGMA: 8 + MIN_SIZE_TRAIN: !!python/object/apply:eval ["[int(x * 0.1 * 640) for x in range(5, 16)]"] + MIN_SIZE_TRAIN_SAMPLING: "choice" + MIN_SIZE_TEST: 640 + MAX_SIZE_TRAIN: 960 + MAX_SIZE_TEST: 640 + CROP: + ENABLED: True + TYPE: "absolute" + SIZE: (640, 640) diff --git a/approach/ovod/detectron2/projects/Panoptic-DeepLab/configs/Cityscapes-PanopticSegmentation/Base-PanopticDeepLab-OS16.yaml b/approach/ovod/detectron2/projects/Panoptic-DeepLab/configs/Cityscapes-PanopticSegmentation/Base-PanopticDeepLab-OS16.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7379980fdace160f385f0647e95325830b6bfd7 --- /dev/null +++ b/approach/ovod/detectron2/projects/Panoptic-DeepLab/configs/Cityscapes-PanopticSegmentation/Base-PanopticDeepLab-OS16.yaml @@ -0,0 +1,65 @@ +MODEL: + META_ARCHITECTURE: "PanopticDeepLab" + BACKBONE: + FREEZE_AT: 0 + RESNETS: + OUT_FEATURES: ["res2", "res3", "res5"] + RES5_DILATION: 2 + SEM_SEG_HEAD: + NAME: "PanopticDeepLabSemSegHead" + IN_FEATURES: ["res2", "res3", "res5"] + PROJECT_FEATURES: ["res2", "res3"] + PROJECT_CHANNELS: [32, 64] + ASPP_CHANNELS: 256 + ASPP_DILATIONS: [6, 12, 18] + ASPP_DROPOUT: 0.1 + HEAD_CHANNELS: 256 + CONVS_DIM: 256 + COMMON_STRIDE: 4 + NUM_CLASSES: 19 + LOSS_TYPE: "hard_pixel_mining" + NORM: "SyncBN" + INS_EMBED_HEAD: + NAME: "PanopticDeepLabInsEmbedHead" + IN_FEATURES: ["res2", "res3", "res5"] + PROJECT_FEATURES: ["res2", "res3"] + PROJECT_CHANNELS: [32, 64] + ASPP_CHANNELS: 256 + ASPP_DILATIONS: [6, 12, 18] + ASPP_DROPOUT: 0.1 + HEAD_CHANNELS: 32 + CONVS_DIM: 128 + COMMON_STRIDE: 4 + NORM: "SyncBN" + CENTER_LOSS_WEIGHT: 200.0 + OFFSET_LOSS_WEIGHT: 0.01 + PANOPTIC_DEEPLAB: + STUFF_AREA: 2048 + CENTER_THRESHOLD: 0.1 + NMS_KERNEL: 7 + TOP_K_INSTANCE: 200 +DATASETS: + TRAIN: ("cityscapes_fine_panoptic_train",) + TEST: ("cityscapes_fine_panoptic_val",) +SOLVER: + OPTIMIZER: "ADAM" + BASE_LR: 0.001 + WEIGHT_DECAY: 0.0 + WEIGHT_DECAY_NORM: 0.0 + WEIGHT_DECAY_BIAS: 0.0 + MAX_ITER: 60000 + LR_SCHEDULER_NAME: "WarmupPolyLR" + IMS_PER_BATCH: 32 +INPUT: + MIN_SIZE_TRAIN: (512, 640, 704, 832, 896, 1024, 1152, 1216, 1344, 1408, 1536, 1664, 1728, 1856, 1920, 2048) + MIN_SIZE_TRAIN_SAMPLING: "choice" + MIN_SIZE_TEST: 1024 + MAX_SIZE_TRAIN: 4096 + MAX_SIZE_TEST: 2048 + CROP: + ENABLED: True + TYPE: "absolute" + SIZE: (1024, 2048) +DATALOADER: + NUM_WORKERS: 10 +VERSION: 2 diff --git a/approach/ovod/detectron2/projects/Panoptic-DeepLab/configs/Cityscapes-PanopticSegmentation/panoptic_deeplab_R_52_os16_mg124_poly_90k_bs32_crop_512_1024.yaml b/approach/ovod/detectron2/projects/Panoptic-DeepLab/configs/Cityscapes-PanopticSegmentation/panoptic_deeplab_R_52_os16_mg124_poly_90k_bs32_crop_512_1024.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fde902bb2a87ccaf2c6fea4e79be4144ca44e239 --- /dev/null +++ b/approach/ovod/detectron2/projects/Panoptic-DeepLab/configs/Cityscapes-PanopticSegmentation/panoptic_deeplab_R_52_os16_mg124_poly_90k_bs32_crop_512_1024.yaml @@ -0,0 +1,20 @@ +_BASE_: Base-PanopticDeepLab-OS16.yaml +MODEL: + WEIGHTS: "detectron2://DeepLab/R-52.pkl" + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.120, 57.375] + BACKBONE: + NAME: "build_resnet_deeplab_backbone" + RESNETS: + DEPTH: 50 + NORM: "SyncBN" + RES5_MULTI_GRID: [1, 2, 4] + STEM_TYPE: "deeplab" + STEM_OUT_CHANNELS: 128 + STRIDE_IN_1X1: False +SOLVER: + MAX_ITER: 90000 +INPUT: + FORMAT: "RGB" + CROP: + SIZE: (512, 1024) diff --git a/approach/ovod/detectron2/projects/Panoptic-DeepLab/configs/Cityscapes-PanopticSegmentation/panoptic_deeplab_R_52_os16_mg124_poly_90k_bs32_crop_512_1024_dsconv.yaml b/approach/ovod/detectron2/projects/Panoptic-DeepLab/configs/Cityscapes-PanopticSegmentation/panoptic_deeplab_R_52_os16_mg124_poly_90k_bs32_crop_512_1024_dsconv.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e314204c9b464993d92d3b4d95e2aa9b287b938 --- /dev/null +++ b/approach/ovod/detectron2/projects/Panoptic-DeepLab/configs/Cityscapes-PanopticSegmentation/panoptic_deeplab_R_52_os16_mg124_poly_90k_bs32_crop_512_1024_dsconv.yaml @@ -0,0 +1,24 @@ +_BASE_: Base-PanopticDeepLab-OS16.yaml +MODEL: + WEIGHTS: "detectron2://DeepLab/R-52.pkl" + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.120, 57.375] + BACKBONE: + NAME: "build_resnet_deeplab_backbone" + RESNETS: + DEPTH: 50 + NORM: "SyncBN" + RES5_MULTI_GRID: [1, 2, 4] + STEM_TYPE: "deeplab" + STEM_OUT_CHANNELS: 128 + STRIDE_IN_1X1: False + PANOPTIC_DEEPLAB: + USE_DEPTHWISE_SEPARABLE_CONV: True + SEM_SEG_HEAD: + USE_DEPTHWISE_SEPARABLE_CONV: True +SOLVER: + MAX_ITER: 90000 +INPUT: + FORMAT: "RGB" + CROP: + SIZE: (512, 1024) diff --git a/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/__init__.py b/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8d3c980643bbd385594850bfbffa84cd1412c162 --- /dev/null +++ b/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .config import add_panoptic_deeplab_config +from .dataset_mapper import PanopticDeeplabDatasetMapper +from .panoptic_seg import ( + PanopticDeepLab, + INS_EMBED_BRANCHES_REGISTRY, + build_ins_embed_branch, + PanopticDeepLabSemSegHead, + PanopticDeepLabInsEmbedHead, +) diff --git a/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/config.py b/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/config.py new file mode 100644 index 0000000000000000000000000000000000000000..5aa2d280c66dbccc9ff8c3ccf39ccfbfc1eaa430 --- /dev/null +++ b/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/config.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +from detectron2.config import CfgNode as CN +from detectron2.projects.deeplab import add_deeplab_config + + +def add_panoptic_deeplab_config(cfg): + """ + Add config for Panoptic-DeepLab. + """ + # Reuse DeepLab config. + add_deeplab_config(cfg) + # Target generation parameters. + cfg.INPUT.GAUSSIAN_SIGMA = 10 + cfg.INPUT.IGNORE_STUFF_IN_OFFSET = True + cfg.INPUT.SMALL_INSTANCE_AREA = 4096 + cfg.INPUT.SMALL_INSTANCE_WEIGHT = 3 + cfg.INPUT.IGNORE_CROWD_IN_SEMANTIC = False + # Optimizer type. + cfg.SOLVER.OPTIMIZER = "ADAM" + # Panoptic-DeepLab semantic segmentation head. + # We add an extra convolution before predictor. + cfg.MODEL.SEM_SEG_HEAD.HEAD_CHANNELS = 256 + cfg.MODEL.SEM_SEG_HEAD.LOSS_TOP_K = 0.2 + # Panoptic-DeepLab instance segmentation head. + cfg.MODEL.INS_EMBED_HEAD = CN() + cfg.MODEL.INS_EMBED_HEAD.NAME = "PanopticDeepLabInsEmbedHead" + cfg.MODEL.INS_EMBED_HEAD.IN_FEATURES = ["res2", "res3", "res5"] + cfg.MODEL.INS_EMBED_HEAD.PROJECT_FEATURES = ["res2", "res3"] + cfg.MODEL.INS_EMBED_HEAD.PROJECT_CHANNELS = [32, 64] + cfg.MODEL.INS_EMBED_HEAD.ASPP_CHANNELS = 256 + cfg.MODEL.INS_EMBED_HEAD.ASPP_DILATIONS = [6, 12, 18] + cfg.MODEL.INS_EMBED_HEAD.ASPP_DROPOUT = 0.1 + # We add an extra convolution before predictor. + cfg.MODEL.INS_EMBED_HEAD.HEAD_CHANNELS = 32 + cfg.MODEL.INS_EMBED_HEAD.CONVS_DIM = 128 + cfg.MODEL.INS_EMBED_HEAD.COMMON_STRIDE = 4 + cfg.MODEL.INS_EMBED_HEAD.NORM = "SyncBN" + cfg.MODEL.INS_EMBED_HEAD.CENTER_LOSS_WEIGHT = 200.0 + cfg.MODEL.INS_EMBED_HEAD.OFFSET_LOSS_WEIGHT = 0.01 + # Panoptic-DeepLab post-processing setting. + cfg.MODEL.PANOPTIC_DEEPLAB = CN() + # Stuff area limit, ignore stuff region below this number. + cfg.MODEL.PANOPTIC_DEEPLAB.STUFF_AREA = 2048 + cfg.MODEL.PANOPTIC_DEEPLAB.CENTER_THRESHOLD = 0.1 + cfg.MODEL.PANOPTIC_DEEPLAB.NMS_KERNEL = 7 + cfg.MODEL.PANOPTIC_DEEPLAB.TOP_K_INSTANCE = 200 + # If set to False, Panoptic-DeepLab will not evaluate instance segmentation. + cfg.MODEL.PANOPTIC_DEEPLAB.PREDICT_INSTANCES = True + cfg.MODEL.PANOPTIC_DEEPLAB.USE_DEPTHWISE_SEPARABLE_CONV = False + # This is the padding parameter for images with various sizes. ASPP layers + # requires input images to be divisible by the average pooling size and we + # can use `MODEL.PANOPTIC_DEEPLAB.SIZE_DIVISIBILITY` to pad all images to + # a fixed resolution (e.g. 640x640 for COCO) to avoid having a image size + # that is not divisible by ASPP average pooling size. + cfg.MODEL.PANOPTIC_DEEPLAB.SIZE_DIVISIBILITY = -1 + # Only evaluates network speed (ignores post-processing). + cfg.MODEL.PANOPTIC_DEEPLAB.BENCHMARK_NETWORK_SPEED = False diff --git a/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/dataset_mapper.py b/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/dataset_mapper.py new file mode 100644 index 0000000000000000000000000000000000000000..53272c726af810efc248f2428dda7ca7271fcd00 --- /dev/null +++ b/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/dataset_mapper.py @@ -0,0 +1,116 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import logging +import numpy as np +from typing import Callable, List, Union +import torch +from panopticapi.utils import rgb2id + +from detectron2.config import configurable +from detectron2.data import MetadataCatalog +from detectron2.data import detection_utils as utils +from detectron2.data import transforms as T + +from .target_generator import PanopticDeepLabTargetGenerator + +__all__ = ["PanopticDeeplabDatasetMapper"] + + +class PanopticDeeplabDatasetMapper: + """ + The callable currently does the following: + + 1. Read the image from "file_name" and label from "pan_seg_file_name" + 2. Applies random scale, crop and flip transforms to image and label + 3. Prepare data to Tensor and generate training targets from label + """ + + @configurable + def __init__( + self, + *, + augmentations: List[Union[T.Augmentation, T.Transform]], + image_format: str, + panoptic_target_generator: Callable, + ): + """ + NOTE: this interface is experimental. + + Args: + augmentations: a list of augmentations or deterministic transforms to apply + image_format: an image format supported by :func:`detection_utils.read_image`. + panoptic_target_generator: a callable that takes "panoptic_seg" and + "segments_info" to generate training targets for the model. + """ + # fmt: off + self.augmentations = T.AugmentationList(augmentations) + self.image_format = image_format + # fmt: on + logger = logging.getLogger(__name__) + logger.info("Augmentations used in training: " + str(augmentations)) + + self.panoptic_target_generator = panoptic_target_generator + + @classmethod + def from_config(cls, cfg): + augs = [ + T.ResizeShortestEdge( + cfg.INPUT.MIN_SIZE_TRAIN, + cfg.INPUT.MAX_SIZE_TRAIN, + cfg.INPUT.MIN_SIZE_TRAIN_SAMPLING, + ) + ] + if cfg.INPUT.CROP.ENABLED: + augs.append(T.RandomCrop(cfg.INPUT.CROP.TYPE, cfg.INPUT.CROP.SIZE)) + augs.append(T.RandomFlip()) + + # Assume always applies to the training set. + dataset_names = cfg.DATASETS.TRAIN + meta = MetadataCatalog.get(dataset_names[0]) + panoptic_target_generator = PanopticDeepLabTargetGenerator( + ignore_label=meta.ignore_label, + thing_ids=list(meta.thing_dataset_id_to_contiguous_id.values()), + sigma=cfg.INPUT.GAUSSIAN_SIGMA, + ignore_stuff_in_offset=cfg.INPUT.IGNORE_STUFF_IN_OFFSET, + small_instance_area=cfg.INPUT.SMALL_INSTANCE_AREA, + small_instance_weight=cfg.INPUT.SMALL_INSTANCE_WEIGHT, + ignore_crowd_in_semantic=cfg.INPUT.IGNORE_CROWD_IN_SEMANTIC, + ) + + ret = { + "augmentations": augs, + "image_format": cfg.INPUT.FORMAT, + "panoptic_target_generator": panoptic_target_generator, + } + return ret + + def __call__(self, dataset_dict): + """ + Args: + dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. + + Returns: + dict: a format that builtin models in detectron2 accept + """ + dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below + # Load image. + image = utils.read_image(dataset_dict["file_name"], format=self.image_format) + utils.check_image_size(dataset_dict, image) + # Panoptic label is encoded in RGB image. + pan_seg_gt = utils.read_image(dataset_dict.pop("pan_seg_file_name"), "RGB") + + # Reuses semantic transform for panoptic labels. + aug_input = T.AugInput(image, sem_seg=pan_seg_gt) + _ = self.augmentations(aug_input) + image, pan_seg_gt = aug_input.image, aug_input.sem_seg + + # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, + # but not efficient on large generic data structures due to the use of pickle & mp.Queue. + # Therefore it's important to use torch.Tensor. + dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1))) + + # Generates training targets for Panoptic-DeepLab. + targets = self.panoptic_target_generator(rgb2id(pan_seg_gt), dataset_dict["segments_info"]) + dataset_dict.update(targets) + + return dataset_dict diff --git a/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/panoptic_seg.py b/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/panoptic_seg.py new file mode 100644 index 0000000000000000000000000000000000000000..c12ca74e3b281e74e8893c87d2ba7e2b60931c65 --- /dev/null +++ b/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/panoptic_seg.py @@ -0,0 +1,572 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import numpy as np +from typing import Callable, Dict, List, Union +import fvcore.nn.weight_init as weight_init +import torch +from torch import nn +from torch.nn import functional as F + +from detectron2.config import configurable +from detectron2.data import MetadataCatalog +from detectron2.layers import Conv2d, DepthwiseSeparableConv2d, ShapeSpec, get_norm +from detectron2.modeling import ( + META_ARCH_REGISTRY, + SEM_SEG_HEADS_REGISTRY, + build_backbone, + build_sem_seg_head, +) +from detectron2.modeling.postprocessing import sem_seg_postprocess +from detectron2.projects.deeplab import DeepLabV3PlusHead +from detectron2.projects.deeplab.loss import DeepLabCE +from detectron2.structures import BitMasks, ImageList, Instances +from detectron2.utils.registry import Registry + +from .post_processing import get_panoptic_segmentation + +__all__ = ["PanopticDeepLab", "INS_EMBED_BRANCHES_REGISTRY", "build_ins_embed_branch"] + + +INS_EMBED_BRANCHES_REGISTRY = Registry("INS_EMBED_BRANCHES") +INS_EMBED_BRANCHES_REGISTRY.__doc__ = """ +Registry for instance embedding branches, which make instance embedding +predictions from feature maps. +""" + + +@META_ARCH_REGISTRY.register() +class PanopticDeepLab(nn.Module): + """ + Main class for panoptic segmentation architectures. + """ + + def __init__(self, cfg): + super().__init__() + self.backbone = build_backbone(cfg) + self.sem_seg_head = build_sem_seg_head(cfg, self.backbone.output_shape()) + self.ins_embed_head = build_ins_embed_branch(cfg, self.backbone.output_shape()) + self.register_buffer("pixel_mean", torch.tensor(cfg.MODEL.PIXEL_MEAN).view(-1, 1, 1), False) + self.register_buffer("pixel_std", torch.tensor(cfg.MODEL.PIXEL_STD).view(-1, 1, 1), False) + self.meta = MetadataCatalog.get(cfg.DATASETS.TRAIN[0]) + self.stuff_area = cfg.MODEL.PANOPTIC_DEEPLAB.STUFF_AREA + self.threshold = cfg.MODEL.PANOPTIC_DEEPLAB.CENTER_THRESHOLD + self.nms_kernel = cfg.MODEL.PANOPTIC_DEEPLAB.NMS_KERNEL + self.top_k = cfg.MODEL.PANOPTIC_DEEPLAB.TOP_K_INSTANCE + self.predict_instances = cfg.MODEL.PANOPTIC_DEEPLAB.PREDICT_INSTANCES + self.use_depthwise_separable_conv = cfg.MODEL.PANOPTIC_DEEPLAB.USE_DEPTHWISE_SEPARABLE_CONV + assert ( + cfg.MODEL.SEM_SEG_HEAD.USE_DEPTHWISE_SEPARABLE_CONV + == cfg.MODEL.PANOPTIC_DEEPLAB.USE_DEPTHWISE_SEPARABLE_CONV + ) + self.size_divisibility = cfg.MODEL.PANOPTIC_DEEPLAB.SIZE_DIVISIBILITY + self.benchmark_network_speed = cfg.MODEL.PANOPTIC_DEEPLAB.BENCHMARK_NETWORK_SPEED + + @property + def device(self): + return self.pixel_mean.device + + def forward(self, batched_inputs): + """ + Args: + batched_inputs: a list, batched outputs of :class:`DatasetMapper`. + Each item in the list contains the inputs for one image. + For now, each item in the list is a dict that contains: + * "image": Tensor, image in (C, H, W) format. + * "sem_seg": semantic segmentation ground truth + * "center": center points heatmap ground truth + * "offset": pixel offsets to center points ground truth + * Other information that's included in the original dicts, such as: + "height", "width" (int): the output resolution of the model (may be different + from input resolution), used in inference. + Returns: + list[dict]: + each dict is the results for one image. The dict contains the following keys: + + * "panoptic_seg", "sem_seg": see documentation + :doc:`/tutorials/models` for the standard output format + * "instances": available if ``predict_instances is True``. see documentation + :doc:`/tutorials/models` for the standard output format + """ + images = [x["image"].to(self.device) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + # To avoid error in ASPP layer when input has different size. + size_divisibility = ( + self.size_divisibility + if self.size_divisibility > 0 + else self.backbone.size_divisibility + ) + images = ImageList.from_tensors(images, size_divisibility) + + features = self.backbone(images.tensor) + + losses = {} + if "sem_seg" in batched_inputs[0]: + targets = [x["sem_seg"].to(self.device) for x in batched_inputs] + targets = ImageList.from_tensors( + targets, size_divisibility, self.sem_seg_head.ignore_value + ).tensor + if "sem_seg_weights" in batched_inputs[0]: + # The default D2 DatasetMapper may not contain "sem_seg_weights" + # Avoid error in testing when default DatasetMapper is used. + weights = [x["sem_seg_weights"].to(self.device) for x in batched_inputs] + weights = ImageList.from_tensors(weights, size_divisibility).tensor + else: + weights = None + else: + targets = None + weights = None + sem_seg_results, sem_seg_losses = self.sem_seg_head(features, targets, weights) + losses.update(sem_seg_losses) + + if "center" in batched_inputs[0] and "offset" in batched_inputs[0]: + center_targets = [x["center"].to(self.device) for x in batched_inputs] + center_targets = ImageList.from_tensors( + center_targets, size_divisibility + ).tensor.unsqueeze(1) + center_weights = [x["center_weights"].to(self.device) for x in batched_inputs] + center_weights = ImageList.from_tensors(center_weights, size_divisibility).tensor + + offset_targets = [x["offset"].to(self.device) for x in batched_inputs] + offset_targets = ImageList.from_tensors(offset_targets, size_divisibility).tensor + offset_weights = [x["offset_weights"].to(self.device) for x in batched_inputs] + offset_weights = ImageList.from_tensors(offset_weights, size_divisibility).tensor + else: + center_targets = None + center_weights = None + + offset_targets = None + offset_weights = None + + center_results, offset_results, center_losses, offset_losses = self.ins_embed_head( + features, center_targets, center_weights, offset_targets, offset_weights + ) + losses.update(center_losses) + losses.update(offset_losses) + + if self.training: + return losses + + if self.benchmark_network_speed: + return [] + + processed_results = [] + for sem_seg_result, center_result, offset_result, input_per_image, image_size in zip( + sem_seg_results, center_results, offset_results, batched_inputs, images.image_sizes + ): + height = input_per_image.get("height") + width = input_per_image.get("width") + r = sem_seg_postprocess(sem_seg_result, image_size, height, width) + c = sem_seg_postprocess(center_result, image_size, height, width) + o = sem_seg_postprocess(offset_result, image_size, height, width) + # Post-processing to get panoptic segmentation. + panoptic_image, _ = get_panoptic_segmentation( + r.argmax(dim=0, keepdim=True), + c, + o, + thing_ids=self.meta.thing_dataset_id_to_contiguous_id.values(), + label_divisor=self.meta.label_divisor, + stuff_area=self.stuff_area, + void_label=-1, + threshold=self.threshold, + nms_kernel=self.nms_kernel, + top_k=self.top_k, + ) + # For semantic segmentation evaluation. + processed_results.append({"sem_seg": r}) + panoptic_image = panoptic_image.squeeze(0) + semantic_prob = F.softmax(r, dim=0) + # For panoptic segmentation evaluation. + processed_results[-1]["panoptic_seg"] = (panoptic_image, None) + # For instance segmentation evaluation. + if self.predict_instances: + instances = [] + panoptic_image_cpu = panoptic_image.cpu().numpy() + for panoptic_label in np.unique(panoptic_image_cpu): + if panoptic_label == -1: + continue + pred_class = panoptic_label // self.meta.label_divisor + isthing = pred_class in list( + self.meta.thing_dataset_id_to_contiguous_id.values() + ) + # Get instance segmentation results. + if isthing: + instance = Instances((height, width)) + # Evaluation code takes continuous id starting from 0 + instance.pred_classes = torch.tensor( + [pred_class], device=panoptic_image.device + ) + mask = panoptic_image == panoptic_label + instance.pred_masks = mask.unsqueeze(0) + # Average semantic probability + sem_scores = semantic_prob[pred_class, ...] + sem_scores = torch.mean(sem_scores[mask]) + # Center point probability + mask_indices = torch.nonzero(mask).float() + center_y, center_x = ( + torch.mean(mask_indices[:, 0]), + torch.mean(mask_indices[:, 1]), + ) + center_scores = c[0, int(center_y.item()), int(center_x.item())] + # Confidence score is semantic prob * center prob. + instance.scores = torch.tensor( + [sem_scores * center_scores], device=panoptic_image.device + ) + # Get bounding boxes + instance.pred_boxes = BitMasks(instance.pred_masks).get_bounding_boxes() + instances.append(instance) + if len(instances) > 0: + processed_results[-1]["instances"] = Instances.cat(instances) + + return processed_results + + +@SEM_SEG_HEADS_REGISTRY.register() +class PanopticDeepLabSemSegHead(DeepLabV3PlusHead): + """ + A semantic segmentation head described in :paper:`Panoptic-DeepLab`. + """ + + @configurable + def __init__( + self, + input_shape: Dict[str, ShapeSpec], + *, + decoder_channels: List[int], + norm: Union[str, Callable], + head_channels: int, + loss_weight: float, + loss_type: str, + loss_top_k: float, + ignore_value: int, + num_classes: int, + **kwargs, + ): + """ + NOTE: this interface is experimental. + + Args: + input_shape (ShapeSpec): shape of the input feature + decoder_channels (list[int]): a list of output channels of each + decoder stage. It should have the same length as "input_shape" + (each element in "input_shape" corresponds to one decoder stage). + norm (str or callable): normalization for all conv layers. + head_channels (int): the output channels of extra convolutions + between decoder and predictor. + loss_weight (float): loss weight. + loss_top_k: (float): setting the top k% hardest pixels for + "hard_pixel_mining" loss. + loss_type, ignore_value, num_classes: the same as the base class. + """ + super().__init__( + input_shape, + decoder_channels=decoder_channels, + norm=norm, + ignore_value=ignore_value, + **kwargs, + ) + assert self.decoder_only + + self.loss_weight = loss_weight + use_bias = norm == "" + # `head` is additional transform before predictor + if self.use_depthwise_separable_conv: + # We use a single 5x5 DepthwiseSeparableConv2d to replace + # 2 3x3 Conv2d since they have the same receptive field. + self.head = DepthwiseSeparableConv2d( + decoder_channels[0], + head_channels, + kernel_size=5, + padding=2, + norm1=norm, + activation1=F.relu, + norm2=norm, + activation2=F.relu, + ) + else: + self.head = nn.Sequential( + Conv2d( + decoder_channels[0], + decoder_channels[0], + kernel_size=3, + padding=1, + bias=use_bias, + norm=get_norm(norm, decoder_channels[0]), + activation=F.relu, + ), + Conv2d( + decoder_channels[0], + head_channels, + kernel_size=3, + padding=1, + bias=use_bias, + norm=get_norm(norm, head_channels), + activation=F.relu, + ), + ) + weight_init.c2_xavier_fill(self.head[0]) + weight_init.c2_xavier_fill(self.head[1]) + self.predictor = Conv2d(head_channels, num_classes, kernel_size=1) + nn.init.normal_(self.predictor.weight, 0, 0.001) + nn.init.constant_(self.predictor.bias, 0) + + if loss_type == "cross_entropy": + self.loss = nn.CrossEntropyLoss(reduction="mean", ignore_index=ignore_value) + elif loss_type == "hard_pixel_mining": + self.loss = DeepLabCE(ignore_label=ignore_value, top_k_percent_pixels=loss_top_k) + else: + raise ValueError("Unexpected loss type: %s" % loss_type) + + @classmethod + def from_config(cls, cfg, input_shape): + ret = super().from_config(cfg, input_shape) + ret["head_channels"] = cfg.MODEL.SEM_SEG_HEAD.HEAD_CHANNELS + ret["loss_top_k"] = cfg.MODEL.SEM_SEG_HEAD.LOSS_TOP_K + return ret + + def forward(self, features, targets=None, weights=None): + """ + Returns: + In training, returns (None, dict of losses) + In inference, returns (CxHxW logits, {}) + """ + y = self.layers(features) + if self.training: + return None, self.losses(y, targets, weights) + else: + y = F.interpolate( + y, scale_factor=self.common_stride, mode="bilinear", align_corners=False + ) + return y, {} + + def layers(self, features): + assert self.decoder_only + y = super().layers(features) + y = self.head(y) + y = self.predictor(y) + return y + + def losses(self, predictions, targets, weights=None): + predictions = F.interpolate( + predictions, scale_factor=self.common_stride, mode="bilinear", align_corners=False + ) + loss = self.loss(predictions, targets, weights) + losses = {"loss_sem_seg": loss * self.loss_weight} + return losses + + +def build_ins_embed_branch(cfg, input_shape): + """ + Build a instance embedding branch from `cfg.MODEL.INS_EMBED_HEAD.NAME`. + """ + name = cfg.MODEL.INS_EMBED_HEAD.NAME + return INS_EMBED_BRANCHES_REGISTRY.get(name)(cfg, input_shape) + + +@INS_EMBED_BRANCHES_REGISTRY.register() +class PanopticDeepLabInsEmbedHead(DeepLabV3PlusHead): + """ + A instance embedding head described in :paper:`Panoptic-DeepLab`. + """ + + @configurable + def __init__( + self, + input_shape: Dict[str, ShapeSpec], + *, + decoder_channels: List[int], + norm: Union[str, Callable], + head_channels: int, + center_loss_weight: float, + offset_loss_weight: float, + **kwargs, + ): + """ + NOTE: this interface is experimental. + + Args: + input_shape (ShapeSpec): shape of the input feature + decoder_channels (list[int]): a list of output channels of each + decoder stage. It should have the same length as "input_shape" + (each element in "input_shape" corresponds to one decoder stage). + norm (str or callable): normalization for all conv layers. + head_channels (int): the output channels of extra convolutions + between decoder and predictor. + center_loss_weight (float): loss weight for center point prediction. + offset_loss_weight (float): loss weight for center offset prediction. + """ + super().__init__(input_shape, decoder_channels=decoder_channels, norm=norm, **kwargs) + assert self.decoder_only + + self.center_loss_weight = center_loss_weight + self.offset_loss_weight = offset_loss_weight + use_bias = norm == "" + # center prediction + # `head` is additional transform before predictor + self.center_head = nn.Sequential( + Conv2d( + decoder_channels[0], + decoder_channels[0], + kernel_size=3, + padding=1, + bias=use_bias, + norm=get_norm(norm, decoder_channels[0]), + activation=F.relu, + ), + Conv2d( + decoder_channels[0], + head_channels, + kernel_size=3, + padding=1, + bias=use_bias, + norm=get_norm(norm, head_channels), + activation=F.relu, + ), + ) + weight_init.c2_xavier_fill(self.center_head[0]) + weight_init.c2_xavier_fill(self.center_head[1]) + self.center_predictor = Conv2d(head_channels, 1, kernel_size=1) + nn.init.normal_(self.center_predictor.weight, 0, 0.001) + nn.init.constant_(self.center_predictor.bias, 0) + + # offset prediction + # `head` is additional transform before predictor + if self.use_depthwise_separable_conv: + # We use a single 5x5 DepthwiseSeparableConv2d to replace + # 2 3x3 Conv2d since they have the same receptive field. + self.offset_head = DepthwiseSeparableConv2d( + decoder_channels[0], + head_channels, + kernel_size=5, + padding=2, + norm1=norm, + activation1=F.relu, + norm2=norm, + activation2=F.relu, + ) + else: + self.offset_head = nn.Sequential( + Conv2d( + decoder_channels[0], + decoder_channels[0], + kernel_size=3, + padding=1, + bias=use_bias, + norm=get_norm(norm, decoder_channels[0]), + activation=F.relu, + ), + Conv2d( + decoder_channels[0], + head_channels, + kernel_size=3, + padding=1, + bias=use_bias, + norm=get_norm(norm, head_channels), + activation=F.relu, + ), + ) + weight_init.c2_xavier_fill(self.offset_head[0]) + weight_init.c2_xavier_fill(self.offset_head[1]) + self.offset_predictor = Conv2d(head_channels, 2, kernel_size=1) + nn.init.normal_(self.offset_predictor.weight, 0, 0.001) + nn.init.constant_(self.offset_predictor.bias, 0) + + self.center_loss = nn.MSELoss(reduction="none") + self.offset_loss = nn.L1Loss(reduction="none") + + @classmethod + def from_config(cls, cfg, input_shape): + if cfg.INPUT.CROP.ENABLED: + assert cfg.INPUT.CROP.TYPE == "absolute" + train_size = cfg.INPUT.CROP.SIZE + else: + train_size = None + decoder_channels = [cfg.MODEL.INS_EMBED_HEAD.CONVS_DIM] * ( + len(cfg.MODEL.INS_EMBED_HEAD.IN_FEATURES) - 1 + ) + [cfg.MODEL.INS_EMBED_HEAD.ASPP_CHANNELS] + ret = dict( + input_shape={ + k: v for k, v in input_shape.items() if k in cfg.MODEL.INS_EMBED_HEAD.IN_FEATURES + }, + project_channels=cfg.MODEL.INS_EMBED_HEAD.PROJECT_CHANNELS, + aspp_dilations=cfg.MODEL.INS_EMBED_HEAD.ASPP_DILATIONS, + aspp_dropout=cfg.MODEL.INS_EMBED_HEAD.ASPP_DROPOUT, + decoder_channels=decoder_channels, + common_stride=cfg.MODEL.INS_EMBED_HEAD.COMMON_STRIDE, + norm=cfg.MODEL.INS_EMBED_HEAD.NORM, + train_size=train_size, + head_channels=cfg.MODEL.INS_EMBED_HEAD.HEAD_CHANNELS, + center_loss_weight=cfg.MODEL.INS_EMBED_HEAD.CENTER_LOSS_WEIGHT, + offset_loss_weight=cfg.MODEL.INS_EMBED_HEAD.OFFSET_LOSS_WEIGHT, + use_depthwise_separable_conv=cfg.MODEL.SEM_SEG_HEAD.USE_DEPTHWISE_SEPARABLE_CONV, + ) + return ret + + def forward( + self, + features, + center_targets=None, + center_weights=None, + offset_targets=None, + offset_weights=None, + ): + """ + Returns: + In training, returns (None, dict of losses) + In inference, returns (CxHxW logits, {}) + """ + center, offset = self.layers(features) + if self.training: + return ( + None, + None, + self.center_losses(center, center_targets, center_weights), + self.offset_losses(offset, offset_targets, offset_weights), + ) + else: + center = F.interpolate( + center, scale_factor=self.common_stride, mode="bilinear", align_corners=False + ) + offset = ( + F.interpolate( + offset, scale_factor=self.common_stride, mode="bilinear", align_corners=False + ) + * self.common_stride + ) + return center, offset, {}, {} + + def layers(self, features): + assert self.decoder_only + y = super().layers(features) + # center + center = self.center_head(y) + center = self.center_predictor(center) + # offset + offset = self.offset_head(y) + offset = self.offset_predictor(offset) + return center, offset + + def center_losses(self, predictions, targets, weights): + predictions = F.interpolate( + predictions, scale_factor=self.common_stride, mode="bilinear", align_corners=False + ) + loss = self.center_loss(predictions, targets) * weights + if weights.sum() > 0: + loss = loss.sum() / weights.sum() + else: + loss = loss.sum() * 0 + losses = {"loss_center": loss * self.center_loss_weight} + return losses + + def offset_losses(self, predictions, targets, weights): + predictions = ( + F.interpolate( + predictions, scale_factor=self.common_stride, mode="bilinear", align_corners=False + ) + * self.common_stride + ) + loss = self.offset_loss(predictions, targets) * weights + if weights.sum() > 0: + loss = loss.sum() / weights.sum() + else: + loss = loss.sum() * 0 + losses = {"loss_offset": loss * self.offset_loss_weight} + return losses diff --git a/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/post_processing.py b/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/post_processing.py new file mode 100644 index 0000000000000000000000000000000000000000..194724eb414db073bde87bf482e5c647fa23cde7 --- /dev/null +++ b/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/post_processing.py @@ -0,0 +1,234 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# Reference: https://github.com/bowenc0221/panoptic-deeplab/blob/master/segmentation/model/post_processing/instance_post_processing.py # noqa + +from collections import Counter +import torch +import torch.nn.functional as F + + +def find_instance_center(center_heatmap, threshold=0.1, nms_kernel=3, top_k=None): + """ + Find the center points from the center heatmap. + Args: + center_heatmap: A Tensor of shape [1, H, W] of raw center heatmap output. + threshold: A float, threshold applied to center heatmap score. + nms_kernel: An integer, NMS max pooling kernel size. + top_k: An integer, top k centers to keep. + Returns: + A Tensor of shape [K, 2] where K is the number of center points. The + order of second dim is (y, x). + """ + # Thresholding, setting values below threshold to -1. + center_heatmap = F.threshold(center_heatmap, threshold, -1) + + # NMS + nms_padding = (nms_kernel - 1) // 2 + center_heatmap_max_pooled = F.max_pool2d( + center_heatmap, kernel_size=nms_kernel, stride=1, padding=nms_padding + ) + center_heatmap[center_heatmap != center_heatmap_max_pooled] = -1 + + # Squeeze first two dimensions. + center_heatmap = center_heatmap.squeeze() + assert len(center_heatmap.size()) == 2, "Something is wrong with center heatmap dimension." + + # Find non-zero elements. + if top_k is None: + return torch.nonzero(center_heatmap > 0) + else: + # find top k centers. + top_k_scores, _ = torch.topk(torch.flatten(center_heatmap), top_k) + return torch.nonzero(center_heatmap > top_k_scores[-1].clamp_(min=0)) + + +def group_pixels(center_points, offsets): + """ + Gives each pixel in the image an instance id. + Args: + center_points: A Tensor of shape [K, 2] where K is the number of center points. + The order of second dim is (y, x). + offsets: A Tensor of shape [2, H, W] of raw offset output. The order of + second dim is (offset_y, offset_x). + Returns: + A Tensor of shape [1, H, W] with values in range [1, K], which represents + the center this pixel belongs to. + """ + height, width = offsets.size()[1:] + + # Generates a coordinate map, where each location is the coordinate of + # that location. + y_coord, x_coord = torch.meshgrid( + torch.arange(height, dtype=offsets.dtype, device=offsets.device), + torch.arange(width, dtype=offsets.dtype, device=offsets.device), + ) + coord = torch.cat((y_coord.unsqueeze(0), x_coord.unsqueeze(0)), dim=0) + + center_loc = coord + offsets + center_loc = center_loc.flatten(1).T.unsqueeze_(0) # [1, H*W, 2] + center_points = center_points.unsqueeze(1) # [K, 1, 2] + + # Distance: [K, H*W]. + distance = torch.norm(center_points - center_loc, dim=-1) + + # Finds center with minimum distance at each location, offset by 1, to + # reserve id=0 for stuff. + instance_id = torch.argmin(distance, dim=0).reshape((1, height, width)) + 1 + return instance_id + + +def get_instance_segmentation( + sem_seg, center_heatmap, offsets, thing_seg, thing_ids, threshold=0.1, nms_kernel=3, top_k=None +): + """ + Post-processing for instance segmentation, gets class agnostic instance id. + Args: + sem_seg: A Tensor of shape [1, H, W], predicted semantic label. + center_heatmap: A Tensor of shape [1, H, W] of raw center heatmap output. + offsets: A Tensor of shape [2, H, W] of raw offset output. The order of + second dim is (offset_y, offset_x). + thing_seg: A Tensor of shape [1, H, W], predicted foreground mask, + if not provided, inference from semantic prediction. + thing_ids: A set of ids from contiguous category ids belonging + to thing categories. + threshold: A float, threshold applied to center heatmap score. + nms_kernel: An integer, NMS max pooling kernel size. + top_k: An integer, top k centers to keep. + Returns: + A Tensor of shape [1, H, W] with value 0 represent stuff (not instance) + and other positive values represent different instances. + A Tensor of shape [1, K, 2] where K is the number of center points. + The order of second dim is (y, x). + """ + center_points = find_instance_center( + center_heatmap, threshold=threshold, nms_kernel=nms_kernel, top_k=top_k + ) + if center_points.size(0) == 0: + return torch.zeros_like(sem_seg), center_points.unsqueeze(0) + ins_seg = group_pixels(center_points, offsets) + return thing_seg * ins_seg, center_points.unsqueeze(0) + + +def merge_semantic_and_instance( + sem_seg, ins_seg, semantic_thing_seg, label_divisor, thing_ids, stuff_area, void_label +): + """ + Post-processing for panoptic segmentation, by merging semantic segmentation + label and class agnostic instance segmentation label. + Args: + sem_seg: A Tensor of shape [1, H, W], predicted category id for each pixel. + ins_seg: A Tensor of shape [1, H, W], predicted instance id for each pixel. + semantic_thing_seg: A Tensor of shape [1, H, W], predicted foreground mask. + label_divisor: An integer, used to convert panoptic id = + semantic id * label_divisor + instance_id. + thing_ids: Set, a set of ids from contiguous category ids belonging + to thing categories. + stuff_area: An integer, remove stuff whose area is less tan stuff_area. + void_label: An integer, indicates the region has no confident prediction. + Returns: + A Tensor of shape [1, H, W]. + """ + # In case thing mask does not align with semantic prediction. + pan_seg = torch.zeros_like(sem_seg) + void_label + is_thing = (ins_seg > 0) & (semantic_thing_seg > 0) + + # Keep track of instance id for each class. + class_id_tracker = Counter() + + # Paste thing by majority voting. + instance_ids = torch.unique(ins_seg) + for ins_id in instance_ids: + if ins_id == 0: + continue + # Make sure only do majority voting within `semantic_thing_seg`. + thing_mask = (ins_seg == ins_id) & is_thing + if torch.nonzero(thing_mask).size(0) == 0: + continue + class_id, _ = torch.mode(sem_seg[thing_mask].view(-1)) + class_id_tracker[class_id.item()] += 1 + new_ins_id = class_id_tracker[class_id.item()] + pan_seg[thing_mask] = class_id * label_divisor + new_ins_id + + # Paste stuff to unoccupied area. + class_ids = torch.unique(sem_seg) + for class_id in class_ids: + if class_id.item() in thing_ids: + # thing class + continue + # Calculate stuff area. + stuff_mask = (sem_seg == class_id) & (ins_seg == 0) + if stuff_mask.sum().item() >= stuff_area: + pan_seg[stuff_mask] = class_id * label_divisor + + return pan_seg + + +def get_panoptic_segmentation( + sem_seg, + center_heatmap, + offsets, + thing_ids, + label_divisor, + stuff_area, + void_label, + threshold=0.1, + nms_kernel=7, + top_k=200, + foreground_mask=None, +): + """ + Post-processing for panoptic segmentation. + Args: + sem_seg: A Tensor of shape [1, H, W] of predicted semantic label. + center_heatmap: A Tensor of shape [1, H, W] of raw center heatmap output. + offsets: A Tensor of shape [2, H, W] of raw offset output. The order of + second dim is (offset_y, offset_x). + thing_ids: A set of ids from contiguous category ids belonging + to thing categories. + label_divisor: An integer, used to convert panoptic id = + semantic id * label_divisor + instance_id. + stuff_area: An integer, remove stuff whose area is less tan stuff_area. + void_label: An integer, indicates the region has no confident prediction. + threshold: A float, threshold applied to center heatmap score. + nms_kernel: An integer, NMS max pooling kernel size. + top_k: An integer, top k centers to keep. + foreground_mask: Optional, A Tensor of shape [1, H, W] of predicted + binary foreground mask. If not provided, it will be generated from + sem_seg. + Returns: + A Tensor of shape [1, H, W], int64. + """ + if sem_seg.dim() != 3 and sem_seg.size(0) != 1: + raise ValueError("Semantic prediction with un-supported shape: {}.".format(sem_seg.size())) + if center_heatmap.dim() != 3: + raise ValueError( + "Center prediction with un-supported dimension: {}.".format(center_heatmap.dim()) + ) + if offsets.dim() != 3: + raise ValueError("Offset prediction with un-supported dimension: {}.".format(offsets.dim())) + if foreground_mask is not None: + if foreground_mask.dim() != 3 and foreground_mask.size(0) != 1: + raise ValueError( + "Foreground prediction with un-supported shape: {}.".format(sem_seg.size()) + ) + thing_seg = foreground_mask + else: + # inference from semantic segmentation + thing_seg = torch.zeros_like(sem_seg) + for thing_class in list(thing_ids): + thing_seg[sem_seg == thing_class] = 1 + + instance, center = get_instance_segmentation( + sem_seg, + center_heatmap, + offsets, + thing_seg, + thing_ids, + threshold=threshold, + nms_kernel=nms_kernel, + top_k=top_k, + ) + panoptic = merge_semantic_and_instance( + sem_seg, instance, thing_seg, label_divisor, thing_ids, stuff_area, void_label + ) + + return panoptic, center diff --git a/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/target_generator.py b/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/target_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..a575c672494327e0e13c51de04ceca0f2bddc102 --- /dev/null +++ b/approach/ovod/detectron2/projects/Panoptic-DeepLab/panoptic_deeplab/target_generator.py @@ -0,0 +1,155 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# Reference: https://github.com/bowenc0221/panoptic-deeplab/blob/aa934324b55a34ce95fea143aea1cb7a6dbe04bd/segmentation/data/transforms/target_transforms.py#L11 # noqa +import numpy as np +import torch + + +class PanopticDeepLabTargetGenerator(object): + """ + Generates training targets for Panoptic-DeepLab. + """ + + def __init__( + self, + ignore_label, + thing_ids, + sigma=8, + ignore_stuff_in_offset=False, + small_instance_area=0, + small_instance_weight=1, + ignore_crowd_in_semantic=False, + ): + """ + Args: + ignore_label: Integer, the ignore label for semantic segmentation. + thing_ids: Set, a set of ids from contiguous category ids belonging + to thing categories. + sigma: the sigma for Gaussian kernel. + ignore_stuff_in_offset: Boolean, whether to ignore stuff region when + training the offset branch. + small_instance_area: Integer, indicates largest area for small instances. + small_instance_weight: Integer, indicates semantic loss weights for + small instances. + ignore_crowd_in_semantic: Boolean, whether to ignore crowd region in + semantic segmentation branch, crowd region is ignored in the original + TensorFlow implementation. + """ + self.ignore_label = ignore_label + self.thing_ids = set(thing_ids) + self.ignore_stuff_in_offset = ignore_stuff_in_offset + self.small_instance_area = small_instance_area + self.small_instance_weight = small_instance_weight + self.ignore_crowd_in_semantic = ignore_crowd_in_semantic + + # Generate the default Gaussian image for each center + self.sigma = sigma + size = 6 * sigma + 3 + x = np.arange(0, size, 1, float) + y = x[:, np.newaxis] + x0, y0 = 3 * sigma + 1, 3 * sigma + 1 + self.g = np.exp(-((x - x0) ** 2 + (y - y0) ** 2) / (2 * sigma**2)) + + def __call__(self, panoptic, segments_info): + """Generates the training target. + reference: https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/preparation/createPanopticImgs.py # noqa + reference: https://github.com/facebookresearch/detectron2/blob/main/datasets/prepare_panoptic_fpn.py#L18 # noqa + + Args: + panoptic: numpy.array, panoptic label, we assume it is already + converted from rgb image by panopticapi.utils.rgb2id. + segments_info (list[dict]): see detectron2 documentation of "Use Custom Datasets". + + Returns: + A dictionary with fields: + - sem_seg: Tensor, semantic label, shape=(H, W). + - center: Tensor, center heatmap, shape=(H, W). + - center_points: List, center coordinates, with tuple + (y-coord, x-coord). + - offset: Tensor, offset, shape=(2, H, W), first dim is + (offset_y, offset_x). + - sem_seg_weights: Tensor, loss weight for semantic prediction, + shape=(H, W). + - center_weights: Tensor, ignore region of center prediction, + shape=(H, W), used as weights for center regression 0 is + ignore, 1 is has instance. Multiply this mask to loss. + - offset_weights: Tensor, ignore region of offset prediction, + shape=(H, W), used as weights for offset regression 0 is + ignore, 1 is has instance. Multiply this mask to loss. + """ + height, width = panoptic.shape[0], panoptic.shape[1] + semantic = np.zeros_like(panoptic, dtype=np.uint8) + self.ignore_label + center = np.zeros((height, width), dtype=np.float32) + center_pts = [] + offset = np.zeros((2, height, width), dtype=np.float32) + y_coord, x_coord = np.meshgrid( + np.arange(height, dtype=np.float32), np.arange(width, dtype=np.float32), indexing="ij" + ) + # Generate pixel-wise loss weights + semantic_weights = np.ones_like(panoptic, dtype=np.uint8) + # 0: ignore, 1: has instance + # three conditions for a region to be ignored for instance branches: + # (1) It is labeled as `ignore_label` + # (2) It is crowd region (iscrowd=1) + # (3) (Optional) It is stuff region (for offset branch) + center_weights = np.zeros_like(panoptic, dtype=np.uint8) + offset_weights = np.zeros_like(panoptic, dtype=np.uint8) + for seg in segments_info: + cat_id = seg["category_id"] + if not (self.ignore_crowd_in_semantic and seg["iscrowd"]): + semantic[panoptic == seg["id"]] = cat_id + if not seg["iscrowd"]: + # Ignored regions are not in `segments_info`. + # Handle crowd region. + center_weights[panoptic == seg["id"]] = 1 + if not self.ignore_stuff_in_offset or cat_id in self.thing_ids: + offset_weights[panoptic == seg["id"]] = 1 + if cat_id in self.thing_ids: + # find instance center + mask_index = np.where(panoptic == seg["id"]) + if len(mask_index[0]) == 0: + # the instance is completely cropped + continue + + # Find instance area + ins_area = len(mask_index[0]) + if ins_area < self.small_instance_area: + semantic_weights[panoptic == seg["id"]] = self.small_instance_weight + + center_y, center_x = np.mean(mask_index[0]), np.mean(mask_index[1]) + center_pts.append([center_y, center_x]) + + # generate center heatmap + y, x = int(round(center_y)), int(round(center_x)) + sigma = self.sigma + # upper left + ul = int(np.round(x - 3 * sigma - 1)), int(np.round(y - 3 * sigma - 1)) + # bottom right + br = int(np.round(x + 3 * sigma + 2)), int(np.round(y + 3 * sigma + 2)) + + # start and end indices in default Gaussian image + gaussian_x0, gaussian_x1 = max(0, -ul[0]), min(br[0], width) - ul[0] + gaussian_y0, gaussian_y1 = max(0, -ul[1]), min(br[1], height) - ul[1] + + # start and end indices in center heatmap image + center_x0, center_x1 = max(0, ul[0]), min(br[0], width) + center_y0, center_y1 = max(0, ul[1]), min(br[1], height) + center[center_y0:center_y1, center_x0:center_x1] = np.maximum( + center[center_y0:center_y1, center_x0:center_x1], + self.g[gaussian_y0:gaussian_y1, gaussian_x0:gaussian_x1], + ) + + # generate offset (2, h, w) -> (y-dir, x-dir) + offset[0][mask_index] = center_y - y_coord[mask_index] + offset[1][mask_index] = center_x - x_coord[mask_index] + + center_weights = center_weights[None] + offset_weights = offset_weights[None] + return dict( + sem_seg=torch.as_tensor(semantic.astype("long")), + center=torch.as_tensor(center.astype(np.float32)), + center_points=center_pts, + offset=torch.as_tensor(offset.astype(np.float32)), + sem_seg_weights=torch.as_tensor(semantic_weights.astype(np.float32)), + center_weights=torch.as_tensor(center_weights.astype(np.float32)), + offset_weights=torch.as_tensor(offset_weights.astype(np.float32)), + ) diff --git a/approach/ovod/detectron2/projects/Panoptic-DeepLab/train_net.py b/approach/ovod/detectron2/projects/Panoptic-DeepLab/train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..780764f22fe8f4d52f218748dc64cf6c609e87b9 --- /dev/null +++ b/approach/ovod/detectron2/projects/Panoptic-DeepLab/train_net.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. + +""" +Panoptic-DeepLab Training Script. +This script is a simplified version of the training script in detectron2/tools. +""" + +import os +import torch + +import detectron2.data.transforms as T +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import get_cfg +from detectron2.data import MetadataCatalog, build_detection_train_loader +from detectron2.engine import DefaultTrainer, default_argument_parser, default_setup, launch +from detectron2.evaluation import ( + CityscapesInstanceEvaluator, + CityscapesSemSegEvaluator, + COCOEvaluator, + COCOPanopticEvaluator, + DatasetEvaluators, +) +from detectron2.projects.deeplab import build_lr_scheduler +from detectron2.projects.panoptic_deeplab import ( + PanopticDeeplabDatasetMapper, + add_panoptic_deeplab_config, +) +from detectron2.solver import get_default_optimizer_params +from detectron2.solver.build import maybe_add_gradient_clipping + + +def build_sem_seg_train_aug(cfg): + augs = [ + T.ResizeShortestEdge( + cfg.INPUT.MIN_SIZE_TRAIN, cfg.INPUT.MAX_SIZE_TRAIN, cfg.INPUT.MIN_SIZE_TRAIN_SAMPLING + ) + ] + if cfg.INPUT.CROP.ENABLED: + augs.append(T.RandomCrop(cfg.INPUT.CROP.TYPE, cfg.INPUT.CROP.SIZE)) + augs.append(T.RandomFlip()) + return augs + + +class Trainer(DefaultTrainer): + """ + We use the "DefaultTrainer" which contains a number pre-defined logic for + standard training workflow. They may not work for you, especially if you + are working on a new research project. In that case you can use the cleaner + "SimpleTrainer", or write your own training loop. + """ + + @classmethod + def build_evaluator(cls, cfg, dataset_name, output_folder=None): + """ + Create evaluator(s) for a given dataset. + This uses the special metadata "evaluator_type" associated with each builtin dataset. + For your own dataset, you can simply create an evaluator manually in your + script and do not have to worry about the hacky if-else logic here. + """ + if cfg.MODEL.PANOPTIC_DEEPLAB.BENCHMARK_NETWORK_SPEED: + return None + if output_folder is None: + output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") + evaluator_list = [] + evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type + if evaluator_type in ["cityscapes_panoptic_seg", "coco_panoptic_seg"]: + evaluator_list.append(COCOPanopticEvaluator(dataset_name, output_folder)) + if evaluator_type == "cityscapes_panoptic_seg": + evaluator_list.append(CityscapesSemSegEvaluator(dataset_name)) + evaluator_list.append(CityscapesInstanceEvaluator(dataset_name)) + if evaluator_type == "coco_panoptic_seg": + # `thing_classes` in COCO panoptic metadata includes both thing and + # stuff classes for visualization. COCOEvaluator requires metadata + # which only contains thing classes, thus we map the name of + # panoptic datasets to their corresponding instance datasets. + dataset_name_mapper = { + "coco_2017_val_panoptic": "coco_2017_val", + "coco_2017_val_100_panoptic": "coco_2017_val_100", + } + evaluator_list.append( + COCOEvaluator(dataset_name_mapper[dataset_name], output_dir=output_folder) + ) + if len(evaluator_list) == 0: + raise NotImplementedError( + "no Evaluator for the dataset {} with the type {}".format( + dataset_name, evaluator_type + ) + ) + elif len(evaluator_list) == 1: + return evaluator_list[0] + return DatasetEvaluators(evaluator_list) + + @classmethod + def build_train_loader(cls, cfg): + mapper = PanopticDeeplabDatasetMapper(cfg, augmentations=build_sem_seg_train_aug(cfg)) + return build_detection_train_loader(cfg, mapper=mapper) + + @classmethod + def build_lr_scheduler(cls, cfg, optimizer): + """ + It now calls :func:`detectron2.solver.build_lr_scheduler`. + Overwrite it if you'd like a different scheduler. + """ + return build_lr_scheduler(cfg, optimizer) + + @classmethod + def build_optimizer(cls, cfg, model): + """ + Build an optimizer from config. + """ + params = get_default_optimizer_params( + model, + weight_decay=cfg.SOLVER.WEIGHT_DECAY, + weight_decay_norm=cfg.SOLVER.WEIGHT_DECAY_NORM, + ) + + optimizer_type = cfg.SOLVER.OPTIMIZER + if optimizer_type == "SGD": + return maybe_add_gradient_clipping(cfg, torch.optim.SGD)( + params, + cfg.SOLVER.BASE_LR, + momentum=cfg.SOLVER.MOMENTUM, + nesterov=cfg.SOLVER.NESTEROV, + ) + elif optimizer_type == "ADAM": + return maybe_add_gradient_clipping(cfg, torch.optim.Adam)(params, cfg.SOLVER.BASE_LR) + else: + raise NotImplementedError(f"no optimizer type {optimizer_type}") + + +def setup(args): + """ + Create configs and perform basic setups. + """ + cfg = get_cfg() + add_panoptic_deeplab_config(cfg) + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + cfg.freeze() + default_setup(cfg, args) + return cfg + + +def main(args): + cfg = setup(args) + + if args.eval_only: + model = Trainer.build_model(cfg) + DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( + cfg.MODEL.WEIGHTS, resume=args.resume + ) + res = Trainer.test(cfg, model) + return res + + trainer = Trainer(cfg) + trainer.resume_or_load(resume=args.resume) + return trainer.train() + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + print("Command Line Args:", args) + launch( + main, + args.num_gpus, + num_machines=args.num_machines, + machine_rank=args.machine_rank, + dist_url=args.dist_url, + args=(args,), + ) diff --git a/approach/ovod/detectron2/projects/PointRend/README.md b/approach/ovod/detectron2/projects/PointRend/README.md new file mode 100644 index 0000000000000000000000000000000000000000..79d75d506c6f5db710044d3c1cd2583027ac3dbe --- /dev/null +++ b/approach/ovod/detectron2/projects/PointRend/README.md @@ -0,0 +1,167 @@ +# PointRend: Image Segmentation as Rendering + +Alexander Kirillov, Yuxin Wu, Kaiming He, Ross Girshick + +[[`arXiv`](https://arxiv.org/abs/1912.08193)] [[`BibTeX`](#CitingPointRend)] + +
+ +

+ +In this repository, we release code for PointRend in Detectron2. PointRend can be flexibly applied to both instance and semantic segmentation tasks by building on top of existing state-of-the-art models. + +## Quick start and visualization + +This [Colab Notebook](https://colab.research.google.com/drive/1isGPL5h5_cKoPPhVL9XhMokRtHDvmMVL) tutorial contains examples of PointRend usage and visualizations of its point sampling stages. + +## Training + +To train a model with 8 GPUs run: +```bash +cd /path/to/detectron2/projects/PointRend +python train_net.py --config-file configs/InstanceSegmentation/pointrend_rcnn_R_50_FPN_1x_coco.yaml --num-gpus 8 +``` + +## Evaluation + +Model evaluation can be done similarly: +```bash +cd /path/to/detectron2/projects/PointRend +python train_net.py --config-file configs/InstanceSegmentation/pointrend_rcnn_R_50_FPN_1x_coco.yaml --eval-only MODEL.WEIGHTS /path/to/model_checkpoint +``` + +# Pretrained Models + +## Instance Segmentation +#### COCO + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Mask
head		Backbonelr
sched		Output
resolution		mask
AP		mask
AP*		model iddownload
PointRendR50-FPN1×224×22436.239.7164254221model | metrics
PointRendR50-FPN3×224×22438.341.6164955410model | metrics
PointRendR101-FPN3×224×22440.143.8model | metrics
PointRendX101-FPN3×224×22441.144.7model | metrics
+ +AP* is COCO mask AP evaluated against the higher-quality LVIS annotations; see the paper for details. +Run `python detectron2/datasets/prepare_cocofied_lvis.py` to prepare GT files for AP* evaluation. +Since LVIS annotations are not exhaustive, `lvis-api` and not `cocoapi` should be used to evaluate AP*. + +#### Cityscapes +Cityscapes model is trained with ImageNet pretraining. + + + + + + + + + + + + + + + + + + + + +
Mask
head		Backbonelr
sched		Output
resolution		mask
AP		model iddownload
PointRendR50-FPN1×224×22435.9164255101model | metrics
+ + +## Semantic Segmentation + +#### Cityscapes +Cityscapes model is trained with ImageNet pretraining. + + + + + + + + + + + + + + + + + + +
MethodBackboneOutput
resolution		mIoUmodel iddownload
SemanticFPN + PointRendR101-FPN1024×204878.9202576688model | metrics
+ +## Citing PointRend + +If you use PointRend, please use the following BibTeX entry. + +```BibTeX +@InProceedings{kirillov2019pointrend, + title={{PointRend}: Image Segmentation as Rendering}, + author={Alexander Kirillov and Yuxin Wu and Kaiming He and Ross Girshick}, + journal={ArXiv:1912.08193}, + year={2019} +} +``` + +## Citing Implicit PointRend + +If you use Implicit PointRend, please use the following BibTeX entry. + +```BibTeX +@InProceedings{cheng2021pointly, + title={Pointly-Supervised Instance Segmentation, + author={Bowen Cheng and Omkar Parkhi and Alexander Kirillov}, + journal={ArXiv}, + year={2021} +} +``` diff --git a/approach/ovod/detectron2/projects/PointRend/train_net.py b/approach/ovod/detectron2/projects/PointRend/train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..9ae6f1a9b3ac12e59d42eafc680e2887973872d3 --- /dev/null +++ b/approach/ovod/detectron2/projects/PointRend/train_net.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. + +""" +PointRend Training Script. + +This script is a simplified version of the training script in detectron2/tools. +""" + +import os + +import detectron2.data.transforms as T +import detectron2.utils.comm as comm +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import get_cfg +from detectron2.data import DatasetMapper, MetadataCatalog, build_detection_train_loader +from detectron2.engine import DefaultTrainer, default_argument_parser, default_setup, launch +from detectron2.evaluation import ( + CityscapesInstanceEvaluator, + CityscapesSemSegEvaluator, + COCOEvaluator, + DatasetEvaluators, + LVISEvaluator, + SemSegEvaluator, + verify_results, +) +from detectron2.projects.point_rend import ColorAugSSDTransform, add_pointrend_config + + +def build_sem_seg_train_aug(cfg): + augs = [ + T.ResizeShortestEdge( + cfg.INPUT.MIN_SIZE_TRAIN, cfg.INPUT.MAX_SIZE_TRAIN, cfg.INPUT.MIN_SIZE_TRAIN_SAMPLING + ) + ] + if cfg.INPUT.CROP.ENABLED: + augs.append( + T.RandomCrop_CategoryAreaConstraint( + cfg.INPUT.CROP.TYPE, + cfg.INPUT.CROP.SIZE, + cfg.INPUT.CROP.SINGLE_CATEGORY_MAX_AREA, + cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE, + ) + ) + if cfg.INPUT.COLOR_AUG_SSD: + augs.append(ColorAugSSDTransform(img_format=cfg.INPUT.FORMAT)) + augs.append(T.RandomFlip()) + return augs + + +class Trainer(DefaultTrainer): + """ + We use the "DefaultTrainer" which contains a number pre-defined logic for + standard training workflow. They may not work for you, especially if you + are working on a new research project. In that case you can use the cleaner + "SimpleTrainer", or write your own training loop. + """ + + @classmethod + def build_evaluator(cls, cfg, dataset_name, output_folder=None): + """ + Create evaluator(s) for a given dataset. + This uses the special metadata "evaluator_type" associated with each builtin dataset. + For your own dataset, you can simply create an evaluator manually in your + script and do not have to worry about the hacky if-else logic here. + """ + if output_folder is None: + output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") + evaluator_list = [] + evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type + if evaluator_type == "lvis": + return LVISEvaluator(dataset_name, output_dir=output_folder) + if evaluator_type == "coco": + return COCOEvaluator(dataset_name, output_dir=output_folder) + if evaluator_type == "sem_seg": + return SemSegEvaluator( + dataset_name, + distributed=True, + output_dir=output_folder, + ) + if evaluator_type == "cityscapes_instance": + return CityscapesInstanceEvaluator(dataset_name) + if evaluator_type == "cityscapes_sem_seg": + return CityscapesSemSegEvaluator(dataset_name) + if len(evaluator_list) == 0: + raise NotImplementedError( + "no Evaluator for the dataset {} with the type {}".format( + dataset_name, evaluator_type + ) + ) + if len(evaluator_list) == 1: + return evaluator_list[0] + return DatasetEvaluators(evaluator_list) + + @classmethod + def build_train_loader(cls, cfg): + if "SemanticSegmentor" in cfg.MODEL.META_ARCHITECTURE: + mapper = DatasetMapper(cfg, is_train=True, augmentations=build_sem_seg_train_aug(cfg)) + else: + mapper = None + return build_detection_train_loader(cfg, mapper=mapper) + + +def setup(args): + """ + Create configs and perform basic setups. + """ + cfg = get_cfg() + add_pointrend_config(cfg) + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + cfg.freeze() + default_setup(cfg, args) + return cfg + + +def main(args): + cfg = setup(args) + + if args.eval_only: + model = Trainer.build_model(cfg) + DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( + cfg.MODEL.WEIGHTS, resume=args.resume + ) + res = Trainer.test(cfg, model) + if comm.is_main_process(): + verify_results(cfg, res) + return res + + trainer = Trainer(cfg) + trainer.resume_or_load(resume=args.resume) + return trainer.train() + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + print("Command Line Args:", args) + launch( + main, + args.num_gpus, + num_machines=args.num_machines, + machine_rank=args.machine_rank, + dist_url=args.dist_url, + args=(args,), + ) diff --git a/approach/ovod/detectron2/projects/PointSup/README.md b/approach/ovod/detectron2/projects/PointSup/README.md new file mode 100644 index 0000000000000000000000000000000000000000..75ce084530d192a522824d01b98a474d77863e68 --- /dev/null +++ b/approach/ovod/detectron2/projects/PointSup/README.md @@ -0,0 +1,41 @@ +# Pointly-Supervised Instance Segmentation + +Bowen Cheng, Omkar Parkhi, Alexander Kirillov + +[[`arXiv`](https://arxiv.org/abs/2104.06404)] [[`Project`](https://bowenc0221.github.io/point-sup)] [[`BibTeX`](#CitingPointSup)] + +
+ +

+ +## Data preparation +Please follow these steps to prepare your datasets: +1. Follow official Detectron2 instruction to prepare COCO dataset. Set up `DETECTRON2_DATASETS` environment variable to the location of your Detectron2 dataset. +2. Generate 10-points annotations for COCO by running: `python tools/prepare_coco_point_annotations_without_masks.py 10` + +## Training + +To train a model with 8 GPUs run: +```bash +python train_net.py --config-file configs/mask_rcnn_R_50_FPN_3x_point_sup_point_aug_coco.yaml --num-gpus 8 +``` + +## Evaluation + +Model evaluation can be done similarly: +```bash +python train_net.py --config-file configs/mask_rcnn_R_50_FPN_3x_point_sup_point_aug_coco.yaml --eval-only MODEL.WEIGHTS /path/to/model_checkpoint +``` + +## Citing Pointly-Supervised Instance Segmentation + +If you use PointSup, please use the following BibTeX entry. + +```BibTeX +@article{cheng2021pointly, + title={Pointly-Supervised Instance Segmentation}, + author={Bowen Cheng and Omkar Parkhi and Alexander Kirillov}, + journal={arXiv}, + year={2021} +} +``` diff --git a/approach/ovod/detectron2/projects/PointSup/configs/implicit_pointrend_R_50_FPN_3x_point_sup_point_aug_coco.yaml b/approach/ovod/detectron2/projects/PointSup/configs/implicit_pointrend_R_50_FPN_3x_point_sup_point_aug_coco.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b3d4272c6f8a3820c8d354bfb3c915ccdebfc4a --- /dev/null +++ b/approach/ovod/detectron2/projects/PointSup/configs/implicit_pointrend_R_50_FPN_3x_point_sup_point_aug_coco.yaml @@ -0,0 +1,9 @@ +_BASE_: "../../PointRend/configs/InstanceSegmentation/implicit_pointrend_R_50_FPN_3x_coco.yaml" +MODEL: + ROI_MASK_HEAD: + NAME: "ImplicitPointRendPointSupHead" +INPUT: + POINT_SUP: True + SAMPLE_POINTS: 5 +DATASETS: + TRAIN: ("coco_2017_train_points_n10_v1_without_masks",) diff --git a/approach/ovod/detectron2/projects/PointSup/configs/mask_rcnn_R_50_FPN_3x_point_sup_coco.yaml b/approach/ovod/detectron2/projects/PointSup/configs/mask_rcnn_R_50_FPN_3x_point_sup_coco.yaml new file mode 100644 index 0000000000000000000000000000000000000000..157e3844ef68779cda3579bee5d8c132826c9fba --- /dev/null +++ b/approach/ovod/detectron2/projects/PointSup/configs/mask_rcnn_R_50_FPN_3x_point_sup_coco.yaml @@ -0,0 +1,15 @@ +_BASE_: "../../../configs/Base-RCNN-FPN.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + MASK_ON: True + RESNETS: + DEPTH: 50 + ROI_MASK_HEAD: + NAME: "MaskRCNNConvUpsamplePointSupHead" +INPUT: + POINT_SUP: True +DATASETS: + TRAIN: ("coco_2017_train_points_n10_v1_without_masks",) +SOLVER: + STEPS: (210000, 250000) + MAX_ITER: 270000 diff --git a/approach/ovod/detectron2/projects/PointSup/configs/mask_rcnn_R_50_FPN_3x_point_sup_point_aug_coco.yaml b/approach/ovod/detectron2/projects/PointSup/configs/mask_rcnn_R_50_FPN_3x_point_sup_point_aug_coco.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b11224d595bed88238e02caeb4833b0b1d7b286 --- /dev/null +++ b/approach/ovod/detectron2/projects/PointSup/configs/mask_rcnn_R_50_FPN_3x_point_sup_point_aug_coco.yaml @@ -0,0 +1,3 @@ +_BASE_: "mask_rcnn_R_50_FPN_3x_point_sup_coco.yaml" +INPUT: + SAMPLE_POINTS: 5 diff --git a/approach/ovod/detectron2/projects/PointSup/point_sup/__init__.py b/approach/ovod/detectron2/projects/PointSup/point_sup/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..510e3814ac1bb273b48804191b4a7c1272ea9a9b --- /dev/null +++ b/approach/ovod/detectron2/projects/PointSup/point_sup/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +from . import register_point_annotations +from .config import add_point_sup_config +from .dataset_mapper import PointSupDatasetMapper +from .mask_head import MaskRCNNConvUpsamplePointSupHead +from .point_utils import get_point_coords_from_point_annotation diff --git a/approach/ovod/detectron2/projects/PointSup/point_sup/config.py b/approach/ovod/detectron2/projects/PointSup/point_sup/config.py new file mode 100644 index 0000000000000000000000000000000000000000..5e00b786cf6055a0cda664f143c1fac56a3c6d11 --- /dev/null +++ b/approach/ovod/detectron2/projects/PointSup/point_sup/config.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + + +def add_point_sup_config(cfg): + """ + Add config for point supervision. + """ + # Use point annotation + cfg.INPUT.POINT_SUP = False + # Sample only part of points in each iteration. + # Default: 0, use all available points. + cfg.INPUT.SAMPLE_POINTS = 0 diff --git a/approach/ovod/detectron2/projects/PointSup/point_sup/dataset_mapper.py b/approach/ovod/detectron2/projects/PointSup/point_sup/dataset_mapper.py new file mode 100644 index 0000000000000000000000000000000000000000..52b9bd4ce19d51e07f98aa9adf36c41f6ddc22af --- /dev/null +++ b/approach/ovod/detectron2/projects/PointSup/point_sup/dataset_mapper.py @@ -0,0 +1,125 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import copy +import logging +import numpy as np +from typing import List, Union +import torch + +import detectron2.data.detection_utils as utils +import detectron2.data.transforms as T +from detectron2.config import configurable + +from .detection_utils import annotations_to_instances, transform_instance_annotations + +__all__ = [ + "PointSupDatasetMapper", +] + + +class PointSupDatasetMapper: + """ + The callable currently does the following: + 1. Read the image from "file_name" + 2. Applies transforms to the image and annotations + 3. Prepare data and annotations to Tensor and :class:`Instances` + """ + + @configurable + def __init__( + self, + is_train: bool, + *, + augmentations: List[Union[T.Augmentation, T.Transform]], + image_format: str, + # Extra data augmentation for point supervision + sample_points: int = 0, + ): + """ + NOTE: this interface is experimental. + + Args: + is_train: whether it's used in training or inference + augmentations: a list of augmentations or deterministic transforms to apply + image_format: an image format supported by :func:`detection_utils.read_image`. + sample_points: subsample points at each iteration + """ + # fmt: off + self.is_train = is_train + self.augmentations = T.AugmentationList(augmentations) + self.image_format = image_format + self.sample_points = sample_points + # fmt: on + logger = logging.getLogger(__name__) + mode = "training" if is_train else "inference" + logger.info(f"[DatasetMapper] Augmentations used in {mode}: {augmentations}") + logger.info(f"Point Augmentations used in {mode}: sample {sample_points} points") + + @classmethod + def from_config(cls, cfg, is_train: bool = True): + augs = utils.build_augmentation(cfg, is_train) + if cfg.INPUT.CROP.ENABLED and is_train: + raise ValueError("Crop augmentation not supported to point supervision.") + + ret = { + "is_train": is_train, + "augmentations": augs, + "image_format": cfg.INPUT.FORMAT, + "sample_points": cfg.INPUT.SAMPLE_POINTS, + } + + return ret + + def __call__(self, dataset_dict): + """ + Args: + dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. + Returns: + dict: a format that builtin models in detectron2 accept + """ + dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below + image = utils.read_image(dataset_dict["file_name"], format=self.image_format) + utils.check_image_size(dataset_dict, image) + + aug_input = T.AugInput(image) + transforms = self.augmentations(aug_input) + image = aug_input.image + + image_shape = image.shape[:2] # h, w + # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, + # but not efficient on large generic data structures due to the use of pickle & mp.Queue. + # Therefore it's important to use torch.Tensor. + dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1))) + + if not self.is_train: + dataset_dict.pop("annotations", None) + return dataset_dict + + if "annotations" in dataset_dict: + # Maps points from the closed interval [0, image_size - 1] on discrete + # image coordinates to the half-open interval [x1, x2) on continuous image + # coordinates. We use the continuous-discrete conversion from Heckbert + # 1990 ("What is the coordinate of a pixel?"): d = floor(c) and c = d + 0.5, + # where d is a discrete coordinate and c is a continuous coordinate. + for ann in dataset_dict["annotations"]: + point_coords_wrt_image = np.array(ann["point_coords"]).astype(np.float) + point_coords_wrt_image = point_coords_wrt_image + 0.5 + ann["point_coords"] = point_coords_wrt_image + + annos = [ + # also need to transform point coordinates + transform_instance_annotations( + obj, + transforms, + image_shape, + ) + for obj in dataset_dict.pop("annotations") + if obj.get("iscrowd", 0) == 0 + ] + instances = annotations_to_instances( + annos, + image_shape, + sample_points=self.sample_points, + ) + + dataset_dict["instances"] = utils.filter_empty_instances(instances) + return dataset_dict diff --git a/approach/ovod/detectron2/projects/PointSup/point_sup/detection_utils.py b/approach/ovod/detectron2/projects/PointSup/point_sup/detection_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3f95d9449277fc55e93121582f6c6a6396dc833d --- /dev/null +++ b/approach/ovod/detectron2/projects/PointSup/point_sup/detection_utils.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +import numpy as np +import torch + +# fmt: off +from detectron2.data.detection_utils import \ + annotations_to_instances as base_annotations_to_instances +from detectron2.data.detection_utils import \ + transform_instance_annotations as base_transform_instance_annotations + +# fmt: on + + +def annotations_to_instances(annos, image_size, sample_points=0): + """ + Create an :class:`Instances` object used by the models, + from instance annotations in the dataset dict. + + Args: + annos (list[dict]): a list of instance annotations in one image, each + element for one instance. + image_size (tuple): height, width + sample_points (int): subsample points at each iteration + + Returns: + Instances: + It will contain fields "gt_boxes", "gt_classes", + "gt_point_coords", "gt_point_labels", if they can be obtained from `annos`. + This is the format that builtin models with point supervision expect. + """ + target = base_annotations_to_instances(annos, image_size) + + assert ("point_coords" in annos[0]) == ("point_labels" in annos[0]) + + if len(annos) and "point_labels" in annos[0]: + point_coords = [] + point_labels = [] + for i, _ in enumerate(annos): + # Already in the image coordinate system + point_coords_wrt_image = np.array(annos[i]["point_coords"]) + point_labels_wrt_image = np.array(annos[i]["point_labels"]) + + if sample_points > 0: + random_indices = np.random.choice( + point_coords_wrt_image.shape[0], + sample_points, + replace=point_coords_wrt_image.shape[0] < sample_points, + ).astype(int) + point_coords_wrt_image = point_coords_wrt_image[random_indices] + point_labels_wrt_image = point_labels_wrt_image[random_indices] + assert point_coords_wrt_image.shape[0] == point_labels_wrt_image.size + + point_coords.append(point_coords_wrt_image) + point_labels.append(point_labels_wrt_image) + + point_coords = torch.stack([torch.from_numpy(x) for x in point_coords]) + point_labels = torch.stack([torch.from_numpy(x) for x in point_labels]) + target.gt_point_coords = point_coords + target.gt_point_labels = point_labels + + return target + + +def transform_instance_annotations( + annotation, transforms, image_size, *, keypoint_hflip_indices=None +): + """ + Apply transforms to box, and point annotations of a single instance. + It will use `transforms.apply_box` for the box, and + `transforms.apply_coords` for points. + Args: + annotation (dict): dict of instance annotations for a single instance. + It will be modified in-place. + transforms (TransformList or list[Transform]): + image_size (tuple): the height, width of the transformed image + keypoint_hflip_indices (ndarray[int]): see `create_keypoint_hflip_indices`. + Returns: + dict: + the same input dict with fields "bbox", "point_coords", "point_labels" + transformed according to `transforms`. + The "bbox_mode" field will be set to XYXY_ABS. + """ + annotation = base_transform_instance_annotations( + annotation, transforms, image_size, keypoint_hflip_indices + ) + + assert ("point_coords" in annotation) == ("point_labels" in annotation) + if "point_coords" in annotation and "point_labels" in annotation: + point_coords = annotation["point_coords"] + point_labels = np.array(annotation["point_labels"]).astype(np.float) + point_coords = transforms.apply_coords(point_coords) + + # Set all out-of-boundary points to "unlabeled" + inside = (point_coords >= np.array([0, 0])) & (point_coords <= np.array(image_size[::-1])) + inside = inside.all(axis=1) + point_labels[~inside] = -1 + + annotation["point_coords"] = point_coords + annotation["point_labels"] = point_labels + + return annotation diff --git a/approach/ovod/detectron2/projects/PointSup/point_sup/mask_head.py b/approach/ovod/detectron2/projects/PointSup/point_sup/mask_head.py new file mode 100644 index 0000000000000000000000000000000000000000..81c21f55009b1891c4684e2eaa8fee0f144b0a54 --- /dev/null +++ b/approach/ovod/detectron2/projects/PointSup/point_sup/mask_head.py @@ -0,0 +1,77 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import numpy as np +from typing import Any, List + +from detectron2.modeling import ROI_MASK_HEAD_REGISTRY +from detectron2.modeling.roi_heads.mask_head import MaskRCNNConvUpsampleHead, mask_rcnn_inference +from detectron2.projects.point_rend import ImplicitPointRendMaskHead +from detectron2.projects.point_rend.point_features import point_sample +from detectron2.projects.point_rend.point_head import roi_mask_point_loss +from detectron2.structures import Instances + +from .point_utils import get_point_coords_from_point_annotation + +__all__ = [ + "ImplicitPointRendPointSupHead", + "MaskRCNNConvUpsamplePointSupHead", +] + + +@ROI_MASK_HEAD_REGISTRY.register() +class MaskRCNNConvUpsamplePointSupHead(MaskRCNNConvUpsampleHead): + """ + A mask head with several conv layers, plus an upsample layer (with `ConvTranspose2d`). + Predictions are made with a final 1x1 conv layer. + + The difference with `MaskRCNNConvUpsampleHead` is that this head is trained + with point supervision. Please use the `MaskRCNNConvUpsampleHead` if you want + to train the model with mask supervision. + """ + + def forward(self, x, instances: List[Instances]) -> Any: + """ + Args: + x: input region feature(s) provided by :class:`ROIHeads`. + instances (list[Instances]): contains the boxes & labels corresponding + to the input features. + Exact format is up to its caller to decide. + Typically, this is the foreground instances in training, with + "proposal_boxes" field and other gt annotations. + In inference, it contains boxes that are already predicted. + Returns: + A dict of losses in training. The predicted "instances" in inference. + """ + x = self.layers(x) + if self.training: + N, C, H, W = x.shape + assert H == W + + proposal_boxes = [x.proposal_boxes for x in instances] + assert N == np.sum(len(x) for x in proposal_boxes) + + if N == 0: + return {"loss_mask": x.sum() * 0} + + # Training with point supervision + point_coords, point_labels = get_point_coords_from_point_annotation(instances) + + mask_logits = point_sample( + x, + point_coords, + align_corners=False, + ) + + return {"loss_mask": roi_mask_point_loss(mask_logits, instances, point_labels)} + else: + mask_rcnn_inference(x, instances) + return instances + + +@ROI_MASK_HEAD_REGISTRY.register() +class ImplicitPointRendPointSupHead(ImplicitPointRendMaskHead): + def _uniform_sample_train_points(self, instances): + assert self.training + # Please keep in mind that "gt_masks" is not used in this mask head. + point_coords, point_labels = get_point_coords_from_point_annotation(instances) + + return point_coords, point_labels diff --git a/approach/ovod/detectron2/projects/PointSup/point_sup/point_utils.py b/approach/ovod/detectron2/projects/PointSup/point_sup/point_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..eed876ea9e0127c584c008bd5aab3e16e2c8c66a --- /dev/null +++ b/approach/ovod/detectron2/projects/PointSup/point_sup/point_utils.py @@ -0,0 +1,77 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import torch + +from detectron2.layers import cat + + +def get_point_coords_from_point_annotation(instances): + """ + Load point coords and their corresponding labels from point annotation. + + Args: + instances (list[Instances]): A list of N Instances, where N is the number of images + in the batch. These instances are in 1:1 + correspondence with the pred_mask_logits. The ground-truth labels (class, box, mask, + ...) associated with each instance are stored in fields. + Returns: + point_coords (Tensor): A tensor of shape (N, P, 2) that contains the coordinates of P + sampled points. + point_labels (Tensor): A tensor of shape (N, P) that contains the labels of P + sampled points. `point_labels` takes 3 possible values: + - 0: the point belongs to background + - 1: the point belongs to the object + - -1: the point is ignored during training + """ + point_coords_list = [] + point_labels_list = [] + for instances_per_image in instances: + if len(instances_per_image) == 0: + continue + point_coords = instances_per_image.gt_point_coords.to(torch.float32) + point_labels = instances_per_image.gt_point_labels.to(torch.float32).clone() + proposal_boxes_per_image = instances_per_image.proposal_boxes.tensor + + # Convert point coordinate system, ground truth points are in image coord. + point_coords_wrt_box = get_point_coords_wrt_box(proposal_boxes_per_image, point_coords) + + # Ignore points that are outside predicted boxes. + point_ignores = ( + (point_coords_wrt_box[:, :, 0] < 0) + | (point_coords_wrt_box[:, :, 0] > 1) + | (point_coords_wrt_box[:, :, 1] < 0) + | (point_coords_wrt_box[:, :, 1] > 1) + ) + point_labels[point_ignores] = -1 + + point_coords_list.append(point_coords_wrt_box) + point_labels_list.append(point_labels) + + return ( + cat(point_coords_list, dim=0), + cat(point_labels_list, dim=0), + ) + + +def get_point_coords_wrt_box(boxes_coords, point_coords): + """ + Convert image-level absolute coordinates to box-normalized [0, 1] x [0, 1] point cooordinates. + Args: + boxes_coords (Tensor): A tensor of shape (R, 4) that contains bounding boxes. + coordinates. + point_coords (Tensor): A tensor of shape (R, P, 2) that contains + image-normalized coordinates of P sampled points. + Returns: + point_coords_wrt_box (Tensor): A tensor of shape (R, P, 2) that contains + [0, 1] x [0, 1] box-normalized coordinates of the P sampled points. + """ + with torch.no_grad(): + point_coords_wrt_box = point_coords.clone() + point_coords_wrt_box[:, :, 0] -= boxes_coords[:, None, 0] + point_coords_wrt_box[:, :, 1] -= boxes_coords[:, None, 1] + point_coords_wrt_box[:, :, 0] = point_coords_wrt_box[:, :, 0] / ( + boxes_coords[:, None, 2] - boxes_coords[:, None, 0] + ) + point_coords_wrt_box[:, :, 1] = point_coords_wrt_box[:, :, 1] / ( + boxes_coords[:, None, 3] - boxes_coords[:, None, 1] + ) + return point_coords_wrt_box diff --git a/approach/ovod/detectron2/projects/PointSup/point_sup/register_point_annotations.py b/approach/ovod/detectron2/projects/PointSup/point_sup/register_point_annotations.py new file mode 100644 index 0000000000000000000000000000000000000000..32f2bb45e864e5be9d002f4d07badb91700ace4b --- /dev/null +++ b/approach/ovod/detectron2/projects/PointSup/point_sup/register_point_annotations.py @@ -0,0 +1,69 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import logging +import os + +from detectron2.data import DatasetCatalog, MetadataCatalog +from detectron2.data.datasets.builtin import _get_builtin_metadata +from detectron2.data.datasets.coco import load_coco_json + +logger = logging.getLogger(__name__) + + +# COCO dataset +def register_coco_instances_with_points(name, metadata, json_file, image_root): + """ + Register a dataset in COCO's json annotation format for + instance segmentation with point annotation. + + The point annotation json does not have "segmentation" field, instead, + it has "point_coords" and "point_labels" fields. + + Args: + name (str): the name that identifies a dataset, e.g. "coco_2014_train". + metadata (dict): extra metadata associated with this dataset. You can + leave it as an empty dict. + json_file (str): path to the json instance annotation file. + image_root (str or path-like): directory which contains all the images. + """ + assert isinstance(name, str), name + assert isinstance(json_file, (str, os.PathLike)), json_file + assert isinstance(image_root, (str, os.PathLike)), image_root + # 1. register a function which returns dicts + DatasetCatalog.register( + name, lambda: load_coco_json(json_file, image_root, name, ["point_coords", "point_labels"]) + ) + + # 2. Optionally, add metadata about this dataset, + # since they might be useful in evaluation, visualization or logging + MetadataCatalog.get(name).set( + json_file=json_file, image_root=image_root, evaluator_type="coco", **metadata + ) + + +_PREDEFINED_SPLITS_COCO = {} +_PREDEFINED_SPLITS_COCO["coco"] = { + # point annotations without masks + "coco_2017_train_points_n10_v1_without_masks": ( + "coco/train2017", + "coco/annotations/instances_train2017_n10_v1_without_masks.json", + ), +} + + +def register_all_coco_train_points(root): + for dataset_name, splits_per_dataset in _PREDEFINED_SPLITS_COCO.items(): + for key, (image_root, json_file) in splits_per_dataset.items(): + # Assume pre-defined datasets live in `./datasets`. + register_coco_instances_with_points( + key, + _get_builtin_metadata(dataset_name), + os.path.join(root, json_file) if "://" not in json_file else json_file, + os.path.join(root, image_root), + ) + + +# True for open source; +# Internally at fb, we register them elsewhere +if __name__.endswith(".register_point_annotations"): + _root = os.getenv("DETECTRON2_DATASETS", "datasets") + register_all_coco_train_points(_root) diff --git a/approach/ovod/detectron2/projects/PointSup/tools/prepare_coco_point_annotations_without_masks.py b/approach/ovod/detectron2/projects/PointSup/tools/prepare_coco_point_annotations_without_masks.py new file mode 100644 index 0000000000000000000000000000000000000000..e4aee2aedf2e62e2357f278417ac58c6b4ff264e --- /dev/null +++ b/approach/ovod/detectron2/projects/PointSup/tools/prepare_coco_point_annotations_without_masks.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +import copy +import json +import numpy as np +import os +import sys +import pycocotools.mask as mask_utils + +from detectron2.utils.env import seed_all_rng +from detectron2.utils.file_io import PathManager + + +def get_point_annotations(input_filename, output_filename, num_points_per_instance): + with PathManager.open(input_filename, "r") as f: + coco_json = json.load(f) + + coco_annos = coco_json.pop("annotations") + coco_points_json = copy.deepcopy(coco_json) + + imgs = {} + for img in coco_json["images"]: + imgs[img["id"]] = img + + new_annos = [] + for ann in coco_annos: + # convert mask + t = imgs[ann["image_id"]] + h, w = t["height"], t["width"] + segm = ann.pop("segmentation") + if type(segm) == list: + # polygon -- a single object might consist of multiple parts + # we merge all parts into one mask rle code + rles = mask_utils.frPyObjects(segm, h, w) + rle = mask_utils.merge(rles) + elif type(segm["counts"]) == list: + # uncompressed RLE + rle = mask_utils.frPyObjects(segm, h, w) + else: + # rle + rle = segm + mask = mask_utils.decode(rle) + new_ann = copy.deepcopy(ann) + # sample points in image coordinates + box = ann["bbox"] + point_coords_wrt_image = np.random.rand(num_points_per_instance, 2) + point_coords_wrt_image[:, 0] = point_coords_wrt_image[:, 0] * box[2] + point_coords_wrt_image[:, 1] = point_coords_wrt_image[:, 1] * box[3] + point_coords_wrt_image[:, 0] += box[0] + point_coords_wrt_image[:, 1] += box[1] + # round to integer coordinates + point_coords_wrt_image = np.floor(point_coords_wrt_image).astype(int) + # get labels + assert (point_coords_wrt_image >= 0).all(), (point_coords_wrt_image, mask.shape) + assert (point_coords_wrt_image[:, 0] < w).all(), (point_coords_wrt_image, mask.shape) + assert (point_coords_wrt_image[:, 1] < h).all(), (point_coords_wrt_image, mask.shape) + point_labels = mask[point_coords_wrt_image[:, 1], point_coords_wrt_image[:, 0]] + # store new annotations + new_ann["point_coords"] = point_coords_wrt_image.tolist() + new_ann["point_labels"] = point_labels.tolist() + new_annos.append(new_ann) + coco_points_json["annotations"] = new_annos + + with PathManager.open(output_filename, "w") as f: + json.dump(coco_points_json, f) + + print("{} is modified and stored in {}.".format(input_filename, output_filename)) + + +if __name__ == "__main__": + """ + Generate point-based supervision for COCO dataset. + + Usage: + python tools/prepare_coco_point_annotations_without_masks.py \ + NUM_POINTS_PER_INSTANCE NUM_VERSIONS_WITH_DIFFERENT_SEED + + Example to generate point-based COCO dataset with 10 points per instance: + python tools/prepare_coco_point_annotations_without_masks.py 10 + """ + + # Fix random seed + seed_all_rng(12345) + + assert len(sys.argv) >= 2, "Please provide number of points to sample per instance" + dataset_dir = os.path.join(os.getenv("DETECTRON2_DATASETS", "datasets"), "coco/annotations") + num_points_per_instance = int(sys.argv[1]) + if len(sys.argv) == 3: + repeat = int(sys.argv[2]) + else: + repeat = 1 + s = "instances_train2017" + for version in range(repeat): + print( + "Start sampling {} points per instance for annotations {}.".format( + num_points_per_instance, s + ) + ) + get_point_annotations( + os.path.join(dataset_dir, "{}.json".format(s)), + os.path.join( + dataset_dir, + "{}_n{}_v{}_without_masks.json".format(s, num_points_per_instance, version + 1), + ), + num_points_per_instance, + ) diff --git a/approach/ovod/detectron2/projects/PointSup/train_net.py b/approach/ovod/detectron2/projects/PointSup/train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..0fe970a8fe6da4ea6b2b124d1ee1dc66c38ebc56 --- /dev/null +++ b/approach/ovod/detectron2/projects/PointSup/train_net.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +""" +Point supervision Training Script. + +This script is a simplified version of the training script in detectron2/tools. +""" + +import os + +import detectron2.utils.comm as comm +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import get_cfg +from detectron2.data import MetadataCatalog, build_detection_train_loader +from detectron2.engine import DefaultTrainer, default_argument_parser, default_setup, launch +from detectron2.evaluation import COCOEvaluator, DatasetEvaluators, verify_results +from detectron2.projects.point_rend import add_pointrend_config +from detectron2.utils.logger import setup_logger + +from point_sup import PointSupDatasetMapper, add_point_sup_config + + +class Trainer(DefaultTrainer): + """ + We use the "DefaultTrainer" which contains pre-defined default logic for + standard training workflow. They may not work for you, especially if you + are working on a new research project. In that case you can write your + own training loop. You can use "tools/plain_train_net.py" as an example. + """ + + @classmethod + def build_evaluator(cls, cfg, dataset_name, output_folder=None): + """ + Create evaluator(s) for a given dataset. + This uses the special metadata "evaluator_type" associated with each builtin dataset. + For your own dataset, you can simply create an evaluator manually in your + script and do not have to worry about the hacky if-else logic here. + """ + if output_folder is None: + output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") + evaluator_list = [] + evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type + if evaluator_type == "coco": + evaluator_list.append(COCOEvaluator(dataset_name, output_dir=output_folder)) + if len(evaluator_list) == 0: + raise NotImplementedError( + "no Evaluator for the dataset {} with the type {}".format( + dataset_name, evaluator_type + ) + ) + elif len(evaluator_list) == 1: + return evaluator_list[0] + return DatasetEvaluators(evaluator_list) + + @classmethod + def build_train_loader(cls, cfg): + if cfg.INPUT.POINT_SUP: + mapper = PointSupDatasetMapper(cfg, is_train=True) + else: + mapper = None + return build_detection_train_loader(cfg, mapper=mapper) + + +def setup(args): + """ + Create configs and perform basic setups. + """ + cfg = get_cfg() + add_pointrend_config(cfg) + add_point_sup_config(cfg) + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + cfg.freeze() + default_setup(cfg, args) + # Setup logger for "point_sup" module + setup_logger(output=cfg.OUTPUT_DIR, distributed_rank=comm.get_rank(), name="point_sup") + return cfg + + +def main(args): + cfg = setup(args) + + if args.eval_only: + model = Trainer.build_model(cfg) + DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( + cfg.MODEL.WEIGHTS, resume=args.resume + ) + res = Trainer.test(cfg, model) + if cfg.TEST.AUG.ENABLED: + res.update(Trainer.test_with_TTA(cfg, model)) + if comm.is_main_process(): + verify_results(cfg, res) + return res + + """ + If you'd like to do anything fancier than the standard training logic, + consider writing your own training loop (see plain_train_net.py) or + subclassing the trainer. + """ + trainer = Trainer(cfg) + trainer.resume_or_load(resume=args.resume) + return trainer.train() + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + print("Command Line Args:", args) + launch( + main, + args.num_gpus, + num_machines=args.num_machines, + machine_rank=args.machine_rank, + dist_url=args.dist_url, + args=(args,), + ) diff --git a/approach/ovod/detectron2/projects/Rethinking-BatchNorm/README.md b/approach/ovod/detectron2/projects/Rethinking-BatchNorm/README.md new file mode 100644 index 0000000000000000000000000000000000000000..42c5c68fb4837043df62ff398f15fe0326f96e1c --- /dev/null +++ b/approach/ovod/detectron2/projects/Rethinking-BatchNorm/README.md @@ -0,0 +1,36 @@ +# Rethinking "Batch" in BatchNorm + +We provide configs that reproduce detection experiments in the paper [Rethinking "Batch" in BatchNorm](https://arxiv.org/abs/2105.07576). + +All configs can be trained with: + +``` +../../tools/lazyconfig_train_net.py --config-file configs/X.py --num-gpus 8 +``` + +## Mask R-CNN + +* `mask_rcnn_BNhead.py`, `mask_rcnn_BNhead_batch_stats.py`: + Mask R-CNN with BatchNorm in the head. See Table 3 in the paper. + +* `mask_rcnn_BNhead_shuffle.py`: Mask R-CNN with cross-GPU shuffling of head inputs. + See Figure 9 and Table 6 in the paper. + +* `mask_rcnn_SyncBNhead.py`: Mask R-CNN with cross-GPU SyncBatchNorm in the head. + It matches Table 6 in the paper. + +## RetinaNet + +* `retinanet_SyncBNhead.py`: RetinaNet with SyncBN in head, a straightforward implementation + which matches row 3 of Table 5. + +* `retinanet_SyncBNhead_SharedTraining.py`: RetinaNet with SyncBN in head, normalizing + all 5 feature levels together. Match row 1 of Table 5. + +The script `retinanet-eval-domain-specific.py` evaluates a checkpoint after recomputing +domain-specific statistics. Running it with +``` +./retinanet-eval-domain-specific.py checkpoint.pth +``` +on a model produced by the above two configs, can produce results that match row 4 and +row 2 of Table 5. diff --git a/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/mask_rcnn_BNhead.py b/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/mask_rcnn_BNhead.py new file mode 100644 index 0000000000000000000000000000000000000000..336c133e0e34ee82674d595ef98d1844f801fa4f --- /dev/null +++ b/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/mask_rcnn_BNhead.py @@ -0,0 +1,18 @@ +from detectron2.model_zoo import get_config + +model = get_config("common/models/mask_rcnn_fpn.py").model + +model.backbone.bottom_up.freeze_at = 2 + +model.roi_heads.box_head.conv_norm = model.roi_heads.mask_head.conv_norm = "BN" +# 4conv1fc head +model.roi_heads.box_head.conv_dims = [256, 256, 256, 256] +model.roi_heads.box_head.fc_dims = [1024] + +dataloader = get_config("common/data/coco.py").dataloader +lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_3x +optimizer = get_config("common/optim.py").SGD +train = get_config("common/train.py").train + +train.init_checkpoint = "detectron2://ImageNetPretrained/MSRA/R-50.pkl" +train.max_iter = 270000 # 3x for batchsize = 16 diff --git a/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/mask_rcnn_BNhead_batch_stats.py b/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/mask_rcnn_BNhead_batch_stats.py new file mode 100644 index 0000000000000000000000000000000000000000..872e17c8a9aa000250a0a61613ddb3e3886f9991 --- /dev/null +++ b/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/mask_rcnn_BNhead_batch_stats.py @@ -0,0 +1,20 @@ +from torch.nn import BatchNorm2d +from torch.nn import functional as F + + +class BatchNormBatchStat(BatchNorm2d): + """ + BN that uses batch stat in inference + """ + + def forward(self, input): + if self.training: + return super().forward(input) + return F.batch_norm(input, None, None, self.weight, self.bias, True, 1.0, self.eps) + + +# After training with the base config, it's sufficient to load its model with +# this config only for inference -- because the training-time behavior is identical. +from .mask_rcnn_BNhead import model, dataloader, lr_multiplier, optimizer, train + +model.roi_heads.box_head.conv_norm = model.roi_heads.mask_head.conv_norm = BatchNormBatchStat diff --git a/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/mask_rcnn_BNhead_shuffle.py b/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/mask_rcnn_BNhead_shuffle.py new file mode 100644 index 0000000000000000000000000000000000000000..5117a7dad0f952af02580e5373a7be52b749ee86 --- /dev/null +++ b/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/mask_rcnn_BNhead_shuffle.py @@ -0,0 +1,74 @@ +import math +import torch +import torch.distributed as dist + +from detectron2.modeling.roi_heads import FastRCNNConvFCHead, MaskRCNNConvUpsampleHead +from detectron2.utils import comm +from fvcore.nn.distributed import differentiable_all_gather + + +def concat_all_gather(input): + bs_int = input.shape[0] + size_list = comm.all_gather(bs_int) + max_size = max(size_list) + max_shape = (max_size,) + input.shape[1:] + + padded_input = input.new_zeros(max_shape) + padded_input[:bs_int] = input + all_inputs = differentiable_all_gather(padded_input) + inputs = [x[:sz] for sz, x in zip(size_list, all_inputs)] + return inputs, size_list + + +def batch_shuffle(x): + # gather from all gpus + batch_size_this = x.shape[0] + all_xs, batch_size_all = concat_all_gather(x) + all_xs_concat = torch.cat(all_xs, dim=0) + total_bs = sum(batch_size_all) + + rank = dist.get_rank() + assert batch_size_all[rank] == batch_size_this + + idx_range = (sum(batch_size_all[:rank]), sum(batch_size_all[: rank + 1])) + + # random shuffle index + idx_shuffle = torch.randperm(total_bs, device=x.device) + # broadcast to all gpus + dist.broadcast(idx_shuffle, src=0) + + # index for restoring + idx_unshuffle = torch.argsort(idx_shuffle) + + # shuffled index for this gpu + splits = torch.split(idx_shuffle, math.ceil(total_bs / dist.get_world_size())) + if len(splits) > rank: + idx_this = splits[rank] + else: + idx_this = idx_shuffle.new_zeros([0]) + return all_xs_concat[idx_this], idx_unshuffle[idx_range[0] : idx_range[1]] + + +def batch_unshuffle(x, idx_unshuffle): + all_x, _ = concat_all_gather(x) + x_gather = torch.cat(all_x, dim=0) + return x_gather[idx_unshuffle] + + +def wrap_shuffle(module_type, method): + def new_method(self, x): + if self.training: + x, idx = batch_shuffle(x) + x = getattr(module_type, method)(self, x) + if self.training: + x = batch_unshuffle(x, idx) + return x + + return type(module_type.__name__ + "WithShuffle", (module_type,), {method: new_method}) + + +from .mask_rcnn_BNhead import model, dataloader, lr_multiplier, optimizer, train + + +model.roi_heads.box_head._target_ = wrap_shuffle(FastRCNNConvFCHead, "forward") +model.roi_heads.mask_head._target_ = wrap_shuffle(MaskRCNNConvUpsampleHead, "layers") diff --git a/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/mask_rcnn_SyncBNhead.py b/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/mask_rcnn_SyncBNhead.py new file mode 100644 index 0000000000000000000000000000000000000000..5f05da03514a4ee6aa37d6bc3e678873ead73c61 --- /dev/null +++ b/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/mask_rcnn_SyncBNhead.py @@ -0,0 +1,3 @@ +from .mask_rcnn_BNhead import model, dataloader, lr_multiplier, optimizer, train + +model.roi_heads.box_head.conv_norm = model.roi_heads.mask_head.conv_norm = "SyncBN" diff --git a/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/retinanet_SyncBNhead.py b/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/retinanet_SyncBNhead.py new file mode 100644 index 0000000000000000000000000000000000000000..222dfddffb1f9bedf87f4c345534045b29e2d8ee --- /dev/null +++ b/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/retinanet_SyncBNhead.py @@ -0,0 +1,19 @@ +from detectron2.model_zoo import get_config +from torch import nn + +model = get_config("common/models/retinanet.py").model +model.backbone.bottom_up.freeze_at = 2 + +# The head will overwrite string "SyncBN" to use domain-specific BN, so we +# provide a class here to use shared BN in training. +model.head.norm = nn.SyncBatchNorm2d + +dataloader = get_config("common/data/coco.py").dataloader +lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_3x +optimizer = get_config("common/optim.py").SGD +train = get_config("common/train.py").train + +optimizer.lr = 0.01 + +train.init_checkpoint = "detectron2://ImageNetPretrained/MSRA/R-50.pkl" +train.max_iter = 270000 # 3x for batchsize = 16 diff --git a/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/retinanet_SyncBNhead_SharedTraining.py b/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/retinanet_SyncBNhead_SharedTraining.py new file mode 100644 index 0000000000000000000000000000000000000000..3f146009d04aad2fca08d970569a4d76d46c9bd2 --- /dev/null +++ b/approach/ovod/detectron2/projects/Rethinking-BatchNorm/configs/retinanet_SyncBNhead_SharedTraining.py @@ -0,0 +1,32 @@ +from typing import List +import torch +from torch import Tensor, nn + +from detectron2.modeling.meta_arch.retinanet import RetinaNetHead + + +def apply_sequential(inputs, modules): + for mod in modules: + if isinstance(mod, (nn.BatchNorm2d, nn.SyncBatchNorm)): + # for BN layer, normalize all inputs together + shapes = [i.shape for i in inputs] + spatial_sizes = [s[2] * s[3] for s in shapes] + x = [i.flatten(2) for i in inputs] + x = torch.cat(x, dim=2).unsqueeze(3) + x = mod(x).split(spatial_sizes, dim=2) + inputs = [i.view(s) for s, i in zip(shapes, x)] + else: + inputs = [mod(i) for i in inputs] + return inputs + + +class RetinaNetHead_SharedTrainingBN(RetinaNetHead): + def forward(self, features: List[Tensor]): + logits = apply_sequential(features, list(self.cls_subnet) + [self.cls_score]) + bbox_reg = apply_sequential(features, list(self.bbox_subnet) + [self.bbox_pred]) + return logits, bbox_reg + + +from .retinanet_SyncBNhead import model, dataloader, lr_multiplier, optimizer, train + +model.head._target_ = RetinaNetHead_SharedTrainingBN diff --git a/approach/ovod/detectron2/projects/Rethinking-BatchNorm/retinanet-eval-domain-specific.py b/approach/ovod/detectron2/projects/Rethinking-BatchNorm/retinanet-eval-domain-specific.py new file mode 100644 index 0000000000000000000000000000000000000000..49a74adf1f286135c5551d9b31e722169f23b8f0 --- /dev/null +++ b/approach/ovod/detectron2/projects/Rethinking-BatchNorm/retinanet-eval-domain-specific.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +import sys +import torch +from fvcore.nn.precise_bn import update_bn_stats + +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import LazyConfig, instantiate +from detectron2.evaluation import inference_on_dataset +from detectron2.layers import CycleBatchNormList +from detectron2.utils.events import EventStorage +from detectron2.utils.logger import setup_logger + +logger = setup_logger() +setup_logger(name="fvcore") + + +if __name__ == "__main__": + checkpoint = sys.argv[1] + cfg = LazyConfig.load_rel("./configs/retinanet_SyncBNhead.py") + model = cfg.model + model.head.norm = lambda c: CycleBatchNormList(len(model.head_in_features), num_features=c) + model = instantiate(model) + model.cuda() + DetectionCheckpointer(model).load(checkpoint) + + cfg.dataloader.train.total_batch_size = 8 + logger.info("Running PreciseBN ...") + with EventStorage(), torch.no_grad(): + update_bn_stats(model, instantiate(cfg.dataloader.train), 500) + + logger.info("Running evaluation ...") + inference_on_dataset( + model, instantiate(cfg.dataloader.test), instantiate(cfg.dataloader.evaluator) + ) diff --git a/approach/ovod/detectron2/projects/TensorMask/README.md b/approach/ovod/detectron2/projects/TensorMask/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e81307c4c9be8d1cb2fd27b716531f4ebcd9ae5c --- /dev/null +++ b/approach/ovod/detectron2/projects/TensorMask/README.md @@ -0,0 +1,63 @@ + +# TensorMask in Detectron2 +**A Foundation for Dense Object Segmentation** + +Xinlei Chen, Ross Girshick, Kaiming He, Piotr Dollár + +[[`arXiv`](https://arxiv.org/abs/1903.12174)] [[`BibTeX`](#CitingTensorMask)] + +
+ +
+ +In this repository, we release code for TensorMask in Detectron2. +TensorMask is a dense sliding-window instance segmentation framework that, for the first time, achieves results close to the well-developed Mask R-CNN framework -- both qualitatively and quantitatively. It establishes a conceptually complementary direction for object instance segmentation research. + +## Installation +First install Detectron2 following the [documentation](https://detectron2.readthedocs.io/tutorials/install.html) and +[setup the dataset](../../datasets). Then compile the TensorMask-specific op (`swap_align2nat`): +```bash +pip install -e /path/to/detectron2/projects/TensorMask +``` + +## Training + +To train a model, run: +```bash +python /path/to/detectron2/projects/TensorMask/train_net.py --config-file +``` + +For example, to launch TensorMask BiPyramid training (1x schedule) with ResNet-50 backbone on 8 GPUs, +one should execute: +```bash +python /path/to/detectron2/projects/TensorMask/train_net.py --config-file configs/tensormask_R_50_FPN_1x.yaml --num-gpus 8 +``` + +## Evaluation + +Model evaluation can be done similarly (6x schedule with scale augmentation): +```bash +python /path/to/detectron2/projects/TensorMask/train_net.py --config-file configs/tensormask_R_50_FPN_6x.yaml --eval-only MODEL.WEIGHTS /path/to/model_checkpoint +``` + +# Pretrained Models + +| Backbone | lr sched | AP box | AP mask | download | +| -------- | -------- | -- | --- | -------- | +| R50 | 1x | 37.6 | 32.4 | model \|  metrics | +| R50 | 6x | 41.4 | 35.8 | model \|  metrics | + + +## Citing TensorMask + +If you use TensorMask, please use the following BibTeX entry. + +``` +@InProceedings{chen2019tensormask, + title={Tensormask: A Foundation for Dense Object Segmentation}, + author={Chen, Xinlei and Girshick, Ross and He, Kaiming and Doll{\'a}r, Piotr}, + journal={The International Conference on Computer Vision (ICCV)}, + year={2019} +} +``` + diff --git a/approach/ovod/detectron2/projects/TensorMask/setup.py b/approach/ovod/detectron2/projects/TensorMask/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..f6980e0dd2d2d239faed11e1474e1a8394c9b843 --- /dev/null +++ b/approach/ovod/detectron2/projects/TensorMask/setup.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. + +import glob +import os +from setuptools import find_packages, setup +import torch +from torch.utils.cpp_extension import CUDA_HOME, CppExtension, CUDAExtension + + +def get_extensions(): + this_dir = os.path.dirname(os.path.abspath(__file__)) + extensions_dir = os.path.join(this_dir, "tensormask", "layers", "csrc") + + main_source = os.path.join(extensions_dir, "vision.cpp") + sources = glob.glob(os.path.join(extensions_dir, "**", "*.cpp")) + source_cuda = glob.glob(os.path.join(extensions_dir, "**", "*.cu")) + glob.glob( + os.path.join(extensions_dir, "*.cu") + ) + + sources = [main_source] + sources + + extension = CppExtension + + extra_compile_args = {"cxx": []} + define_macros = [] + + if (torch.cuda.is_available() and CUDA_HOME is not None) or os.getenv("FORCE_CUDA", "0") == "1": + extension = CUDAExtension + sources += source_cuda + define_macros += [("WITH_CUDA", None)] + extra_compile_args["nvcc"] = [ + "-DCUDA_HAS_FP16=1", + "-D__CUDA_NO_HALF_OPERATORS__", + "-D__CUDA_NO_HALF_CONVERSIONS__", + "-D__CUDA_NO_HALF2_OPERATORS__", + ] + + # It's better if pytorch can do this by default .. + CC = os.environ.get("CC", None) + if CC is not None: + extra_compile_args["nvcc"].append("-ccbin={}".format(CC)) + + sources = [os.path.join(extensions_dir, s) for s in sources] + + include_dirs = [extensions_dir] + + ext_modules = [ + extension( + "tensormask._C", + sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + ) + ] + + return ext_modules + + +setup( + name="tensormask", + version="0.1", + author="FAIR", + packages=find_packages(exclude=("configs", "tests")), + python_requires=">=3.7", + ext_modules=get_extensions(), + cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension}, +) diff --git a/approach/ovod/detectron2/projects/TensorMask/train_net.py b/approach/ovod/detectron2/projects/TensorMask/train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..dc77a64d7e0f8b2b0385a8f7842fa1efe6d5edfb --- /dev/null +++ b/approach/ovod/detectron2/projects/TensorMask/train_net.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. + +""" +TensorMask Training Script. + +This script is a simplified version of the training script in detectron2/tools. +""" + +import os + +import detectron2.utils.comm as comm +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import get_cfg +from detectron2.engine import DefaultTrainer, default_argument_parser, default_setup, launch +from detectron2.evaluation import COCOEvaluator, verify_results + +from tensormask import add_tensormask_config + + +class Trainer(DefaultTrainer): + @classmethod + def build_evaluator(cls, cfg, dataset_name, output_folder=None): + if output_folder is None: + output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") + return COCOEvaluator(dataset_name, output_dir=output_folder) + + +def setup(args): + """ + Create configs and perform basic setups. + """ + cfg = get_cfg() + add_tensormask_config(cfg) + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + cfg.freeze() + default_setup(cfg, args) + return cfg + + +def main(args): + cfg = setup(args) + + if args.eval_only: + model = Trainer.build_model(cfg) + DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( + cfg.MODEL.WEIGHTS, resume=args.resume + ) + res = Trainer.test(cfg, model) + if comm.is_main_process(): + verify_results(cfg, res) + return res + + trainer = Trainer(cfg) + trainer.resume_or_load(resume=args.resume) + return trainer.train() + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + print("Command Line Args:", args) + launch( + main, + args.num_gpus, + num_machines=args.num_machines, + machine_rank=args.machine_rank, + dist_url=args.dist_url, + args=(args,), + ) diff --git a/approach/ovod/detectron2/projects/TridentNet/README.md b/approach/ovod/detectron2/projects/TridentNet/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4b7a90102d008a498e93dff595a09206be5269e7 --- /dev/null +++ b/approach/ovod/detectron2/projects/TridentNet/README.md @@ -0,0 +1,60 @@ + +# TridentNet in Detectron2 +**Scale-Aware Trident Networks for Object Detection** + +Yanghao Li\*, Yuntao Chen\*, Naiyan Wang, Zhaoxiang Zhang + +[[`TridentNet`](https://github.com/TuSimple/simpledet/tree/master/models/tridentnet)] [[`arXiv`](https://arxiv.org/abs/1901.01892)] [[`BibTeX`](#CitingTridentNet)] + +
+ +
+ +In this repository, we implement TridentNet-Fast in Detectron2. +Trident Network (TridentNet) aims to generate scale-specific feature maps with a uniform representational power. We construct a parallel multi-branch architecture in which each branch shares the same transformation parameters but with different receptive fields. TridentNet-Fast is a fast approximation version of TridentNet that could achieve significant improvements without any additional parameters and computational cost. + +## Training + +To train a model, run +```bash +python /path/to/detectron2/projects/TridentNet/train_net.py --config-file +``` + +For example, to launch end-to-end TridentNet training with ResNet-50 backbone on 8 GPUs, +one should execute: +```bash +python /path/to/detectron2/projects/TridentNet/train_net.py --config-file configs/tridentnet_fast_R_50_C4_1x.yaml --num-gpus 8 +``` + +## Evaluation + +Model evaluation can be done similarly: +```bash +python /path/to/detectron2/projects/TridentNet/train_net.py --config-file configs/tridentnet_fast_R_50_C4_1x.yaml --eval-only MODEL.WEIGHTS model.pth +``` + +## Results on MS-COCO in Detectron2 + +|Model|Backbone|Head|lr sched|AP|AP50|AP75|APs|APm|APl|download| +|-----|--------|----|--------|--|----|----|---|---|---|--------| +|Faster|R50-C4|C5-512ROI|1X|35.7|56.1|38.0|19.2|40.9|48.7|model \| metrics| +|TridentFast|R50-C4|C5-128ROI|1X|38.0|58.1|40.8|19.5|42.2|54.6|model \| metrics| +|Faster|R50-C4|C5-512ROI|3X|38.4|58.7|41.3|20.7|42.7|53.1|model \| metrics| +|TridentFast|R50-C4|C5-128ROI|3X|40.6|60.8|43.6|23.4|44.7|57.1|model \| metrics| +|Faster|R101-C4|C5-512ROI|3X|41.1|61.4|44.0|22.2|45.5|55.9|model \| metrics| +|TridentFast|R101-C4|C5-128ROI|3X|43.6|63.4|47.0|24.3|47.8|60.0|model \| metrics| + + +## Citing TridentNet + +If you use TridentNet, please use the following BibTeX entry. + +``` +@InProceedings{li2019scale, + title={Scale-Aware Trident Networks for Object Detection}, + author={Li, Yanghao and Chen, Yuntao and Wang, Naiyan and Zhang, Zhaoxiang}, + journal={The International Conference on Computer Vision (ICCV)}, + year={2019} +} +``` + diff --git a/approach/ovod/detectron2/projects/TridentNet/configs/Base-TridentNet-Fast-C4.yaml b/approach/ovod/detectron2/projects/TridentNet/configs/Base-TridentNet-Fast-C4.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c3d80797ba9ae63a5669ccbd74a0d2006fee3b7 --- /dev/null +++ b/approach/ovod/detectron2/projects/TridentNet/configs/Base-TridentNet-Fast-C4.yaml @@ -0,0 +1,29 @@ +MODEL: + META_ARCHITECTURE: "GeneralizedRCNN" + BACKBONE: + NAME: "build_trident_resnet_backbone" + ROI_HEADS: + NAME: "TridentRes5ROIHeads" + POSITIVE_FRACTION: 0.5 + BATCH_SIZE_PER_IMAGE: 128 + PROPOSAL_APPEND_GT: False + PROPOSAL_GENERATOR: + NAME: "TridentRPN" + RPN: + POST_NMS_TOPK_TRAIN: 500 + TRIDENT: + NUM_BRANCH: 3 + BRANCH_DILATIONS: [1, 2, 3] + TEST_BRANCH_IDX: 1 + TRIDENT_STAGE: "res4" +DATASETS: + TRAIN: ("coco_2017_train",) + TEST: ("coco_2017_val",) +SOLVER: + IMS_PER_BATCH: 16 + BASE_LR: 0.02 + STEPS: (60000, 80000) + MAX_ITER: 90000 +INPUT: + MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800) +VERSION: 2 diff --git a/approach/ovod/detectron2/projects/TridentNet/configs/tridentnet_fast_R_101_C4_3x.yaml b/approach/ovod/detectron2/projects/TridentNet/configs/tridentnet_fast_R_101_C4_3x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc83c2f9e7b7653c8982e657b5f116abe6ad6e1f --- /dev/null +++ b/approach/ovod/detectron2/projects/TridentNet/configs/tridentnet_fast_R_101_C4_3x.yaml @@ -0,0 +1,9 @@ +_BASE_: "Base-TridentNet-Fast-C4.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-101.pkl" + MASK_ON: False + RESNETS: + DEPTH: 101 +SOLVER: + STEPS: (210000, 250000) + MAX_ITER: 270000 diff --git a/approach/ovod/detectron2/projects/TridentNet/configs/tridentnet_fast_R_50_C4_1x.yaml b/approach/ovod/detectron2/projects/TridentNet/configs/tridentnet_fast_R_50_C4_1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fda2cb6622d732c0f70d74d567c26182a9a41c44 --- /dev/null +++ b/approach/ovod/detectron2/projects/TridentNet/configs/tridentnet_fast_R_50_C4_1x.yaml @@ -0,0 +1,6 @@ +_BASE_: "Base-TridentNet-Fast-C4.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + MASK_ON: False + RESNETS: + DEPTH: 50 diff --git a/approach/ovod/detectron2/projects/TridentNet/configs/tridentnet_fast_R_50_C4_3x.yaml b/approach/ovod/detectron2/projects/TridentNet/configs/tridentnet_fast_R_50_C4_3x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebf89d03ea043810b02e71ecc2c1711c250e161c --- /dev/null +++ b/approach/ovod/detectron2/projects/TridentNet/configs/tridentnet_fast_R_50_C4_3x.yaml @@ -0,0 +1,9 @@ +_BASE_: "Base-TridentNet-Fast-C4.yaml" +MODEL: + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + MASK_ON: False + RESNETS: + DEPTH: 50 +SOLVER: + STEPS: (210000, 250000) + MAX_ITER: 270000 diff --git a/approach/ovod/detectron2/projects/TridentNet/train_net.py b/approach/ovod/detectron2/projects/TridentNet/train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..143289a10514cb87059f62425d79aa3812bc0c98 --- /dev/null +++ b/approach/ovod/detectron2/projects/TridentNet/train_net.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. + +""" +TridentNet Training Script. + +This script is a simplified version of the training script in detectron2/tools. +""" + +import os + +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import get_cfg +from detectron2.engine import DefaultTrainer, default_argument_parser, default_setup, launch +from detectron2.evaluation import COCOEvaluator + +from tridentnet import add_tridentnet_config + + +class Trainer(DefaultTrainer): + @classmethod + def build_evaluator(cls, cfg, dataset_name, output_folder=None): + if output_folder is None: + output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") + return COCOEvaluator(dataset_name, output_dir=output_folder) + + +def setup(args): + """ + Create configs and perform basic setups. + """ + cfg = get_cfg() + add_tridentnet_config(cfg) + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + cfg.freeze() + default_setup(cfg, args) + return cfg + + +def main(args): + cfg = setup(args) + + if args.eval_only: + model = Trainer.build_model(cfg) + DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( + cfg.MODEL.WEIGHTS, resume=args.resume + ) + res = Trainer.test(cfg, model) + return res + + trainer = Trainer(cfg) + trainer.resume_or_load(resume=args.resume) + return trainer.train() + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + print("Command Line Args:", args) + launch( + main, + args.num_gpus, + num_machines=args.num_machines, + machine_rank=args.machine_rank, + dist_url=args.dist_url, + args=(args,), + ) diff --git a/approach/ovod/detectron2/projects/TridentNet/tridentnet/__init__.py b/approach/ovod/detectron2/projects/TridentNet/tridentnet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..abaa9579051e7ef5ee7f388b9d59b5962440155c --- /dev/null +++ b/approach/ovod/detectron2/projects/TridentNet/tridentnet/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .config import add_tridentnet_config +from .trident_backbone import ( + TridentBottleneckBlock, + build_trident_resnet_backbone, + make_trident_stage, +) +from .trident_rpn import TridentRPN +from .trident_rcnn import TridentRes5ROIHeads, TridentStandardROIHeads diff --git a/approach/ovod/detectron2/projects/TridentNet/tridentnet/config.py b/approach/ovod/detectron2/projects/TridentNet/tridentnet/config.py new file mode 100644 index 0000000000000000000000000000000000000000..4b8732a43f6974ec60168652bf08e382ddc9c941 --- /dev/null +++ b/approach/ovod/detectron2/projects/TridentNet/tridentnet/config.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +from detectron2.config import CfgNode as CN + + +def add_tridentnet_config(cfg): + """ + Add config for tridentnet. + """ + _C = cfg + + _C.MODEL.TRIDENT = CN() + + # Number of branches for TridentNet. + _C.MODEL.TRIDENT.NUM_BRANCH = 3 + # Specify the dilations for each branch. + _C.MODEL.TRIDENT.BRANCH_DILATIONS = [1, 2, 3] + # Specify the stage for applying trident blocks. Default stage is Res4 according to the + # TridentNet paper. + _C.MODEL.TRIDENT.TRIDENT_STAGE = "res4" + # Specify the test branch index TridentNet Fast inference: + # - use -1 to aggregate results of all branches during inference. + # - otherwise, only using specified branch for fast inference. Recommended setting is + # to use the middle branch. + _C.MODEL.TRIDENT.TEST_BRANCH_IDX = 1 diff --git a/approach/ovod/detectron2/projects/TridentNet/tridentnet/trident_backbone.py b/approach/ovod/detectron2/projects/TridentNet/tridentnet/trident_backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..7789bd219b01d452e876ad2ad7f811502719465c --- /dev/null +++ b/approach/ovod/detectron2/projects/TridentNet/tridentnet/trident_backbone.py @@ -0,0 +1,220 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import fvcore.nn.weight_init as weight_init +import torch +import torch.nn.functional as F + +from detectron2.layers import Conv2d, FrozenBatchNorm2d, get_norm +from detectron2.modeling import BACKBONE_REGISTRY, ResNet, ResNetBlockBase +from detectron2.modeling.backbone.resnet import BasicStem, BottleneckBlock, DeformBottleneckBlock + +from .trident_conv import TridentConv + +__all__ = ["TridentBottleneckBlock", "make_trident_stage", "build_trident_resnet_backbone"] + + +class TridentBottleneckBlock(ResNetBlockBase): + def __init__( + self, + in_channels, + out_channels, + *, + bottleneck_channels, + stride=1, + num_groups=1, + norm="BN", + stride_in_1x1=False, + num_branch=3, + dilations=(1, 2, 3), + concat_output=False, + test_branch_idx=-1, + ): + """ + Args: + num_branch (int): the number of branches in TridentNet. + dilations (tuple): the dilations of multiple branches in TridentNet. + concat_output (bool): if concatenate outputs of multiple branches in TridentNet. + Use 'True' for the last trident block. + """ + super().__init__(in_channels, out_channels, stride) + + assert num_branch == len(dilations) + + self.num_branch = num_branch + self.concat_output = concat_output + self.test_branch_idx = test_branch_idx + + if in_channels != out_channels: + self.shortcut = Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=stride, + bias=False, + norm=get_norm(norm, out_channels), + ) + else: + self.shortcut = None + + stride_1x1, stride_3x3 = (stride, 1) if stride_in_1x1 else (1, stride) + + self.conv1 = Conv2d( + in_channels, + bottleneck_channels, + kernel_size=1, + stride=stride_1x1, + bias=False, + norm=get_norm(norm, bottleneck_channels), + ) + + self.conv2 = TridentConv( + bottleneck_channels, + bottleneck_channels, + kernel_size=3, + stride=stride_3x3, + paddings=dilations, + bias=False, + groups=num_groups, + dilations=dilations, + num_branch=num_branch, + test_branch_idx=test_branch_idx, + norm=get_norm(norm, bottleneck_channels), + ) + + self.conv3 = Conv2d( + bottleneck_channels, + out_channels, + kernel_size=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + + for layer in [self.conv1, self.conv2, self.conv3, self.shortcut]: + if layer is not None: # shortcut can be None + weight_init.c2_msra_fill(layer) + + def forward(self, x): + num_branch = self.num_branch if self.training or self.test_branch_idx == -1 else 1 + if not isinstance(x, list): + x = [x] * num_branch + out = [self.conv1(b) for b in x] + out = [F.relu_(b) for b in out] + + out = self.conv2(out) + out = [F.relu_(b) for b in out] + + out = [self.conv3(b) for b in out] + + if self.shortcut is not None: + shortcut = [self.shortcut(b) for b in x] + else: + shortcut = x + + out = [out_b + shortcut_b for out_b, shortcut_b in zip(out, shortcut)] + out = [F.relu_(b) for b in out] + if self.concat_output: + out = torch.cat(out) + return out + + +def make_trident_stage(block_class, num_blocks, **kwargs): + """ + Create a resnet stage by creating many blocks for TridentNet. + """ + concat_output = [False] * (num_blocks - 1) + [True] + kwargs["concat_output_per_block"] = concat_output + return ResNet.make_stage(block_class, num_blocks, **kwargs) + + +@BACKBONE_REGISTRY.register() +def build_trident_resnet_backbone(cfg, input_shape): + """ + Create a ResNet instance from config for TridentNet. + + Returns: + ResNet: a :class:`ResNet` instance. + """ + # need registration of new blocks/stems? + norm = cfg.MODEL.RESNETS.NORM + stem = BasicStem( + in_channels=input_shape.channels, + out_channels=cfg.MODEL.RESNETS.STEM_OUT_CHANNELS, + norm=norm, + ) + freeze_at = cfg.MODEL.BACKBONE.FREEZE_AT + + if freeze_at >= 1: + for p in stem.parameters(): + p.requires_grad = False + stem = FrozenBatchNorm2d.convert_frozen_batchnorm(stem) + + # fmt: off + out_features = cfg.MODEL.RESNETS.OUT_FEATURES + depth = cfg.MODEL.RESNETS.DEPTH + num_groups = cfg.MODEL.RESNETS.NUM_GROUPS + width_per_group = cfg.MODEL.RESNETS.WIDTH_PER_GROUP + bottleneck_channels = num_groups * width_per_group + in_channels = cfg.MODEL.RESNETS.STEM_OUT_CHANNELS + out_channels = cfg.MODEL.RESNETS.RES2_OUT_CHANNELS + stride_in_1x1 = cfg.MODEL.RESNETS.STRIDE_IN_1X1 + res5_dilation = cfg.MODEL.RESNETS.RES5_DILATION + deform_on_per_stage = cfg.MODEL.RESNETS.DEFORM_ON_PER_STAGE + deform_modulated = cfg.MODEL.RESNETS.DEFORM_MODULATED + deform_num_groups = cfg.MODEL.RESNETS.DEFORM_NUM_GROUPS + num_branch = cfg.MODEL.TRIDENT.NUM_BRANCH + branch_dilations = cfg.MODEL.TRIDENT.BRANCH_DILATIONS + trident_stage = cfg.MODEL.TRIDENT.TRIDENT_STAGE + test_branch_idx = cfg.MODEL.TRIDENT.TEST_BRANCH_IDX + # fmt: on + assert res5_dilation in {1, 2}, "res5_dilation cannot be {}.".format(res5_dilation) + + num_blocks_per_stage = {50: [3, 4, 6, 3], 101: [3, 4, 23, 3], 152: [3, 8, 36, 3]}[depth] + + stages = [] + + res_stage_idx = {"res2": 2, "res3": 3, "res4": 4, "res5": 5} + out_stage_idx = [res_stage_idx[f] for f in out_features] + trident_stage_idx = res_stage_idx[trident_stage] + max_stage_idx = max(out_stage_idx) + for idx, stage_idx in enumerate(range(2, max_stage_idx + 1)): + dilation = res5_dilation if stage_idx == 5 else 1 + first_stride = 1 if idx == 0 or (stage_idx == 5 and dilation == 2) else 2 + stage_kargs = { + "num_blocks": num_blocks_per_stage[idx], + "stride_per_block": [first_stride] + [1] * (num_blocks_per_stage[idx] - 1), + "in_channels": in_channels, + "bottleneck_channels": bottleneck_channels, + "out_channels": out_channels, + "num_groups": num_groups, + "norm": norm, + "stride_in_1x1": stride_in_1x1, + "dilation": dilation, + } + if stage_idx == trident_stage_idx: + assert not deform_on_per_stage[ + idx + ], "Not support deformable conv in Trident blocks yet." + stage_kargs["block_class"] = TridentBottleneckBlock + stage_kargs["num_branch"] = num_branch + stage_kargs["dilations"] = branch_dilations + stage_kargs["test_branch_idx"] = test_branch_idx + stage_kargs.pop("dilation") + elif deform_on_per_stage[idx]: + stage_kargs["block_class"] = DeformBottleneckBlock + stage_kargs["deform_modulated"] = deform_modulated + stage_kargs["deform_num_groups"] = deform_num_groups + else: + stage_kargs["block_class"] = BottleneckBlock + blocks = ( + make_trident_stage(**stage_kargs) + if stage_idx == trident_stage_idx + else ResNet.make_stage(**stage_kargs) + ) + in_channels = out_channels + out_channels *= 2 + bottleneck_channels *= 2 + + if freeze_at >= stage_idx: + for block in blocks: + block.freeze() + stages.append(blocks) + return ResNet(stem, stages, out_features=out_features) diff --git a/approach/ovod/detectron2/projects/TridentNet/tridentnet/trident_conv.py b/approach/ovod/detectron2/projects/TridentNet/tridentnet/trident_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..18d5b0b9d73f2da263e7e026a82c62231a88d279 --- /dev/null +++ b/approach/ovod/detectron2/projects/TridentNet/tridentnet/trident_conv.py @@ -0,0 +1,107 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import torch +from torch import nn +from torch.nn import functional as F +from torch.nn.modules.utils import _pair + +from detectron2.layers.wrappers import _NewEmptyTensorOp + + +class TridentConv(nn.Module): + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + paddings=0, + dilations=1, + groups=1, + num_branch=1, + test_branch_idx=-1, + bias=False, + norm=None, + activation=None, + ): + super(TridentConv, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.num_branch = num_branch + self.stride = _pair(stride) + self.groups = groups + self.with_bias = bias + if isinstance(paddings, int): + paddings = [paddings] * self.num_branch + if isinstance(dilations, int): + dilations = [dilations] * self.num_branch + self.paddings = [_pair(padding) for padding in paddings] + self.dilations = [_pair(dilation) for dilation in dilations] + self.test_branch_idx = test_branch_idx + self.norm = norm + self.activation = activation + + assert len({self.num_branch, len(self.paddings), len(self.dilations)}) == 1 + + self.weight = nn.Parameter( + torch.Tensor(out_channels, in_channels // groups, *self.kernel_size) + ) + if bias: + self.bias = nn.Parameter(torch.Tensor(out_channels)) + else: + self.bias = None + + nn.init.kaiming_uniform_(self.weight, nonlinearity="relu") + if self.bias is not None: + nn.init.constant_(self.bias, 0) + + def forward(self, inputs): + num_branch = self.num_branch if self.training or self.test_branch_idx == -1 else 1 + assert len(inputs) == num_branch + + if inputs[0].numel() == 0: + output_shape = [ + (i + 2 * p - (di * (k - 1) + 1)) // s + 1 + for i, p, di, k, s in zip( + inputs[0].shape[-2:], self.padding, self.dilation, self.kernel_size, self.stride + ) + ] + output_shape = [input[0].shape[0], self.weight.shape[0]] + output_shape + return [_NewEmptyTensorOp.apply(input, output_shape) for input in inputs] + + if self.training or self.test_branch_idx == -1: + outputs = [ + F.conv2d(input, self.weight, self.bias, self.stride, padding, dilation, self.groups) + for input, dilation, padding in zip(inputs, self.dilations, self.paddings) + ] + else: + outputs = [ + F.conv2d( + inputs[0], + self.weight, + self.bias, + self.stride, + self.paddings[self.test_branch_idx], + self.dilations[self.test_branch_idx], + self.groups, + ) + ] + + if self.norm is not None: + outputs = [self.norm(x) for x in outputs] + if self.activation is not None: + outputs = [self.activation(x) for x in outputs] + return outputs + + def extra_repr(self): + tmpstr = "in_channels=" + str(self.in_channels) + tmpstr += ", out_channels=" + str(self.out_channels) + tmpstr += ", kernel_size=" + str(self.kernel_size) + tmpstr += ", num_branch=" + str(self.num_branch) + tmpstr += ", test_branch_idx=" + str(self.test_branch_idx) + tmpstr += ", stride=" + str(self.stride) + tmpstr += ", paddings=" + str(self.paddings) + tmpstr += ", dilations=" + str(self.dilations) + tmpstr += ", groups=" + str(self.groups) + tmpstr += ", bias=" + str(self.with_bias) + return tmpstr diff --git a/approach/ovod/detectron2/projects/TridentNet/tridentnet/trident_rcnn.py b/approach/ovod/detectron2/projects/TridentNet/tridentnet/trident_rcnn.py new file mode 100644 index 0000000000000000000000000000000000000000..fc22c712c84f96813fb275931ad4e350ee1f3bfd --- /dev/null +++ b/approach/ovod/detectron2/projects/TridentNet/tridentnet/trident_rcnn.py @@ -0,0 +1,116 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from detectron2.layers import batched_nms +from detectron2.modeling import ROI_HEADS_REGISTRY, StandardROIHeads +from detectron2.modeling.roi_heads.roi_heads import Res5ROIHeads +from detectron2.structures import Instances + + +def merge_branch_instances(instances, num_branch, nms_thresh, topk_per_image): + """ + Merge detection results from different branches of TridentNet. + Return detection results by applying non-maximum suppression (NMS) on bounding boxes + and keep the unsuppressed boxes and other instances (e.g mask) if any. + + Args: + instances (list[Instances]): A list of N * num_branch instances that store detection + results. Contain N images and each image has num_branch instances. + num_branch (int): Number of branches used for merging detection results for each image. + nms_thresh (float): The threshold to use for box non-maximum suppression. Value in [0, 1]. + topk_per_image (int): The number of top scoring detections to return. Set < 0 to return + all detections. + + Returns: + results: (list[Instances]): A list of N instances, one for each image in the batch, + that stores the topk most confidence detections after merging results from multiple + branches. + """ + if num_branch == 1: + return instances + + batch_size = len(instances) // num_branch + results = [] + for i in range(batch_size): + instance = Instances.cat([instances[i + batch_size * j] for j in range(num_branch)]) + + # Apply per-class NMS + keep = batched_nms( + instance.pred_boxes.tensor, instance.scores, instance.pred_classes, nms_thresh + ) + keep = keep[:topk_per_image] + result = instance[keep] + + results.append(result) + + return results + + +@ROI_HEADS_REGISTRY.register() +class TridentRes5ROIHeads(Res5ROIHeads): + """ + The TridentNet ROIHeads in a typical "C4" R-CNN model. + See :class:`Res5ROIHeads`. + """ + + def __init__(self, cfg, input_shape): + super().__init__(cfg, input_shape) + + self.num_branch = cfg.MODEL.TRIDENT.NUM_BRANCH + self.trident_fast = cfg.MODEL.TRIDENT.TEST_BRANCH_IDX != -1 + + def forward(self, images, features, proposals, targets=None): + """ + See :class:`Res5ROIHeads.forward`. + """ + num_branch = self.num_branch if self.training or not self.trident_fast else 1 + all_targets = targets * num_branch if targets is not None else None + pred_instances, losses = super().forward(images, features, proposals, all_targets) + del images, all_targets, targets + + if self.training: + return pred_instances, losses + else: + pred_instances = merge_branch_instances( + pred_instances, + num_branch, + self.box_predictor.test_nms_thresh, + self.box_predictor.test_topk_per_image, + ) + + return pred_instances, {} + + +@ROI_HEADS_REGISTRY.register() +class TridentStandardROIHeads(StandardROIHeads): + """ + The `StandardROIHeads` for TridentNet. + See :class:`StandardROIHeads`. + """ + + def __init__(self, cfg, input_shape): + super(TridentStandardROIHeads, self).__init__(cfg, input_shape) + + self.num_branch = cfg.MODEL.TRIDENT.NUM_BRANCH + self.trident_fast = cfg.MODEL.TRIDENT.TEST_BRANCH_IDX != -1 + + def forward(self, images, features, proposals, targets=None): + """ + See :class:`Res5ROIHeads.forward`. + """ + # Use 1 branch if using trident_fast during inference. + num_branch = self.num_branch if self.training or not self.trident_fast else 1 + # Duplicate targets for all branches in TridentNet. + all_targets = targets * num_branch if targets is not None else None + pred_instances, losses = super().forward(images, features, proposals, all_targets) + del images, all_targets, targets + + if self.training: + return pred_instances, losses + else: + pred_instances = merge_branch_instances( + pred_instances, + num_branch, + self.box_predictor.test_nms_thresh, + self.box_predictor.test_topk_per_image, + ) + + return pred_instances, {} diff --git a/approach/ovod/detectron2/projects/TridentNet/tridentnet/trident_rpn.py b/approach/ovod/detectron2/projects/TridentNet/tridentnet/trident_rpn.py new file mode 100644 index 0000000000000000000000000000000000000000..f95fbbf8ea59ad014f3337c47d41b5410f2c9d45 --- /dev/null +++ b/approach/ovod/detectron2/projects/TridentNet/tridentnet/trident_rpn.py @@ -0,0 +1,32 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import torch + +from detectron2.modeling import PROPOSAL_GENERATOR_REGISTRY +from detectron2.modeling.proposal_generator.rpn import RPN +from detectron2.structures import ImageList + + +@PROPOSAL_GENERATOR_REGISTRY.register() +class TridentRPN(RPN): + """ + Trident RPN subnetwork. + """ + + def __init__(self, cfg, input_shape): + super(TridentRPN, self).__init__(cfg, input_shape) + + self.num_branch = cfg.MODEL.TRIDENT.NUM_BRANCH + self.trident_fast = cfg.MODEL.TRIDENT.TEST_BRANCH_IDX != -1 + + def forward(self, images, features, gt_instances=None): + """ + See :class:`RPN.forward`. + """ + num_branch = self.num_branch if self.training or not self.trident_fast else 1 + # Duplicate images and gt_instances for all branches in TridentNet. + all_images = ImageList( + torch.cat([images.tensor] * num_branch), images.image_sizes * num_branch + ) + all_gt_instances = gt_instances * num_branch if gt_instances is not None else None + + return super(TridentRPN, self).forward(all_images, features, all_gt_instances) diff --git a/approach/ovod/detectron2/projects/ViTDet/README.md b/approach/ovod/detectron2/projects/ViTDet/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0a525e00e643017fc971566931936f1573d9b47c --- /dev/null +++ b/approach/ovod/detectron2/projects/ViTDet/README.md @@ -0,0 +1,364 @@ +# ViTDet: Exploring Plain Vision Transformer Backbones for Object Detection + +Yanghao Li, Hanzi Mao, Ross Girshick†, Kaiming He† + +[[`arXiv`](https://arxiv.org/abs/2203.16527)] [[`BibTeX`](#CitingViTDet)] + +In this repository, we provide configs and models in Detectron2 for ViTDet as well as MViTv2 and Swin backbones with our implementation and settings as described in [ViTDet](https://arxiv.org/abs/2203.16527) paper. + + +## Pretrained Models + +### COCO + +#### Mask R-CNN + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Namepre-traintrain
time
(s/im)		inference
time
(s/im)		train
mem
(GB)		box
AP		mask
AP		model iddownload
ViTDet, ViT-BIN1K, MAE0.3140.07910.951.645.9325346929model
ViTDet, ViT-LIN1K, MAE0.6030.12520.955.549.2325599698model
ViTDet, ViT-HIN1K, MAE1.0980.17831.556.750.2329145471model
+ +#### Cascade Mask R-CNN + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Namepre-traintrain
time
(s/im)		inference
time
(s/im)		train
mem
(GB)		box
AP		mask
AP		model iddownload
Swin-BIN21K, sup0.3890.0778.753.946.2342979038model
Swin-LIN21K, sup0.5080.09712.655.047.2342979186model
MViTv2-BIN21K, sup0.4750.0908.955.648.1325820315model
MViTv2-LIN21K, sup0.8440.15719.755.748.3325607715model
MViTv2-HIN21K, sup1.6550.28518.4*55.948.3326187358model
ViTDet, ViT-BIN1K, MAE0.3620.08912.354.046.7325358525model
ViTDet, ViT-LIN1K, MAE0.6430.14222.357.650.0328021305model
ViTDet, ViT-HIN1K, MAE1.1370.19632.958.751.0328730692model
+ + +### LVIS + +#### Mask R-CNN + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Namepre-traintrain
time
(s/im)		inference
time
(s/im)		train
mem
(GB)		box
AP		mask
AP		model iddownload
ViTDet, ViT-BIN1K, MAE0.3170.08514.440.238.2329225748model
ViTDet, ViT-LIN1K, MAE0.5760.13724.746.143.6329211570model
ViTDet, ViT-HIN1K, MAE1.0590.18635.349.146.0332434656model
+ +#### Cascade Mask R-CNN + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Namepre-traintrain
time
(s/im)		inference
time
(s/im)		train
mem
(GB)		box
AP		mask
AP		model iddownload
Swin-BIN21K, sup0.3680.09011.544.039.6329222304model
Swin-LIN21K, sup0.4860.10513.846.041.4329222724model
MViTv2-BIN21K, sup0.4750.10011.846.342.0329477206model
MViTv2-LIN21K, sup0.8440.17221.049.444.2329661552model
MViTv2-HIN21K, sup1.6610.29021.3*49.544.1330445165model
ViTDet, ViT-BIN1K, MAE0.3560.09915.243.038.9329226874model
ViTDet, ViT-LIN1K, MAE0.6290.15024.949.244.5329042206model
ViTDet, ViT-HIN1K, MAE1.1000.20435.551.546.6332552778model
+ +Note: Unlike the system-level comparisons in the paper, these models use a lower resolution (1024 instead of 1280) and standard NMS (instead of soft NMS). As a result, they have slightly lower box and mask AP. + +We observed higher variance on LVIS evalution results compared to COCO. For example, the standard deviations of box AP and mask AP were 0.30% (compared to 0.10% on COCO) when we trained ViTDet, ViT-B five times with varying random seeds. + +The above models were trained and measured on 8-node with 64 NVIDIA A100 GPUs in total. *: Activation checkpointing is used. + + +## Training +All configs can be trained with: + +``` +../../tools/lazyconfig_train_net.py --config-file configs/path/to/config.py +``` +By default, we use 64 GPUs with batch size as 64 for training. + +## Evaluation +Model evaluation can be done similarly: +``` +../../tools/lazyconfig_train_net.py --config-file configs/path/to/config.py --eval-only train.init_checkpoint=/path/to/model_checkpoint +``` + + +## Citing ViTDet + +If you use ViTDet, please use the following BibTeX entry. + +```BibTeX +@article{li2022exploring, + title={Exploring plain vision transformer backbones for object detection}, + author={Li, Yanghao and Mao, Hanzi and Girshick, Ross and He, Kaiming}, + journal={arXiv preprint arXiv:2203.16527}, + year={2022} +} +``` diff --git a/approach/ovod/detectron2/projects/ViTDet/configs/LVIS/mask_rcnn_vitdet_l_100ep.py b/approach/ovod/detectron2/projects/ViTDet/configs/LVIS/mask_rcnn_vitdet_l_100ep.py new file mode 100644 index 0000000000000000000000000000000000000000..de0981acb1363530e08ff405d57ecfc5923cf792 --- /dev/null +++ b/approach/ovod/detectron2/projects/ViTDet/configs/LVIS/mask_rcnn_vitdet_l_100ep.py @@ -0,0 +1,24 @@ +from functools import partial + +from detectron2.modeling.backbone.vit import get_vit_lr_decay_rate + +from .mask_rcnn_vitdet_b_100ep import ( + dataloader, + lr_multiplier, + model, + train, + optimizer, +) + +train.init_checkpoint = "detectron2://ImageNetPretrained/MAE/mae_pretrain_vit_large.pth" + +model.backbone.net.embed_dim = 1024 +model.backbone.net.depth = 24 +model.backbone.net.num_heads = 16 +model.backbone.net.drop_path_rate = 0.4 +# 5, 11, 17, 23 for global attention +model.backbone.net.window_block_indexes = ( + list(range(0, 5)) + list(range(6, 11)) + list(range(12, 17)) + list(range(18, 23)) +) + +optimizer.params.lr_factor_func = partial(get_vit_lr_decay_rate, lr_decay_rate=0.8, num_layers=24) diff --git a/approach/ovod/detectron2/tests/README.md b/approach/ovod/detectron2/tests/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f560384045ab4f6bc2beabef1170308fca117eb3 --- /dev/null +++ b/approach/ovod/detectron2/tests/README.md @@ -0,0 +1,9 @@ +## Unit Tests + +To run the unittests, do: +``` +cd detectron2 +python -m unittest discover -v -s ./tests +``` + +There are also end-to-end inference & training tests, in [dev/run_*_tests.sh](../dev). diff --git a/approach/ovod/detectron2/tests/__init__.py b/approach/ovod/detectron2/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9020c2df23e2af280b7bb168b996ae9eaf312eb8 --- /dev/null +++ b/approach/ovod/detectron2/tests/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Facebook, Inc. and its affiliates. diff --git a/approach/ovod/detectron2/tests/config/dir1/bad_import.py b/approach/ovod/detectron2/tests/config/dir1/bad_import.py new file mode 100644 index 0000000000000000000000000000000000000000..d7452c4dfc211223c946f22df7a2eb6bdc2cd829 --- /dev/null +++ b/approach/ovod/detectron2/tests/config/dir1/bad_import.py @@ -0,0 +1,2 @@ +# import from directory is not allowed +from . import dir1a diff --git a/approach/ovod/detectron2/tests/config/dir1/bad_import2.py b/approach/ovod/detectron2/tests/config/dir1/bad_import2.py new file mode 100644 index 0000000000000000000000000000000000000000..085a4dfa84a28b92f7d515e1911ac2cc12cbbf7d --- /dev/null +++ b/approach/ovod/detectron2/tests/config/dir1/bad_import2.py @@ -0,0 +1 @@ +from .does_not_exist import x diff --git a/approach/ovod/detectron2/tests/config/dir1/dir1_a.py b/approach/ovod/detectron2/tests/config/dir1/dir1_a.py new file mode 100644 index 0000000000000000000000000000000000000000..a939955124556355524f48c0f0c16abb07cfc4c4 --- /dev/null +++ b/approach/ovod/detectron2/tests/config/dir1/dir1_a.py @@ -0,0 +1,3 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +dir1a_str = "base_a_1" +dir1a_dict = {"a": 1, "b": 2} diff --git a/approach/ovod/detectron2/tests/config/dir1/dir1_b.py b/approach/ovod/detectron2/tests/config/dir1/dir1_b.py new file mode 100644 index 0000000000000000000000000000000000000000..2dcb54cb1054c5d80ccc823af21f13b9ebbcf1a3 --- /dev/null +++ b/approach/ovod/detectron2/tests/config/dir1/dir1_b.py @@ -0,0 +1,11 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from detectron2.config import LazyConfig + +# equivalent to relative import +dir1a_str, dir1a_dict = LazyConfig.load_rel("dir1_a.py", ("dir1a_str", "dir1a_dict")) + +dir1b_str = dir1a_str + "_from_b" +dir1b_dict = dir1a_dict + +# Every import is a reload: not modified by other config files +assert dir1a_dict.a == 1 diff --git a/approach/ovod/detectron2/tests/config/dir1/load_rel.py b/approach/ovod/detectron2/tests/config/dir1/load_rel.py new file mode 100644 index 0000000000000000000000000000000000000000..22d10db7fe28ad66819aeb8e991f129301095ea1 --- /dev/null +++ b/approach/ovod/detectron2/tests/config/dir1/load_rel.py @@ -0,0 +1,5 @@ +# test that load_rel can work +from detectron2.config import LazyConfig + +x = LazyConfig.load_rel("dir1_a.py", "dir1a_dict") +assert x["a"] == 1 diff --git a/approach/ovod/detectron2/tests/export/test_c10.py b/approach/ovod/detectron2/tests/export/test_c10.py new file mode 100644 index 0000000000000000000000000000000000000000..55076abd15beb50b1774f0b5fe399b22d7cc630f --- /dev/null +++ b/approach/ovod/detectron2/tests/export/test_c10.py @@ -0,0 +1,25 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import unittest + +try: + # Caffe2 used to be included in PyTorch, but since PyTorch 1.10+, + # it is not included in pre-built packages. This is a safety BC check + from detectron2.config import get_cfg + from detectron2.export.c10 import Caffe2RPN + from detectron2.layers import ShapeSpec +except ImportError: + raise unittest.SkipTest( + f"PyTorch does not have Caffe2 support. Skipping all tests in {__name__}" + ) from None + + +class TestCaffe2RPN(unittest.TestCase): + def test_instantiation(self): + cfg = get_cfg() + cfg.MODEL.RPN.BBOX_REG_WEIGHTS = (1, 1, 1, 1, 1) + input_shapes = {"res4": ShapeSpec(channels=256, stride=4)} + rpn = Caffe2RPN(cfg, input_shapes) + assert rpn is not None + cfg.MODEL.RPN.BBOX_REG_WEIGHTS = (10, 10, 5, 5, 1) + with self.assertRaises(AssertionError): + rpn = Caffe2RPN(cfg, input_shapes) diff --git a/approach/ovod/detectron2/tests/test_checkpoint.py b/approach/ovod/detectron2/tests/test_checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..ab0bfbd590e0c37e377cc93bf99c7c005fc4bdd6 --- /dev/null +++ b/approach/ovod/detectron2/tests/test_checkpoint.py @@ -0,0 +1,49 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import unittest +from collections import OrderedDict +import torch +from torch import nn + +from detectron2.checkpoint.c2_model_loading import align_and_update_state_dicts +from detectron2.utils.logger import setup_logger + + +class TestCheckpointer(unittest.TestCase): + def setUp(self): + setup_logger() + + def create_complex_model(self): + m = nn.Module() + m.block1 = nn.Module() + m.block1.layer1 = nn.Linear(2, 3) + m.layer2 = nn.Linear(3, 2) + m.res = nn.Module() + m.res.layer2 = nn.Linear(3, 2) + + state_dict = OrderedDict() + state_dict["layer1.weight"] = torch.rand(3, 2) + state_dict["layer1.bias"] = torch.rand(3) + state_dict["layer2.weight"] = torch.rand(2, 3) + state_dict["layer2.bias"] = torch.rand(2) + state_dict["res.layer2.weight"] = torch.rand(2, 3) + state_dict["res.layer2.bias"] = torch.rand(2) + return m, state_dict + + def test_complex_model_loaded(self): + for add_data_parallel in [False, True]: + model, state_dict = self.create_complex_model() + if add_data_parallel: + model = nn.DataParallel(model) + model_sd = model.state_dict() + + sd_to_load = align_and_update_state_dicts(model_sd, state_dict) + model.load_state_dict(sd_to_load) + for loaded, stored in zip(model_sd.values(), state_dict.values()): + # different tensor references + self.assertFalse(id(loaded) == id(stored)) + # same content + self.assertTrue(loaded.to(stored).equal(stored)) + + +if __name__ == "__main__": + unittest.main() diff --git a/approach/ovod/detectron2/tests/test_events.py b/approach/ovod/detectron2/tests/test_events.py new file mode 100644 index 0000000000000000000000000000000000000000..c1b03e4d1a703a417a83c2805be1ca15a4e458ed --- /dev/null +++ b/approach/ovod/detectron2/tests/test_events.py @@ -0,0 +1,64 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import json +import os +import tempfile +import unittest + +from detectron2.utils.events import CommonMetricPrinter, EventStorage, JSONWriter + + +class TestEventWriter(unittest.TestCase): + def testScalar(self): + with tempfile.TemporaryDirectory( + prefix="detectron2_tests" + ) as dir, EventStorage() as storage: + json_file = os.path.join(dir, "test.json") + writer = JSONWriter(json_file) + for k in range(60): + storage.put_scalar("key", k, smoothing_hint=False) + if (k + 1) % 20 == 0: + writer.write() + storage.step() + writer.close() + with open(json_file) as f: + data = [json.loads(l) for l in f] + self.assertTrue([int(k["key"]) for k in data] == [19, 39, 59]) + + def testScalarMismatchedPeriod(self): + with tempfile.TemporaryDirectory( + prefix="detectron2_tests" + ) as dir, EventStorage() as storage: + json_file = os.path.join(dir, "test.json") + + writer = JSONWriter(json_file) + for k in range(60): + if k % 17 == 0: # write in a differnt period + storage.put_scalar("key2", k, smoothing_hint=False) + storage.put_scalar("key", k, smoothing_hint=False) + if (k + 1) % 20 == 0: + writer.write() + storage.step() + writer.close() + with open(json_file) as f: + data = [json.loads(l) for l in f] + self.assertTrue([int(k.get("key2", 0)) for k in data] == [17, 0, 34, 0, 51, 0]) + self.assertTrue([int(k.get("key", 0)) for k in data] == [0, 19, 0, 39, 0, 59]) + self.assertTrue([int(k["iteration"]) for k in data] == [17, 19, 34, 39, 51, 59]) + + def testPrintETA(self): + with EventStorage() as s: + p1 = CommonMetricPrinter(10) + p2 = CommonMetricPrinter() + + s.put_scalar("time", 1.0) + s.step() + s.put_scalar("time", 1.0) + s.step() + + with self.assertLogs("detectron2.utils.events") as logs: + p1.write() + self.assertIn("eta", logs.output[0]) + + with self.assertLogs("detectron2.utils.events") as logs: + p2.write() + self.assertNotIn("eta", logs.output[0]) diff --git a/approach/ovod/detectron2/tests/test_export_caffe2.py b/approach/ovod/detectron2/tests/test_export_caffe2.py new file mode 100644 index 0000000000000000000000000000000000000000..58e9f681c356d05e3d03b06b603721ed51840c5c --- /dev/null +++ b/approach/ovod/detectron2/tests/test_export_caffe2.py @@ -0,0 +1,62 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# -*- coding: utf-8 -*- + +import copy +import os +import tempfile +import unittest +import torch +from torch.hub import _check_module_exists + +from detectron2 import model_zoo +from detectron2.utils.logger import setup_logger +from detectron2.utils.testing import get_sample_coco_image + +try: + # Caffe2 used to be included in PyTorch, but since PyTorch 1.10+, + # Caffe2 is not included in pre-built packages. This is a safety BC check + from detectron2.export import Caffe2Model, Caffe2Tracer +except ImportError: + raise unittest.SkipTest( + f"PyTorch does not have Caffe2 support. Skipping all tests in {__name__}" + ) from None + + +# TODO: this test requires manifold access, see: T88318502 +# Running it on CircleCI causes crash, not sure why. +@unittest.skipIf(os.environ.get("CIRCLECI"), "Caffe2 tests crash on CircleCI.") +@unittest.skipIf(not _check_module_exists("onnx"), "ONNX not installed.") +class TestCaffe2Export(unittest.TestCase): + def setUp(self): + setup_logger() + + def _test_model(self, config_path, device="cpu"): + cfg = model_zoo.get_config(config_path) + cfg.MODEL.DEVICE = device + model = model_zoo.get(config_path, trained=True, device=device) + + inputs = [{"image": get_sample_coco_image()}] + tracer = Caffe2Tracer(cfg, model, copy.deepcopy(inputs)) + + with tempfile.TemporaryDirectory(prefix="detectron2_unittest") as d: + if not os.environ.get("CI"): + # This requires onnx, which is not yet available on public CI + c2_model = tracer.export_caffe2() + c2_model.save_protobuf(d) + c2_model.save_graph(os.path.join(d, "test.svg"), inputs=copy.deepcopy(inputs)) + + c2_model = Caffe2Model.load_protobuf(d) + c2_model(inputs)[0]["instances"] + + ts_model = tracer.export_torchscript() + ts_model.save(os.path.join(d, "model.ts")) + + def testMaskRCNN(self): + self._test_model("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml") + + @unittest.skipIf(not torch.cuda.is_available(), "CUDA not available") + def testMaskRCNNGPU(self): + self._test_model("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml", device="cuda") + + def testRetinaNet(self): + self._test_model("COCO-Detection/retinanet_R_50_FPN_3x.yaml") diff --git a/approach/ovod/detectron2/tests/test_export_onnx.py b/approach/ovod/detectron2/tests/test_export_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..ffab02ae4824f9865482def3fb770ce0a7b9daa1 --- /dev/null +++ b/approach/ovod/detectron2/tests/test_export_onnx.py @@ -0,0 +1,181 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import io +import unittest +import warnings +import torch +from torch.hub import _check_module_exists + +from detectron2 import model_zoo +from detectron2.config import get_cfg +from detectron2.export import STABLE_ONNX_OPSET_VERSION +from detectron2.export.flatten import TracingAdapter +from detectron2.modeling import build_model +from detectron2.utils.testing import ( + _pytorch1111_symbolic_opset9_repeat_interleave, + _pytorch1111_symbolic_opset9_to, + get_sample_coco_image, + register_custom_op_onnx_export, + skipIfOnCPUCI, + skipIfUnsupportedMinOpsetVersion, + skipIfUnsupportedMinTorchVersion, + unregister_custom_op_onnx_export, +) + + +@unittest.skipIf(not _check_module_exists("onnx"), "ONNX not installed.") +@skipIfUnsupportedMinTorchVersion("1.10") +class TestONNXTracingExport(unittest.TestCase): + def testMaskRCNNFPN(self): + def inference_func(model, images): + with warnings.catch_warnings(record=True): + inputs = [{"image": image} for image in images] + inst = model.inference(inputs, do_postprocess=False)[0] + return [{"instances": inst}] + + self._test_model_zoo_from_config_path( + "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml", inference_func + ) + + @skipIfOnCPUCI + def testMaskRCNNC4(self): + def inference_func(model, image): + inputs = [{"image": image}] + return model.inference(inputs, do_postprocess=False)[0] + + self._test_model_zoo_from_config_path( + "COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x.yaml", inference_func + ) + + @skipIfOnCPUCI + def testCascadeRCNN(self): + def inference_func(model, image): + inputs = [{"image": image}] + return model.inference(inputs, do_postprocess=False)[0] + + self._test_model_zoo_from_config_path( + "Misc/cascade_mask_rcnn_R_50_FPN_3x.yaml", inference_func + ) + + def testRetinaNet(self): + def inference_func(model, image): + return model.forward([{"image": image}])[0]["instances"] + + self._test_model_zoo_from_config_path( + "COCO-Detection/retinanet_R_50_FPN_3x.yaml", inference_func + ) + + @skipIfOnCPUCI + def testMaskRCNNFPN_batched(self): + def inference_func(model, image1, image2): + inputs = [{"image": image1}, {"image": image2}] + return model.inference(inputs, do_postprocess=False) + + self._test_model_zoo_from_config_path( + "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml", inference_func, batch=2 + ) + + @skipIfUnsupportedMinOpsetVersion(16, STABLE_ONNX_OPSET_VERSION) + @skipIfUnsupportedMinTorchVersion("1.11.1") + def testMaskRCNNFPN_with_postproc(self): + def inference_func(model, image): + inputs = [{"image": image, "height": image.shape[1], "width": image.shape[2]}] + return model.inference(inputs, do_postprocess=True)[0]["instances"] + + self._test_model_zoo_from_config_path( + "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml", + inference_func, + opset_version=STABLE_ONNX_OPSET_VERSION, + ) + + ################################################################################ + # Testcase internals - DO NOT add tests below this point + ################################################################################ + + def setUp(self): + register_custom_op_onnx_export("::to", _pytorch1111_symbolic_opset9_to, 9, "1.11.1") + register_custom_op_onnx_export( + "::repeat_interleave", + _pytorch1111_symbolic_opset9_repeat_interleave, + 9, + "1.11.1", + ) + + def tearDown(self): + unregister_custom_op_onnx_export("::to", 9, "1.11.1") + unregister_custom_op_onnx_export("::repeat_interleave", 9, "1.11.1") + + def _test_model( + self, + model, + inputs, + inference_func=None, + opset_version=STABLE_ONNX_OPSET_VERSION, + save_onnx_graph_path=None, + **export_kwargs, + ): + import onnx # isort:skip + + f = io.BytesIO() + adapter_model = TracingAdapter(model, inputs, inference_func) + adapter_model.eval() + with torch.no_grad(): + try: + torch.onnx.enable_log() + except AttributeError: + # Older ONNX versions does not have this API + pass + torch.onnx.export( + adapter_model, + adapter_model.flattened_inputs, + f, + training=torch.onnx.TrainingMode.EVAL, + opset_version=opset_version, + verbose=True, + **export_kwargs, + ) + onnx_model = onnx.load_from_string(f.getvalue()) + assert onnx_model is not None + if save_onnx_graph_path: + onnx.save(onnx_model, save_onnx_graph_path) + + def _test_model_zoo_from_config_path( + self, + config_path, + inference_func, + batch=1, + opset_version=STABLE_ONNX_OPSET_VERSION, + save_onnx_graph_path=None, + **export_kwargs, + ): + model = model_zoo.get(config_path, trained=True) + image = get_sample_coco_image() + inputs = tuple(image.clone() for _ in range(batch)) + return self._test_model( + model, inputs, inference_func, opset_version, save_onnx_graph_path, **export_kwargs + ) + + def _test_model_from_config_path( + self, + config_path, + inference_func, + batch=1, + opset_version=STABLE_ONNX_OPSET_VERSION, + save_onnx_graph_path=None, + **export_kwargs, + ): + from projects.PointRend import point_rend # isort:skip + + cfg = get_cfg() + cfg.DATALOADER.NUM_WORKERS = 0 + point_rend.add_pointrend_config(cfg) + cfg.merge_from_file(config_path) + cfg.freeze() + + model = build_model(cfg) + + image = get_sample_coco_image() + inputs = tuple(image.clone() for _ in range(batch)) + return self._test_model( + model, inputs, inference_func, opset_version, save_onnx_graph_path, **export_kwargs + ) diff --git a/approach/ovod/detectron2/tests/test_export_torchscript.py b/approach/ovod/detectron2/tests/test_export_torchscript.py new file mode 100644 index 0000000000000000000000000000000000000000..b9905a6632669750337941ae2f96afa976145541 --- /dev/null +++ b/approach/ovod/detectron2/tests/test_export_torchscript.py @@ -0,0 +1,336 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import copy +import glob +import json +import os +import random +import tempfile +import unittest +import zipfile +import torch +from torch import Tensor, nn + +from detectron2 import model_zoo +from detectron2.config import get_cfg +from detectron2.config.instantiate import dump_dataclass, instantiate +from detectron2.export import dump_torchscript_IR, scripting_with_instances +from detectron2.export.flatten import TracingAdapter, flatten_to_tuple +from detectron2.export.torchscript_patch import patch_builtin_len +from detectron2.layers import ShapeSpec +from detectron2.modeling import build_backbone +from detectron2.modeling.postprocessing import detector_postprocess +from detectron2.modeling.roi_heads import KRCNNConvDeconvUpsampleHead +from detectron2.structures import Boxes, Instances +from detectron2.utils.env import TORCH_VERSION +from detectron2.utils.testing import ( + assert_instances_allclose, + convert_scripted_instances, + get_sample_coco_image, + random_boxes, + skipIfOnCPUCI, +) + + +""" +https://detectron2.readthedocs.io/tutorials/deployment.html +contains some explanations of this file. +""" + + +class TestScripting(unittest.TestCase): + def testMaskRCNNFPN(self): + self._test_rcnn_model("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml") + + @skipIfOnCPUCI + def testMaskRCNNC4(self): + self._test_rcnn_model("COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x.yaml") + + def testRetinaNet(self): + self._test_retinanet_model("COCO-Detection/retinanet_R_50_FPN_3x.yaml") + + def _test_rcnn_model(self, config_path): + model = model_zoo.get(config_path, trained=True) + model.eval() + + fields = { + "proposal_boxes": Boxes, + "objectness_logits": Tensor, + "pred_boxes": Boxes, + "scores": Tensor, + "pred_classes": Tensor, + "pred_masks": Tensor, + } + script_model = scripting_with_instances(model, fields) + + # Test that batch inference with different shapes are supported + image = get_sample_coco_image() + small_image = nn.functional.interpolate(image, scale_factor=0.5) + inputs = [{"image": image}, {"image": small_image}] + with torch.no_grad(): + instance = model.inference(inputs, do_postprocess=False)[0] + scripted_instance = script_model.inference(inputs, do_postprocess=False)[0] + assert_instances_allclose(instance, scripted_instance) + + def _test_retinanet_model(self, config_path): + model = model_zoo.get(config_path, trained=True) + model.eval() + + fields = { + "pred_boxes": Boxes, + "scores": Tensor, + "pred_classes": Tensor, + } + script_model = scripting_with_instances(model, fields) + + img = get_sample_coco_image() + inputs = [{"image": img}] * 2 + with torch.no_grad(): + instance = model(inputs)[0]["instances"] + scripted_instance = convert_scripted_instances(script_model(inputs)[0]) + scripted_instance = detector_postprocess(scripted_instance, img.shape[1], img.shape[2]) + assert_instances_allclose(instance, scripted_instance) + # Note that the model currently cannot be saved and loaded into a new process: + # https://github.com/pytorch/pytorch/issues/46944 + + +# TODO: this test requires manifold access, see: T88318502 +class TestTracing(unittest.TestCase): + def testMaskRCNNFPN(self): + def inference_func(model, image): + inputs = [{"image": image}] + return model.inference(inputs, do_postprocess=False)[0] + + self._test_model("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml", inference_func) + + def testMaskRCNNFPN_with_postproc(self): + def inference_func(model, image): + inputs = [{"image": image, "height": image.shape[1], "width": image.shape[2]}] + return model.inference(inputs, do_postprocess=True)[0]["instances"] + + self._test_model("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml", inference_func) + + @skipIfOnCPUCI + def testMaskRCNNC4(self): + def inference_func(model, image): + inputs = [{"image": image}] + return model.inference(inputs, do_postprocess=False)[0] + + self._test_model("COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x.yaml", inference_func) + + @skipIfOnCPUCI + def testCascadeRCNN(self): + def inference_func(model, image): + inputs = [{"image": image}] + return model.inference(inputs, do_postprocess=False)[0] + + self._test_model("Misc/cascade_mask_rcnn_R_50_FPN_3x.yaml", inference_func) + + # bug fixed by https://github.com/pytorch/pytorch/pull/67734 + @unittest.skipIf(TORCH_VERSION == (1, 10) and os.environ.get("CI"), "1.10 has bugs.") + def testRetinaNet(self): + def inference_func(model, image): + return model.forward([{"image": image}])[0]["instances"] + + self._test_model("COCO-Detection/retinanet_R_50_FPN_3x.yaml", inference_func) + + def _check_torchscript_no_hardcoded_device(self, jitfile, extract_dir, device): + zipfile.ZipFile(jitfile).extractall(extract_dir) + dir_path = os.path.join(extract_dir, os.path.splitext(os.path.basename(jitfile))[0]) + error_files = [] + for f in glob.glob(f"{dir_path}/code/**/*.py", recursive=True): + content = open(f).read() + if device in content: + error_files.append((f, content)) + if len(error_files): + msg = "\n".join(f"{f}\n{content}" for f, content in error_files) + raise ValueError(f"Found device '{device}' in following files:\n{msg}") + + def _get_device_casting_test_cases(self, model): + # Indexing operation can causes hardcoded device type before 1.10 + if not TORCH_VERSION >= (1, 10) or torch.cuda.device_count() == 0: + return [None] + + testing_devices = ["cpu", "cuda:0"] + if torch.cuda.device_count() > 1: + testing_devices.append(f"cuda:{torch.cuda.device_count() - 1}") + assert str(model.device) in testing_devices + testing_devices.remove(str(model.device)) + testing_devices = [None] + testing_devices # test no casting first + + return testing_devices + + def _test_model(self, config_path, inference_func, batch=1): + model = model_zoo.get(config_path, trained=True) + image = get_sample_coco_image() + inputs = tuple(image.clone() for _ in range(batch)) + + wrapper = TracingAdapter(model, inputs, inference_func) + wrapper.eval() + with torch.no_grad(): + # trace with smaller images, and the trace must still work + trace_inputs = tuple( + nn.functional.interpolate(image, scale_factor=random.uniform(0.5, 0.7)) + for _ in range(batch) + ) + traced_model = torch.jit.trace(wrapper, trace_inputs) + + testing_devices = self._get_device_casting_test_cases(model) + # save and load back the model in order to show traceback of TorchScript + with tempfile.TemporaryDirectory(prefix="detectron2_test") as d: + basename = "model" + jitfile = f"{d}/{basename}.jit" + torch.jit.save(traced_model, jitfile) + traced_model = torch.jit.load(jitfile) + + if any(device and "cuda" in device for device in testing_devices): + self._check_torchscript_no_hardcoded_device(jitfile, d, "cuda") + + for device in testing_devices: + print(f"Testing casting to {device} for inference (traced on {model.device}) ...") + with torch.no_grad(): + outputs = inference_func(copy.deepcopy(model).to(device), *inputs) + traced_outputs = wrapper.outputs_schema(traced_model.to(device)(*inputs)) + if batch > 1: + for output, traced_output in zip(outputs, traced_outputs): + assert_instances_allclose(output, traced_output, size_as_tensor=True) + else: + assert_instances_allclose(outputs, traced_outputs, size_as_tensor=True) + + @skipIfOnCPUCI + def testMaskRCNNFPN_batched(self): + def inference_func(model, image1, image2): + inputs = [{"image": image1}, {"image": image2}] + return model.inference(inputs, do_postprocess=False) + + self._test_model( + "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml", inference_func, batch=2 + ) + + def testKeypointHead(self): + class M(nn.Module): + def __init__(self): + super().__init__() + self.model = KRCNNConvDeconvUpsampleHead( + ShapeSpec(channels=4, height=14, width=14), num_keypoints=17, conv_dims=(4,) + ) + + def forward(self, x, predbox1, predbox2): + inst = [ + Instances((100, 100), pred_boxes=Boxes(predbox1)), + Instances((100, 100), pred_boxes=Boxes(predbox2)), + ] + ret = self.model(x, inst) + return tuple(x.pred_keypoints for x in ret) + + model = M() + model.eval() + + def gen_input(num1, num2): + feat = torch.randn((num1 + num2, 4, 14, 14)) + box1 = random_boxes(num1) + box2 = random_boxes(num2) + return feat, box1, box2 + + with torch.no_grad(), patch_builtin_len(): + trace = torch.jit.trace(model, gen_input(15, 15), check_trace=False) + + inputs = gen_input(12, 10) + trace_outputs = trace(*inputs) + true_outputs = model(*inputs) + for trace_output, true_output in zip(trace_outputs, true_outputs): + self.assertTrue(torch.allclose(trace_output, true_output)) + + +class TestTorchscriptUtils(unittest.TestCase): + # TODO: add test to dump scripting + def test_dump_IR_tracing(self): + cfg = get_cfg() + cfg.MODEL.RESNETS.DEPTH = 18 + cfg.MODEL.RESNETS.RES2_OUT_CHANNELS = 64 + + class Mod(nn.Module): + def forward(self, x): + return tuple(self.m(x).values()) + + model = Mod() + model.m = build_backbone(cfg) + model.eval() + + with torch.no_grad(): + ts_model = torch.jit.trace(model, (torch.rand(2, 3, 224, 224),)) + + with tempfile.TemporaryDirectory(prefix="detectron2_test") as d: + dump_torchscript_IR(ts_model, d) + # check that the files are created + for name in ["model_ts_code", "model_ts_IR", "model_ts_IR_inlined", "model"]: + fname = os.path.join(d, name + ".txt") + self.assertTrue(os.stat(fname).st_size > 0, fname) + + def test_dump_IR_function(self): + @torch.jit.script + def gunc(x, y): + return x + y + + def func(x, y): + return x + y + gunc(x, y) + + ts_model = torch.jit.trace(func, (torch.rand(3), torch.rand(3))) + with tempfile.TemporaryDirectory(prefix="detectron2_test") as d: + dump_torchscript_IR(ts_model, d) + for name in ["model_ts_code", "model_ts_IR", "model_ts_IR_inlined"]: + fname = os.path.join(d, name + ".txt") + self.assertTrue(os.stat(fname).st_size > 0, fname) + + def test_flatten_basic(self): + obj = [3, ([5, 6], {"name": [7, 9], "name2": 3})] + res, schema = flatten_to_tuple(obj) + self.assertEqual(res, (3, 5, 6, 7, 9, 3)) + new_obj = schema(res) + self.assertEqual(new_obj, obj) + + _, new_schema = flatten_to_tuple(new_obj) + self.assertEqual(schema, new_schema) # test __eq__ + self._check_schema(schema) + + def _check_schema(self, schema): + dumped_schema = dump_dataclass(schema) + # Check that the schema is json-serializable + # Although in reality you might want to use yaml because it often has many levels + json.dumps(dumped_schema) + + # Check that the schema can be deserialized + new_schema = instantiate(dumped_schema) + self.assertEqual(schema, new_schema) + + def test_flatten_instances_boxes(self): + inst = Instances( + torch.tensor([5, 8]), pred_masks=torch.tensor([3]), pred_boxes=Boxes(torch.ones((1, 4))) + ) + obj = [3, ([5, 6], inst)] + res, schema = flatten_to_tuple(obj) + self.assertEqual(res[:3], (3, 5, 6)) + for r, expected in zip(res[3:], (inst.pred_boxes.tensor, inst.pred_masks, inst.image_size)): + self.assertIs(r, expected) + new_obj = schema(res) + assert_instances_allclose(new_obj[1][1], inst, rtol=0.0, size_as_tensor=True) + + self._check_schema(schema) + + def test_allow_non_tensor(self): + data = (torch.tensor([5, 8]), 3) # contains non-tensor + + class M(nn.Module): + def forward(self, input, number): + return input + + model = M() + with self.assertRaisesRegex(ValueError, "must only contain tensors"): + adap = TracingAdapter(model, data, allow_non_tensor=False) + + adap = TracingAdapter(model, data, allow_non_tensor=True) + _ = adap(*adap.flattened_inputs) + + newdata = (data[0].clone(),) + with self.assertRaisesRegex(ValueError, "cannot generalize"): + _ = adap(*newdata) diff --git a/approach/ovod/detectron2/tests/test_model_zoo.py b/approach/ovod/detectron2/tests/test_model_zoo.py new file mode 100644 index 0000000000000000000000000000000000000000..e3360a74864e0c00ed92ffbc8531c8d36e8be379 --- /dev/null +++ b/approach/ovod/detectron2/tests/test_model_zoo.py @@ -0,0 +1,50 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import unittest + +from detectron2 import model_zoo +from detectron2.config import instantiate +from detectron2.modeling import FPN, GeneralizedRCNN + +logger = logging.getLogger(__name__) + + +class TestModelZoo(unittest.TestCase): + def test_get_returns_model(self): + model = model_zoo.get("Misc/scratch_mask_rcnn_R_50_FPN_3x_gn.yaml", trained=False) + self.assertIsInstance(model, GeneralizedRCNN) + self.assertIsInstance(model.backbone, FPN) + + def test_get_invalid_model(self): + self.assertRaises(RuntimeError, model_zoo.get, "Invalid/config.yaml") + + def test_get_url(self): + url = model_zoo.get_checkpoint_url("Misc/scratch_mask_rcnn_R_50_FPN_3x_gn.yaml") + self.assertEqual( + url, + "https://dl.fbaipublicfiles.com/detectron2/Misc/scratch_mask_rcnn_R_50_FPN_3x_gn/138602908/model_final_01ca85.pkl", # noqa + ) + url2 = model_zoo.get_checkpoint_url("Misc/scratch_mask_rcnn_R_50_FPN_3x_gn.py") + self.assertEqual(url, url2) + + def _build_lazy_model(self, name): + cfg = model_zoo.get_config("common/models/" + name) + instantiate(cfg.model) + + def test_mask_rcnn_fpn(self): + self._build_lazy_model("mask_rcnn_fpn.py") + + def test_mask_rcnn_c4(self): + self._build_lazy_model("mask_rcnn_c4.py") + + def test_panoptic_fpn(self): + self._build_lazy_model("panoptic_fpn.py") + + def test_schedule(self): + cfg = model_zoo.get_config("common/coco_schedule.py") + for _, v in cfg.items(): + instantiate(v) + + +if __name__ == "__main__": + unittest.main() diff --git a/approach/ovod/detectron2/tests/test_scheduler.py b/approach/ovod/detectron2/tests/test_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..5649a4a2e167f44a734cfcc3ec86ab3a22bfc1b0 --- /dev/null +++ b/approach/ovod/detectron2/tests/test_scheduler.py @@ -0,0 +1,158 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import math +import numpy as np +from unittest import TestCase +import torch +from fvcore.common.param_scheduler import ( + CosineParamScheduler, + MultiStepParamScheduler, + StepWithFixedGammaParamScheduler, +) +from torch import nn + +from detectron2.solver import LRMultiplier, WarmupParamScheduler, build_lr_scheduler + + +class TestScheduler(TestCase): + def test_warmup_multistep(self): + p = nn.Parameter(torch.zeros(0)) + opt = torch.optim.SGD([p], lr=5) + + multiplier = WarmupParamScheduler( + MultiStepParamScheduler( + [1, 0.1, 0.01, 0.001], + milestones=[10, 15, 20], + num_updates=30, + ), + 0.001, + 5 / 30, + ) + sched = LRMultiplier(opt, multiplier, 30) + # This is an equivalent of: + # sched = WarmupMultiStepLR( + # opt, milestones=[10, 15, 20], gamma=0.1, warmup_factor=0.001, warmup_iters=5) + + p.sum().backward() + opt.step() + + lrs = [0.005] + for _ in range(30): + sched.step() + lrs.append(opt.param_groups[0]["lr"]) + self.assertTrue(np.allclose(lrs[:5], [0.005, 1.004, 2.003, 3.002, 4.001])) + self.assertTrue(np.allclose(lrs[5:10], 5.0)) + self.assertTrue(np.allclose(lrs[10:15], 0.5)) + self.assertTrue(np.allclose(lrs[15:20], 0.05)) + self.assertTrue(np.allclose(lrs[20:], 0.005)) + + def test_warmup_cosine(self): + p = nn.Parameter(torch.zeros(0)) + opt = torch.optim.SGD([p], lr=5) + multiplier = WarmupParamScheduler( + CosineParamScheduler(1, 0), + 0.001, + 5 / 30, + ) + sched = LRMultiplier(opt, multiplier, 30) + + p.sum().backward() + opt.step() + self.assertEqual(opt.param_groups[0]["lr"], 0.005) + lrs = [0.005] + + for _ in range(30): + sched.step() + lrs.append(opt.param_groups[0]["lr"]) + for idx, lr in enumerate(lrs): + expected_cosine = 2.5 * (1.0 + math.cos(math.pi * idx / 30)) + if idx >= 5: + self.assertAlmostEqual(lr, expected_cosine) + else: + self.assertNotAlmostEqual(lr, expected_cosine) + + def test_warmup_cosine_end_value(self): + from detectron2.config import CfgNode, get_cfg + + def _test_end_value(cfg_dict): + cfg = get_cfg() + cfg.merge_from_other_cfg(CfgNode(cfg_dict)) + + p = nn.Parameter(torch.zeros(0)) + opt = torch.optim.SGD([p], lr=cfg.SOLVER.BASE_LR) + + scheduler = build_lr_scheduler(cfg, opt) + + p.sum().backward() + opt.step() + self.assertEqual( + opt.param_groups[0]["lr"], cfg.SOLVER.BASE_LR * cfg.SOLVER.WARMUP_FACTOR + ) + + lrs = [] + for _ in range(cfg.SOLVER.MAX_ITER): + scheduler.step() + lrs.append(opt.param_groups[0]["lr"]) + + self.assertAlmostEqual(lrs[-1], cfg.SOLVER.BASE_LR_END) + + _test_end_value( + { + "SOLVER": { + "LR_SCHEDULER_NAME": "WarmupCosineLR", + "MAX_ITER": 100, + "WARMUP_ITERS": 10, + "WARMUP_FACTOR": 0.1, + "BASE_LR": 5.0, + "BASE_LR_END": 0.0, + } + } + ) + + _test_end_value( + { + "SOLVER": { + "LR_SCHEDULER_NAME": "WarmupCosineLR", + "MAX_ITER": 100, + "WARMUP_ITERS": 10, + "WARMUP_FACTOR": 0.1, + "BASE_LR": 5.0, + "BASE_LR_END": 0.5, + } + } + ) + + def test_warmup_stepwithfixedgamma(self): + p = nn.Parameter(torch.zeros(0)) + opt = torch.optim.SGD([p], lr=5) + + multiplier = WarmupParamScheduler( + StepWithFixedGammaParamScheduler( + base_value=1.0, + gamma=0.1, + num_decays=4, + num_updates=30, + ), + 0.001, + 5 / 30, + rescale_interval=True, + ) + sched = LRMultiplier(opt, multiplier, 30) + + p.sum().backward() + opt.step() + + lrs = [0.005] + for _ in range(29): + sched.step() + lrs.append(opt.param_groups[0]["lr"]) + self.assertTrue(np.allclose(lrs[:5], [0.005, 1.004, 2.003, 3.002, 4.001])) + self.assertTrue(np.allclose(lrs[5:10], 5.0)) + self.assertTrue(np.allclose(lrs[10:15], 0.5)) + self.assertTrue(np.allclose(lrs[15:20], 0.05)) + self.assertTrue(np.allclose(lrs[20:25], 0.005)) + self.assertTrue(np.allclose(lrs[25:], 0.0005)) + + # Calling sche.step() after the last training iteration is done will trigger IndexError + with self.assertRaises(IndexError, msg="list index out of range"): + sched.step() diff --git a/approach/ovod/detectron2/tests/test_solver.py b/approach/ovod/detectron2/tests/test_solver.py new file mode 100644 index 0000000000000000000000000000000000000000..6b3ae84c00b789df071ab5d12bae42d991df1d0b --- /dev/null +++ b/approach/ovod/detectron2/tests/test_solver.py @@ -0,0 +1,66 @@ +import unittest + +from detectron2.solver.build import _expand_param_groups, reduce_param_groups + + +class TestOptimizer(unittest.TestCase): + def testExpandParamsGroups(self): + params = [ + { + "params": ["p1", "p2", "p3", "p4"], + "lr": 1.0, + "weight_decay": 3.0, + }, + { + "params": ["p2", "p3", "p5"], + "lr": 2.0, + "momentum": 2.0, + }, + { + "params": ["p1"], + "weight_decay": 4.0, + }, + ] + out = _expand_param_groups(params) + gt = [ + dict(params=["p1"], lr=1.0, weight_decay=4.0), # noqa + dict(params=["p2"], lr=2.0, weight_decay=3.0, momentum=2.0), # noqa + dict(params=["p3"], lr=2.0, weight_decay=3.0, momentum=2.0), # noqa + dict(params=["p4"], lr=1.0, weight_decay=3.0), # noqa + dict(params=["p5"], lr=2.0, momentum=2.0), # noqa + ] + self.assertEqual(out, gt) + + def testReduceParamGroups(self): + params = [ + dict(params=["p1"], lr=1.0, weight_decay=4.0), # noqa + dict(params=["p2", "p6"], lr=2.0, weight_decay=3.0, momentum=2.0), # noqa + dict(params=["p3"], lr=2.0, weight_decay=3.0, momentum=2.0), # noqa + dict(params=["p4"], lr=1.0, weight_decay=3.0), # noqa + dict(params=["p5"], lr=2.0, momentum=2.0), # noqa + ] + gt_groups = [ + { + "lr": 1.0, + "weight_decay": 4.0, + "params": ["p1"], + }, + { + "lr": 2.0, + "weight_decay": 3.0, + "momentum": 2.0, + "params": ["p2", "p6", "p3"], + }, + { + "lr": 1.0, + "weight_decay": 3.0, + "params": ["p4"], + }, + { + "lr": 2.0, + "momentum": 2.0, + "params": ["p5"], + }, + ] + out = reduce_param_groups(params) + self.assertEqual(out, gt_groups) diff --git a/approach/ovod/detectron2/tools/benchmark.py b/approach/ovod/detectron2/tools/benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..c2d673fab1cfbc7ab55244b52c714d0c7404ecc2 --- /dev/null +++ b/approach/ovod/detectron2/tools/benchmark.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +""" +A script to benchmark builtin models. + +Note: this script has an extra dependency of psutil. +""" + +import itertools +import logging +import psutil +import torch +import tqdm +from fvcore.common.timer import Timer +from torch.nn.parallel import DistributedDataParallel + +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import LazyConfig, get_cfg, instantiate +from detectron2.data import ( + DatasetFromList, + build_detection_test_loader, + build_detection_train_loader, +) +from detectron2.data.benchmark import DataLoaderBenchmark +from detectron2.engine import AMPTrainer, SimpleTrainer, default_argument_parser, hooks, launch +from detectron2.modeling import build_model +from detectron2.solver import build_optimizer +from detectron2.utils import comm +from detectron2.utils.collect_env import collect_env_info +from detectron2.utils.events import CommonMetricPrinter +from detectron2.utils.logger import setup_logger + +logger = logging.getLogger("detectron2") + + +def setup(args): + if args.config_file.endswith(".yaml"): + cfg = get_cfg() + cfg.merge_from_file(args.config_file) + cfg.SOLVER.BASE_LR = 0.001 # Avoid NaNs. Not useful in this script anyway. + cfg.merge_from_list(args.opts) + cfg.freeze() + else: + cfg = LazyConfig.load(args.config_file) + cfg = LazyConfig.apply_overrides(cfg, args.opts) + setup_logger(distributed_rank=comm.get_rank()) + return cfg + + +def create_data_benchmark(cfg, args): + if args.config_file.endswith(".py"): + dl_cfg = cfg.dataloader.train + dl_cfg._target_ = DataLoaderBenchmark + return instantiate(dl_cfg) + else: + kwargs = build_detection_train_loader.from_config(cfg) + kwargs.pop("aspect_ratio_grouping", None) + kwargs["_target_"] = DataLoaderBenchmark + return instantiate(kwargs) + + +def RAM_msg(): + vram = psutil.virtual_memory() + return "RAM Usage: {:.2f}/{:.2f} GB".format( + (vram.total - vram.available) / 1024**3, vram.total / 1024**3 + ) + + +def benchmark_data(args): + cfg = setup(args) + logger.info("After spawning " + RAM_msg()) + + benchmark = create_data_benchmark(cfg, args) + benchmark.benchmark_distributed(250, 10) + # test for a few more rounds + for k in range(10): + logger.info(f"Iteration {k} " + RAM_msg()) + benchmark.benchmark_distributed(250, 1) + + +def benchmark_data_advanced(args): + # benchmark dataloader with more details to help analyze performance bottleneck + cfg = setup(args) + benchmark = create_data_benchmark(cfg, args) + + if comm.get_rank() == 0: + benchmark.benchmark_dataset(100) + benchmark.benchmark_mapper(100) + benchmark.benchmark_workers(100, warmup=10) + benchmark.benchmark_IPC(100, warmup=10) + if comm.get_world_size() > 1: + benchmark.benchmark_distributed(100) + logger.info("Rerun ...") + benchmark.benchmark_distributed(100) + + +def benchmark_train(args): + cfg = setup(args) + model = build_model(cfg) + logger.info("Model:\n{}".format(model)) + if comm.get_world_size() > 1: + model = DistributedDataParallel( + model, device_ids=[comm.get_local_rank()], broadcast_buffers=False + ) + optimizer = build_optimizer(cfg, model) + checkpointer = DetectionCheckpointer(model, optimizer=optimizer) + checkpointer.load(cfg.MODEL.WEIGHTS) + + cfg.defrost() + cfg.DATALOADER.NUM_WORKERS = 2 + data_loader = build_detection_train_loader(cfg) + dummy_data = list(itertools.islice(data_loader, 100)) + + def f(): + data = DatasetFromList(dummy_data, copy=False, serialize=False) + while True: + yield from data + + max_iter = 400 + trainer = (AMPTrainer if cfg.SOLVER.AMP.ENABLED else SimpleTrainer)(model, f(), optimizer) + trainer.register_hooks( + [ + hooks.IterationTimer(), + hooks.PeriodicWriter([CommonMetricPrinter(max_iter)]), + hooks.TorchProfiler( + lambda trainer: trainer.iter == max_iter - 1, cfg.OUTPUT_DIR, save_tensorboard=True + ), + ] + ) + trainer.train(1, max_iter) + + +@torch.no_grad() +def benchmark_eval(args): + cfg = setup(args) + if args.config_file.endswith(".yaml"): + model = build_model(cfg) + DetectionCheckpointer(model).load(cfg.MODEL.WEIGHTS) + + cfg.defrost() + cfg.DATALOADER.NUM_WORKERS = 0 + data_loader = build_detection_test_loader(cfg, cfg.DATASETS.TEST[0]) + else: + model = instantiate(cfg.model) + model.to(cfg.train.device) + DetectionCheckpointer(model).load(cfg.train.init_checkpoint) + + cfg.dataloader.num_workers = 0 + data_loader = instantiate(cfg.dataloader.test) + + model.eval() + logger.info("Model:\n{}".format(model)) + dummy_data = DatasetFromList(list(itertools.islice(data_loader, 100)), copy=False) + + def f(): + while True: + yield from dummy_data + + for k in range(5): # warmup + model(dummy_data[k]) + + max_iter = 300 + timer = Timer() + with tqdm.tqdm(total=max_iter) as pbar: + for idx, d in enumerate(f()): + if idx == max_iter: + break + model(d) + pbar.update() + logger.info("{} iters in {} seconds.".format(max_iter, timer.seconds())) + + +if __name__ == "__main__": + parser = default_argument_parser() + parser.add_argument("--task", choices=["train", "eval", "data", "data_advanced"], required=True) + args = parser.parse_args() + assert not args.eval_only + + logger.info("Environment info:\n" + collect_env_info()) + if "data" in args.task: + print("Initial " + RAM_msg()) + if args.task == "data": + f = benchmark_data + if args.task == "data_advanced": + f = benchmark_data_advanced + elif args.task == "train": + """ + Note: training speed may not be representative. + The training cost of a R-CNN model varies with the content of the data + and the quality of the model. + """ + f = benchmark_train + elif args.task == "eval": + f = benchmark_eval + # only benchmark single-GPU inference. + assert args.num_gpus == 1 and args.num_machines == 1 + launch(f, args.num_gpus, args.num_machines, args.machine_rank, args.dist_url, args=(args,)) diff --git a/approach/ovod/detectron2/tools/convert-torchvision-to-d2.py b/approach/ovod/detectron2/tools/convert-torchvision-to-d2.py new file mode 100644 index 0000000000000000000000000000000000000000..4b827d960cca69657e98bd89a9aa5623a847099d --- /dev/null +++ b/approach/ovod/detectron2/tools/convert-torchvision-to-d2.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. + +import pickle as pkl +import sys +import torch + +""" +Usage: + # download one of the ResNet{18,34,50,101,152} models from torchvision: + wget https://download.pytorch.org/models/resnet50-19c8e357.pth -O r50.pth + # run the conversion + ./convert-torchvision-to-d2.py r50.pth r50.pkl + + # Then, use r50.pkl with the following changes in config: + +MODEL: + WEIGHTS: "/path/to/r50.pkl" + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.120, 57.375] + RESNETS: + DEPTH: 50 + STRIDE_IN_1X1: False +INPUT: + FORMAT: "RGB" + + These models typically produce slightly worse results than the + pre-trained ResNets we use in official configs, which are the + original ResNet models released by MSRA. +""" + +if __name__ == "__main__": + input = sys.argv[1] + + obj = torch.load(input, map_location="cpu") + + newmodel = {} + for k in list(obj.keys()): + old_k = k + if "layer" not in k: + k = "stem." + k + for t in [1, 2, 3, 4]: + k = k.replace("layer{}".format(t), "res{}".format(t + 1)) + for t in [1, 2, 3]: + k = k.replace("bn{}".format(t), "conv{}.norm".format(t)) + k = k.replace("downsample.0", "shortcut") + k = k.replace("downsample.1", "shortcut.norm") + print(old_k, "->", k) + newmodel[k] = obj.pop(old_k).detach().numpy() + + res = {"model": newmodel, "__author__": "torchvision", "matching_heuristics": True} + + with open(sys.argv[2], "wb") as f: + pkl.dump(res, f) + if obj: + print("Unconverted keys:", obj.keys()) diff --git a/approach/ovod/detectron2/tools/deploy/CMakeLists.txt b/approach/ovod/detectron2/tools/deploy/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..80dae12500af4c7e7e6cfc5b7b3a5800782956c3 --- /dev/null +++ b/approach/ovod/detectron2/tools/deploy/CMakeLists.txt @@ -0,0 +1,15 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# See https://pytorch.org/tutorials/advanced/cpp_frontend.html +cmake_minimum_required(VERSION 3.12 FATAL_ERROR) +project(torchscript_mask_rcnn) + +find_package(Torch REQUIRED) +find_package(OpenCV REQUIRED) +find_package(TorchVision REQUIRED) # needed by export-method=tracing/scripting + +add_executable(torchscript_mask_rcnn torchscript_mask_rcnn.cpp) +target_link_libraries( + torchscript_mask_rcnn + -Wl,--no-as-needed TorchVision::TorchVision -Wl,--as-needed + "${TORCH_LIBRARIES}" ${OpenCV_LIBS}) +set_property(TARGET torchscript_mask_rcnn PROPERTY CXX_STANDARD 14) diff --git a/approach/ovod/detectron2/tools/deploy/README.md b/approach/ovod/detectron2/tools/deploy/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e33cbeb54c003a5738da68c838fdaa4e0d218501 --- /dev/null +++ b/approach/ovod/detectron2/tools/deploy/README.md @@ -0,0 +1,66 @@ +See [deployment tutorial](https://detectron2.readthedocs.io/tutorials/deployment.html) +for some high-level background about deployment. + +This directory contains the following examples: + +1. An example script `export_model.py` + that exports a detectron2 model for deployment using different methods and formats. + +2. A C++ example that runs inference with Mask R-CNN model in TorchScript format. + +## Build +Deployment depends on libtorch and OpenCV. Some require more dependencies: + +* Running TorchScript-format models produced by `--export-method=caffe2_tracing` requires libtorch + to be built with caffe2 enabled. +* Running TorchScript-format models produced by `--export-method=tracing/scripting` requires libtorchvision (C++ library of torchvision). + +All methods are supported in one C++ file that requires all the above dependencies. +Adjust it and remove code you don't need. +As a reference, we provide a [Dockerfile](../../docker/deploy.Dockerfile) that installs all the above dependencies and builds the C++ example. + +## Use + +We show a few example commands to export and execute a Mask R-CNN model in C++. + +* `export-method=tracing, format=torchscript`: +``` +./export_model.py --config-file ../../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml \ + --output ./output --export-method tracing --format torchscript \ + MODEL.WEIGHTS detectron2://COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/model_final_f10217.pkl \ + MODEL.DEVICE cuda + +./build/torchscript_mask_rcnn output/model.ts input.jpg tracing +``` + +* `export-method=scripting, format=torchscript`: +``` +./export_model.py --config-file ../../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml \ + --output ./output --export-method scripting --format torchscript \ + MODEL.WEIGHTS detectron2://COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/model_final_f10217.pkl \ + +./build/torchscript_mask_rcnn output/model.ts input.jpg scripting +``` + +* `export-method=caffe2_tracing, format=torchscript`: + +``` +./export_model.py --config-file ../../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml \ + --output ./output --export-method caffe2_tracing --format torchscript \ + MODEL.WEIGHTS detectron2://COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/model_final_f10217.pkl \ + +./build/torchscript_mask_rcnn output/model.ts input.jpg caffe2_tracing +``` + + +## Notes: + +1. Tracing/Caffe2-tracing requires valid weights & sample inputs. + Therefore the above commands require pre-trained models and [COCO dataset](https://detectron2.readthedocs.io/tutorials/builtin_datasets.html). + You can modify the script to obtain sample inputs in other ways instead of from COCO. + +2. `--run-eval` is implemented only for tracing mode + to evaluate the exported model using the dataset in the config. + It's recommended to always verify the accuracy in case the conversion is not successful. + Evaluation can be slow if model is exported to CPU or dataset is too large ("coco_2017_val_100" is a small subset of COCO useful for evaluation). + `caffe2_tracing` accuracy may be slightly different (within 0.1 AP) from original model due to numerical precisions between different runtime. diff --git a/approach/ovod/detectron2/tools/deploy/export_model.py b/approach/ovod/detectron2/tools/deploy/export_model.py new file mode 100644 index 0000000000000000000000000000000000000000..f507dffe56a4121756874186eacdc9be0cbcdee1 --- /dev/null +++ b/approach/ovod/detectron2/tools/deploy/export_model.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +import argparse +import os +from typing import Dict, List, Tuple +import torch +from torch import Tensor, nn + +import detectron2.data.transforms as T +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import get_cfg +from detectron2.data import build_detection_test_loader, detection_utils +from detectron2.evaluation import COCOEvaluator, inference_on_dataset, print_csv_format +from detectron2.export import ( + STABLE_ONNX_OPSET_VERSION, + TracingAdapter, + dump_torchscript_IR, + scripting_with_instances, +) +from detectron2.modeling import GeneralizedRCNN, RetinaNet, build_model +from detectron2.modeling.postprocessing import detector_postprocess +from detectron2.projects.point_rend import add_pointrend_config +from detectron2.structures import Boxes +from detectron2.utils.env import TORCH_VERSION +from detectron2.utils.file_io import PathManager +from detectron2.utils.logger import setup_logger + + +def setup_cfg(args): + cfg = get_cfg() + # cuda context is initialized before creating dataloader, so we don't fork anymore + cfg.DATALOADER.NUM_WORKERS = 0 + add_pointrend_config(cfg) + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + cfg.freeze() + return cfg + + +def export_caffe2_tracing(cfg, torch_model, inputs): + from detectron2.export import Caffe2Tracer + + tracer = Caffe2Tracer(cfg, torch_model, inputs) + if args.format == "caffe2": + caffe2_model = tracer.export_caffe2() + caffe2_model.save_protobuf(args.output) + # draw the caffe2 graph + caffe2_model.save_graph(os.path.join(args.output, "model.svg"), inputs=inputs) + return caffe2_model + elif args.format == "onnx": + import onnx + + onnx_model = tracer.export_onnx() + onnx.save(onnx_model, os.path.join(args.output, "model.onnx")) + elif args.format == "torchscript": + ts_model = tracer.export_torchscript() + with PathManager.open(os.path.join(args.output, "model.ts"), "wb") as f: + torch.jit.save(ts_model, f) + dump_torchscript_IR(ts_model, args.output) + + +# experimental. API not yet final +def export_scripting(torch_model): + assert TORCH_VERSION >= (1, 8) + fields = { + "proposal_boxes": Boxes, + "objectness_logits": Tensor, + "pred_boxes": Boxes, + "scores": Tensor, + "pred_classes": Tensor, + "pred_masks": Tensor, + "pred_keypoints": torch.Tensor, + "pred_keypoint_heatmaps": torch.Tensor, + } + assert args.format == "torchscript", "Scripting only supports torchscript format." + + class ScriptableAdapterBase(nn.Module): + # Use this adapter to workaround https://github.com/pytorch/pytorch/issues/46944 + # by not retuning instances but dicts. Otherwise the exported model is not deployable + def __init__(self): + super().__init__() + self.model = torch_model + self.eval() + + if isinstance(torch_model, GeneralizedRCNN): + + class ScriptableAdapter(ScriptableAdapterBase): + def forward(self, inputs: Tuple[Dict[str, torch.Tensor]]) -> List[Dict[str, Tensor]]: + instances = self.model.inference(inputs, do_postprocess=False) + return [i.get_fields() for i in instances] + + else: + + class ScriptableAdapter(ScriptableAdapterBase): + def forward(self, inputs: Tuple[Dict[str, torch.Tensor]]) -> List[Dict[str, Tensor]]: + instances = self.model(inputs) + return [i.get_fields() for i in instances] + + ts_model = scripting_with_instances(ScriptableAdapter(), fields) + with PathManager.open(os.path.join(args.output, "model.ts"), "wb") as f: + torch.jit.save(ts_model, f) + dump_torchscript_IR(ts_model, args.output) + # TODO inference in Python now missing postprocessing glue code + return None + + +# experimental. API not yet final +def export_tracing(torch_model, inputs): + assert TORCH_VERSION >= (1, 8) + image = inputs[0]["image"] + inputs = [{"image": image}] # remove other unused keys + + if isinstance(torch_model, GeneralizedRCNN): + + def inference(model, inputs): + # use do_postprocess=False so it returns ROI mask + inst = model.inference(inputs, do_postprocess=False)[0] + return [{"instances": inst}] + + else: + inference = None # assume that we just call the model directly + + traceable_model = TracingAdapter(torch_model, inputs, inference) + + if args.format == "torchscript": + ts_model = torch.jit.trace(traceable_model, (image,)) + with PathManager.open(os.path.join(args.output, "model.ts"), "wb") as f: + torch.jit.save(ts_model, f) + dump_torchscript_IR(ts_model, args.output) + elif args.format == "onnx": + with PathManager.open(os.path.join(args.output, "model.onnx"), "wb") as f: + torch.onnx.export(traceable_model, (image,), f, opset_version=STABLE_ONNX_OPSET_VERSION) + logger.info("Inputs schema: " + str(traceable_model.inputs_schema)) + logger.info("Outputs schema: " + str(traceable_model.outputs_schema)) + + if args.format != "torchscript": + return None + if not isinstance(torch_model, (GeneralizedRCNN, RetinaNet)): + return None + + def eval_wrapper(inputs): + """ + The exported model does not contain the final resize step, which is typically + unused in deployment but needed for evaluation. We add it manually here. + """ + input = inputs[0] + instances = traceable_model.outputs_schema(ts_model(input["image"]))[0]["instances"] + postprocessed = detector_postprocess(instances, input["height"], input["width"]) + return [{"instances": postprocessed}] + + return eval_wrapper + + +def get_sample_inputs(args): + + if args.sample_image is None: + # get a first batch from dataset + data_loader = build_detection_test_loader(cfg, cfg.DATASETS.TEST[0]) + first_batch = next(iter(data_loader)) + return first_batch + else: + # get a sample data + original_image = detection_utils.read_image(args.sample_image, format=cfg.INPUT.FORMAT) + # Do same preprocessing as DefaultPredictor + aug = T.ResizeShortestEdge( + [cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST], cfg.INPUT.MAX_SIZE_TEST + ) + height, width = original_image.shape[:2] + image = aug.get_transform(original_image).apply_image(original_image) + image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) + + inputs = {"image": image, "height": height, "width": width} + + # Sample ready + sample_inputs = [inputs] + return sample_inputs + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Export a model for deployment.") + parser.add_argument( + "--format", + choices=["caffe2", "onnx", "torchscript"], + help="output format", + default="torchscript", + ) + parser.add_argument( + "--export-method", + choices=["caffe2_tracing", "tracing", "scripting"], + help="Method to export models", + default="tracing", + ) + parser.add_argument("--config-file", default="", metavar="FILE", help="path to config file") + parser.add_argument("--sample-image", default=None, type=str, help="sample image for input") + parser.add_argument("--run-eval", action="store_true") + parser.add_argument("--output", help="output directory for the converted model") + parser.add_argument( + "opts", + help="Modify config options using the command-line", + default=None, + nargs=argparse.REMAINDER, + ) + args = parser.parse_args() + logger = setup_logger() + logger.info("Command line arguments: " + str(args)) + PathManager.mkdirs(args.output) + # Disable re-specialization on new shapes. Otherwise --run-eval will be slow + torch._C._jit_set_bailout_depth(1) + + cfg = setup_cfg(args) + + # create a torch model + torch_model = build_model(cfg) + DetectionCheckpointer(torch_model).resume_or_load(cfg.MODEL.WEIGHTS) + torch_model.eval() + + # convert and save model + if args.export_method == "caffe2_tracing": + sample_inputs = get_sample_inputs(args) + exported_model = export_caffe2_tracing(cfg, torch_model, sample_inputs) + elif args.export_method == "scripting": + exported_model = export_scripting(torch_model) + elif args.export_method == "tracing": + sample_inputs = get_sample_inputs(args) + exported_model = export_tracing(torch_model, sample_inputs) + + # run evaluation with the converted model + if args.run_eval: + assert exported_model is not None, ( + "Python inference is not yet implemented for " + f"export_method={args.export_method}, format={args.format}." + ) + logger.info("Running evaluation ... this takes a long time if you export to CPU.") + dataset = cfg.DATASETS.TEST[0] + data_loader = build_detection_test_loader(cfg, dataset) + # NOTE: hard-coded evaluator. change to the evaluator for your dataset + evaluator = COCOEvaluator(dataset, output_dir=args.output) + metrics = inference_on_dataset(exported_model, data_loader, evaluator) + print_csv_format(metrics) + logger.info("Success.") diff --git a/approach/ovod/detectron2/tools/deploy/torchscript_mask_rcnn.cpp b/approach/ovod/detectron2/tools/deploy/torchscript_mask_rcnn.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fd6e1e9f82652a1d4d221447cd140ab675f312b2 --- /dev/null +++ b/approach/ovod/detectron2/tools/deploy/torchscript_mask_rcnn.cpp @@ -0,0 +1,188 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +// @lint-ignore-every CLANGTIDY +// This is an example code that demonstrates how to run inference +// with a torchscript format Mask R-CNN model exported by ./export_model.py +// using export method=tracing, caffe2_tracing & scripting. + +#include +#include +#include + +#include +#include +#include +#include + +// only needed for export_method=tracing +#include // @oss-only +// @fb-only: #include + +using namespace std; + +c10::IValue get_caffe2_tracing_inputs(cv::Mat& img, c10::Device device) { + const int height = img.rows; + const int width = img.cols; + // FPN models require divisibility of 32. + // Tracing mode does padding inside the graph, but caffe2_tracing does not. + assert(height % 32 == 0 && width % 32 == 0); + const int channels = 3; + + auto input = + torch::from_blob(img.data, {1, height, width, channels}, torch::kUInt8); + // NHWC to NCHW + input = input.to(device, torch::kFloat).permute({0, 3, 1, 2}).contiguous(); + + std::array im_info_data{height * 1.0f, width * 1.0f, 1.0f}; + auto im_info = + torch::from_blob(im_info_data.data(), {1, 3}).clone().to(device); + return std::make_tuple(input, im_info); +} + +c10::IValue get_tracing_inputs(cv::Mat& img, c10::Device device) { + const int height = img.rows; + const int width = img.cols; + const int channels = 3; + + auto input = + torch::from_blob(img.data, {height, width, channels}, torch::kUInt8); + // HWC to CHW + input = input.to(device, torch::kFloat).permute({2, 0, 1}).contiguous(); + return input; +} + +// create a Tuple[Dict[str, Tensor]] which is the input type of scripted model +c10::IValue get_scripting_inputs(cv::Mat& img, c10::Device device) { + const int height = img.rows; + const int width = img.cols; + const int channels = 3; + + auto img_tensor = + torch::from_blob(img.data, {height, width, channels}, torch::kUInt8); + // HWC to CHW + img_tensor = + img_tensor.to(device, torch::kFloat).permute({2, 0, 1}).contiguous(); + auto dic = c10::Dict(); + dic.insert("image", img_tensor); + return std::make_tuple(dic); +} + +c10::IValue +get_inputs(std::string export_method, cv::Mat& img, c10::Device device) { + // Given an image, create inputs in the format required by the model. + if (export_method == "tracing") + return get_tracing_inputs(img, device); + if (export_method == "caffe2_tracing") + return get_caffe2_tracing_inputs(img, device); + if (export_method == "scripting") + return get_scripting_inputs(img, device); + abort(); +} + +struct MaskRCNNOutputs { + at::Tensor pred_boxes, pred_classes, pred_masks, scores; + int num_instances() const { + return pred_boxes.sizes()[0]; + } +}; + +MaskRCNNOutputs get_outputs(std::string export_method, c10::IValue outputs) { + // Given outputs of the model, extract tensors from it to turn into a + // common MaskRCNNOutputs format. + if (export_method == "tracing") { + auto out_tuple = outputs.toTuple()->elements(); + // They are ordered alphabetically by their field name in Instances + return MaskRCNNOutputs{ + out_tuple[0].toTensor(), + out_tuple[1].toTensor(), + out_tuple[2].toTensor(), + out_tuple[3].toTensor()}; + } + if (export_method == "caffe2_tracing") { + auto out_tuple = outputs.toTuple()->elements(); + // A legacy order used by caffe2 models + return MaskRCNNOutputs{ + out_tuple[0].toTensor(), + out_tuple[2].toTensor(), + out_tuple[3].toTensor(), + out_tuple[1].toTensor()}; + } + if (export_method == "scripting") { + // With the ScriptableAdapter defined in export_model.py, the output is + // List[Dict[str, Any]]. + auto out_dict = outputs.toList().get(0).toGenericDict(); + return MaskRCNNOutputs{ + out_dict.at("pred_boxes").toTensor(), + out_dict.at("pred_classes").toTensor(), + out_dict.at("pred_masks").toTensor(), + out_dict.at("scores").toTensor()}; + } + abort(); +} + +int main(int argc, const char* argv[]) { + if (argc != 4) { + cerr << R"xx( +Usage: + ./torchscript_mask_rcnn model.ts input.jpg EXPORT_METHOD + + EXPORT_METHOD can be "tracing", "caffe2_tracing" or "scripting". +)xx"; + return 1; + } + std::string image_file = argv[2]; + std::string export_method = argv[3]; + assert( + export_method == "caffe2_tracing" || export_method == "tracing" || + export_method == "scripting"); + + torch::jit::FusionStrategy strat = {{torch::jit::FusionBehavior::DYNAMIC, 1}}; + torch::jit::setFusionStrategy(strat); + torch::autograd::AutoGradMode guard(false); + auto module = torch::jit::load(argv[1]); + + assert(module.buffers().size() > 0); + // Assume that the entire model is on the same device. + // We just put input to this device. + auto device = (*begin(module.buffers())).device(); + + cv::Mat input_img = cv::imread(image_file, cv::IMREAD_COLOR); + auto inputs = get_inputs(export_method, input_img, device); + + // Run the network + auto output = module.forward({inputs}); + if (device.is_cuda()) + c10::cuda::getCurrentCUDAStream().synchronize(); + + // run 3 more times to benchmark + int N_benchmark = 3, N_warmup = 1; + auto start_time = chrono::high_resolution_clock::now(); + for (int i = 0; i < N_benchmark + N_warmup; ++i) { + if (i == N_warmup) + start_time = chrono::high_resolution_clock::now(); + output = module.forward({inputs}); + if (device.is_cuda()) + c10::cuda::getCurrentCUDAStream().synchronize(); + } + auto end_time = chrono::high_resolution_clock::now(); + auto ms = chrono::duration_cast(end_time - start_time) + .count(); + cout << "Latency (should vary with different inputs): " + << ms * 1.0 / 1e6 / N_benchmark << " seconds" << endl; + + // Parse Mask R-CNN outputs + auto rcnn_outputs = get_outputs(export_method, output); + cout << "Number of detected objects: " << rcnn_outputs.num_instances() + << endl; + + cout << "pred_boxes: " << rcnn_outputs.pred_boxes.toString() << " " + << rcnn_outputs.pred_boxes.sizes() << endl; + cout << "scores: " << rcnn_outputs.scores.toString() << " " + << rcnn_outputs.scores.sizes() << endl; + cout << "pred_classes: " << rcnn_outputs.pred_classes.toString() << " " + << rcnn_outputs.pred_classes.sizes() << endl; + cout << "pred_masks: " << rcnn_outputs.pred_masks.toString() << " " + << rcnn_outputs.pred_masks.sizes() << endl; + + cout << rcnn_outputs.pred_boxes << endl; + return 0; +} diff --git a/approach/ovod/detectron2/tools/lightning_train_net.py b/approach/ovod/detectron2/tools/lightning_train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..423f540035898dc31c33a4d2785b090691b920c7 --- /dev/null +++ b/approach/ovod/detectron2/tools/lightning_train_net.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. +# Lightning Trainer should be considered beta at this point +# We have confirmed that training and validation run correctly and produce correct results +# Depending on how you launch the trainer, there are issues with processes terminating correctly +# This module is still dependent on D2 logging, but could be transferred to use Lightning logging + +import logging +import os +import time +import weakref +from collections import OrderedDict +from typing import Any, Dict, List + +import detectron2.utils.comm as comm +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import get_cfg +from detectron2.data import build_detection_test_loader, build_detection_train_loader +from detectron2.engine import ( + DefaultTrainer, + SimpleTrainer, + default_argument_parser, + default_setup, + default_writers, + hooks, +) +from detectron2.evaluation import print_csv_format +from detectron2.evaluation.testing import flatten_results_dict +from detectron2.modeling import build_model +from detectron2.solver import build_lr_scheduler, build_optimizer +from detectron2.utils.events import EventStorage +from detectron2.utils.logger import setup_logger + +import pytorch_lightning as pl # type: ignore +from pytorch_lightning import LightningDataModule, LightningModule +from train_net import build_evaluator + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("detectron2") + + +class TrainingModule(LightningModule): + def __init__(self, cfg): + super().__init__() + if not logger.isEnabledFor(logging.INFO): # setup_logger is not called for d2 + setup_logger() + self.cfg = DefaultTrainer.auto_scale_workers(cfg, comm.get_world_size()) + self.storage: EventStorage = None + self.model = build_model(self.cfg) + + self.start_iter = 0 + self.max_iter = cfg.SOLVER.MAX_ITER + + def on_save_checkpoint(self, checkpoint: Dict[str, Any]) -> None: + checkpoint["iteration"] = self.storage.iter + + def on_load_checkpoint(self, checkpointed_state: Dict[str, Any]) -> None: + self.start_iter = checkpointed_state["iteration"] + self.storage.iter = self.start_iter + + def setup(self, stage: str): + if self.cfg.MODEL.WEIGHTS: + self.checkpointer = DetectionCheckpointer( + # Assume you want to save checkpoints together with logs/statistics + self.model, + self.cfg.OUTPUT_DIR, + ) + logger.info(f"Load model weights from checkpoint: {self.cfg.MODEL.WEIGHTS}.") + # Only load weights, use lightning checkpointing if you want to resume + self.checkpointer.load(self.cfg.MODEL.WEIGHTS) + + self.iteration_timer = hooks.IterationTimer() + self.iteration_timer.before_train() + self.data_start = time.perf_counter() + self.writers = None + + def training_step(self, batch, batch_idx): + data_time = time.perf_counter() - self.data_start + # Need to manually enter/exit since trainer may launch processes + # This ideally belongs in setup, but setup seems to run before processes are spawned + if self.storage is None: + self.storage = EventStorage(0) + self.storage.__enter__() + self.iteration_timer.trainer = weakref.proxy(self) + self.iteration_timer.before_step() + self.writers = ( + default_writers(self.cfg.OUTPUT_DIR, self.max_iter) + if comm.is_main_process() + else {} + ) + + loss_dict = self.model(batch) + SimpleTrainer.write_metrics(loss_dict, data_time) + + opt = self.optimizers() + self.storage.put_scalar( + "lr", opt.param_groups[self._best_param_group_id]["lr"], smoothing_hint=False + ) + self.iteration_timer.after_step() + self.storage.step() + # A little odd to put before step here, but it's the best way to get a proper timing + self.iteration_timer.before_step() + + if self.storage.iter % 20 == 0: + for writer in self.writers: + writer.write() + return sum(loss_dict.values()) + + def training_step_end(self, training_step_outpus): + self.data_start = time.perf_counter() + return training_step_outpus + + def training_epoch_end(self, training_step_outputs): + self.iteration_timer.after_train() + if comm.is_main_process(): + self.checkpointer.save("model_final") + for writer in self.writers: + writer.write() + writer.close() + self.storage.__exit__(None, None, None) + + def _process_dataset_evaluation_results(self) -> OrderedDict: + results = OrderedDict() + for idx, dataset_name in enumerate(self.cfg.DATASETS.TEST): + results[dataset_name] = self._evaluators[idx].evaluate() + if comm.is_main_process(): + print_csv_format(results[dataset_name]) + + if len(results) == 1: + results = list(results.values())[0] + return results + + def _reset_dataset_evaluators(self): + self._evaluators = [] + for dataset_name in self.cfg.DATASETS.TEST: + evaluator = build_evaluator(self.cfg, dataset_name) + evaluator.reset() + self._evaluators.append(evaluator) + + def on_validation_epoch_start(self, _outputs): + self._reset_dataset_evaluators() + + def validation_epoch_end(self, _outputs): + results = self._process_dataset_evaluation_results(_outputs) + + flattened_results = flatten_results_dict(results) + for k, v in flattened_results.items(): + try: + v = float(v) + except Exception as e: + raise ValueError( + "[EvalHook] eval_function should return a nested dict of float. " + "Got '{}: {}' instead.".format(k, v) + ) from e + self.storage.put_scalars(**flattened_results, smoothing_hint=False) + + def validation_step(self, batch, batch_idx: int, dataloader_idx: int = 0) -> None: + if not isinstance(batch, List): + batch = [batch] + outputs = self.model(batch) + self._evaluators[dataloader_idx].process(batch, outputs) + + def configure_optimizers(self): + optimizer = build_optimizer(self.cfg, self.model) + self._best_param_group_id = hooks.LRScheduler.get_best_param_group_id(optimizer) + scheduler = build_lr_scheduler(self.cfg, optimizer) + return [optimizer], [{"scheduler": scheduler, "interval": "step"}] + + +class DataModule(LightningDataModule): + def __init__(self, cfg): + super().__init__() + self.cfg = DefaultTrainer.auto_scale_workers(cfg, comm.get_world_size()) + + def train_dataloader(self): + return build_detection_train_loader(self.cfg) + + def val_dataloader(self): + dataloaders = [] + for dataset_name in self.cfg.DATASETS.TEST: + dataloaders.append(build_detection_test_loader(self.cfg, dataset_name)) + return dataloaders + + +def main(args): + cfg = setup(args) + train(cfg, args) + + +def train(cfg, args): + trainer_params = { + # training loop is bounded by max steps, use a large max_epochs to make + # sure max_steps is met first + "max_epochs": 10**8, + "max_steps": cfg.SOLVER.MAX_ITER, + "val_check_interval": cfg.TEST.EVAL_PERIOD if cfg.TEST.EVAL_PERIOD > 0 else 10**8, + "num_nodes": args.num_machines, + "gpus": args.num_gpus, + "num_sanity_val_steps": 0, + } + if cfg.SOLVER.AMP.ENABLED: + trainer_params["precision"] = 16 + + last_checkpoint = os.path.join(cfg.OUTPUT_DIR, "last.ckpt") + if args.resume: + # resume training from checkpoint + trainer_params["resume_from_checkpoint"] = last_checkpoint + logger.info(f"Resuming training from checkpoint: {last_checkpoint}.") + + trainer = pl.Trainer(**trainer_params) + logger.info(f"start to train with {args.num_machines} nodes and {args.num_gpus} GPUs") + + module = TrainingModule(cfg) + data_module = DataModule(cfg) + if args.eval_only: + logger.info("Running inference") + trainer.validate(module, data_module) + else: + logger.info("Running training") + trainer.fit(module, data_module) + + +def setup(args): + """ + Create configs and perform basic setups. + """ + cfg = get_cfg() + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + cfg.freeze() + default_setup(cfg, args) + return cfg + + +if __name__ == "__main__": + parser = default_argument_parser() + args = parser.parse_args() + logger.info("Command Line Args:", args) + main(args) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/.github/CODE_OF_CONDUCT.md b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000000000000000000000000000000000..0f7ad8bfc173eac554f0b6ef7c684861e8014bbe --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,5 @@ +# Code of Conduct + +Facebook has adopted a Code of Conduct that we expect project participants to adhere to. +Please read the [full text](https://code.fb.com/codeofconduct/) +so that you can understand what actions will and will not be tolerated. diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/.github/CONTRIBUTING.md b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/CONTRIBUTING.md new file mode 100644 index 0000000000000000000000000000000000000000..9bab709cae689ba3b92dd52f7fbcc0c6926f4a38 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/CONTRIBUTING.md @@ -0,0 +1,68 @@ +# Contributing to detectron2 + +## Issues +We use GitHub issues to track public bugs and questions. +Please make sure to follow one of the +[issue templates](https://github.com/facebookresearch/detectron2/issues/new/choose) +when reporting any issues. + +Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe +disclosure of security bugs. In those cases, please go through the process +outlined on that page and do not file a public issue. + +## Pull Requests +We actively welcome pull requests. + +However, if you're adding any significant features (e.g. > 50 lines), please +make sure to discuss with maintainers about your motivation and proposals in an issue +before sending a PR. This is to save your time so you don't spend time on a PR that we'll not accept. + +We do not always accept new features, and we take the following +factors into consideration: + +1. Whether the same feature can be achieved without modifying detectron2. + Detectron2 is designed so that you can implement many extensions from the outside, e.g. + those in [projects](https://github.com/facebookresearch/detectron2/tree/master/projects). + * If some part of detectron2 is not extensible enough, you can also bring up a more general issue to + improve it. Such feature request may be useful to more users. +2. Whether the feature is potentially useful to a large audience (e.g. an impactful detection paper, a popular dataset, + a significant speedup, a widely useful utility), + or only to a small portion of users (e.g., a less-known paper, an improvement not in the object + detection field, a trick that's not very popular in the community, code to handle a non-standard type of data) + * Adoption of additional models, datasets, new task are by default not added to detectron2 before they + receive significant popularity in the community. + We sometimes accept such features in `projects/`, or as a link in `projects/README.md`. +3. Whether the proposed solution has a good design / interface. This can be discussed in the issue prior to PRs, or + in the form of a draft PR. +4. Whether the proposed solution adds extra mental/practical overhead to users who don't + need such feature. +5. Whether the proposed solution breaks existing APIs. + +To add a feature to an existing function/class `Func`, there are always two approaches: +(1) add new arguments to `Func`; (2) write a new `Func_with_new_feature`. +To meet the above criteria, we often prefer approach (2), because: + +1. It does not involve modifying or potentially breaking existing code. +2. It does not add overhead to users who do not need the new feature. +3. Adding new arguments to a function/class is not scalable w.r.t. all the possible new research ideas in the future. + +When sending a PR, please do: + +1. If a PR contains multiple orthogonal changes, split it to several PRs. +2. If you've added code that should be tested, add tests. +3. For PRs that need experiments (e.g. adding a new model or new methods), + you don't need to update model zoo, but do provide experiment results in the description of the PR. +4. If APIs are changed, update the documentation. +5. We use the [Google style docstrings](https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html) in python. +6. Make sure your code lints with `./dev/linter.sh`. + + +## Contributor License Agreement ("CLA") +In order to accept your pull request, we need you to submit a CLA. You only need +to do this once to work on any of Facebook's open source projects. + +Complete your CLA here: + +## License +By contributing to detectron2, you agree that your contributions will be licensed +under the LICENSE file in the root directory of this source tree. diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/.github/Detectron2-Logo-Horz.svg b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/Detectron2-Logo-Horz.svg new file mode 100644 index 0000000000000000000000000000000000000000..eb2d643ddd940cd8bdb5eaad093029969ff2364c --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/Detectron2-Logo-Horz.svg @@ -0,0 +1 @@ +Detectron2-Logo-Horz \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE.md b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000000000000000000000000000000000000..5e8aaa2d3722e7e73a3d94b2b7dfc4f751d7a240 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,5 @@ + +Please select an issue template from +https://github.com/facebookresearch/detectron2/issues/new/choose . + +Otherwise your issue will be closed. diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE/bugs.md b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE/bugs.md new file mode 100644 index 0000000000000000000000000000000000000000..d0235c708ab6b0cdadb5865110e9e8c22ca313aa --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE/bugs.md @@ -0,0 +1,38 @@ +--- +name: "🐛 Bugs" +about: Report bugs in detectron2 +title: Please read & provide the following + +--- + +## Instructions To Reproduce the 🐛 Bug: +1. Full runnable code or full changes you made: +``` +If making changes to the project itself, please use output of the following command: +git rev-parse HEAD; git diff + + +``` +2. What exact command you run: +3. __Full logs__ or other relevant observations: +``` + +``` +4. please simplify the steps as much as possible so they do not require additional resources to + run, such as a private dataset. + +## Expected behavior: + +If there are no obvious error in "full logs" provided above, +please tell us the expected behavior. + +## Environment: + +Provide your environment information using the following command: +``` +wget -nc -q https://github.com/facebookresearch/detectron2/raw/main/detectron2/utils/collect_env.py && python collect_env.py +``` + +If your issue looks like an installation issue / environment issue, +please first try to solve it yourself with the instructions in +https://detectron2.readthedocs.io/tutorials/install.html#common-installation-issues diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE/config.yml b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..c60c2e14309be9a93293a64e7481f2a91385f76a --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,17 @@ +# require an issue template to be chosen +blank_issues_enabled: false + +contact_links: + - name: How-To / All Other Questions + url: https://github.com/facebookresearch/detectron2/discussions + about: Use "github discussions" for community support on general questions that don't belong to the above issue categories + - name: Detectron2 Documentation + url: https://detectron2.readthedocs.io/index.html + about: Check if your question is answered in tutorials or API docs + +# Unexpected behaviors & bugs are split to two templates. +# When they are one template, users think "it's not a bug" and don't choose the template. +# +# But the file name is still "unexpected-problems-bugs.md" so that old references +# to this issue template still works. +# It's ok since this template should be a superset of "bugs.md" (unexpected behaviors is a superset of bugs) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE/documentation.md b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE/documentation.md new file mode 100644 index 0000000000000000000000000000000000000000..88214d62e5228639491e019c78bb4171d535cdd1 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE/documentation.md @@ -0,0 +1,14 @@ +--- +name: "\U0001F4DA Documentation Issue" +about: Report a problem about existing documentation, comments, website or tutorials. +labels: documentation + +--- + +## 📚 Documentation Issue + +This issue category is for problems about existing documentation, not for asking how-to questions. + +* Provide a link to an existing documentation/comment/tutorial: + +* How should the above documentation/comment/tutorial improve: diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE/feature-request.md b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 0000000000000000000000000000000000000000..03a1e93d7293948042120b875af8be0c6964e59c --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,31 @@ +--- +name: "\U0001F680Feature Request" +about: Suggest an improvement or new feature +labels: enhancement + +--- + +## 🚀 Feature +A clear and concise description of the feature proposal. + +## Motivation & Examples + +Tell us why the feature is useful. + +Describe what the feature would look like, if it is implemented. +Best demonstrated using **code examples** in addition to words. + +## Note + +We only consider adding new features if they are relevant to many users. + +If you request implementation of research papers -- we only consider papers that have enough significance and prevalance in the object detection field. + +We do not take requests for most projects in the `projects/` directory, because they are research code release that is mainly for other researchers to reproduce results. + +"Make X faster/accurate" is not a valid feature request. "Implement a concrete feature that can make X faster/accurate" can be a valid feature request. + +Instead of adding features inside detectron2, +you can implement many features by [extending detectron2](https://detectron2.readthedocs.io/tutorials/extend.html). +The [projects/](https://github.com/facebookresearch/detectron2/tree/main/projects/) directory contains many of such examples. + diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE/unexpected-problems-bugs.md b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE/unexpected-problems-bugs.md new file mode 100644 index 0000000000000000000000000000000000000000..5db8f22415ff5c857ce83fb0d3de68211f775080 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/ISSUE_TEMPLATE/unexpected-problems-bugs.md @@ -0,0 +1,44 @@ +--- +name: "😩 Unexpected behaviors" +about: Report unexpected behaviors when using detectron2 +title: Please read & provide the following + +--- + +If you do not know the root cause of the problem, please post according to this template: + +## Instructions To Reproduce the Issue: + +Check https://stackoverflow.com/help/minimal-reproducible-example for how to ask good questions. +Simplify the steps to reproduce the issue using suggestions from the above link, and provide them below: + +1. Full runnable code or full changes you made: +``` +If making changes to the project itself, please use output of the following command: +git rev-parse HEAD; git diff + + +``` +2. What exact command you run: +3. __Full logs__ or other relevant observations: +``` + +``` + +## Expected behavior: + +If there are no obvious crash in "full logs" provided above, +please tell us the expected behavior. + +If you expect a model to converge / work better, we do not help with such issues, unless +a model fails to reproduce the results in detectron2 model zoo, or proves existence of bugs. + +## Environment: + +Paste the output of the following command: +``` +wget -nc -nv https://github.com/facebookresearch/detectron2/raw/main/detectron2/utils/collect_env.py && python collect_env.py +``` + +If your issue looks like an installation issue / environment issue, +please first check common issues in https://detectron2.readthedocs.io/tutorials/install.html#common-installation-issues diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/.github/pull_request_template.md b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/pull_request_template.md new file mode 100644 index 0000000000000000000000000000000000000000..d71729baee1ec324ab9db6e7562965cf9e2a091b --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/pull_request_template.md @@ -0,0 +1,10 @@ +Thanks for your contribution! + +If you're sending a large PR (e.g., >100 lines), +please open an issue first about the feature / bug, and indicate how you want to contribute. + +We do not always accept features. +See https://detectron2.readthedocs.io/notes/contributing.html#pull-requests about how we handle PRs. + +Before submitting a PR, please run `dev/linter.sh` to lint the code. + diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/.github/workflows/check-template.yml b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/workflows/check-template.yml new file mode 100644 index 0000000000000000000000000000000000000000..3caed9df3caa50c0d3b606e4a56a1959c463b710 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/workflows/check-template.yml @@ -0,0 +1,86 @@ +name: Check issue template + +on: + issues: + types: [opened] + +jobs: + check-template: + runs-on: ubuntu-latest + # comment this out when testing with https://github.com/nektos/act + if: ${{ github.repository_owner == 'facebookresearch' }} + steps: + - uses: actions/checkout@v2 + - uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + // Arguments available: + // - github: A pre-authenticated octokit/rest.js client + // - context: An object containing the context of the workflow run + // - core: A reference to the @actions/core package + // - io: A reference to the @actions/io package + const fs = require('fs'); + const editDistance = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/levenshtein.js`).getEditDistance + issue = await github.issues.get({ + owner: context.issue.owner, + repo: context.issue.repo, + issue_number: context.issue.number, + }); + const hasLabel = issue.data.labels.length > 0; + if (hasLabel || issue.state === "closed") { + // don't require template on them + core.debug("Issue " + issue.data.title + " was skipped."); + return; + } + + sameAsTemplate = function(filename, body) { + let tmpl = fs.readFileSync(`.github/ISSUE_TEMPLATE/${filename}`, 'utf8'); + tmpl = tmpl.toLowerCase().split("---").slice(2).join("").trim(); + tmpl = tmpl.replace(/(\r\n|\n|\r)/gm, ""); + let bodyr = body.replace(/(\r\n|\n|\r)/gm, ""); + let dist = editDistance(tmpl, bodyr); + return dist < 8; + }; + + checkFail = async function(msg) { + core.info("Processing '" + issue.data.title + "' with message: " + msg); + await github.issues.addLabels({ + owner: context.issue.owner, + repo: context.issue.repo, + issue_number: context.issue.number, + labels: ["needs-more-info"], + }); + await github.issues.createComment({ + owner: context.issue.owner, + repo: context.issue.repo, + issue_number: context.issue.number, + body: msg, + }); + }; + + const body = issue.data.body.toLowerCase().trim(); + + if (sameAsTemplate("bugs.md", body) || sameAsTemplate("unexpected-problems-bugs.md", body)) { + await checkFail(` + We found that not enough information is provided about this issue. + Please provide details following the [issue template](https://github.com/facebookresearch/detectron2/issues/new/choose).`) + return; + } + + const hasInstructions = body.indexOf("reproduce") != -1; + const hasEnvironment = (body.indexOf("environment") != -1) || (body.indexOf("colab") != -1) || (body.indexOf("docker") != -1); + if (hasInstructions && hasEnvironment) { + core.debug("Issue " + issue.data.title + " follows template."); + return; + } + + let message = "You've chosen to report an unexpected problem or bug. Unless you already know the root cause of it, please include details about it by filling the [issue template](https://github.com/facebookresearch/detectron2/issues/new/choose).\n"; + message += "The following information is missing: "; + if (!hasInstructions) { + message += "\"Instructions To Reproduce the Issue and __Full__ Logs\"; "; + } + if (!hasEnvironment) { + message += "\"Your Environment\"; "; + } + await checkFail(message); diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/.github/workflows/levenshtein.js b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/workflows/levenshtein.js new file mode 100644 index 0000000000000000000000000000000000000000..67a5e3613c0072d124035ee8933a23de2105cfe3 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/workflows/levenshtein.js @@ -0,0 +1,44 @@ +/* +Copyright (c) 2011 Andrei Mackenzie + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +// Compute the edit distance between the two given strings +exports.getEditDistance = function(a, b){ + if(a.length == 0) return b.length; + if(b.length == 0) return a.length; + + var matrix = []; + + // increment along the first column of each row + var i; + for(i = 0; i <= b.length; i++){ + matrix[i] = [i]; + } + + // increment each column in the first row + var j; + for(j = 0; j <= a.length; j++){ + matrix[0][j] = j; + } + + // Fill in the rest of the matrix + for(i = 1; i <= b.length; i++){ + for(j = 1; j <= a.length; j++){ + if(b.charAt(i-1) == a.charAt(j-1)){ + matrix[i][j] = matrix[i-1][j-1]; + } else { + matrix[i][j] = Math.min(matrix[i-1][j-1] + 1, // substitution + Math.min(matrix[i][j-1] + 1, // insertion + matrix[i-1][j] + 1)); // deletion + } + } + } + + return matrix[b.length][a.length]; +}; diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/.github/workflows/needs-reply.yml b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/workflows/needs-reply.yml new file mode 100644 index 0000000000000000000000000000000000000000..4affabd3498290a752fab6d848fc667758bedaf2 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/workflows/needs-reply.yml @@ -0,0 +1,98 @@ +name: Close/Lock issues after inactivity + +on: + schedule: + - cron: "0 0 * * *" + +jobs: + close-issues-needs-more-info: + runs-on: ubuntu-latest + if: ${{ github.repository_owner == 'facebookresearch' }} + steps: + - name: Close old issues that need reply + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + # Modified from https://github.com/dwieeb/needs-reply + script: | + // Arguments available: + // - github: A pre-authenticated octokit/rest.js client + // - context: An object containing the context of the workflow run + // - core: A reference to the @actions/core package + // - io: A reference to the @actions/io package + const kLabelToCheck = "needs-more-info"; + const kInvalidLabel = "invalid/unrelated"; + const kDaysBeforeClose = 7; + const kMessage = "Requested information was not provided in 7 days, so we're closing this issue.\n\nPlease open new issue if information becomes available. Otherwise, use [github discussions](https://github.com/facebookresearch/detectron2/discussions) for free-form discussions." + + issues = await github.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: kLabelToCheck, + sort: 'updated', + direction: 'asc', + per_page: 30, + page: 1, + }); + issues = issues.data; + if (issues.length === 0) { + core.info('No more issues found to process. Exiting.'); + return; + } + for (const issue of issues) { + if (!!issue.pull_request) + continue; + core.info(`Processing issue #${issue.number}`); + + let updatedAt = new Date(issue.updated_at).getTime(); + const numComments = issue.comments; + const comments = await github.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + per_page: 30, + page: Math.floor((numComments - 1) / 30) + 1, // the last page + }); + const lastComments = comments.data + .map(l => new Date(l.created_at).getTime()) + .sort(); + if (lastComments.length > 0) { + updatedAt = lastComments[lastComments.length - 1]; + } + + const now = new Date().getTime(); + const daysSinceUpdated = (now - updatedAt) / 1000 / 60 / 60 / 24; + + if (daysSinceUpdated < kDaysBeforeClose) { + core.info(`Skipping #${issue.number} because it has been updated in the last ${daysSinceUpdated} days`); + continue; + } + core.info(`Closing #${issue.number} because it has not been updated in the last ${daysSinceUpdated} days`); + await github.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + body: kMessage, + }); + const newLabels = numComments <= 2 ? [kInvalidLabel, kLabelToCheck] : issue.labels; + await github.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue.number, + labels: newLabels, + state: 'closed', + }); + } + + lock-issues-after-closed: + runs-on: ubuntu-latest + if: ${{ github.repository_owner == 'facebookresearch' }} + steps: + - name: Lock closed issues that have no activity for a while + uses: dessant/lock-threads@v2 + with: + github-token: ${{ github.token }} + issue-lock-inactive-days: '300' + process-only: 'issues' + issue-exclude-labels: 'enhancement,bug,documentation' diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/.github/workflows/remove-needs-reply.yml b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/workflows/remove-needs-reply.yml new file mode 100644 index 0000000000000000000000000000000000000000..1f000b28ca27ef9c219d197f95251be1cb8c0979 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/workflows/remove-needs-reply.yml @@ -0,0 +1,25 @@ +name: Remove needs-more-info label + +on: + issue_comment: + types: [created] + issues: + types: [edited] + +jobs: + remove-needs-more-info-label: + runs-on: ubuntu-latest + # 1. issue_comment events could include PR comment, filter them out + # 2. Only trigger action if event was produced by the original author + if: ${{ !github.event.issue.pull_request && github.event.sender.login == github.event.issue.user.login }} + steps: + - name: Remove needs-more-info label + uses: octokit/request-action@v2.x + continue-on-error: true + with: + route: DELETE /repos/:repository/issues/:issue/labels/:label + repository: ${{ github.repository }} + issue: ${{ github.event.issue.number }} + label: needs-more-info + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/.github/workflows/workflow.yml b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/workflows/workflow.yml new file mode 100644 index 0000000000000000000000000000000000000000..6085b32a503d264b0339b48a717ce7bde151f69c --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/.github/workflows/workflow.yml @@ -0,0 +1,81 @@ +name: CI +on: [push, pull_request] + +# Run linter with github actions for quick feedbacks. +# Run macos tests with github actions. Linux (CPU & GPU) tests currently runs on CircleCI +jobs: + linter: + runs-on: ubuntu-latest + # run on PRs, or commits to facebookresearch (not internal) + if: ${{ github.repository_owner == 'facebookresearch' || github.event_name == 'pull_request' }} + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.6 + uses: actions/setup-python@v2 + with: + python-version: 3.6 + - name: Install dependencies + # flake8-bugbear flake8-comprehensions are useful but not available internally + run: | + python -m pip install --upgrade pip + python -m pip install flake8==3.8.1 isort==4.3.21 + python -m pip install black==21.4b2 + flake8 --version + - name: Lint + run: | + echo "Running isort" + isort -c -sp . + echo "Running black" + black -l 100 --check . + echo "Running flake8" + flake8 . + + macos_tests: + runs-on: macos-latest + # run on PRs, or commits to facebookresearch (not internal) + if: ${{ github.repository_owner == 'facebookresearch' || github.event_name == 'pull_request' }} + strategy: + fail-fast: false + matrix: + torch: ["1.8", "1.9", "1.10"] + include: + - torch: "1.8" + torchvision: 0.9 + - torch: "1.9" + torchvision: "0.10" + - torch: "1.10" + torchvision: "0.11.1" + env: + # point datasets to ~/.torch so it's cached by CI + DETECTRON2_DATASETS: ~/.torch/datasets + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up Python 3.6 + uses: actions/setup-python@v2 + with: + python-version: 3.6 + - name: Cache dependencies + uses: actions/cache@v2 + with: + path: | + ${{ env.pythonLocation }}/lib/python3.6/site-packages + ~/.torch + key: ${{ runner.os }}-torch${{ matrix.torch }}-${{ hashFiles('setup.py') }}-20210420 + + - name: Install dependencies + run: | + python -m pip install -U pip + python -m pip install ninja opencv-python-headless onnx pytest-xdist + python -m pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html + # install from github to get latest; install iopath first since fvcore depends on it + python -m pip install -U 'git+https://github.com/facebookresearch/iopath' + python -m pip install -U 'git+https://github.com/facebookresearch/fvcore' + + - name: Build and install + run: | + CC=clang CXX=clang++ python -m pip install -e .[all] + python -m detectron2.utils.collect_env + ./datasets/prepare_for_tests.sh + - name: Run unittests + run: python -m pytest -n 4 --durations=15 -v tests/ diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/.gitignore b/approach/ovod/mm-ovod/third_party/CenterNet2/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..e045ffa557a8ca047d61386c8018b9fe965953bf --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/.gitignore @@ -0,0 +1,58 @@ +third_party/detectron2 +slurm* +# output dir +output +instant_test_output +inference_test_output + + +*.png +*.json +*.diff +# *.jpg +!/projects/DensePose/doc/images/*.jpg + +# compilation and distribution +__pycache__ +_ext +*.pyc +*.pyd +*.so +*.dll +*.egg-info/ +build/ +dist/ +wheels/ + +# pytorch/python/numpy formats +*.pth +*.pkl +*.npy +*.ts +model_ts*.txt + +# ipython/jupyter notebooks +*.ipynb +**/.ipynb_checkpoints/ + +# Editor temporaries +*.swn +*.swo +*.swp +*~ + +# editor settings +.idea +.vscode +_darcs + +# project dirs +/detectron2/model_zoo/configs +/datasets/* +!/datasets/*.* +!/datasets/lvis/ +/datasets/lvis/* +!/datasets/lvis/lvis_v1_train_cat_info.json +/projects/*/datasets +/models +/snippet diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/LICENSE b/approach/ovod/mm-ovod/third_party/CenterNet2/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..cd1b070674331757508398d99c830664dce6eaec --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/LICENSE @@ -0,0 +1,202 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, +and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by +the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all +other entities that control, are controlled by, or are under common +control with that entity. For the purposes of this definition, +"control" means (i) the power, direct or indirect, to cause the +direction or management of such entity, whether by contract or +otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity +exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, +including but not limited to software source code, documentation +source, and configuration files. + +"Object" form shall mean any form resulting from mechanical +transformation or translation of a Source form, including but +not limited to compiled object code, generated documentation, +and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or +Object form, made available under the License, as indicated by a +copyright notice that is included in or attached to the work +(an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object +form, that is based on (or derived from) the Work and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. For the purposes +of this License, Derivative Works shall not include works that remain +separable from, or merely link (or bind by name) to the interfaces of, +the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including +the original version of the Work and any modifications or additions +to that Work or Derivative Works thereof, that is intentionally +submitted to Licensor for inclusion in the Work by the copyright owner +or by an individual or Legal Entity authorized to submit on behalf of +the copyright owner. For the purposes of this definition, "submitted" +means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, +and issue tracking systems that are managed by, or on behalf of, the +Licensor for the purpose of discussing and improving the Work, but +excluding communication that is conspicuously marked or otherwise +designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity +on behalf of whom a Contribution has been received by Licensor and +subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the +Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except as stated in this section) patent license to make, have made, +use, offer to sell, sell, import, and otherwise transfer the Work, +where such license applies only to those patent claims licensable +by such Contributor that are necessarily infringed by their +Contribution(s) alone or by combination of their Contribution(s) +with the Work to which such Contribution(s) was submitted. If You +institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work +or a Contribution incorporated within the Work constitutes direct +or contributory patent infringement, then any patent licenses +granted to You under this License for that Work shall terminate +as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the +Work or Derivative Works thereof in any medium, with or without +modifications, and in Source or Object form, provided that You +meet the following conditions: + +(a) You must give any other recipients of the Work or +Derivative Works a copy of this License; and + +(b) You must cause any modified files to carry prominent notices +stating that You changed the files; and + +(c) You must retain, in the Source form of any Derivative Works +that You distribute, all copyright, patent, trademark, and +attribution notices from the Source form of the Work, +excluding those notices that do not pertain to any part of +the Derivative Works; and + +(d) If the Work includes a "NOTICE" text file as part of its +distribution, then any Derivative Works that You distribute must +include a readable copy of the attribution notices contained +within such NOTICE file, excluding those notices that do not +pertain to any part of the Derivative Works, in at least one +of the following places: within a NOTICE text file distributed +as part of the Derivative Works; within the Source form or +documentation, if provided along with the Derivative Works; or, +within a display generated by the Derivative Works, if and +wherever such third-party notices normally appear. The contents +of the NOTICE file are for informational purposes only and +do not modify the License. You may add Your own attribution +notices within Derivative Works that You distribute, alongside +or as an addendum to the NOTICE text from the Work, provided +that such additional attribution notices cannot be construed +as modifying the License. + +You may add Your own copyright statement to Your modifications and +may provide additional or different license terms and conditions +for use, reproduction, or distribution of Your modifications, or +for any such Derivative Works as a whole, provided Your use, +reproduction, and distribution of the Work otherwise complies with +the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, +any Contribution intentionally submitted for inclusion in the Work +by You to the Licensor shall be under the terms and conditions of +this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify +the terms of any separate license agreement you may have executed +with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade +names, trademarks, service marks, or product names of the Licensor, +except as required for reasonable and customary use in describing the +origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or +agreed to in writing, Licensor provides the Work (and each +Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied, including, without limitation, any warranties or conditions +of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A +PARTICULAR PURPOSE. You are solely responsible for determining the +appropriateness of using or redistributing the Work and assume any +risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, +whether in tort (including negligence), contract, or otherwise, +unless required by applicable law (such as deliberate and grossly +negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, +incidental, or consequential damages of any character arising as a +result of this License or out of the use or inability to use the +Work (including but not limited to damages for loss of goodwill, +work stoppage, computer failure or malfunction, or any and all +other commercial damages or losses), even if such Contributor +has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing +the Work or Derivative Works thereof, You may choose to offer, +and charge a fee for, acceptance of support, warranty, indemnity, +or other liability obligations and/or rights consistent with this +License. However, in accepting such obligations, You may act only +on Your own behalf and on Your sole responsibility, not on behalf +of any other Contributor, and only if You agree to indemnify, +defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason +of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following +boilerplate notice, with the fields enclosed by brackets "[]" +replaced with your own identifying information. (Don't include +the brackets!) The text should be enclosed in the appropriate +comment syntax for the file format. We also recommend that a +file or class name and description of purpose be included on the +same "printed page" as the copyright notice for easier +identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/README.md b/approach/ovod/mm-ovod/third_party/CenterNet2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ccbf8818f1c6576fcec8072835e5dddb1b9a2ed --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/README.md @@ -0,0 +1,81 @@ +# Probabilistic two-stage detection +Two-stage object detectors that use class-agnostic one-stage detectors as the proposal network. + + +

+ +> [**Probabilistic two-stage detection**](http://arxiv.org/abs/2103.07461), +> Xingyi Zhou, Vladlen Koltun, Philipp Krähenbühl, +> *arXiv technical report ([arXiv 2103.07461](http://arxiv.org/abs/2103.07461))* + +Contact: [zhouxy@cs.utexas.edu](mailto:zhouxy@cs.utexas.edu). Any questions or discussions are welcomed! + +## Summary + +- Two-stage CenterNet: First stage estimates object probabilities, second stage conditionally classifies objects. + +- Resulting detector is faster and more accurate than both traditional two-stage detectors (fewer proposals required), and one-stage detectors (lighter first stage head). + +- Our best model achieves 56.4 mAP on COCO test-dev. + +- This repo also includes a detectron2-based CenterNet implementation with better accuracy (42.5 mAP at 70FPS) and a new FPN version of CenterNet (40.2 mAP with Res50_1x). + +## Main results + +All models are trained with multi-scale training, and tested with a single scale. The FPS is tested on a Titan RTX GPU. +More models and details can be found in the [MODEL_ZOO](docs/MODEL_ZOO.md). + +#### COCO + +| Model | COCO val mAP | FPS | +|-------------------------------------------|---------------|-------| +| CenterNet-S4_DLA_8x | 42.5 | 71 | +| CenterNet2_R50_1x | 42.9 | 24 | +| CenterNet2_X101-DCN_2x | 49.9 | 8 | +| CenterNet2_R2-101-DCN-BiFPN_4x+4x_1560_ST | 56.1 | 5 | +| CenterNet2_DLA-BiFPN-P5_24x_ST | 49.2 | 38 | + + +#### LVIS + +| Model | val mAP box | +| ------------------------- | ----------- | +| CenterNet2_R50_1x | 26.5 | +| CenterNet2_FedLoss_R50_1x | 28.3 | + + +#### Objects365 + +| Model | val mAP | +|-------------------------------------------|----------| +| CenterNet2_R50_1x | 22.6 | + +## Installation + +Our project is developed on [detectron2](https://github.com/facebookresearch/detectron2). Please follow the official detectron2 [installation](https://github.com/facebookresearch/detectron2/blob/master/INSTALL.md). + +We use the default detectron2 demo script. To run inference on an image folder using our pre-trained model, run + +~~~ +python demo.py --config-file configs/CenterNet2_R50_1x.yaml --input path/to/image/ --opts MODEL.WEIGHTS models/CenterNet2_R50_1x.pth +~~~ + +## Benchmark evaluation and training + +Please check detectron2 [GETTING_STARTED.md](https://github.com/facebookresearch/detectron2/blob/master/GETTING_STARTED.md) for running evaluation and training. Our config files are under `configs` and the pre-trained models are in the [MODEL_ZOO](docs/MODEL_ZOO.md). + + +## License + +Our code is under [Apache 2.0 license](LICENSE). `centernet/modeling/backbone/bifpn_fcos.py` are from [AdelaiDet](https://github.com/aim-uofa/AdelaiDet), which follows the original [non-commercial license](https://github.com/aim-uofa/AdelaiDet/blob/master/LICENSE). + +## Citation + +If you find this project useful for your research, please use the following BibTeX entry. + + @inproceedings{zhou2021probablistic, + title={Probabilistic two-stage detection}, + author={Zhou, Xingyi and Koltun, Vladlen and Kr{\"a}henb{\"u}hl, Philipp}, + booktitle={arXiv preprint arXiv:2103.07461}, + year={2021} + } diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/__init__.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e17db317d9ba31b9f027bc922332c1ca8b039128 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/__init__.py @@ -0,0 +1,14 @@ +from .modeling.meta_arch.centernet_detector import CenterNetDetector +from .modeling.dense_heads.centernet import CenterNet +from .modeling.roi_heads.custom_roi_heads import CustomROIHeads, CustomCascadeROIHeads + +from .modeling.backbone.fpn_p5 import build_p67_resnet_fpn_backbone +from .modeling.backbone.dla import build_dla_backbone +from .modeling.backbone.dlafpn import build_dla_fpn3_backbone +from .modeling.backbone.bifpn import build_resnet_bifpn_backbone +from .modeling.backbone.bifpn_fcos import build_fcos_resnet_bifpn_backbone +from .modeling.backbone.res2net import build_p67_res2net_fpn_backbone + +from .data.datasets.objects365 import categories_v1 +from .data.datasets.coco import _PREDEFINED_SPLITS_COCO +from .data.datasets import nuimages diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/config.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/config.py new file mode 100644 index 0000000000000000000000000000000000000000..82c44fa640cba12fa41ab7d369a2e84405942e8c --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/config.py @@ -0,0 +1,88 @@ +from detectron2.config import CfgNode as CN + +def add_centernet_config(cfg): + _C = cfg + + _C.MODEL.CENTERNET = CN() + _C.MODEL.CENTERNET.NUM_CLASSES = 80 + _C.MODEL.CENTERNET.IN_FEATURES = ["p3", "p4", "p5", "p6", "p7"] + _C.MODEL.CENTERNET.FPN_STRIDES = [8, 16, 32, 64, 128] + _C.MODEL.CENTERNET.PRIOR_PROB = 0.01 + _C.MODEL.CENTERNET.INFERENCE_TH = 0.05 + _C.MODEL.CENTERNET.CENTER_NMS = False + _C.MODEL.CENTERNET.NMS_TH_TRAIN = 0.6 + _C.MODEL.CENTERNET.NMS_TH_TEST = 0.6 + _C.MODEL.CENTERNET.PRE_NMS_TOPK_TRAIN = 1000 + _C.MODEL.CENTERNET.POST_NMS_TOPK_TRAIN = 100 + _C.MODEL.CENTERNET.PRE_NMS_TOPK_TEST = 1000 + _C.MODEL.CENTERNET.POST_NMS_TOPK_TEST = 100 + _C.MODEL.CENTERNET.NORM = "GN" + _C.MODEL.CENTERNET.USE_DEFORMABLE = False + _C.MODEL.CENTERNET.NUM_CLS_CONVS = 4 + _C.MODEL.CENTERNET.NUM_BOX_CONVS = 4 + _C.MODEL.CENTERNET.NUM_SHARE_CONVS = 0 + _C.MODEL.CENTERNET.LOC_LOSS_TYPE = 'giou' + _C.MODEL.CENTERNET.SIGMOID_CLAMP = 1e-4 + _C.MODEL.CENTERNET.HM_MIN_OVERLAP = 0.8 + _C.MODEL.CENTERNET.MIN_RADIUS = 4 + _C.MODEL.CENTERNET.SOI = [[0, 80], [64, 160], [128, 320], [256, 640], [512, 10000000]] + _C.MODEL.CENTERNET.POS_WEIGHT = 1. + _C.MODEL.CENTERNET.NEG_WEIGHT = 1. + _C.MODEL.CENTERNET.REG_WEIGHT = 2. + _C.MODEL.CENTERNET.HM_FOCAL_BETA = 4 + _C.MODEL.CENTERNET.HM_FOCAL_ALPHA = 0.25 + _C.MODEL.CENTERNET.LOSS_GAMMA = 2.0 + _C.MODEL.CENTERNET.WITH_AGN_HM = False + _C.MODEL.CENTERNET.ONLY_PROPOSAL = False + _C.MODEL.CENTERNET.AS_PROPOSAL = False + _C.MODEL.CENTERNET.IGNORE_HIGH_FP = -1. + _C.MODEL.CENTERNET.MORE_POS = False + _C.MODEL.CENTERNET.MORE_POS_THRESH = 0.2 + _C.MODEL.CENTERNET.MORE_POS_TOPK = 9 + _C.MODEL.CENTERNET.NOT_NORM_REG = True + _C.MODEL.CENTERNET.NOT_NMS = False + _C.MODEL.CENTERNET.NO_REDUCE = False + + _C.MODEL.ROI_BOX_HEAD.USE_SIGMOID_CE = False + _C.MODEL.ROI_BOX_HEAD.PRIOR_PROB = 0.01 + _C.MODEL.ROI_BOX_HEAD.USE_EQL_LOSS = False + _C.MODEL.ROI_BOX_HEAD.CAT_FREQ_PATH = \ + 'datasets/lvis/lvis_v1_train_cat_info.json' + _C.MODEL.ROI_BOX_HEAD.EQL_FREQ_CAT = 200 + _C.MODEL.ROI_BOX_HEAD.USE_FED_LOSS = False + _C.MODEL.ROI_BOX_HEAD.FED_LOSS_NUM_CAT = 50 + _C.MODEL.ROI_BOX_HEAD.FED_LOSS_FREQ_WEIGHT = 0.5 + _C.MODEL.ROI_BOX_HEAD.MULT_PROPOSAL_SCORE = False + + _C.MODEL.BIFPN = CN() + _C.MODEL.BIFPN.NUM_LEVELS = 5 + _C.MODEL.BIFPN.NUM_BIFPN = 6 + _C.MODEL.BIFPN.NORM = 'GN' + _C.MODEL.BIFPN.OUT_CHANNELS = 160 + _C.MODEL.BIFPN.SEPARABLE_CONV = False + + _C.MODEL.DLA = CN() + _C.MODEL.DLA.OUT_FEATURES = ['dla2'] + _C.MODEL.DLA.USE_DLA_UP = True + _C.MODEL.DLA.NUM_LAYERS = 34 + _C.MODEL.DLA.MS_OUTPUT = False + _C.MODEL.DLA.NORM = 'BN' + _C.MODEL.DLA.DLAUP_IN_FEATURES = ['dla3', 'dla4', 'dla5'] + _C.MODEL.DLA.DLAUP_NODE = 'conv' + + _C.SOLVER.RESET_ITER = False + _C.SOLVER.TRAIN_ITER = -1 + + _C.INPUT.CUSTOM_AUG = '' + _C.INPUT.TRAIN_SIZE = 640 + _C.INPUT.TEST_SIZE = 640 + _C.INPUT.SCALE_RANGE = (0.1, 2.) + # 'default' for fixed short/ long edge, 'square' for max size=INPUT.SIZE + _C.INPUT.TEST_INPUT_TYPE = 'default' + _C.INPUT.NOT_CLAMP_BOX = False + + _C.DEBUG = False + _C.SAVE_DEBUG = False + _C.SAVE_PTH = False + _C.VIS_THRESH = 0.3 + _C.DEBUG_SHOW_NAME = False diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/bifpn.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/bifpn.py new file mode 100644 index 0000000000000000000000000000000000000000..565e2940ad0e4c43ec2172d4a79a9bd72adef09e --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/bifpn.py @@ -0,0 +1,425 @@ +# Modified from https://github.com/rwightman/efficientdet-pytorch/blob/master/effdet/efficientdet.py +# The original file is under Apache-2.0 License +import math +from os.path import join +import numpy as np +from collections import OrderedDict +from typing import List + +import torch +from torch import nn +import torch.utils.model_zoo as model_zoo +import torch.nn.functional as F +import fvcore.nn.weight_init as weight_init + +from detectron2.layers import ShapeSpec, Conv2d +from detectron2.modeling.backbone.resnet import build_resnet_backbone +from detectron2.modeling.backbone.build import BACKBONE_REGISTRY +from detectron2.layers.batch_norm import get_norm +from detectron2.modeling.backbone import Backbone +from .dlafpn import dla34 + +def get_fpn_config(base_reduction=8): + """BiFPN config with sum.""" + p = { + 'nodes': [ + {'reduction': base_reduction << 3, 'inputs_offsets': [3, 4]}, + {'reduction': base_reduction << 2, 'inputs_offsets': [2, 5]}, + {'reduction': base_reduction << 1, 'inputs_offsets': [1, 6]}, + {'reduction': base_reduction, 'inputs_offsets': [0, 7]}, + {'reduction': base_reduction << 1, 'inputs_offsets': [1, 7, 8]}, + {'reduction': base_reduction << 2, 'inputs_offsets': [2, 6, 9]}, + {'reduction': base_reduction << 3, 'inputs_offsets': [3, 5, 10]}, + {'reduction': base_reduction << 4, 'inputs_offsets': [4, 11]}, + ], + 'weight_method': 'fastattn', + } + return p + + +def swish(x, inplace: bool = False): + """Swish - Described in: https://arxiv.org/abs/1710.05941 + """ + return x.mul_(x.sigmoid()) if inplace else x.mul(x.sigmoid()) + + +class Swish(nn.Module): + def __init__(self, inplace: bool = False): + super(Swish, self).__init__() + self.inplace = inplace + + def forward(self, x): + return swish(x, self.inplace) + + +class SequentialAppend(nn.Sequential): + def __init__(self, *args): + super(SequentialAppend, self).__init__(*args) + + def forward(self, x): + for module in self: + x.append(module(x)) + return x + + +class SequentialAppendLast(nn.Sequential): + def __init__(self, *args): + super(SequentialAppendLast, self).__init__(*args) + + # def forward(self, x: List[torch.Tensor]): + def forward(self, x): + for module in self: + x.append(module(x[-1])) + return x + + +class ConvBnAct2d(nn.Module): + def __init__(self, in_channels, out_channels, kernel_size, stride=1, dilation=1, padding='', bias=False, + norm='', act_layer=Swish): + super(ConvBnAct2d, self).__init__() + # self.conv = create_conv2d( + # in_channels, out_channels, kernel_size, stride=stride, dilation=dilation, padding=padding, bias=bias) + self.conv = Conv2d( + in_channels, out_channels, kernel_size=kernel_size, stride=stride, + padding=kernel_size // 2, bias=(norm == '')) + self.bn = get_norm(norm, out_channels) + self.act = None if act_layer is None else act_layer(inplace=True) + + def forward(self, x): + x = self.conv(x) + if self.bn is not None: + x = self.bn(x) + if self.act is not None: + x = self.act(x) + return x + + +class SeparableConv2d(nn.Module): + """ Separable Conv + """ + def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, dilation=1, padding='', bias=False, + channel_multiplier=1.0, pw_kernel_size=1, act_layer=Swish, + norm=''): + super(SeparableConv2d, self).__init__() + + # self.conv_dw = create_conv2d( + # in_channels, int(in_channels * channel_multiplier), kernel_size, + # stride=stride, dilation=dilation, padding=padding, depthwise=True) + + self.conv_dw = Conv2d( + in_channels, int(in_channels * channel_multiplier), + kernel_size=kernel_size, stride=stride, padding=kernel_size // 2, bias=bias, + groups=out_channels) + # print('conv_dw', kernel_size, stride) + # self.conv_pw = create_conv2d( + # int(in_channels * channel_multiplier), out_channels, pw_kernel_size, padding=padding, bias=bias) + + self.conv_pw = Conv2d( + int(in_channels * channel_multiplier), out_channels, + kernel_size=pw_kernel_size, padding=pw_kernel_size // 2, bias=(norm=='')) + # print('conv_pw', pw_kernel_size) + + self.bn = get_norm(norm, out_channels) + self.act = None if act_layer is None else act_layer(inplace=True) + + def forward(self, x): + x = self.conv_dw(x) + x = self.conv_pw(x) + if self.bn is not None: + x = self.bn(x) + if self.act is not None: + x = self.act(x) + return x + + +class ResampleFeatureMap(nn.Sequential): + def __init__(self, in_channels, out_channels, reduction_ratio=1., pad_type='', pooling_type='max', + norm='', apply_bn=False, conv_after_downsample=False, + redundant_bias=False): + super(ResampleFeatureMap, self).__init__() + pooling_type = pooling_type or 'max' + self.in_channels = in_channels + self.out_channels = out_channels + self.reduction_ratio = reduction_ratio + self.conv_after_downsample = conv_after_downsample + + conv = None + if in_channels != out_channels: + conv = ConvBnAct2d( + in_channels, out_channels, kernel_size=1, padding=pad_type, + norm=norm if apply_bn else '', + bias=not apply_bn or redundant_bias, act_layer=None) + + if reduction_ratio > 1: + stride_size = int(reduction_ratio) + if conv is not None and not self.conv_after_downsample: + self.add_module('conv', conv) + self.add_module( + 'downsample', + # create_pool2d( + # pooling_type, kernel_size=stride_size + 1, stride=stride_size, padding=pad_type) + # nn.MaxPool2d(kernel_size=stride_size + 1, stride=stride_size, padding=pad_type) + nn.MaxPool2d(kernel_size=stride_size, stride=stride_size) + ) + if conv is not None and self.conv_after_downsample: + self.add_module('conv', conv) + else: + if conv is not None: + self.add_module('conv', conv) + if reduction_ratio < 1: + scale = int(1 // reduction_ratio) + self.add_module('upsample', nn.UpsamplingNearest2d(scale_factor=scale)) + + +class FpnCombine(nn.Module): + def __init__(self, feature_info, fpn_config, fpn_channels, inputs_offsets, target_reduction, pad_type='', + pooling_type='max', norm='', apply_bn_for_resampling=False, + conv_after_downsample=False, redundant_bias=False, weight_method='attn'): + super(FpnCombine, self).__init__() + self.inputs_offsets = inputs_offsets + self.weight_method = weight_method + + self.resample = nn.ModuleDict() + for idx, offset in enumerate(inputs_offsets): + in_channels = fpn_channels + if offset < len(feature_info): + in_channels = feature_info[offset]['num_chs'] + input_reduction = feature_info[offset]['reduction'] + else: + node_idx = offset - len(feature_info) + # print('node_idx, len', node_idx, len(fpn_config['nodes'])) + input_reduction = fpn_config['nodes'][node_idx]['reduction'] + reduction_ratio = target_reduction / input_reduction + self.resample[str(offset)] = ResampleFeatureMap( + in_channels, fpn_channels, reduction_ratio=reduction_ratio, pad_type=pad_type, + pooling_type=pooling_type, norm=norm, + apply_bn=apply_bn_for_resampling, conv_after_downsample=conv_after_downsample, + redundant_bias=redundant_bias) + + if weight_method == 'attn' or weight_method == 'fastattn': + # WSM + self.edge_weights = nn.Parameter(torch.ones(len(inputs_offsets)), requires_grad=True) + else: + self.edge_weights = None + + def forward(self, x): + dtype = x[0].dtype + nodes = [] + for offset in self.inputs_offsets: + input_node = x[offset] + input_node = self.resample[str(offset)](input_node) + nodes.append(input_node) + + if self.weight_method == 'attn': + normalized_weights = torch.softmax(self.edge_weights.type(dtype), dim=0) + x = torch.stack(nodes, dim=-1) * normalized_weights + elif self.weight_method == 'fastattn': + edge_weights = nn.functional.relu(self.edge_weights.type(dtype)) + weights_sum = torch.sum(edge_weights) + x = torch.stack( + [(nodes[i] * edge_weights[i]) / (weights_sum + 0.0001) for i in range(len(nodes))], dim=-1) + elif self.weight_method == 'sum': + x = torch.stack(nodes, dim=-1) + else: + raise ValueError('unknown weight_method {}'.format(self.weight_method)) + x = torch.sum(x, dim=-1) + return x + + +class BiFpnLayer(nn.Module): + def __init__(self, feature_info, fpn_config, fpn_channels, num_levels=5, pad_type='', + pooling_type='max', norm='', act_layer=Swish, + apply_bn_for_resampling=False, conv_after_downsample=True, conv_bn_relu_pattern=False, + separable_conv=True, redundant_bias=False): + super(BiFpnLayer, self).__init__() + self.fpn_config = fpn_config + self.num_levels = num_levels + self.conv_bn_relu_pattern = False + + self.feature_info = [] + self.fnode = SequentialAppend() + for i, fnode_cfg in enumerate(fpn_config['nodes']): + # logging.debug('fnode {} : {}'.format(i, fnode_cfg)) + # print('fnode {} : {}'.format(i, fnode_cfg)) + fnode_layers = OrderedDict() + + # combine features + reduction = fnode_cfg['reduction'] + fnode_layers['combine'] = FpnCombine( + feature_info, fpn_config, fpn_channels, fnode_cfg['inputs_offsets'], target_reduction=reduction, + pad_type=pad_type, pooling_type=pooling_type, norm=norm, + apply_bn_for_resampling=apply_bn_for_resampling, conv_after_downsample=conv_after_downsample, + redundant_bias=redundant_bias, weight_method=fpn_config['weight_method']) + self.feature_info.append(dict(num_chs=fpn_channels, reduction=reduction)) + + # after combine ops + after_combine = OrderedDict() + if not conv_bn_relu_pattern: + after_combine['act'] = act_layer(inplace=True) + conv_bias = redundant_bias + conv_act = None + else: + conv_bias = False + conv_act = act_layer + conv_kwargs = dict( + in_channels=fpn_channels, out_channels=fpn_channels, kernel_size=3, padding=pad_type, + bias=conv_bias, norm=norm, act_layer=conv_act) + after_combine['conv'] = SeparableConv2d(**conv_kwargs) if separable_conv else ConvBnAct2d(**conv_kwargs) + fnode_layers['after_combine'] = nn.Sequential(after_combine) + + self.fnode.add_module(str(i), nn.Sequential(fnode_layers)) + + self.feature_info = self.feature_info[-num_levels::] + + def forward(self, x): + x = self.fnode(x) + return x[-self.num_levels::] + + +class BiFPN(Backbone): + def __init__( + self, cfg, bottom_up, in_features, out_channels, norm='', + num_levels=5, num_bifpn=4, separable_conv=False, + ): + super(BiFPN, self).__init__() + assert isinstance(bottom_up, Backbone) + + # Feature map strides and channels from the bottom up network (e.g. ResNet) + input_shapes = bottom_up.output_shape() + in_strides = [input_shapes[f].stride for f in in_features] + in_channels = [input_shapes[f].channels for f in in_features] + + self.num_levels = num_levels + self.num_bifpn = num_bifpn + self.bottom_up = bottom_up + self.in_features = in_features + self._size_divisibility = 128 + levels = [int(math.log2(s)) for s in in_strides] + self._out_feature_strides = { + "p{}".format(int(math.log2(s))): s for s in in_strides} + if len(in_features) < num_levels: + for l in range(num_levels - len(in_features)): + s = l + levels[-1] + self._out_feature_strides["p{}".format(s + 1)] = 2 ** (s + 1) + self._out_features = list(sorted(self._out_feature_strides.keys())) + self._out_feature_channels = {k: out_channels for k in self._out_features} + + # print('self._out_feature_strides', self._out_feature_strides) + # print('self._out_feature_channels', self._out_feature_channels) + + feature_info = [ + {'num_chs': in_channels[level], 'reduction': in_strides[level]} \ + for level in range(len(self.in_features)) + ] + # self.config = config + fpn_config = get_fpn_config() + self.resample = SequentialAppendLast() + for level in range(num_levels): + if level < len(feature_info): + in_chs = in_channels[level] # feature_info[level]['num_chs'] + reduction = in_strides[level] # feature_info[level]['reduction'] + else: + # Adds a coarser level by downsampling the last feature map + reduction_ratio = 2 + self.resample.add_module(str(level), ResampleFeatureMap( + in_channels=in_chs, + out_channels=out_channels, + pad_type='same', + pooling_type=None, + norm=norm, + reduction_ratio=reduction_ratio, + apply_bn=True, + conv_after_downsample=False, + redundant_bias=False, + )) + in_chs = out_channels + reduction = int(reduction * reduction_ratio) + feature_info.append(dict(num_chs=in_chs, reduction=reduction)) + + self.cell = nn.Sequential() + for rep in range(self.num_bifpn): + # logging.debug('building cell {}'.format(rep)) + # print('building cell {}'.format(rep)) + fpn_layer = BiFpnLayer( + feature_info=feature_info, + fpn_config=fpn_config, + fpn_channels=out_channels, + num_levels=self.num_levels, + pad_type='same', + pooling_type=None, + norm=norm, + act_layer=Swish, + separable_conv=separable_conv, + apply_bn_for_resampling=True, + conv_after_downsample=False, + conv_bn_relu_pattern=False, + redundant_bias=False, + ) + self.cell.add_module(str(rep), fpn_layer) + feature_info = fpn_layer.feature_info + # import pdb; pdb.set_trace() + + @property + def size_divisibility(self): + return self._size_divisibility + + def forward(self, x): + # print('input shapes', x.shape) + bottom_up_features = self.bottom_up(x) + x = [bottom_up_features[f] for f in self.in_features] + assert len(self.resample) == self.num_levels - len(x) + x = self.resample(x) + shapes = [xx.shape for xx in x] + # print('resample shapes', shapes) + x = self.cell(x) + out = {f: xx for f, xx in zip(self._out_features, x)} + # import pdb; pdb.set_trace() + return out + + +@BACKBONE_REGISTRY.register() +def build_resnet_bifpn_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + bottom_up = build_resnet_backbone(cfg, input_shape) + in_features = cfg.MODEL.FPN.IN_FEATURES + backbone = BiFPN( + cfg=cfg, + bottom_up=bottom_up, + in_features=in_features, + out_channels=cfg.MODEL.BIFPN.OUT_CHANNELS, + norm=cfg.MODEL.BIFPN.NORM, + num_levels=cfg.MODEL.BIFPN.NUM_LEVELS, + num_bifpn=cfg.MODEL.BIFPN.NUM_BIFPN, + separable_conv=cfg.MODEL.BIFPN.SEPARABLE_CONV, + ) + return backbone + +@BACKBONE_REGISTRY.register() +def build_p37_dla_bifpn_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + bottom_up = dla34(cfg) + in_features = cfg.MODEL.FPN.IN_FEATURES + assert cfg.MODEL.BIFPN.NUM_LEVELS == 5 + + backbone = BiFPN( + cfg=cfg, + bottom_up=bottom_up, + in_features=in_features, + out_channels=cfg.MODEL.BIFPN.OUT_CHANNELS, + norm=cfg.MODEL.BIFPN.NORM, + num_levels=cfg.MODEL.BIFPN.NUM_LEVELS, + num_bifpn=cfg.MODEL.BIFPN.NUM_BIFPN, + separable_conv=cfg.MODEL.BIFPN.SEPARABLE_CONV, + ) + return backbone diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/bifpn_fcos.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/bifpn_fcos.py new file mode 100644 index 0000000000000000000000000000000000000000..17f2904ccad484f380b64efc668b9090d047d15e --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/bifpn_fcos.py @@ -0,0 +1,469 @@ +# This file is modified from https://github.com/aim-uofa/AdelaiDet/blob/master/adet/modeling/backbone/bifpn.py +# The original file is under 2-clause BSD License for academic use, and *non-commercial use*. +import torch +import torch.nn.functional as F +from torch import nn + +from detectron2.layers import Conv2d, ShapeSpec, get_norm + +from detectron2.modeling.backbone import Backbone, build_resnet_backbone +from detectron2.modeling import BACKBONE_REGISTRY +from .dlafpn import dla34 + +__all__ = [] + + +def swish(x): + return x * x.sigmoid() + + +def split_name(name): + for i, c in enumerate(name): + if not c.isalpha(): + return name[:i], int(name[i:]) + raise ValueError() + + +class FeatureMapResampler(nn.Module): + def __init__(self, in_channels, out_channels, stride, norm=""): + super(FeatureMapResampler, self).__init__() + if in_channels != out_channels: + self.reduction = Conv2d( + in_channels, out_channels, kernel_size=1, + bias=(norm == ""), + norm=get_norm(norm, out_channels), + activation=None + ) + else: + self.reduction = None + + assert stride <= 2 + self.stride = stride + + def forward(self, x): + if self.reduction is not None: + x = self.reduction(x) + + if self.stride == 2: + x = F.max_pool2d( + x, kernel_size=self.stride + 1, + stride=self.stride, padding=1 + ) + elif self.stride == 1: + pass + else: + raise NotImplementedError() + return x + + +class BackboneWithTopLevels(Backbone): + def __init__(self, backbone, out_channels, num_top_levels, norm=""): + super(BackboneWithTopLevels, self).__init__() + self.backbone = backbone + backbone_output_shape = backbone.output_shape() + + self._out_feature_channels = {name: shape.channels for name, shape in backbone_output_shape.items()} + self._out_feature_strides = {name: shape.stride for name, shape in backbone_output_shape.items()} + self._out_features = list(self._out_feature_strides.keys()) + + last_feature_name = max(self._out_feature_strides.keys(), key=lambda x: split_name(x)[1]) + self.last_feature_name = last_feature_name + self.num_top_levels = num_top_levels + + last_channels = self._out_feature_channels[last_feature_name] + last_stride = self._out_feature_strides[last_feature_name] + + prefix, suffix = split_name(last_feature_name) + prev_channels = last_channels + for i in range(num_top_levels): + name = prefix + str(suffix + i + 1) + self.add_module(name, FeatureMapResampler( + prev_channels, out_channels, 2, norm + )) + prev_channels = out_channels + + self._out_feature_channels[name] = out_channels + self._out_feature_strides[name] = last_stride * 2 ** (i + 1) + self._out_features.append(name) + + def forward(self, x): + outputs = self.backbone(x) + last_features = outputs[self.last_feature_name] + prefix, suffix = split_name(self.last_feature_name) + + x = last_features + for i in range(self.num_top_levels): + name = prefix + str(suffix + i + 1) + x = self.__getattr__(name)(x) + outputs[name] = x + + return outputs + + +class SingleBiFPN(Backbone): + """ + This module implements Feature Pyramid Network. + It creates pyramid features built on top of some input feature maps. + """ + + def __init__( + self, in_channels_list, out_channels, norm="" + ): + """ + Args: + bottom_up (Backbone): module representing the bottom up subnetwork. + Must be a subclass of :class:`Backbone`. The multi-scale feature + maps generated by the bottom up network, and listed in `in_features`, + are used to generate FPN levels. + in_features (list[str]): names of the input feature maps coming + from the backbone to which FPN is attached. For example, if the + backbone produces ["res2", "res3", "res4"], any *contiguous* sublist + of these may be used; order must be from high to low resolution. + out_channels (int): number of channels in the output feature maps. + norm (str): the normalization to use. + """ + super(SingleBiFPN, self).__init__() + + self.out_channels = out_channels + # build 5-levels bifpn + if len(in_channels_list) == 5: + self.nodes = [ + {'feat_level': 3, 'inputs_offsets': [3, 4]}, + {'feat_level': 2, 'inputs_offsets': [2, 5]}, + {'feat_level': 1, 'inputs_offsets': [1, 6]}, + {'feat_level': 0, 'inputs_offsets': [0, 7]}, + {'feat_level': 1, 'inputs_offsets': [1, 7, 8]}, + {'feat_level': 2, 'inputs_offsets': [2, 6, 9]}, + {'feat_level': 3, 'inputs_offsets': [3, 5, 10]}, + {'feat_level': 4, 'inputs_offsets': [4, 11]}, + ] + elif len(in_channels_list) == 3: + self.nodes = [ + {'feat_level': 1, 'inputs_offsets': [1, 2]}, + {'feat_level': 0, 'inputs_offsets': [0, 3]}, + {'feat_level': 1, 'inputs_offsets': [1, 3, 4]}, + {'feat_level': 2, 'inputs_offsets': [2, 5]}, + ] + else: + raise NotImplementedError + + node_info = [_ for _ in in_channels_list] + + num_output_connections = [0 for _ in in_channels_list] + for fnode in self.nodes: + feat_level = fnode["feat_level"] + inputs_offsets = fnode["inputs_offsets"] + inputs_offsets_str = "_".join(map(str, inputs_offsets)) + for input_offset in inputs_offsets: + num_output_connections[input_offset] += 1 + + in_channels = node_info[input_offset] + if in_channels != out_channels: + lateral_conv = Conv2d( + in_channels, + out_channels, + kernel_size=1, + norm=get_norm(norm, out_channels) + ) + self.add_module( + "lateral_{}_f{}".format(input_offset, feat_level), lateral_conv + ) + node_info.append(out_channels) + num_output_connections.append(0) + + # generate attention weights + name = "weights_f{}_{}".format(feat_level, inputs_offsets_str) + self.__setattr__(name, nn.Parameter( + torch.ones(len(inputs_offsets), dtype=torch.float32), + requires_grad=True + )) + + # generate convolutions after combination + name = "outputs_f{}_{}".format(feat_level, inputs_offsets_str) + self.add_module(name, Conv2d( + out_channels, + out_channels, + kernel_size=3, + padding=1, + norm=get_norm(norm, out_channels), + bias=(norm == "") + )) + + def forward(self, feats): + """ + Args: + input (dict[str->Tensor]): mapping feature map name (e.g., "p5") to + feature map tensor for each feature level in high to low resolution order. + Returns: + dict[str->Tensor]: + mapping from feature map name to FPN feature map tensor + in high to low resolution order. Returned feature names follow the FPN + paper convention: "p", where stage has stride = 2 ** stage e.g., + ["n2", "n3", ..., "n6"]. + """ + feats = [_ for _ in feats] + num_levels = len(feats) + num_output_connections = [0 for _ in feats] + for fnode in self.nodes: + feat_level = fnode["feat_level"] + inputs_offsets = fnode["inputs_offsets"] + inputs_offsets_str = "_".join(map(str, inputs_offsets)) + input_nodes = [] + _, _, target_h, target_w = feats[feat_level].size() + for input_offset in inputs_offsets: + num_output_connections[input_offset] += 1 + input_node = feats[input_offset] + + # reduction + if input_node.size(1) != self.out_channels: + name = "lateral_{}_f{}".format(input_offset, feat_level) + input_node = self.__getattr__(name)(input_node) + + # maybe downsample + _, _, h, w = input_node.size() + if h > target_h and w > target_w: + height_stride_size = int((h - 1) // target_h + 1) + width_stride_size = int((w - 1) // target_w + 1) + assert height_stride_size == width_stride_size == 2 + input_node = F.max_pool2d( + input_node, kernel_size=(height_stride_size + 1, width_stride_size + 1), + stride=(height_stride_size, width_stride_size), padding=1 + ) + elif h <= target_h and w <= target_w: + if h < target_h or w < target_w: + input_node = F.interpolate( + input_node, + size=(target_h, target_w), + mode="nearest" + ) + else: + raise NotImplementedError() + input_nodes.append(input_node) + + # attention + name = "weights_f{}_{}".format(feat_level, inputs_offsets_str) + weights = F.relu(self.__getattr__(name)) + norm_weights = weights / (weights.sum() + 0.0001) + + new_node = torch.stack(input_nodes, dim=-1) + new_node = (norm_weights * new_node).sum(dim=-1) + new_node = swish(new_node) + + name = "outputs_f{}_{}".format(feat_level, inputs_offsets_str) + feats.append(self.__getattr__(name)(new_node)) + + num_output_connections.append(0) + + output_feats = [] + for idx in range(num_levels): + for i, fnode in enumerate(reversed(self.nodes)): + if fnode['feat_level'] == idx: + output_feats.append(feats[-1 - i]) + break + else: + raise ValueError() + return output_feats + + +class BiFPN(Backbone): + """ + This module implements Feature Pyramid Network. + It creates pyramid features built on top of some input feature maps. + """ + + def __init__( + self, bottom_up, in_features, out_channels, num_top_levels, num_repeats, norm="" + ): + """ + Args: + bottom_up (Backbone): module representing the bottom up subnetwork. + Must be a subclass of :class:`Backbone`. The multi-scale feature + maps generated by the bottom up network, and listed in `in_features`, + are used to generate FPN levels. + in_features (list[str]): names of the input feature maps coming + from the backbone to which FPN is attached. For example, if the + backbone produces ["res2", "res3", "res4"], any *contiguous* sublist + of these may be used; order must be from high to low resolution. + out_channels (int): number of channels in the output feature maps. + num_top_levels (int): the number of the top levels (p6 or p7). + num_repeats (int): the number of repeats of BiFPN. + norm (str): the normalization to use. + """ + super(BiFPN, self).__init__() + assert isinstance(bottom_up, Backbone) + + # add extra feature levels (i.e., 6 and 7) + self.bottom_up = BackboneWithTopLevels( + bottom_up, out_channels, + num_top_levels, norm + ) + bottom_up_output_shapes = self.bottom_up.output_shape() + + in_features = sorted(in_features, key=lambda x: split_name(x)[1]) + self._size_divisibility = 128 #bottom_up_output_shapes[in_features[-1]].stride + self.out_channels = out_channels + self.min_level = split_name(in_features[0])[1] + + # add the names for top blocks + prefix, last_suffix = split_name(in_features[-1]) + for i in range(num_top_levels): + in_features.append(prefix + str(last_suffix + i + 1)) + self.in_features = in_features + + # generate output features + self._out_features = ["p{}".format(split_name(name)[1]) for name in in_features] + self._out_feature_strides = { + out_name: bottom_up_output_shapes[in_name].stride + for out_name, in_name in zip(self._out_features, in_features) + } + self._out_feature_channels = {k: out_channels for k in self._out_features} + + # build bifpn + self.repeated_bifpn = nn.ModuleList() + for i in range(num_repeats): + if i == 0: + in_channels_list = [ + bottom_up_output_shapes[name].channels for name in in_features + ] + else: + in_channels_list = [ + self._out_feature_channels[name] for name in self._out_features + ] + self.repeated_bifpn.append(SingleBiFPN( + in_channels_list, out_channels, norm + )) + + @property + def size_divisibility(self): + return self._size_divisibility + + def forward(self, x): + """ + Args: + input (dict[str->Tensor]): mapping feature map name (e.g., "p5") to + feature map tensor for each feature level in high to low resolution order. + Returns: + dict[str->Tensor]: + mapping from feature map name to FPN feature map tensor + in high to low resolution order. Returned feature names follow the FPN + paper convention: "p", where stage has stride = 2 ** stage e.g., + ["n2", "n3", ..., "n6"]. + """ + bottom_up_features = self.bottom_up(x) + feats = [bottom_up_features[f] for f in self.in_features] + + for bifpn in self.repeated_bifpn: + feats = bifpn(feats) + + return dict(zip(self._out_features, feats)) + + +def _assert_strides_are_log2_contiguous(strides): + """ + Assert that each stride is 2x times its preceding stride, i.e. "contiguous in log2". + """ + for i, stride in enumerate(strides[1:], 1): + assert stride == 2 * strides[i - 1], "Strides {} {} are not log2 contiguous".format( + stride, strides[i - 1] + ) + + +@BACKBONE_REGISTRY.register() +def build_fcos_resnet_bifpn_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + bottom_up = build_resnet_backbone(cfg, input_shape) + in_features = cfg.MODEL.FPN.IN_FEATURES + out_channels = cfg.MODEL.BIFPN.OUT_CHANNELS + num_repeats = cfg.MODEL.BIFPN.NUM_BIFPN + top_levels = 2 + + backbone = BiFPN( + bottom_up=bottom_up, + in_features=in_features, + out_channels=out_channels, + num_top_levels=top_levels, + num_repeats=num_repeats, + norm=cfg.MODEL.BIFPN.NORM + ) + return backbone + + + +@BACKBONE_REGISTRY.register() +def build_p35_fcos_resnet_bifpn_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + bottom_up = build_resnet_backbone(cfg, input_shape) + in_features = cfg.MODEL.FPN.IN_FEATURES + out_channels = cfg.MODEL.BIFPN.OUT_CHANNELS + num_repeats = cfg.MODEL.BIFPN.NUM_BIFPN + top_levels = 0 + + backbone = BiFPN( + bottom_up=bottom_up, + in_features=in_features, + out_channels=out_channels, + num_top_levels=top_levels, + num_repeats=num_repeats, + norm=cfg.MODEL.BIFPN.NORM + ) + return backbone + + +@BACKBONE_REGISTRY.register() +def build_p35_fcos_dla_bifpn_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + bottom_up = dla34(cfg) + in_features = cfg.MODEL.FPN.IN_FEATURES + out_channels = cfg.MODEL.BIFPN.OUT_CHANNELS + num_repeats = cfg.MODEL.BIFPN.NUM_BIFPN + top_levels = 0 + + backbone = BiFPN( + bottom_up=bottom_up, + in_features=in_features, + out_channels=out_channels, + num_top_levels=top_levels, + num_repeats=num_repeats, + norm=cfg.MODEL.BIFPN.NORM + ) + return backbone + +@BACKBONE_REGISTRY.register() +def build_p37_fcos_dla_bifpn_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + bottom_up = dla34(cfg) + in_features = cfg.MODEL.FPN.IN_FEATURES + out_channels = cfg.MODEL.BIFPN.OUT_CHANNELS + num_repeats = cfg.MODEL.BIFPN.NUM_BIFPN + assert cfg.MODEL.BIFPN.NUM_LEVELS == 5 + top_levels = 2 + + backbone = BiFPN( + bottom_up=bottom_up, + in_features=in_features, + out_channels=out_channels, + num_top_levels=top_levels, + num_repeats=num_repeats, + norm=cfg.MODEL.BIFPN.NORM + ) + return backbone \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/dla.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/dla.py new file mode 100644 index 0000000000000000000000000000000000000000..9f15f840355571b6d02d5534fa8a9b6b8cb22c70 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/dla.py @@ -0,0 +1,479 @@ +import numpy as np +import math +from os.path import join +import fvcore.nn.weight_init as weight_init +import torch +import torch.nn.functional as F +from torch import nn +import torch.utils.model_zoo as model_zoo + +from detectron2.modeling.backbone.resnet import ( + BasicStem, BottleneckBlock, DeformBottleneckBlock) +from detectron2.layers import ( + Conv2d, + DeformConv, + FrozenBatchNorm2d, + ModulatedDeformConv, + ShapeSpec, + get_norm, +) + +from detectron2.modeling.backbone.backbone import Backbone +from detectron2.modeling.backbone.build import BACKBONE_REGISTRY +from detectron2.modeling.backbone.fpn import FPN + +__all__ = [ + "BottleneckBlock", + "DeformBottleneckBlock", + "BasicStem", +] + +DCNV1 = False + +HASH = { + 34: 'ba72cf86', + 60: '24839fc4', +} + +def get_model_url(data, name, hash): + return join('http://dl.yf.io/dla/models', data, '{}-{}.pth'.format(name, hash)) + +class BasicBlock(nn.Module): + def __init__(self, inplanes, planes, stride=1, dilation=1, norm='BN'): + super(BasicBlock, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=3, + stride=stride, padding=dilation, + bias=False, dilation=dilation) + self.bn1 = get_norm(norm, planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, + stride=1, padding=dilation, + bias=False, dilation=dilation) + self.bn2 = get_norm(norm, planes) + self.stride = stride + + def forward(self, x, residual=None): + if residual is None: + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + out += residual + out = self.relu(out) + + return out + +class Bottleneck(nn.Module): + expansion = 2 + + def __init__(self, inplanes, planes, stride=1, dilation=1, norm='BN'): + super(Bottleneck, self).__init__() + expansion = Bottleneck.expansion + bottle_planes = planes // expansion + self.conv1 = nn.Conv2d(inplanes, bottle_planes, + kernel_size=1, bias=False) + self.bn1 = get_norm(norm, bottle_planes) + self.conv2 = nn.Conv2d(bottle_planes, bottle_planes, kernel_size=3, + stride=stride, padding=dilation, + bias=False, dilation=dilation) + self.bn2 = get_norm(norm, bottle_planes) + self.conv3 = nn.Conv2d(bottle_planes, planes, + kernel_size=1, bias=False) + self.bn3 = get_norm(norm, planes) + self.relu = nn.ReLU(inplace=True) + self.stride = stride + + def forward(self, x, residual=None): + if residual is None: + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + out += residual + out = self.relu(out) + + return out + +class Root(nn.Module): + def __init__(self, in_channels, out_channels, kernel_size, residual, norm='BN'): + super(Root, self).__init__() + self.conv = nn.Conv2d( + in_channels, out_channels, 1, + stride=1, bias=False, padding=(kernel_size - 1) // 2) + self.bn = get_norm(norm, out_channels) + self.relu = nn.ReLU(inplace=True) + self.residual = residual + + def forward(self, *x): + children = x + x = self.conv(torch.cat(x, 1)) + x = self.bn(x) + if self.residual: + x += children[0] + x = self.relu(x) + + return x + + +class Tree(nn.Module): + def __init__(self, levels, block, in_channels, out_channels, stride=1, + level_root=False, root_dim=0, root_kernel_size=1, + dilation=1, root_residual=False, norm='BN'): + super(Tree, self).__init__() + if root_dim == 0: + root_dim = 2 * out_channels + if level_root: + root_dim += in_channels + if levels == 1: + self.tree1 = block(in_channels, out_channels, stride, + dilation=dilation, norm=norm) + self.tree2 = block(out_channels, out_channels, 1, + dilation=dilation, norm=norm) + else: + self.tree1 = Tree(levels - 1, block, in_channels, out_channels, + stride, root_dim=0, + root_kernel_size=root_kernel_size, + dilation=dilation, root_residual=root_residual, + norm=norm) + self.tree2 = Tree(levels - 1, block, out_channels, out_channels, + root_dim=root_dim + out_channels, + root_kernel_size=root_kernel_size, + dilation=dilation, root_residual=root_residual, + norm=norm) + if levels == 1: + self.root = Root(root_dim, out_channels, root_kernel_size, + root_residual, norm=norm) + self.level_root = level_root + self.root_dim = root_dim + self.downsample = None + self.project = None + self.levels = levels + if stride > 1: + self.downsample = nn.MaxPool2d(stride, stride=stride) + if in_channels != out_channels: + self.project = nn.Sequential( + nn.Conv2d(in_channels, out_channels, + kernel_size=1, stride=1, bias=False), + get_norm(norm, out_channels) + ) + + def forward(self, x, residual=None, children=None): + children = [] if children is None else children + bottom = self.downsample(x) if self.downsample else x + residual = self.project(bottom) if self.project else bottom + if self.level_root: + children.append(bottom) + x1 = self.tree1(x, residual) + if self.levels == 1: + x2 = self.tree2(x1) + x = self.root(x2, x1, *children) + else: + children.append(x1) + x = self.tree2(x1, children=children) + return x + +class DLA(nn.Module): + def __init__(self, num_layers, levels, channels, + block=BasicBlock, residual_root=False, norm='BN'): + """ + Args: + """ + super(DLA, self).__init__() + self.norm = norm + self.channels = channels + self.base_layer = nn.Sequential( + nn.Conv2d(3, channels[0], kernel_size=7, stride=1, + padding=3, bias=False), + get_norm(self.norm, channels[0]), + nn.ReLU(inplace=True)) + self.level0 = self._make_conv_level( + channels[0], channels[0], levels[0]) + self.level1 = self._make_conv_level( + channels[0], channels[1], levels[1], stride=2) + self.level2 = Tree(levels[2], block, channels[1], channels[2], 2, + level_root=False, + root_residual=residual_root, norm=norm) + self.level3 = Tree(levels[3], block, channels[2], channels[3], 2, + level_root=True, root_residual=residual_root, + norm=norm) + self.level4 = Tree(levels[4], block, channels[3], channels[4], 2, + level_root=True, root_residual=residual_root, + norm=norm) + self.level5 = Tree(levels[5], block, channels[4], channels[5], 2, + level_root=True, root_residual=residual_root, + norm=norm) + self.load_pretrained_model( + data='imagenet', name='dla{}'.format(num_layers), + hash=HASH[num_layers]) + + def load_pretrained_model(self, data, name, hash): + model_url = get_model_url(data, name, hash) + model_weights = model_zoo.load_url(model_url) + num_classes = len(model_weights[list(model_weights.keys())[-1]]) + self.fc = nn.Conv2d( + self.channels[-1], num_classes, + kernel_size=1, stride=1, padding=0, bias=True) + print('Loading pretrained') + self.load_state_dict(model_weights, strict=False) + + def _make_conv_level(self, inplanes, planes, convs, stride=1, dilation=1): + modules = [] + for i in range(convs): + modules.extend([ + nn.Conv2d(inplanes, planes, kernel_size=3, + stride=stride if i == 0 else 1, + padding=dilation, bias=False, dilation=dilation), + get_norm(self.norm, planes), + nn.ReLU(inplace=True)]) + inplanes = planes + return nn.Sequential(*modules) + + def forward(self, x): + y = [] + x = self.base_layer(x) + for i in range(6): + x = getattr(self, 'level{}'.format(i))(x) + y.append(x) + return y + + +def fill_up_weights(up): + w = up.weight.data + f = math.ceil(w.size(2) / 2) + c = (2 * f - 1 - f % 2) / (2. * f) + for i in range(w.size(2)): + for j in range(w.size(3)): + w[0, 0, i, j] = \ + (1 - math.fabs(i / f - c)) * (1 - math.fabs(j / f - c)) + for c in range(1, w.size(0)): + w[c, 0, :, :] = w[0, 0, :, :] + + +class _DeformConv(nn.Module): + def __init__(self, chi, cho, norm='BN'): + super(_DeformConv, self).__init__() + self.actf = nn.Sequential( + get_norm(norm, cho), + nn.ReLU(inplace=True) + ) + if DCNV1: + self.offset = Conv2d( + chi, 18, kernel_size=3, stride=1, + padding=1, dilation=1) + self.conv = DeformConv( + chi, cho, kernel_size=(3,3), stride=1, padding=1, + dilation=1, deformable_groups=1) + else: + self.offset = Conv2d( + chi, 27, kernel_size=3, stride=1, + padding=1, dilation=1) + self.conv = ModulatedDeformConv( + chi, cho, kernel_size=3, stride=1, padding=1, + dilation=1, deformable_groups=1) + nn.init.constant_(self.offset.weight, 0) + nn.init.constant_(self.offset.bias, 0) + + def forward(self, x): + if DCNV1: + offset = self.offset(x) + x = self.conv(x, offset) + else: + offset_mask = self.offset(x) + offset_x, offset_y, mask = torch.chunk(offset_mask, 3, dim=1) + offset = torch.cat((offset_x, offset_y), dim=1) + mask = mask.sigmoid() + x = self.conv(x, offset, mask) + x = self.actf(x) + return x + + +class IDAUp(nn.Module): + def __init__(self, o, channels, up_f, norm='BN'): + super(IDAUp, self).__init__() + for i in range(1, len(channels)): + c = channels[i] + f = int(up_f[i]) + proj = _DeformConv(c, o, norm=norm) + node = _DeformConv(o, o, norm=norm) + + up = nn.ConvTranspose2d(o, o, f * 2, stride=f, + padding=f // 2, output_padding=0, + groups=o, bias=False) + fill_up_weights(up) + + setattr(self, 'proj_' + str(i), proj) + setattr(self, 'up_' + str(i), up) + setattr(self, 'node_' + str(i), node) + + + def forward(self, layers, startp, endp): + for i in range(startp + 1, endp): + upsample = getattr(self, 'up_' + str(i - startp)) + project = getattr(self, 'proj_' + str(i - startp)) + layers[i] = upsample(project(layers[i])) + node = getattr(self, 'node_' + str(i - startp)) + layers[i] = node(layers[i] + layers[i - 1]) + + +class DLAUp(nn.Module): + def __init__(self, startp, channels, scales, in_channels=None, norm='BN'): + super(DLAUp, self).__init__() + self.startp = startp + if in_channels is None: + in_channels = channels + self.channels = channels + channels = list(channels) + scales = np.array(scales, dtype=int) + for i in range(len(channels) - 1): + j = -i - 2 + setattr(self, 'ida_{}'.format(i), + IDAUp(channels[j], in_channels[j:], + scales[j:] // scales[j], norm=norm)) + scales[j + 1:] = scales[j] + in_channels[j + 1:] = [channels[j] for _ in channels[j + 1:]] + + def forward(self, layers): + out = [layers[-1]] # start with 32 + for i in range(len(layers) - self.startp - 1): + ida = getattr(self, 'ida_{}'.format(i)) + ida(layers, len(layers) -i - 2, len(layers)) + out.insert(0, layers[-1]) + return out + +DLA_CONFIGS = { + 34: ([1, 1, 1, 2, 2, 1], [16, 32, 64, 128, 256, 512], BasicBlock), + 60: ([1, 1, 1, 2, 3, 1], [16, 32, 128, 256, 512, 1024], Bottleneck) +} + + +class DLASeg(Backbone): + def __init__(self, num_layers, out_features, use_dla_up=True, + ms_output=False, norm='BN'): + super(DLASeg, self).__init__() + # depth = 34 + levels, channels, Block = DLA_CONFIGS[num_layers] + self.base = DLA(num_layers=num_layers, + levels=levels, channels=channels, block=Block, norm=norm) + down_ratio = 4 + self.first_level = int(np.log2(down_ratio)) + self.ms_output = ms_output + self.last_level = 5 if not self.ms_output else 6 + channels = self.base.channels + scales = [2 ** i for i in range(len(channels[self.first_level:]))] + self.use_dla_up = use_dla_up + if self.use_dla_up: + self.dla_up = DLAUp( + self.first_level, channels[self.first_level:], scales, + norm=norm) + out_channel = channels[self.first_level] + if not self.ms_output: # stride 4 DLA + self.ida_up = IDAUp( + out_channel, channels[self.first_level:self.last_level], + [2 ** i for i in range(self.last_level - self.first_level)], + norm=norm) + self._out_features = out_features + self._out_feature_channels = { + 'dla{}'.format(i): channels[i] for i in range(6)} + self._out_feature_strides = { + 'dla{}'.format(i): 2 ** i for i in range(6)} + self._size_divisibility = 32 + + @property + def size_divisibility(self): + return self._size_divisibility + + def forward(self, x): + x = self.base(x) + if self.use_dla_up: + x = self.dla_up(x) + if not self.ms_output: # stride 4 dla + y = [] + for i in range(self.last_level - self.first_level): + y.append(x[i].clone()) + self.ida_up(y, 0, len(y)) + ret = {} + for i in range(self.last_level - self.first_level): + out_feature = 'dla{}'.format(i) + if out_feature in self._out_features: + ret[out_feature] = y[i] + else: + ret = {} + st = self.first_level if self.use_dla_up else 0 + for i in range(self.last_level - st): + out_feature = 'dla{}'.format(i + st) + if out_feature in self._out_features: + ret[out_feature] = x[i] + + return ret + + +@BACKBONE_REGISTRY.register() +def build_dla_backbone(cfg, input_shape): + """ + Create a ResNet instance from config. + + Returns: + ResNet: a :class:`ResNet` instance. + """ + return DLASeg( + out_features=cfg.MODEL.DLA.OUT_FEATURES, + num_layers=cfg.MODEL.DLA.NUM_LAYERS, + use_dla_up=cfg.MODEL.DLA.USE_DLA_UP, + ms_output=cfg.MODEL.DLA.MS_OUTPUT, + norm=cfg.MODEL.DLA.NORM) + +class LastLevelP6P7(nn.Module): + """ + This module is used in RetinaNet to generate extra layers, P6 and P7 from + C5 feature. + """ + + def __init__(self, in_channels, out_channels): + super().__init__() + self.num_levels = 2 + self.in_feature = "dla5" + self.p6 = nn.Conv2d(in_channels, out_channels, 3, 2, 1) + self.p7 = nn.Conv2d(out_channels, out_channels, 3, 2, 1) + for module in [self.p6, self.p7]: + weight_init.c2_xavier_fill(module) + + def forward(self, c5): + p6 = self.p6(c5) + p7 = self.p7(F.relu(p6)) + return [p6, p7] + +@BACKBONE_REGISTRY.register() +def build_retinanet_dla_fpn_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + bottom_up = build_dla_backbone(cfg, input_shape) + in_features = cfg.MODEL.FPN.IN_FEATURES + out_channels = cfg.MODEL.FPN.OUT_CHANNELS + in_channels_p6p7 = bottom_up.output_shape()['dla5'].channels + backbone = FPN( + bottom_up=bottom_up, + in_features=in_features, + out_channels=out_channels, + norm=cfg.MODEL.FPN.NORM, + top_block=LastLevelP6P7(in_channels_p6p7, out_channels), + fuse_type=cfg.MODEL.FPN.FUSE_TYPE, + ) + return backbone diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/dlafpn.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/dlafpn.py new file mode 100644 index 0000000000000000000000000000000000000000..2a33c66bf3d5b97bf882eaf0b80de012151a62b4 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/dlafpn.py @@ -0,0 +1,493 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# this file is from https://github.com/ucbdrive/dla/blob/master/dla.py. + +import math +from os.path import join +import numpy as np + +import torch +from torch import nn +import torch.utils.model_zoo as model_zoo +import torch.nn.functional as F +import fvcore.nn.weight_init as weight_init + +from detectron2.modeling.backbone import FPN +from detectron2.layers import ShapeSpec, ModulatedDeformConv, Conv2d +from detectron2.modeling.backbone.build import BACKBONE_REGISTRY +from detectron2.layers.batch_norm import get_norm +from detectron2.modeling.backbone import Backbone + +WEB_ROOT = 'http://dl.yf.io/dla/models' + + +def get_model_url(data, name, hash): + return join( + 'http://dl.yf.io/dla/models', data, '{}-{}.pth'.format(name, hash)) + + +def conv3x3(in_planes, out_planes, stride=1): + "3x3 convolution with padding" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + + +class BasicBlock(nn.Module): + def __init__(self, cfg, inplanes, planes, stride=1, dilation=1): + super(BasicBlock, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=3, + stride=stride, padding=dilation, + bias=False, dilation=dilation) + self.bn1 = get_norm(cfg.MODEL.DLA.NORM, planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, + stride=1, padding=dilation, + bias=False, dilation=dilation) + self.bn2 = get_norm(cfg.MODEL.DLA.NORM, planes) + self.stride = stride + + def forward(self, x, residual=None): + if residual is None: + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + out += residual + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 2 + + def __init__(self, cfg, inplanes, planes, stride=1, dilation=1): + super(Bottleneck, self).__init__() + expansion = Bottleneck.expansion + bottle_planes = planes // expansion + self.conv1 = nn.Conv2d(inplanes, bottle_planes, + kernel_size=1, bias=False) + self.bn1 = get_norm(cfg.MODEL.DLA.NORM, bottle_planes) + self.conv2 = nn.Conv2d(bottle_planes, bottle_planes, kernel_size=3, + stride=stride, padding=dilation, + bias=False, dilation=dilation) + self.bn2 = get_norm(cfg.MODEL.DLA.NORM, bottle_planes) + self.conv3 = nn.Conv2d(bottle_planes, planes, + kernel_size=1, bias=False) + self.bn3 = get_norm(cfg.MODEL.DLA.NORM, planes) + self.relu = nn.ReLU(inplace=True) + self.stride = stride + + def forward(self, x, residual=None): + if residual is None: + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + out += residual + out = self.relu(out) + + return out + + +class Root(nn.Module): + def __init__(self, cfg, in_channels, out_channels, kernel_size, residual): + super(Root, self).__init__() + self.conv = nn.Conv2d( + in_channels, out_channels, kernel_size, + stride=1, bias=False, padding=(kernel_size - 1) // 2) + self.bn = get_norm(cfg.MODEL.DLA.NORM, out_channels) + self.relu = nn.ReLU(inplace=True) + self.residual = residual + + def forward(self, *x): + children = x + x = self.conv(torch.cat(x, 1)) + x = self.bn(x) + if self.residual: + x += children[0] + x = self.relu(x) + + return x + + +class Tree(nn.Module): + def __init__(self, cfg, levels, block, in_channels, out_channels, stride=1, + level_root=False, root_dim=0, root_kernel_size=1, + dilation=1, root_residual=False): + super(Tree, self).__init__() + if root_dim == 0: + root_dim = 2 * out_channels + if level_root: + root_dim += in_channels + if levels == 1: + self.tree1 = block(cfg, in_channels, out_channels, stride, + dilation=dilation) + self.tree2 = block(cfg, out_channels, out_channels, 1, + dilation=dilation) + else: + self.tree1 = Tree(cfg, levels - 1, block, in_channels, out_channels, + stride, root_dim=0, + root_kernel_size=root_kernel_size, + dilation=dilation, root_residual=root_residual) + self.tree2 = Tree(cfg, levels - 1, block, out_channels, out_channels, + root_dim=root_dim + out_channels, + root_kernel_size=root_kernel_size, + dilation=dilation, root_residual=root_residual) + if levels == 1: + self.root = Root(cfg, root_dim, out_channels, root_kernel_size, + root_residual) + self.level_root = level_root + self.root_dim = root_dim + self.downsample = None + self.project = None + self.levels = levels + if stride > 1: + self.downsample = nn.MaxPool2d(stride, stride=stride) + if in_channels != out_channels: + self.project = nn.Sequential( + nn.Conv2d(in_channels, out_channels, + kernel_size=1, stride=1, bias=False), + get_norm(cfg.MODEL.DLA.NORM, out_channels) + ) + + def forward(self, x, residual=None, children=None): + if self.training and residual is not None: + x = x + residual.sum() * 0.0 + children = [] if children is None else children + bottom = self.downsample(x) if self.downsample else x + residual = self.project(bottom) if self.project else bottom + if self.level_root: + children.append(bottom) + x1 = self.tree1(x, residual) + if self.levels == 1: + x2 = self.tree2(x1) + x = self.root(x2, x1, *children) + else: + children.append(x1) + x = self.tree2(x1, children=children) + return x + + +class DLA(Backbone): + def __init__(self, cfg, levels, channels, block=BasicBlock, residual_root=False): + super(DLA, self).__init__() + self.cfg = cfg + self.channels = channels + + self._out_features = ["dla{}".format(i) for i in range(6)] + self._out_feature_channels = {k: channels[i] for i, k in enumerate(self._out_features)} + self._out_feature_strides = {k: 2 ** i for i, k in enumerate(self._out_features)} + + self.base_layer = nn.Sequential( + nn.Conv2d(3, channels[0], kernel_size=7, stride=1, + padding=3, bias=False), + get_norm(cfg.MODEL.DLA.NORM, channels[0]), + nn.ReLU(inplace=True)) + self.level0 = self._make_conv_level( + channels[0], channels[0], levels[0]) + self.level1 = self._make_conv_level( + channels[0], channels[1], levels[1], stride=2) + self.level2 = Tree(cfg, levels[2], block, channels[1], channels[2], 2, + level_root=False, + root_residual=residual_root) + self.level3 = Tree(cfg, levels[3], block, channels[2], channels[3], 2, + level_root=True, root_residual=residual_root) + self.level4 = Tree(cfg, levels[4], block, channels[3], channels[4], 2, + level_root=True, root_residual=residual_root) + self.level5 = Tree(cfg, levels[5], block, channels[4], channels[5], 2, + level_root=True, root_residual=residual_root) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + + self.load_pretrained_model( + data='imagenet', name='dla34', hash='ba72cf86') + + def load_pretrained_model(self, data, name, hash): + model_url = get_model_url(data, name, hash) + model_weights = model_zoo.load_url(model_url) + del model_weights['fc.weight'] + del model_weights['fc.bias'] + print('Loading pretrained DLA!') + self.load_state_dict(model_weights, strict=True) + + def _make_conv_level(self, inplanes, planes, convs, stride=1, dilation=1): + modules = [] + for i in range(convs): + modules.extend([ + nn.Conv2d(inplanes, planes, kernel_size=3, + stride=stride if i == 0 else 1, + padding=dilation, bias=False, dilation=dilation), + get_norm(self.cfg.MODEL.DLA.NORM, planes), + nn.ReLU(inplace=True)]) + inplanes = planes + return nn.Sequential(*modules) + + def forward(self, x): + y = {} + x = self.base_layer(x) + for i in range(6): + name = 'level{}'.format(i) + x = getattr(self, name)(x) + y['dla{}'.format(i)] = x + return y + + +def fill_up_weights(up): + w = up.weight.data + f = math.ceil(w.size(2) / 2) + c = (2 * f - 1 - f % 2) / (2. * f) + for i in range(w.size(2)): + for j in range(w.size(3)): + w[0, 0, i, j] = \ + (1 - math.fabs(i / f - c)) * (1 - math.fabs(j / f - c)) + for c in range(1, w.size(0)): + w[c, 0, :, :] = w[0, 0, :, :] + + +class Conv(nn.Module): + def __init__(self, chi, cho, norm): + super(Conv, self).__init__() + self.conv = nn.Sequential( + nn.Conv2d(chi, cho, kernel_size=1, stride=1, bias=False), + get_norm(norm, cho), + nn.ReLU(inplace=True)) + + def forward(self, x): + return self.conv(x) + + +class DeformConv(nn.Module): + def __init__(self, chi, cho, norm): + super(DeformConv, self).__init__() + self.actf = nn.Sequential( + get_norm(norm, cho), + nn.ReLU(inplace=True) + ) + self.offset = Conv2d( + chi, 27, kernel_size=3, stride=1, + padding=1, dilation=1) + self.conv = ModulatedDeformConv( + chi, cho, kernel_size=3, stride=1, padding=1, + dilation=1, deformable_groups=1) + nn.init.constant_(self.offset.weight, 0) + nn.init.constant_(self.offset.bias, 0) + + def forward(self, x): + offset_mask = self.offset(x) + offset_x, offset_y, mask = torch.chunk(offset_mask, 3, dim=1) + offset = torch.cat((offset_x, offset_y), dim=1) + mask = mask.sigmoid() + x = self.conv(x, offset, mask) + x = self.actf(x) + return x + + +class IDAUp(nn.Module): + def __init__(self, o, channels, up_f, norm='FrozenBN', node_type=Conv): + super(IDAUp, self).__init__() + for i in range(1, len(channels)): + c = channels[i] + f = int(up_f[i]) + proj = node_type(c, o, norm) + node = node_type(o, o, norm) + + up = nn.ConvTranspose2d(o, o, f * 2, stride=f, + padding=f // 2, output_padding=0, + groups=o, bias=False) + fill_up_weights(up) + + setattr(self, 'proj_' + str(i), proj) + setattr(self, 'up_' + str(i), up) + setattr(self, 'node_' + str(i), node) + + + def forward(self, layers, startp, endp): + for i in range(startp + 1, endp): + upsample = getattr(self, 'up_' + str(i - startp)) + project = getattr(self, 'proj_' + str(i - startp)) + layers[i] = upsample(project(layers[i])) + node = getattr(self, 'node_' + str(i - startp)) + layers[i] = node(layers[i] + layers[i - 1]) + + +DLAUP_NODE_MAP = { + 'conv': Conv, + 'dcn': DeformConv, +} + +class DLAUP(Backbone): + def __init__(self, bottom_up, in_features, norm, dlaup_node='conv'): + super(DLAUP, self).__init__() + assert isinstance(bottom_up, Backbone) + self.bottom_up = bottom_up + input_shapes = bottom_up.output_shape() + in_strides = [input_shapes[f].stride for f in in_features] + in_channels = [input_shapes[f].channels for f in in_features] + in_levels = [int(math.log2(input_shapes[f].stride)) for f in in_features] + self.in_features = in_features + out_features = ['dlaup{}'.format(l) for l in in_levels] + self._out_features = out_features + self._out_feature_channels = { + 'dlaup{}'.format(l): in_channels[i] for i, l in enumerate(in_levels)} + self._out_feature_strides = { + 'dlaup{}'.format(l): 2 ** l for l in in_levels} + + print('self._out_features', self._out_features) + print('self._out_feature_channels', self._out_feature_channels) + print('self._out_feature_strides', self._out_feature_strides) + self._size_divisibility = 32 + + node_type = DLAUP_NODE_MAP[dlaup_node] + + self.startp = int(math.log2(in_strides[0])) + self.channels = in_channels + channels = list(in_channels) + scales = np.array([2 ** i for i in range(len(out_features))], dtype=int) + for i in range(len(channels) - 1): + j = -i - 2 + setattr(self, 'ida_{}'.format(i), + IDAUp(channels[j], in_channels[j:], + scales[j:] // scales[j], + norm=norm, + node_type=node_type)) + scales[j + 1:] = scales[j] + in_channels[j + 1:] = [channels[j] for _ in channels[j + 1:]] + + @property + def size_divisibility(self): + return self._size_divisibility + + def forward(self, x): + bottom_up_features = self.bottom_up(x) + layers = [bottom_up_features[f] for f in self.in_features] + out = [layers[-1]] # start with 32 + for i in range(len(layers) - 1): + ida = getattr(self, 'ida_{}'.format(i)) + ida(layers, len(layers) - i - 2, len(layers)) + out.insert(0, layers[-1]) + ret = {} + for k, v in zip(self._out_features, out): + ret[k] = v + # import pdb; pdb.set_trace() + return ret + + +def dla34(cfg, pretrained=None): # DLA-34 + model = DLA(cfg, [1, 1, 1, 2, 2, 1], + [16, 32, 64, 128, 256, 512], + block=BasicBlock) + return model + + +class LastLevelP6P7(nn.Module): + """ + This module is used in RetinaNet to generate extra layers, P6 and P7 from + C5 feature. + """ + + def __init__(self, in_channels, out_channels): + super().__init__() + self.num_levels = 2 + self.in_feature = "dla5" + self.p6 = nn.Conv2d(in_channels, out_channels, 3, 2, 1) + self.p7 = nn.Conv2d(out_channels, out_channels, 3, 2, 1) + for module in [self.p6, self.p7]: + weight_init.c2_xavier_fill(module) + + def forward(self, c5): + p6 = self.p6(c5) + p7 = self.p7(F.relu(p6)) + return [p6, p7] + + +@BACKBONE_REGISTRY.register() +def build_dla_fpn3_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + + depth_to_creator = {"dla34": dla34} + bottom_up = depth_to_creator['dla{}'.format(cfg.MODEL.DLA.NUM_LAYERS)](cfg) + in_features = cfg.MODEL.FPN.IN_FEATURES + out_channels = cfg.MODEL.FPN.OUT_CHANNELS + + backbone = FPN( + bottom_up=bottom_up, + in_features=in_features, + out_channels=out_channels, + norm=cfg.MODEL.FPN.NORM, + top_block=None, + fuse_type=cfg.MODEL.FPN.FUSE_TYPE, + ) + + return backbone + +@BACKBONE_REGISTRY.register() +def build_dla_fpn5_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + + depth_to_creator = {"dla34": dla34} + bottom_up = depth_to_creator['dla{}'.format(cfg.MODEL.DLA.NUM_LAYERS)](cfg) + in_features = cfg.MODEL.FPN.IN_FEATURES + out_channels = cfg.MODEL.FPN.OUT_CHANNELS + in_channels_top = bottom_up.output_shape()['dla5'].channels + + backbone = FPN( + bottom_up=bottom_up, + in_features=in_features, + out_channels=out_channels, + norm=cfg.MODEL.FPN.NORM, + top_block=LastLevelP6P7(in_channels_top, out_channels), + fuse_type=cfg.MODEL.FPN.FUSE_TYPE, + ) + + return backbone + + +@BACKBONE_REGISTRY.register() +def build_dlaup_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + + depth_to_creator = {"dla34": dla34} + bottom_up = depth_to_creator['dla{}'.format(cfg.MODEL.DLA.NUM_LAYERS)](cfg) + + backbone = DLAUP( + bottom_up=bottom_up, + in_features=cfg.MODEL.DLA.DLAUP_IN_FEATURES, + norm=cfg.MODEL.DLA.NORM, + dlaup_node=cfg.MODEL.DLA.DLAUP_NODE, + ) + + return backbone diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/fpn_p5.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/fpn_p5.py new file mode 100644 index 0000000000000000000000000000000000000000..e991f9c7be095e2a40e12c849b35e246cd0344bd --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/fpn_p5.py @@ -0,0 +1,78 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import math +import fvcore.nn.weight_init as weight_init +import torch.nn.functional as F +from torch import nn + +from detectron2.layers import Conv2d, ShapeSpec, get_norm + +from detectron2.modeling.backbone import Backbone +from detectron2.modeling.backbone.fpn import FPN +from detectron2.modeling.backbone.build import BACKBONE_REGISTRY +from detectron2.modeling.backbone.resnet import build_resnet_backbone + + +class LastLevelP6P7_P5(nn.Module): + """ + This module is used in RetinaNet to generate extra layers, P6 and P7 from + C5 feature. + """ + + def __init__(self, in_channels, out_channels): + super().__init__() + self.num_levels = 2 + self.in_feature = "p5" + self.p6 = nn.Conv2d(in_channels, out_channels, 3, 2, 1) + self.p7 = nn.Conv2d(out_channels, out_channels, 3, 2, 1) + for module in [self.p6, self.p7]: + weight_init.c2_xavier_fill(module) + + def forward(self, c5): + p6 = self.p6(c5) + p7 = self.p7(F.relu(p6)) + return [p6, p7] + + +@BACKBONE_REGISTRY.register() +def build_p67_resnet_fpn_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + bottom_up = build_resnet_backbone(cfg, input_shape) + in_features = cfg.MODEL.FPN.IN_FEATURES + out_channels = cfg.MODEL.FPN.OUT_CHANNELS + backbone = FPN( + bottom_up=bottom_up, + in_features=in_features, + out_channels=out_channels, + norm=cfg.MODEL.FPN.NORM, + top_block=LastLevelP6P7_P5(out_channels, out_channels), + fuse_type=cfg.MODEL.FPN.FUSE_TYPE, + ) + return backbone + +@BACKBONE_REGISTRY.register() +def build_p35_resnet_fpn_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + bottom_up = build_resnet_backbone(cfg, input_shape) + in_features = cfg.MODEL.FPN.IN_FEATURES + out_channels = cfg.MODEL.FPN.OUT_CHANNELS + backbone = FPN( + bottom_up=bottom_up, + in_features=in_features, + out_channels=out_channels, + norm=cfg.MODEL.FPN.NORM, + top_block=None, + fuse_type=cfg.MODEL.FPN.FUSE_TYPE, + ) + return backbone \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/res2net.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/res2net.py new file mode 100644 index 0000000000000000000000000000000000000000..1d0d40adb4a300d916deecebd20bcaac08936e6d --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/backbone/res2net.py @@ -0,0 +1,802 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +# This file is modified from https://github.com/Res2Net/Res2Net-detectron2/blob/master/detectron2/modeling/backbone/resnet.py +# The original file is under Apache-2.0 License +import numpy as np +import fvcore.nn.weight_init as weight_init +import torch +import torch.nn.functional as F +from torch import nn + +from detectron2.layers import ( + CNNBlockBase, + Conv2d, + DeformConv, + ModulatedDeformConv, + ShapeSpec, + get_norm, +) + +from detectron2.modeling.backbone import Backbone +from detectron2.modeling.backbone.fpn import FPN +from detectron2.modeling.backbone.build import BACKBONE_REGISTRY +from .fpn_p5 import LastLevelP6P7_P5 +from .bifpn import BiFPN + +__all__ = [ + "ResNetBlockBase", + "BasicBlock", + "BottleneckBlock", + "DeformBottleneckBlock", + "BasicStem", + "ResNet", + "make_stage", + "build_res2net_backbone", +] + + +ResNetBlockBase = CNNBlockBase +""" +Alias for backward compatibiltiy. +""" + + +class BasicBlock(CNNBlockBase): + """ + The basic residual block for ResNet-18 and ResNet-34, with two 3x3 conv layers + and a projection shortcut if needed. + """ + + def __init__(self, in_channels, out_channels, *, stride=1, norm="BN"): + """ + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + stride (int): Stride for the first conv. + norm (str or callable): normalization for all conv layers. + See :func:`layers.get_norm` for supported format. + """ + super().__init__(in_channels, out_channels, stride) + + if in_channels != out_channels: + self.shortcut = Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=stride, + bias=False, + norm=get_norm(norm, out_channels), + ) + else: + self.shortcut = None + + self.conv1 = Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=stride, + padding=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + + self.conv2 = Conv2d( + out_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + + for layer in [self.conv1, self.conv2, self.shortcut]: + if layer is not None: # shortcut can be None + weight_init.c2_msra_fill(layer) + + def forward(self, x): + out = self.conv1(x) + out = F.relu_(out) + out = self.conv2(out) + + if self.shortcut is not None: + shortcut = self.shortcut(x) + else: + shortcut = x + + out += shortcut + out = F.relu_(out) + return out + + +class BottleneckBlock(CNNBlockBase): + """ + The standard bottle2neck residual block used by Res2Net-50, 101 and 152. + """ + + def __init__( + self, + in_channels, + out_channels, + *, + bottleneck_channels, + stride=1, + num_groups=1, + norm="BN", + stride_in_1x1=False, + dilation=1, + basewidth=26, + scale=4, + ): + """ + Args: + bottleneck_channels (int): number of output channels for the 3x3 + "bottleneck" conv layers. + num_groups (int): number of groups for the 3x3 conv layer. + norm (str or callable): normalization for all conv layers. + See :func:`layers.get_norm` for supported format. + stride_in_1x1 (bool): when stride>1, whether to put stride in the + first 1x1 convolution or the bottleneck 3x3 convolution. + dilation (int): the dilation rate of the 3x3 conv layer. + """ + super().__init__(in_channels, out_channels, stride) + + if in_channels != out_channels: + self.shortcut = nn.Sequential( + nn.AvgPool2d(kernel_size=stride, stride=stride, + ceil_mode=True, count_include_pad=False), + Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + ) + else: + self.shortcut = None + + # The original MSRA ResNet models have stride in the first 1x1 conv + # The subsequent fb.torch.resnet and Caffe2 ResNe[X]t implementations have + # stride in the 3x3 conv + stride_1x1, stride_3x3 = (stride, 1) if stride_in_1x1 else (1, stride) + width = bottleneck_channels//scale + + self.conv1 = Conv2d( + in_channels, + bottleneck_channels, + kernel_size=1, + stride=stride_1x1, + bias=False, + norm=get_norm(norm, bottleneck_channels), + ) + if scale == 1: + self.nums = 1 + else: + self.nums = scale -1 + if self.in_channels!=self.out_channels and stride_3x3!=2: + self.pool = nn.AvgPool2d(kernel_size=3, stride = stride_3x3, padding=1) + + convs = [] + bns = [] + for i in range(self.nums): + convs.append(nn.Conv2d( + width, + width, + kernel_size=3, + stride=stride_3x3, + padding=1 * dilation, + bias=False, + groups=num_groups, + dilation=dilation, + )) + bns.append(get_norm(norm, width)) + self.convs = nn.ModuleList(convs) + self.bns = nn.ModuleList(bns) + + self.conv3 = Conv2d( + bottleneck_channels, + out_channels, + kernel_size=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + self.scale = scale + self.width = width + self.in_channels = in_channels + self.out_channels = out_channels + self.stride_3x3 = stride_3x3 + for layer in [self.conv1, self.conv3]: + if layer is not None: # shortcut can be None + weight_init.c2_msra_fill(layer) + if self.shortcut is not None: + for layer in self.shortcut.modules(): + if isinstance(layer, Conv2d): + weight_init.c2_msra_fill(layer) + + for layer in self.convs: + if layer is not None: # shortcut can be None + weight_init.c2_msra_fill(layer) + + # Zero-initialize the last normalization in each residual branch, + # so that at the beginning, the residual branch starts with zeros, + # and each residual block behaves like an identity. + # See Sec 5.1 in "Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour": + # "For BN layers, the learnable scaling coefficient γ is initialized + # to be 1, except for each residual block's last BN + # where γ is initialized to be 0." + + # nn.init.constant_(self.conv3.norm.weight, 0) + # TODO this somehow hurts performance when training GN models from scratch. + # Add it as an option when we need to use this code to train a backbone. + + def forward(self, x): + out = self.conv1(x) + out = F.relu_(out) + + spx = torch.split(out, self.width, 1) + for i in range(self.nums): + if i==0 or self.in_channels!=self.out_channels: + sp = spx[i] + else: + sp = sp + spx[i] + sp = self.convs[i](sp) + sp = F.relu_(self.bns[i](sp)) + if i==0: + out = sp + else: + out = torch.cat((out, sp), 1) + if self.scale!=1 and self.stride_3x3==1: + out = torch.cat((out, spx[self.nums]), 1) + elif self.scale != 1 and self.stride_3x3==2: + out = torch.cat((out, self.pool(spx[self.nums])), 1) + + out = self.conv3(out) + + if self.shortcut is not None: + shortcut = self.shortcut(x) + else: + shortcut = x + + out += shortcut + out = F.relu_(out) + return out + + +class DeformBottleneckBlock(ResNetBlockBase): + """ + Not implemented for res2net yet. + Similar to :class:`BottleneckBlock`, but with deformable conv in the 3x3 convolution. + """ + + def __init__( + self, + in_channels, + out_channels, + *, + bottleneck_channels, + stride=1, + num_groups=1, + norm="BN", + stride_in_1x1=False, + dilation=1, + deform_modulated=False, + deform_num_groups=1, + basewidth=26, + scale=4, + ): + super().__init__(in_channels, out_channels, stride) + self.deform_modulated = deform_modulated + + if in_channels != out_channels: + # self.shortcut = Conv2d( + # in_channels, + # out_channels, + # kernel_size=1, + # stride=stride, + # bias=False, + # norm=get_norm(norm, out_channels), + # ) + self.shortcut = nn.Sequential( + nn.AvgPool2d(kernel_size=stride, stride=stride, + ceil_mode=True, count_include_pad=False), + Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + ) + else: + self.shortcut = None + + stride_1x1, stride_3x3 = (stride, 1) if stride_in_1x1 else (1, stride) + width = bottleneck_channels//scale + + self.conv1 = Conv2d( + in_channels, + bottleneck_channels, + kernel_size=1, + stride=stride_1x1, + bias=False, + norm=get_norm(norm, bottleneck_channels), + ) + + if scale == 1: + self.nums = 1 + else: + self.nums = scale -1 + if self.in_channels!=self.out_channels and stride_3x3!=2: + self.pool = nn.AvgPool2d(kernel_size=3, stride = stride_3x3, padding=1) + + if deform_modulated: + deform_conv_op = ModulatedDeformConv + # offset channels are 2 or 3 (if with modulated) * kernel_size * kernel_size + offset_channels = 27 + else: + deform_conv_op = DeformConv + offset_channels = 18 + + # self.conv2_offset = Conv2d( + # bottleneck_channels, + # offset_channels * deform_num_groups, + # kernel_size=3, + # stride=stride_3x3, + # padding=1 * dilation, + # dilation=dilation, + # ) + # self.conv2 = deform_conv_op( + # bottleneck_channels, + # bottleneck_channels, + # kernel_size=3, + # stride=stride_3x3, + # padding=1 * dilation, + # bias=False, + # groups=num_groups, + # dilation=dilation, + # deformable_groups=deform_num_groups, + # norm=get_norm(norm, bottleneck_channels), + # ) + + conv2_offsets = [] + convs = [] + bns = [] + for i in range(self.nums): + conv2_offsets.append(Conv2d( + width, + offset_channels * deform_num_groups, + kernel_size=3, + stride=stride_3x3, + padding=1 * dilation, + bias=False, + groups=num_groups, + dilation=dilation, + )) + convs.append(deform_conv_op( + width, + width, + kernel_size=3, + stride=stride_3x3, + padding=1 * dilation, + bias=False, + groups=num_groups, + dilation=dilation, + deformable_groups=deform_num_groups, + )) + bns.append(get_norm(norm, width)) + self.conv2_offsets = nn.ModuleList(conv2_offsets) + self.convs = nn.ModuleList(convs) + self.bns = nn.ModuleList(bns) + + self.conv3 = Conv2d( + bottleneck_channels, + out_channels, + kernel_size=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + self.scale = scale + self.width = width + self.in_channels = in_channels + self.out_channels = out_channels + self.stride_3x3 = stride_3x3 + # for layer in [self.conv1, self.conv2, self.conv3, self.shortcut]: + # if layer is not None: # shortcut can be None + # weight_init.c2_msra_fill(layer) + + # nn.init.constant_(self.conv2_offset.weight, 0) + # nn.init.constant_(self.conv2_offset.bias, 0) + for layer in [self.conv1, self.conv3]: + if layer is not None: # shortcut can be None + weight_init.c2_msra_fill(layer) + if self.shortcut is not None: + for layer in self.shortcut.modules(): + if isinstance(layer, Conv2d): + weight_init.c2_msra_fill(layer) + + for layer in self.convs: + if layer is not None: # shortcut can be None + weight_init.c2_msra_fill(layer) + + for layer in self.conv2_offsets: + if layer.weight is not None: + nn.init.constant_(layer.weight, 0) + if layer.bias is not None: + nn.init.constant_(layer.bias, 0) + + def forward(self, x): + out = self.conv1(x) + out = F.relu_(out) + + # if self.deform_modulated: + # offset_mask = self.conv2_offset(out) + # offset_x, offset_y, mask = torch.chunk(offset_mask, 3, dim=1) + # offset = torch.cat((offset_x, offset_y), dim=1) + # mask = mask.sigmoid() + # out = self.conv2(out, offset, mask) + # else: + # offset = self.conv2_offset(out) + # out = self.conv2(out, offset) + # out = F.relu_(out) + + spx = torch.split(out, self.width, 1) + for i in range(self.nums): + if i==0 or self.in_channels!=self.out_channels: + sp = spx[i].contiguous() + else: + sp = sp + spx[i].contiguous() + + # sp = self.convs[i](sp) + if self.deform_modulated: + offset_mask = self.conv2_offsets[i](sp) + offset_x, offset_y, mask = torch.chunk(offset_mask, 3, dim=1) + offset = torch.cat((offset_x, offset_y), dim=1) + mask = mask.sigmoid() + sp = self.convs[i](sp, offset, mask) + else: + offset = self.conv2_offsets[i](sp) + sp = self.convs[i](sp, offset) + sp = F.relu_(self.bns[i](sp)) + if i==0: + out = sp + else: + out = torch.cat((out, sp), 1) + if self.scale!=1 and self.stride_3x3==1: + out = torch.cat((out, spx[self.nums]), 1) + elif self.scale != 1 and self.stride_3x3==2: + out = torch.cat((out, self.pool(spx[self.nums])), 1) + + out = self.conv3(out) + + if self.shortcut is not None: + shortcut = self.shortcut(x) + else: + shortcut = x + + out += shortcut + out = F.relu_(out) + return out + + +def make_stage(block_class, num_blocks, first_stride, *, in_channels, out_channels, **kwargs): + """ + Create a list of blocks just like those in a ResNet stage. + Args: + block_class (type): a subclass of ResNetBlockBase + num_blocks (int): + first_stride (int): the stride of the first block. The other blocks will have stride=1. + in_channels (int): input channels of the entire stage. + out_channels (int): output channels of **every block** in the stage. + kwargs: other arguments passed to the constructor of every block. + Returns: + list[nn.Module]: a list of block module. + """ + assert "stride" not in kwargs, "Stride of blocks in make_stage cannot be changed." + blocks = [] + for i in range(num_blocks): + blocks.append( + block_class( + in_channels=in_channels, + out_channels=out_channels, + stride=first_stride if i == 0 else 1, + **kwargs, + ) + ) + in_channels = out_channels + return blocks + + +class BasicStem(CNNBlockBase): + """ + The standard ResNet stem (layers before the first residual block). + """ + + def __init__(self, in_channels=3, out_channels=64, norm="BN"): + """ + Args: + norm (str or callable): norm after the first conv layer. + See :func:`layers.get_norm` for supported format. + """ + super().__init__(in_channels, out_channels, 4) + self.in_channels = in_channels + self.conv1 = nn.Sequential( + Conv2d( + in_channels, + 32, + kernel_size=3, + stride=2, + padding=1, + bias=False, + ), + get_norm(norm, 32), + nn.ReLU(inplace=True), + Conv2d( + 32, + 32, + kernel_size=3, + stride=1, + padding=1, + bias=False, + ), + get_norm(norm, 32), + nn.ReLU(inplace=True), + Conv2d( + 32, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False, + ), + ) + self.bn1 = get_norm(norm, out_channels) + + for layer in self.conv1: + if isinstance(layer, Conv2d): + weight_init.c2_msra_fill(layer) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = F.relu_(x) + x = F.max_pool2d(x, kernel_size=3, stride=2, padding=1) + return x + + +class ResNet(Backbone): + def __init__(self, stem, stages, num_classes=None, out_features=None): + """ + Args: + stem (nn.Module): a stem module + stages (list[list[CNNBlockBase]]): several (typically 4) stages, + each contains multiple :class:`CNNBlockBase`. + num_classes (None or int): if None, will not perform classification. + Otherwise, will create a linear layer. + out_features (list[str]): name of the layers whose outputs should + be returned in forward. Can be anything in "stem", "linear", or "res2" ... + If None, will return the output of the last layer. + """ + super(ResNet, self).__init__() + self.stem = stem + self.num_classes = num_classes + + current_stride = self.stem.stride + self._out_feature_strides = {"stem": current_stride} + self._out_feature_channels = {"stem": self.stem.out_channels} + + self.stages_and_names = [] + for i, blocks in enumerate(stages): + assert len(blocks) > 0, len(blocks) + for block in blocks: + assert isinstance(block, CNNBlockBase), block + + name = "res" + str(i + 2) + stage = nn.Sequential(*blocks) + + self.add_module(name, stage) + self.stages_and_names.append((stage, name)) + + self._out_feature_strides[name] = current_stride = int( + current_stride * np.prod([k.stride for k in blocks]) + ) + self._out_feature_channels[name] = curr_channels = blocks[-1].out_channels + + if num_classes is not None: + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.linear = nn.Linear(curr_channels, num_classes) + + # Sec 5.1 in "Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour": + # "The 1000-way fully-connected layer is initialized by + # drawing weights from a zero-mean Gaussian with standard deviation of 0.01." + nn.init.normal_(self.linear.weight, std=0.01) + name = "linear" + + if out_features is None: + out_features = [name] + self._out_features = out_features + assert len(self._out_features) + children = [x[0] for x in self.named_children()] + for out_feature in self._out_features: + assert out_feature in children, "Available children: {}".format(", ".join(children)) + + def forward(self, x): + outputs = {} + x = self.stem(x) + if "stem" in self._out_features: + outputs["stem"] = x + for stage, name in self.stages_and_names: + x = stage(x) + if name in self._out_features: + outputs[name] = x + if self.num_classes is not None: + x = self.avgpool(x) + x = torch.flatten(x, 1) + x = self.linear(x) + if "linear" in self._out_features: + outputs["linear"] = x + return outputs + + def output_shape(self): + return { + name: ShapeSpec( + channels=self._out_feature_channels[name], stride=self._out_feature_strides[name] + ) + for name in self._out_features + } + + def freeze(self, freeze_at=0): + """ + Freeze the first several stages of the ResNet. Commonly used in + fine-tuning. + Args: + freeze_at (int): number of stem and stages to freeze. + `1` means freezing the stem. `2` means freezing the stem and + the first stage, etc. + Returns: + nn.Module: this ResNet itself + """ + if freeze_at >= 1: + self.stem.freeze() + for idx, (stage, _) in enumerate(self.stages_and_names, start=2): + if freeze_at >= idx: + for block in stage.children(): + block.freeze() + return self + + +@BACKBONE_REGISTRY.register() +def build_res2net_backbone(cfg, input_shape): + """ + Create a Res2Net instance from config. + Returns: + ResNet: a :class:`ResNet` instance. + """ + # need registration of new blocks/stems? + norm = cfg.MODEL.RESNETS.NORM + stem = BasicStem( + in_channels=input_shape.channels, + out_channels=cfg.MODEL.RESNETS.STEM_OUT_CHANNELS, + norm=norm, + ) + + # fmt: off + freeze_at = cfg.MODEL.BACKBONE.FREEZE_AT + out_features = cfg.MODEL.RESNETS.OUT_FEATURES + depth = cfg.MODEL.RESNETS.DEPTH + num_groups = cfg.MODEL.RESNETS.NUM_GROUPS + width_per_group = cfg.MODEL.RESNETS.WIDTH_PER_GROUP + scale = 4 + bottleneck_channels = num_groups * width_per_group * scale + in_channels = cfg.MODEL.RESNETS.STEM_OUT_CHANNELS + out_channels = cfg.MODEL.RESNETS.RES2_OUT_CHANNELS + stride_in_1x1 = cfg.MODEL.RESNETS.STRIDE_IN_1X1 + res5_dilation = cfg.MODEL.RESNETS.RES5_DILATION + deform_on_per_stage = cfg.MODEL.RESNETS.DEFORM_ON_PER_STAGE + deform_modulated = cfg.MODEL.RESNETS.DEFORM_MODULATED + deform_num_groups = cfg.MODEL.RESNETS.DEFORM_NUM_GROUPS + # fmt: on + assert res5_dilation in {1, 2}, "res5_dilation cannot be {}.".format(res5_dilation) + + num_blocks_per_stage = { + 18: [2, 2, 2, 2], + 34: [3, 4, 6, 3], + 50: [3, 4, 6, 3], + 101: [3, 4, 23, 3], + 152: [3, 8, 36, 3], + }[depth] + + if depth in [18, 34]: + assert out_channels == 64, "Must set MODEL.RESNETS.RES2_OUT_CHANNELS = 64 for R18/R34" + assert not any( + deform_on_per_stage + ), "MODEL.RESNETS.DEFORM_ON_PER_STAGE unsupported for R18/R34" + assert res5_dilation == 1, "Must set MODEL.RESNETS.RES5_DILATION = 1 for R18/R34" + assert num_groups == 1, "Must set MODEL.RESNETS.NUM_GROUPS = 1 for R18/R34" + + stages = [] + + # Avoid creating variables without gradients + # It consumes extra memory and may cause allreduce to fail + out_stage_idx = [{"res2": 2, "res3": 3, "res4": 4, "res5": 5}[f] for f in out_features] + max_stage_idx = max(out_stage_idx) + for idx, stage_idx in enumerate(range(2, max_stage_idx + 1)): + dilation = res5_dilation if stage_idx == 5 else 1 + first_stride = 1 if idx == 0 or (stage_idx == 5 and dilation == 2) else 2 + stage_kargs = { + "num_blocks": num_blocks_per_stage[idx], + "first_stride": first_stride, + "in_channels": in_channels, + "out_channels": out_channels, + "norm": norm, + } + # Use BasicBlock for R18 and R34. + if depth in [18, 34]: + stage_kargs["block_class"] = BasicBlock + else: + stage_kargs["bottleneck_channels"] = bottleneck_channels + stage_kargs["stride_in_1x1"] = stride_in_1x1 + stage_kargs["dilation"] = dilation + stage_kargs["num_groups"] = num_groups + stage_kargs["scale"] = scale + + if deform_on_per_stage[idx]: + stage_kargs["block_class"] = DeformBottleneckBlock + stage_kargs["deform_modulated"] = deform_modulated + stage_kargs["deform_num_groups"] = deform_num_groups + else: + stage_kargs["block_class"] = BottleneckBlock + blocks = make_stage(**stage_kargs) + in_channels = out_channels + out_channels *= 2 + bottleneck_channels *= 2 + stages.append(blocks) + return ResNet(stem, stages, out_features=out_features).freeze(freeze_at) + + +@BACKBONE_REGISTRY.register() +def build_p67_res2net_fpn_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + bottom_up = build_res2net_backbone(cfg, input_shape) + in_features = cfg.MODEL.FPN.IN_FEATURES + out_channels = cfg.MODEL.FPN.OUT_CHANNELS + backbone = FPN( + bottom_up=bottom_up, + in_features=in_features, + out_channels=out_channels, + norm=cfg.MODEL.FPN.NORM, + top_block=LastLevelP6P7_P5(out_channels, out_channels), + fuse_type=cfg.MODEL.FPN.FUSE_TYPE, + ) + return backbone + + +@BACKBONE_REGISTRY.register() +def build_res2net_bifpn_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + bottom_up = build_res2net_backbone(cfg, input_shape) + in_features = cfg.MODEL.FPN.IN_FEATURES + backbone = BiFPN( + cfg=cfg, + bottom_up=bottom_up, + in_features=in_features, + out_channels=cfg.MODEL.BIFPN.OUT_CHANNELS, + norm=cfg.MODEL.BIFPN.NORM, + num_levels=cfg.MODEL.BIFPN.NUM_LEVELS, + num_bifpn=cfg.MODEL.BIFPN.NUM_BIFPN, + separable_conv=cfg.MODEL.BIFPN.SEPARABLE_CONV, + ) + return backbone \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/debug.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/debug.py new file mode 100644 index 0000000000000000000000000000000000000000..0a4437fb5ae7522e46ca6c42ba5fd980df250446 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/debug.py @@ -0,0 +1,283 @@ +import cv2 +import numpy as np +import torch +import torch.nn.functional as F + +COLORS = ((np.random.rand(1300, 3) * 0.4 + 0.6) * 255).astype( + np.uint8).reshape(1300, 1, 1, 3) + +def _get_color_image(heatmap): + heatmap = heatmap.reshape( + heatmap.shape[0], heatmap.shape[1], heatmap.shape[2], 1) + if heatmap.shape[0] == 1: + color_map = (heatmap * np.ones((1, 1, 1, 3), np.uint8) * 255).max( + axis=0).astype(np.uint8) # H, W, 3 + else: + color_map = (heatmap * COLORS[:heatmap.shape[0]]).max(axis=0).astype(np.uint8) # H, W, 3 + + return color_map + +def _blend_image(image, color_map, a=0.7): + color_map = cv2.resize(color_map, (image.shape[1], image.shape[0])) + ret = np.clip(image * (1 - a) + color_map * a, 0, 255).astype(np.uint8) + return ret + +def _blend_image_heatmaps(image, color_maps, a=0.7): + merges = np.zeros((image.shape[0], image.shape[1], 3), np.float32) + for color_map in color_maps: + color_map = cv2.resize(color_map, (image.shape[1], image.shape[0])) + merges = np.maximum(merges, color_map) + ret = np.clip(image * (1 - a) + merges * a, 0, 255).astype(np.uint8) + return ret + +def _decompose_level(x, shapes_per_level, N): + ''' + x: LNHiWi x C + ''' + x = x.view(x.shape[0], -1) + ret = [] + st = 0 + for l in range(len(shapes_per_level)): + ret.append([]) + h = shapes_per_level[l][0].int().item() + w = shapes_per_level[l][1].int().item() + for i in range(N): + ret[l].append(x[st + h * w * i:st + h * w * (i + 1)].view( + h, w, -1).permute(2, 0, 1)) + st += h * w * N + return ret + +def _imagelist_to_tensor(images): + images = [x for x in images] + image_sizes = [x.shape[-2:] for x in images] + h = max([size[0] for size in image_sizes]) + w = max([size[1] for size in image_sizes]) + S = 32 + h, w = ((h - 1) // S + 1) * S, ((w - 1) // S + 1) * S + images = [F.pad(x, (0, w - x.shape[2], 0, h - x.shape[1], 0, 0)) \ + for x in images] + images = torch.stack(images) + return images + + +def _ind2il(ind, shapes_per_level, N): + r = ind + l = 0 + S = 0 + while r - S >= N * shapes_per_level[l][0] * shapes_per_level[l][1]: + S += N * shapes_per_level[l][0] * shapes_per_level[l][1] + l += 1 + i = (r - S) // (shapes_per_level[l][0] * shapes_per_level[l][1]) + return i, l + +def debug_train( + images, gt_instances, flattened_hms, reg_targets, labels, pos_inds, + shapes_per_level, locations, strides): + ''' + images: N x 3 x H x W + flattened_hms: LNHiWi x C + shapes_per_level: L x 2 [(H_i, W_i)] + locations: LNHiWi x 2 + ''' + reg_inds = torch.nonzero( + reg_targets.max(dim=1)[0] > 0).squeeze(1) + N = len(images) + images = _imagelist_to_tensor(images) + repeated_locations = [torch.cat([loc] * N, dim=0) \ + for loc in locations] + locations = torch.cat(repeated_locations, dim=0) + gt_hms = _decompose_level(flattened_hms, shapes_per_level, N) + masks = flattened_hms.new_zeros((flattened_hms.shape[0], 1)) + masks[pos_inds] = 1 + masks = _decompose_level(masks, shapes_per_level, N) + for i in range(len(images)): + image = images[i].detach().cpu().numpy().transpose(1, 2, 0) + color_maps = [] + for l in range(len(gt_hms)): + color_map = _get_color_image( + gt_hms[l][i].detach().cpu().numpy()) + color_maps.append(color_map) + cv2.imshow('gthm_{}'.format(l), color_map) + blend = _blend_image_heatmaps(image.copy(), color_maps) + if gt_instances is not None: + bboxes = gt_instances[i].gt_boxes.tensor + for j in range(len(bboxes)): + bbox = bboxes[j] + cv2.rectangle( + blend, + (int(bbox[0]), int(bbox[1])), + (int(bbox[2]), int(bbox[3])), + (0, 0, 255), 3, cv2.LINE_AA) + + for j in range(len(pos_inds)): + image_id, l = _ind2il(pos_inds[j], shapes_per_level, N) + if image_id != i: + continue + loc = locations[pos_inds[j]] + cv2.drawMarker( + blend, (int(loc[0]), int(loc[1])), (0, 255, 255), + markerSize=(l + 1) * 16) + + for j in range(len(reg_inds)): + image_id, l = _ind2il(reg_inds[j], shapes_per_level, N) + if image_id != i: + continue + ltrb = reg_targets[reg_inds[j]] + ltrb *= strides[l] + loc = locations[reg_inds[j]] + bbox = [(loc[0] - ltrb[0]), (loc[1] - ltrb[1]), + (loc[0] + ltrb[2]), (loc[1] + ltrb[3])] + cv2.rectangle( + blend, + (int(bbox[0]), int(bbox[1])), + (int(bbox[2]), int(bbox[3])), + (255, 0, 0), 1, cv2.LINE_AA) + cv2.circle(blend, (int(loc[0]), int(loc[1])), 2, (255, 0, 0), -1) + + cv2.imshow('blend', blend) + cv2.waitKey() + + +def debug_test( + images, logits_pred, reg_pred, agn_hm_pred=[], preds=[], + vis_thresh=0.3, debug_show_name=False, mult_agn=False): + ''' + images: N x 3 x H x W + class_target: LNHiWi x C + cat_agn_heatmap: LNHiWi + shapes_per_level: L x 2 [(H_i, W_i)] + ''' + N = len(images) + for i in range(len(images)): + image = images[i].detach().cpu().numpy().transpose(1, 2, 0) + result = image.copy().astype(np.uint8) + pred_image = image.copy().astype(np.uint8) + color_maps = [] + L = len(logits_pred) + for l in range(L): + if logits_pred[0] is not None: + stride = min(image.shape[0], image.shape[1]) / min( + logits_pred[l][i].shape[1], logits_pred[l][i].shape[2]) + else: + stride = min(image.shape[0], image.shape[1]) / min( + agn_hm_pred[l][i].shape[1], agn_hm_pred[l][i].shape[2]) + stride = stride if stride < 60 else 64 if stride < 100 else 128 + if logits_pred[0] is not None: + if mult_agn: + logits_pred[l][i] = logits_pred[l][i] * agn_hm_pred[l][i] + color_map = _get_color_image( + logits_pred[l][i].detach().cpu().numpy()) + color_maps.append(color_map) + cv2.imshow('predhm_{}'.format(l), color_map) + + if debug_show_name: + from detectron2.data.datasets.lvis_v1_categories import LVIS_CATEGORIES + cat2name = [x['name'] for x in LVIS_CATEGORIES] + for j in range(len(preds[i].scores) if preds is not None else 0): + if preds[i].scores[j] > vis_thresh: + bbox = preds[i].proposal_boxes[j] \ + if preds[i].has('proposal_boxes') else \ + preds[i].pred_boxes[j] + bbox = bbox.tensor[0].detach().cpu().numpy().astype(np.int32) + cat = int(preds[i].pred_classes[j]) \ + if preds[i].has('pred_classes') else 0 + cl = COLORS[cat, 0, 0] + cv2.rectangle( + pred_image, (int(bbox[0]), int(bbox[1])), + (int(bbox[2]), int(bbox[3])), + (int(cl[0]), int(cl[1]), int(cl[2])), 2, cv2.LINE_AA) + if debug_show_name: + txt = '{}{:.1f}'.format( + cat2name[cat] if cat > 0 else '', + preds[i].scores[j]) + font = cv2.FONT_HERSHEY_SIMPLEX + cat_size = cv2.getTextSize(txt, font, 0.5, 2)[0] + cv2.rectangle( + pred_image, + (int(bbox[0]), int(bbox[1] - cat_size[1] - 2)), + (int(bbox[0] + cat_size[0]), int(bbox[1] - 2)), + (int(cl[0]), int(cl[1]), int(cl[2])), -1) + cv2.putText( + pred_image, txt, (int(bbox[0]), int(bbox[1] - 2)), + font, 0.5, (0, 0, 0), thickness=1, lineType=cv2.LINE_AA) + + + if agn_hm_pred[l] is not None: + agn_hm_ = agn_hm_pred[l][i, 0, :, :, None].detach().cpu().numpy() + agn_hm_ = (agn_hm_ * np.array([255, 255, 255]).reshape( + 1, 1, 3)).astype(np.uint8) + cv2.imshow('agn_hm_{}'.format(l), agn_hm_) + blend = _blend_image_heatmaps(image.copy(), color_maps) + cv2.imshow('blend', blend) + cv2.imshow('preds', pred_image) + cv2.waitKey() + +global cnt +cnt = 0 + +def debug_second_stage(images, instances, proposals=None, vis_thresh=0.3, + save_debug=False, debug_show_name=False): + images = _imagelist_to_tensor(images) + if debug_show_name: + from detectron2.data.datasets.lvis_v1_categories import LVIS_CATEGORIES + cat2name = [x['name'] for x in LVIS_CATEGORIES] + for i in range(len(images)): + image = images[i].detach().cpu().numpy().transpose(1, 2, 0).astype(np.uint8).copy() + if instances[i].has('gt_boxes'): + bboxes = instances[i].gt_boxes.tensor.cpu().numpy() + scores = np.ones(bboxes.shape[0]) + cats = instances[i].gt_classes.cpu().numpy() + else: + bboxes = instances[i].pred_boxes.tensor.cpu().numpy() + scores = instances[i].scores.cpu().numpy() + cats = instances[i].pred_classes.cpu().numpy() + for j in range(len(bboxes)): + if scores[j] > vis_thresh: + bbox = bboxes[j] + cl = COLORS[cats[j], 0, 0] + cl = (int(cl[0]), int(cl[1]), int(cl[2])) + cv2.rectangle( + image, + (int(bbox[0]), int(bbox[1])), + (int(bbox[2]), int(bbox[3])), + cl, 2, cv2.LINE_AA) + if debug_show_name: + cat = cats[j] + txt = '{}{:.1f}'.format( + cat2name[cat] if cat > 0 else '', + scores[j]) + font = cv2.FONT_HERSHEY_SIMPLEX + cat_size = cv2.getTextSize(txt, font, 0.5, 2)[0] + cv2.rectangle( + image, + (int(bbox[0]), int(bbox[1] - cat_size[1] - 2)), + (int(bbox[0] + cat_size[0]), int(bbox[1] - 2)), + (int(cl[0]), int(cl[1]), int(cl[2])), -1) + cv2.putText( + image, txt, (int(bbox[0]), int(bbox[1] - 2)), + font, 0.5, (0, 0, 0), thickness=1, lineType=cv2.LINE_AA) + if proposals is not None: + proposal_image = images[i].detach().cpu().numpy().transpose(1, 2, 0).astype(np.uint8).copy() + bboxes = proposals[i].proposal_boxes.tensor.cpu().numpy() + if proposals[i].has('scores'): + scores = proposals[i].scores.cpu().numpy() + else: + scores = proposals[i].objectness_logits.sigmoid().cpu().numpy() + for j in range(len(bboxes)): + if scores[j] > vis_thresh: + bbox = bboxes[j] + cl = (209, 159, 83) + cv2.rectangle( + proposal_image, + (int(bbox[0]), int(bbox[1])), + (int(bbox[2]), int(bbox[3])), + cl, 2, cv2.LINE_AA) + + cv2.imshow('image', image) + if proposals is not None: + cv2.imshow('proposals', proposal_image) + if save_debug: + global cnt + cnt += 1 + cv2.imwrite('output/save_debug/{}.jpg'.format(cnt), proposal_image) + cv2.waitKey() \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/dense_heads/centernet.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/dense_heads/centernet.py new file mode 100644 index 0000000000000000000000000000000000000000..aef02942aba1d207a76500cfe9b17bb13685e41c --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/dense_heads/centernet.py @@ -0,0 +1,878 @@ + +import math +import json +import copy +from typing import List, Dict +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F + +from detectron2.modeling.proposal_generator.build import PROPOSAL_GENERATOR_REGISTRY +from detectron2.layers import ShapeSpec, cat +from detectron2.structures import Instances, Boxes +from detectron2.modeling import detector_postprocess +from detectron2.utils.comm import get_world_size +from detectron2.config import configurable + +from ..layers.heatmap_focal_loss import heatmap_focal_loss_jit +from ..layers.heatmap_focal_loss import binary_heatmap_focal_loss_jit +from ..layers.iou_loss import IOULoss +from ..layers.ml_nms import ml_nms +from ..debug import debug_train, debug_test +from .utils import reduce_sum, _transpose +from .centernet_head import CenterNetHead + +__all__ = ["CenterNet"] + +INF = 100000000 + +@PROPOSAL_GENERATOR_REGISTRY.register() +class CenterNet(nn.Module): + @configurable + def __init__(self, + # input_shape: Dict[str, ShapeSpec], + in_channels=256, + *, + num_classes=80, + in_features=("p3", "p4", "p5", "p6", "p7"), + strides=(8, 16, 32, 64, 128), + score_thresh=0.05, + hm_min_overlap=0.8, + loc_loss_type='giou', + min_radius=4, + hm_focal_alpha=0.25, + hm_focal_beta=4, + loss_gamma=2.0, + reg_weight=2.0, + not_norm_reg=True, + with_agn_hm=False, + only_proposal=False, + as_proposal=False, + not_nms=False, + pos_weight=1., + neg_weight=1., + sigmoid_clamp=1e-4, + ignore_high_fp=-1., + center_nms=False, + sizes_of_interest=[[0,80],[64,160],[128,320],[256,640],[512,10000000]], + more_pos=False, + more_pos_thresh=0.2, + more_pos_topk=9, + pre_nms_topk_train=1000, + pre_nms_topk_test=1000, + post_nms_topk_train=100, + post_nms_topk_test=100, + nms_thresh_train=0.6, + nms_thresh_test=0.6, + no_reduce=False, + not_clamp_box=False, + debug=False, + vis_thresh=0.5, + pixel_mean=[103.530,116.280,123.675], + pixel_std=[1.0,1.0,1.0], + device='cuda', + centernet_head=None, + ): + super().__init__() + self.num_classes = num_classes + self.in_features = in_features + self.strides = strides + self.score_thresh = score_thresh + self.min_radius = min_radius + self.hm_focal_alpha = hm_focal_alpha + self.hm_focal_beta = hm_focal_beta + self.loss_gamma = loss_gamma + self.reg_weight = reg_weight + self.not_norm_reg = not_norm_reg + self.with_agn_hm = with_agn_hm + self.only_proposal = only_proposal + self.as_proposal = as_proposal + self.not_nms = not_nms + self.pos_weight = pos_weight + self.neg_weight = neg_weight + self.sigmoid_clamp = sigmoid_clamp + self.ignore_high_fp = ignore_high_fp + self.center_nms = center_nms + self.sizes_of_interest = sizes_of_interest + self.more_pos = more_pos + self.more_pos_thresh = more_pos_thresh + self.more_pos_topk = more_pos_topk + self.pre_nms_topk_train = pre_nms_topk_train + self.pre_nms_topk_test = pre_nms_topk_test + self.post_nms_topk_train = post_nms_topk_train + self.post_nms_topk_test = post_nms_topk_test + self.nms_thresh_train = nms_thresh_train + self.nms_thresh_test = nms_thresh_test + self.no_reduce = no_reduce + self.not_clamp_box = not_clamp_box + + self.debug = debug + self.vis_thresh = vis_thresh + if self.center_nms: + self.not_nms = True + self.iou_loss = IOULoss(loc_loss_type) + assert (not self.only_proposal) or self.with_agn_hm + # delta for rendering heatmap + self.delta = (1 - hm_min_overlap) / (1 + hm_min_overlap) + if centernet_head is None: + self.centernet_head = CenterNetHead( + in_channels=in_channels, + num_levels=len(in_features), + with_agn_hm=with_agn_hm, + only_proposal=only_proposal) + else: + self.centernet_head = centernet_head + if self.debug: + pixel_mean = torch.Tensor(pixel_mean).to( + torch.device(device)).view(3, 1, 1) + pixel_std = torch.Tensor(pixel_std).to( + torch.device(device)).view(3, 1, 1) + self.denormalizer = lambda x: x * pixel_std + pixel_mean + + @classmethod + def from_config(cls, cfg, input_shape): + ret = { + # 'input_shape': input_shape, + 'in_channels': input_shape[ + cfg.MODEL.CENTERNET.IN_FEATURES[0]].channels, + 'num_classes': cfg.MODEL.CENTERNET.NUM_CLASSES, + 'in_features': cfg.MODEL.CENTERNET.IN_FEATURES, + 'strides': cfg.MODEL.CENTERNET.FPN_STRIDES, + 'score_thresh': cfg.MODEL.CENTERNET.INFERENCE_TH, + 'loc_loss_type': cfg.MODEL.CENTERNET.LOC_LOSS_TYPE, + 'hm_min_overlap': cfg.MODEL.CENTERNET.HM_MIN_OVERLAP, + 'min_radius': cfg.MODEL.CENTERNET.MIN_RADIUS, + 'hm_focal_alpha': cfg.MODEL.CENTERNET.HM_FOCAL_ALPHA, + 'hm_focal_beta': cfg.MODEL.CENTERNET.HM_FOCAL_BETA, + 'loss_gamma': cfg.MODEL.CENTERNET.LOSS_GAMMA, + 'reg_weight': cfg.MODEL.CENTERNET.REG_WEIGHT, + 'not_norm_reg': cfg.MODEL.CENTERNET.NOT_NORM_REG, + 'with_agn_hm': cfg.MODEL.CENTERNET.WITH_AGN_HM, + 'only_proposal': cfg.MODEL.CENTERNET.ONLY_PROPOSAL, + 'as_proposal': cfg.MODEL.CENTERNET.AS_PROPOSAL, + 'not_nms': cfg.MODEL.CENTERNET.NOT_NMS, + 'pos_weight': cfg.MODEL.CENTERNET.POS_WEIGHT, + 'neg_weight': cfg.MODEL.CENTERNET.NEG_WEIGHT, + 'sigmoid_clamp': cfg.MODEL.CENTERNET.SIGMOID_CLAMP, + 'ignore_high_fp': cfg.MODEL.CENTERNET.IGNORE_HIGH_FP, + 'center_nms': cfg.MODEL.CENTERNET.CENTER_NMS, + 'sizes_of_interest': cfg.MODEL.CENTERNET.SOI, + 'more_pos': cfg.MODEL.CENTERNET.MORE_POS, + 'more_pos_thresh': cfg.MODEL.CENTERNET.MORE_POS_THRESH, + 'more_pos_topk': cfg.MODEL.CENTERNET.MORE_POS_TOPK, + 'pre_nms_topk_train': cfg.MODEL.CENTERNET.PRE_NMS_TOPK_TRAIN, + 'pre_nms_topk_test': cfg.MODEL.CENTERNET.PRE_NMS_TOPK_TEST, + 'post_nms_topk_train': cfg.MODEL.CENTERNET.POST_NMS_TOPK_TRAIN, + 'post_nms_topk_test': cfg.MODEL.CENTERNET.POST_NMS_TOPK_TEST, + 'nms_thresh_train': cfg.MODEL.CENTERNET.NMS_TH_TRAIN, + 'nms_thresh_test': cfg.MODEL.CENTERNET.NMS_TH_TEST, + 'no_reduce': cfg.MODEL.CENTERNET.NO_REDUCE, + 'not_clamp_box': cfg.INPUT.NOT_CLAMP_BOX, + 'debug': cfg.DEBUG, + 'vis_thresh': cfg.VIS_THRESH, + 'pixel_mean': cfg.MODEL.PIXEL_MEAN, + 'pixel_std': cfg.MODEL.PIXEL_STD, + 'device': cfg.MODEL.DEVICE, + 'centernet_head': CenterNetHead( + cfg, [input_shape[f] for f in cfg.MODEL.CENTERNET.IN_FEATURES]), + } + return ret + + + def forward(self, images, features_dict, gt_instances): + features = [features_dict[f] for f in self.in_features] + clss_per_level, reg_pred_per_level, agn_hm_pred_per_level = \ + self.centernet_head(features) + grids = self.compute_grids(features) + shapes_per_level = grids[0].new_tensor( + [(x.shape[2], x.shape[3]) for x in reg_pred_per_level]) + + if not self.training: + return self.inference( + images, clss_per_level, reg_pred_per_level, + agn_hm_pred_per_level, grids) + else: + pos_inds, labels, reg_targets, flattened_hms = \ + self._get_ground_truth( + grids, shapes_per_level, gt_instances) + # logits_pred: M x F, reg_pred: M x 4, agn_hm_pred: M + logits_pred, reg_pred, agn_hm_pred = self._flatten_outputs( + clss_per_level, reg_pred_per_level, agn_hm_pred_per_level) + + if self.more_pos: + # add more pixels as positive if \ + # 1. they are within the center3x3 region of an object + # 2. their regression losses are small (= 0).squeeze(1) + reg_pred = reg_pred[reg_inds] + reg_targets_pos = reg_targets[reg_inds] + reg_weight_map = flattened_hms.max(dim=1)[0] + reg_weight_map = reg_weight_map[reg_inds] + reg_weight_map = reg_weight_map * 0 + 1 \ + if self.not_norm_reg else reg_weight_map + if self.no_reduce: + reg_norm = max(reg_weight_map.sum(), 1) + else: + reg_norm = max(reduce_sum(reg_weight_map.sum()).item() / num_gpus, 1) + + reg_loss = self.reg_weight * self.iou_loss( + reg_pred, reg_targets_pos, reg_weight_map, + reduction='sum') / reg_norm + losses['loss_centernet_loc'] = reg_loss + + if self.with_agn_hm: + cat_agn_heatmap = flattened_hms.max(dim=1)[0] # M + agn_pos_loss, agn_neg_loss = binary_heatmap_focal_loss_jit( + agn_hm_pred.float(), cat_agn_heatmap.float(), pos_inds, + alpha=self.hm_focal_alpha, + beta=self.hm_focal_beta, + gamma=self.loss_gamma, + sigmoid_clamp=self.sigmoid_clamp, + ignore_high_fp=self.ignore_high_fp, + ) + agn_pos_loss = self.pos_weight * agn_pos_loss / num_pos_avg + agn_neg_loss = self.neg_weight * agn_neg_loss / num_pos_avg + losses['loss_centernet_agn_pos'] = agn_pos_loss + losses['loss_centernet_agn_neg'] = agn_neg_loss + + if self.debug: + print('losses', losses) + print('total_num_pos', total_num_pos) + return losses + + + def compute_grids(self, features): + grids = [] + for level, feature in enumerate(features): + h, w = feature.size()[-2:] + shifts_x = torch.arange( + 0, w * self.strides[level], + step=self.strides[level], + dtype=torch.float32, device=feature.device) + shifts_y = torch.arange( + 0, h * self.strides[level], + step=self.strides[level], + dtype=torch.float32, device=feature.device) + shift_y, shift_x = torch.meshgrid(shifts_y, shifts_x) + shift_x = shift_x.reshape(-1) + shift_y = shift_y.reshape(-1) + grids_per_level = torch.stack((shift_x, shift_y), dim=1) + \ + self.strides[level] // 2 + grids.append(grids_per_level) + return grids + + + def _get_ground_truth(self, grids, shapes_per_level, gt_instances): + ''' + Input: + grids: list of tensors [(hl x wl, 2)]_l + shapes_per_level: list of tuples L x 2: + gt_instances: gt instances + Retuen: + pos_inds: N + labels: N + reg_targets: M x 4 + flattened_hms: M x C or M x 1 + N: number of objects in all images + M: number of pixels from all FPN levels + ''' + + # get positive pixel index + if not self.more_pos: + pos_inds, labels = self._get_label_inds( + gt_instances, shapes_per_level) + else: + pos_inds, labels = None, None + heatmap_channels = self.num_classes + L = len(grids) + num_loc_list = [len(loc) for loc in grids] + strides = torch.cat([ + shapes_per_level.new_ones(num_loc_list[l]) * self.strides[l] \ + for l in range(L)]).float() # M + reg_size_ranges = torch.cat([ + shapes_per_level.new_tensor(self.sizes_of_interest[l]).float().view( + 1, 2).expand(num_loc_list[l], 2) for l in range(L)]) # M x 2 + grids = torch.cat(grids, dim=0) # M x 2 + M = grids.shape[0] + + reg_targets = [] + flattened_hms = [] + for i in range(len(gt_instances)): # images + boxes = gt_instances[i].gt_boxes.tensor # N x 4 + area = gt_instances[i].gt_boxes.area() # N + gt_classes = gt_instances[i].gt_classes # N in [0, self.num_classes] + + N = boxes.shape[0] + if N == 0: + reg_targets.append(grids.new_zeros((M, 4)) - INF) + flattened_hms.append( + grids.new_zeros(( + M, 1 if self.only_proposal else heatmap_channels))) + continue + + l = grids[:, 0].view(M, 1) - boxes[:, 0].view(1, N) # M x N + t = grids[:, 1].view(M, 1) - boxes[:, 1].view(1, N) # M x N + r = boxes[:, 2].view(1, N) - grids[:, 0].view(M, 1) # M x N + b = boxes[:, 3].view(1, N) - grids[:, 1].view(M, 1) # M x N + reg_target = torch.stack([l, t, r, b], dim=2) # M x N x 4 + + centers = ((boxes[:, [0, 1]] + boxes[:, [2, 3]]) / 2) # N x 2 + centers_expanded = centers.view(1, N, 2).expand(M, N, 2) # M x N x 2 + strides_expanded = strides.view(M, 1, 1).expand(M, N, 2) + centers_discret = ((centers_expanded / strides_expanded).int() * \ + strides_expanded).float() + strides_expanded / 2 # M x N x 2 + + is_peak = (((grids.view(M, 1, 2).expand(M, N, 2) - \ + centers_discret) ** 2).sum(dim=2) == 0) # M x N + is_in_boxes = reg_target.min(dim=2)[0] > 0 # M x N + is_center3x3 = self.get_center3x3( + grids, centers, strides) & is_in_boxes # M x N + is_cared_in_the_level = self.assign_reg_fpn( + reg_target, reg_size_ranges) # M x N + reg_mask = is_center3x3 & is_cared_in_the_level # M x N + + dist2 = ((grids.view(M, 1, 2).expand(M, N, 2) - \ + centers_expanded) ** 2).sum(dim=2) # M x N + dist2[is_peak] = 0 + radius2 = self.delta ** 2 * 2 * area # N + radius2 = torch.clamp( + radius2, min=self.min_radius ** 2) + weighted_dist2 = dist2 / radius2.view(1, N).expand(M, N) # M x N + reg_target = self._get_reg_targets( + reg_target, weighted_dist2.clone(), reg_mask, area) # M x 4 + + if self.only_proposal: + flattened_hm = self._create_agn_heatmaps_from_dist( + weighted_dist2.clone()) # M x 1 + else: + flattened_hm = self._create_heatmaps_from_dist( + weighted_dist2.clone(), gt_classes, + channels=heatmap_channels) # M x C + + reg_targets.append(reg_target) + flattened_hms.append(flattened_hm) + + # transpose im first training_targets to level first ones + reg_targets = _transpose(reg_targets, num_loc_list) + flattened_hms = _transpose(flattened_hms, num_loc_list) + for l in range(len(reg_targets)): + reg_targets[l] = reg_targets[l] / float(self.strides[l]) + reg_targets = cat([x for x in reg_targets], dim=0) # MB x 4 + flattened_hms = cat([x for x in flattened_hms], dim=0) # MB x C + + return pos_inds, labels, reg_targets, flattened_hms + + + def _get_label_inds(self, gt_instances, shapes_per_level): + ''' + Inputs: + gt_instances: [n_i], sum n_i = N + shapes_per_level: L x 2 [(h_l, w_l)]_L + Returns: + pos_inds: N' + labels: N' + ''' + pos_inds = [] + labels = [] + L = len(self.strides) + B = len(gt_instances) + shapes_per_level = shapes_per_level.long() + loc_per_level = (shapes_per_level[:, 0] * shapes_per_level[:, 1]).long() # L + level_bases = [] + s = 0 + for l in range(L): + level_bases.append(s) + s = s + B * loc_per_level[l] + level_bases = shapes_per_level.new_tensor(level_bases).long() # L + strides_default = shapes_per_level.new_tensor(self.strides).float() # L + for im_i in range(B): + targets_per_im = gt_instances[im_i] + bboxes = targets_per_im.gt_boxes.tensor # n x 4 + n = bboxes.shape[0] + centers = ((bboxes[:, [0, 1]] + bboxes[:, [2, 3]]) / 2) # n x 2 + centers = centers.view(n, 1, 2).expand(n, L, 2).contiguous() + if self.not_clamp_box: + h, w = gt_instances[im_i]._image_size + centers[:, :, 0].clamp_(min=0).clamp_(max=w-1) + centers[:, :, 1].clamp_(min=0).clamp_(max=h-1) + strides = strides_default.view(1, L, 1).expand(n, L, 2) + centers_inds = (centers / strides).long() # n x L x 2 + Ws = shapes_per_level[:, 1].view(1, L).expand(n, L) + pos_ind = level_bases.view(1, L).expand(n, L) + \ + im_i * loc_per_level.view(1, L).expand(n, L) + \ + centers_inds[:, :, 1] * Ws + \ + centers_inds[:, :, 0] # n x L + is_cared_in_the_level = self.assign_fpn_level(bboxes) + pos_ind = pos_ind[is_cared_in_the_level].view(-1) + label = targets_per_im.gt_classes.view( + n, 1).expand(n, L)[is_cared_in_the_level].view(-1) + + pos_inds.append(pos_ind) # n' + labels.append(label) # n' + pos_inds = torch.cat(pos_inds, dim=0).long() + labels = torch.cat(labels, dim=0) + return pos_inds, labels # N, N + + + def assign_fpn_level(self, boxes): + ''' + Inputs: + boxes: n x 4 + size_ranges: L x 2 + Return: + is_cared_in_the_level: n x L + ''' + size_ranges = boxes.new_tensor( + self.sizes_of_interest).view(len(self.sizes_of_interest), 2) # L x 2 + crit = ((boxes[:, 2:] - boxes[:, :2]) **2).sum(dim=1) ** 0.5 / 2 # n + n, L = crit.shape[0], size_ranges.shape[0] + crit = crit.view(n, 1).expand(n, L) + size_ranges_expand = size_ranges.view(1, L, 2).expand(n, L, 2) + is_cared_in_the_level = (crit >= size_ranges_expand[:, :, 0]) & \ + (crit <= size_ranges_expand[:, :, 1]) + return is_cared_in_the_level + + + def assign_reg_fpn(self, reg_targets_per_im, size_ranges): + ''' + TODO (Xingyi): merge it with assign_fpn_level + Inputs: + reg_targets_per_im: M x N x 4 + size_ranges: M x 2 + ''' + crit = ((reg_targets_per_im[:, :, :2] + \ + reg_targets_per_im[:, :, 2:])**2).sum(dim=2) ** 0.5 / 2 # M x N + is_cared_in_the_level = (crit >= size_ranges[:, [0]]) & \ + (crit <= size_ranges[:, [1]]) + return is_cared_in_the_level + + + def _get_reg_targets(self, reg_targets, dist, mask, area): + ''' + reg_targets (M x N x 4): long tensor + dist (M x N) + is_*: M x N + ''' + dist[mask == 0] = INF * 1.0 + min_dist, min_inds = dist.min(dim=1) # M + reg_targets_per_im = reg_targets[ + range(len(reg_targets)), min_inds] # M x N x 4 --> M x 4 + reg_targets_per_im[min_dist == INF] = - INF + return reg_targets_per_im + + + def _create_heatmaps_from_dist(self, dist, labels, channels): + ''' + dist: M x N + labels: N + return: + heatmaps: M x C + ''' + heatmaps = dist.new_zeros((dist.shape[0], channels)) + for c in range(channels): + inds = (labels == c) # N + if inds.int().sum() == 0: + continue + heatmaps[:, c] = torch.exp(-dist[:, inds].min(dim=1)[0]) + zeros = heatmaps[:, c] < 1e-4 + heatmaps[zeros, c] = 0 + return heatmaps + + + def _create_agn_heatmaps_from_dist(self, dist): + ''' + TODO (Xingyi): merge it with _create_heatmaps_from_dist + dist: M x N + return: + heatmaps: M x 1 + ''' + heatmaps = dist.new_zeros((dist.shape[0], 1)) + heatmaps[:, 0] = torch.exp(-dist.min(dim=1)[0]) + zeros = heatmaps < 1e-4 + heatmaps[zeros] = 0 + return heatmaps + + + def _flatten_outputs(self, clss, reg_pred, agn_hm_pred): + # Reshape: (N, F, Hl, Wl) -> (N, Hl, Wl, F) -> (sum_l N*Hl*Wl, F) + clss = cat([x.permute(0, 2, 3, 1).reshape(-1, x.shape[1]) \ + for x in clss], dim=0) if clss[0] is not None else None + reg_pred = cat( + [x.permute(0, 2, 3, 1).reshape(-1, 4) for x in reg_pred], dim=0) + agn_hm_pred = cat([x.permute(0, 2, 3, 1).reshape(-1) \ + for x in agn_hm_pred], dim=0) if self.with_agn_hm else None + return clss, reg_pred, agn_hm_pred + + + def get_center3x3(self, locations, centers, strides): + ''' + Inputs: + locations: M x 2 + centers: N x 2 + strides: M + ''' + M, N = locations.shape[0], centers.shape[0] + locations_expanded = locations.view(M, 1, 2).expand(M, N, 2) # M x N x 2 + centers_expanded = centers.view(1, N, 2).expand(M, N, 2) # M x N x 2 + strides_expanded = strides.view(M, 1, 1).expand(M, N, 2) # M x N + centers_discret = ((centers_expanded / strides_expanded).int() * \ + strides_expanded).float() + strides_expanded / 2 # M x N x 2 + dist_x = (locations_expanded[:, :, 0] - centers_discret[:, :, 0]).abs() + dist_y = (locations_expanded[:, :, 1] - centers_discret[:, :, 1]).abs() + return (dist_x <= strides_expanded[:, :, 0]) & \ + (dist_y <= strides_expanded[:, :, 0]) + + + @torch.no_grad() + def inference(self, images, clss_per_level, reg_pred_per_level, + agn_hm_pred_per_level, grids): + logits_pred = [x.sigmoid() if x is not None else None \ + for x in clss_per_level] + agn_hm_pred_per_level = [x.sigmoid() if x is not None else None \ + for x in agn_hm_pred_per_level] + + if self.only_proposal: + proposals = self.predict_instances( + grids, agn_hm_pred_per_level, reg_pred_per_level, + images.image_sizes, [None for _ in agn_hm_pred_per_level]) + else: + proposals = self.predict_instances( + grids, logits_pred, reg_pred_per_level, + images.image_sizes, agn_hm_pred_per_level) + if self.as_proposal or self.only_proposal: + for p in range(len(proposals)): + proposals[p].proposal_boxes = proposals[p].get('pred_boxes') + proposals[p].objectness_logits = proposals[p].get('scores') + proposals[p].remove('pred_boxes') + + if self.debug: + debug_test( + [self.denormalizer(x) for x in images], + logits_pred, reg_pred_per_level, + agn_hm_pred_per_level, preds=proposals, + vis_thresh=self.vis_thresh, + debug_show_name=False) + return proposals, {} + + + @torch.no_grad() + def predict_instances( + self, grids, logits_pred, reg_pred, image_sizes, agn_hm_pred, + is_proposal=False): + sampled_boxes = [] + for l in range(len(grids)): + sampled_boxes.append(self.predict_single_level( + grids[l], logits_pred[l], reg_pred[l] * self.strides[l], + image_sizes, agn_hm_pred[l], l, is_proposal=is_proposal)) + boxlists = list(zip(*sampled_boxes)) + boxlists = [Instances.cat(boxlist) for boxlist in boxlists] + boxlists = self.nms_and_topK( + boxlists, nms=not self.not_nms) + return boxlists + + + @torch.no_grad() + def predict_single_level( + self, grids, heatmap, reg_pred, image_sizes, agn_hm, level, + is_proposal=False): + N, C, H, W = heatmap.shape + # put in the same format as grids + if self.center_nms: + heatmap_nms = nn.functional.max_pool2d( + heatmap, (3, 3), stride=1, padding=1) + heatmap = heatmap * (heatmap_nms == heatmap).float() + heatmap = heatmap.permute(0, 2, 3, 1) # N x H x W x C + heatmap = heatmap.reshape(N, -1, C) # N x HW x C + box_regression = reg_pred.view(N, 4, H, W).permute(0, 2, 3, 1) # N x H x W x 4 + box_regression = box_regression.reshape(N, -1, 4) + + candidate_inds = heatmap > self.score_thresh # 0.05 + pre_nms_top_n = candidate_inds.view(N, -1).sum(1) # N + pre_nms_topk = self.pre_nms_topk_train if self.training else self.pre_nms_topk_test + pre_nms_top_n = pre_nms_top_n.clamp(max=pre_nms_topk) # N + + if agn_hm is not None: + agn_hm = agn_hm.view(N, 1, H, W).permute(0, 2, 3, 1) + agn_hm = agn_hm.reshape(N, -1) + heatmap = heatmap * agn_hm[:, :, None] + + results = [] + for i in range(N): + per_box_cls = heatmap[i] # HW x C + per_candidate_inds = candidate_inds[i] # n + per_box_cls = per_box_cls[per_candidate_inds] # n + + per_candidate_nonzeros = per_candidate_inds.nonzero() # n + per_box_loc = per_candidate_nonzeros[:, 0] # n + per_class = per_candidate_nonzeros[:, 1] # n + + per_box_regression = box_regression[i] # HW x 4 + per_box_regression = per_box_regression[per_box_loc] # n x 4 + per_grids = grids[per_box_loc] # n x 2 + + per_pre_nms_top_n = pre_nms_top_n[i] # 1 + + if per_candidate_inds.sum().item() > per_pre_nms_top_n.item(): + per_box_cls, top_k_indices = \ + per_box_cls.topk(per_pre_nms_top_n, sorted=False) + per_class = per_class[top_k_indices] + per_box_regression = per_box_regression[top_k_indices] + per_grids = per_grids[top_k_indices] + + detections = torch.stack([ + per_grids[:, 0] - per_box_regression[:, 0], + per_grids[:, 1] - per_box_regression[:, 1], + per_grids[:, 0] + per_box_regression[:, 2], + per_grids[:, 1] + per_box_regression[:, 3], + ], dim=1) # n x 4 + + # avoid invalid boxes in RoI heads + detections[:, 2] = torch.max(detections[:, 2], detections[:, 0] + 0.01) + detections[:, 3] = torch.max(detections[:, 3], detections[:, 1] + 0.01) + boxlist = Instances(image_sizes[i]) + boxlist.scores = torch.sqrt(per_box_cls) \ + if self.with_agn_hm else per_box_cls # n + # import pdb; pdb.set_trace() + boxlist.pred_boxes = Boxes(detections) + boxlist.pred_classes = per_class + results.append(boxlist) + return results + + + @torch.no_grad() + def nms_and_topK(self, boxlists, nms=True): + num_images = len(boxlists) + results = [] + for i in range(num_images): + nms_thresh = self.nms_thresh_train if self.training else \ + self.nms_thresh_test + result = ml_nms(boxlists[i], nms_thresh) if nms else boxlists[i] + if self.debug: + print('#proposals before nms', len(boxlists[i])) + print('#proposals after nms', len(result)) + num_dets = len(result) + post_nms_topk = self.post_nms_topk_train if self.training else \ + self.post_nms_topk_test + if num_dets > post_nms_topk: + cls_scores = result.scores + image_thresh, _ = torch.kthvalue( + cls_scores.float().cpu(), + num_dets - post_nms_topk + 1 + ) + keep = cls_scores >= image_thresh.item() + keep = torch.nonzero(keep).squeeze(1) + result = result[keep] + if self.debug: + print('#proposals after filter', len(result)) + results.append(result) + return results + + + @torch.no_grad() + def _add_more_pos(self, reg_pred, gt_instances, shapes_per_level): + labels, level_masks, c33_inds, c33_masks, c33_regs = \ + self._get_c33_inds(gt_instances, shapes_per_level) + N, L, K = labels.shape[0], len(self.strides), 9 + c33_inds[c33_masks == 0] = 0 + reg_pred_c33 = reg_pred[c33_inds].detach() # N x L x K + invalid_reg = c33_masks == 0 + c33_regs_expand = c33_regs.view(N * L * K, 4).clamp(min=0) + if N > 0: + with torch.no_grad(): + c33_reg_loss = self.iou_loss( + reg_pred_c33.view(N * L * K, 4), + c33_regs_expand, None, + reduction='none').view(N, L, K).detach() # N x L x K + else: + c33_reg_loss = reg_pred_c33.new_zeros((N, L, K)).detach() + c33_reg_loss[invalid_reg] = INF # N x L x K + c33_reg_loss.view(N * L, K)[level_masks.view(N * L), 4] = 0 # real center + c33_reg_loss = c33_reg_loss.view(N, L * K) + if N == 0: + loss_thresh = c33_reg_loss.new_ones((N)).float() + else: + loss_thresh = torch.kthvalue( + c33_reg_loss, self.more_pos_topk, dim=1)[0] # N + loss_thresh[loss_thresh > self.more_pos_thresh] = self.more_pos_thresh # N + new_pos = c33_reg_loss.view(N, L, K) < \ + loss_thresh.view(N, 1, 1).expand(N, L, K) + pos_inds = c33_inds[new_pos].view(-1) # P + labels = labels.view(N, 1, 1).expand(N, L, K)[new_pos].view(-1) + return pos_inds, labels + + + @torch.no_grad() + def _get_c33_inds(self, gt_instances, shapes_per_level): + ''' + TODO (Xingyi): The current implementation is ugly. Refactor. + Get the center (and the 3x3 region near center) locations of each objects + Inputs: + gt_instances: [n_i], sum n_i = N + shapes_per_level: L x 2 [(h_l, w_l)]_L + ''' + labels = [] + level_masks = [] + c33_inds = [] + c33_masks = [] + c33_regs = [] + L = len(self.strides) + B = len(gt_instances) + shapes_per_level = shapes_per_level.long() + loc_per_level = (shapes_per_level[:, 0] * shapes_per_level[:, 1]).long() # L + level_bases = [] + s = 0 + for l in range(L): + level_bases.append(s) + s = s + B * loc_per_level[l] + level_bases = shapes_per_level.new_tensor(level_bases).long() # L + strides_default = shapes_per_level.new_tensor(self.strides).float() # L + K = 9 + dx = shapes_per_level.new_tensor([-1, 0, 1, -1, 0, 1, -1, 0, 1]).long() + dy = shapes_per_level.new_tensor([-1, -1, -1, 0, 0, 0, 1, 1, 1]).long() + for im_i in range(B): + targets_per_im = gt_instances[im_i] + bboxes = targets_per_im.gt_boxes.tensor # n x 4 + n = bboxes.shape[0] + if n == 0: + continue + centers = ((bboxes[:, [0, 1]] + bboxes[:, [2, 3]]) / 2) # n x 2 + centers = centers.view(n, 1, 2).expand(n, L, 2) + + strides = strides_default.view(1, L, 1).expand(n, L, 2) # + centers_inds = (centers / strides).long() # n x L x 2 + center_grids = centers_inds * strides + strides // 2# n x L x 2 + l = center_grids[:, :, 0] - bboxes[:, 0].view(n, 1).expand(n, L) + t = center_grids[:, :, 1] - bboxes[:, 1].view(n, 1).expand(n, L) + r = bboxes[:, 2].view(n, 1).expand(n, L) - center_grids[:, :, 0] + b = bboxes[:, 3].view(n, 1).expand(n, L) - center_grids[:, :, 1] # n x L + reg = torch.stack([l, t, r, b], dim=2) # n x L x 4 + reg = reg / strides_default.view(1, L, 1).expand(n, L, 4).float() + + Ws = shapes_per_level[:, 1].view(1, L).expand(n, L) + Hs = shapes_per_level[:, 0].view(1, L).expand(n, L) + expand_Ws = Ws.view(n, L, 1).expand(n, L, K) + expand_Hs = Hs.view(n, L, 1).expand(n, L, K) + label = targets_per_im.gt_classes.view(n).clone() + mask = reg.min(dim=2)[0] >= 0 # n x L + mask = mask & self.assign_fpn_level(bboxes) + labels.append(label) # n + level_masks.append(mask) # n x L + + Dy = dy.view(1, 1, K).expand(n, L, K) + Dx = dx.view(1, 1, K).expand(n, L, K) + c33_ind = level_bases.view(1, L, 1).expand(n, L, K) + \ + im_i * loc_per_level.view(1, L, 1).expand(n, L, K) + \ + (centers_inds[:, :, 1:2].expand(n, L, K) + Dy) * expand_Ws + \ + (centers_inds[:, :, 0:1].expand(n, L, K) + Dx) # n x L x K + + c33_mask = \ + ((centers_inds[:, :, 1:2].expand(n, L, K) + dy) < expand_Hs) & \ + ((centers_inds[:, :, 1:2].expand(n, L, K) + dy) >= 0) & \ + ((centers_inds[:, :, 0:1].expand(n, L, K) + dx) < expand_Ws) & \ + ((centers_inds[:, :, 0:1].expand(n, L, K) + dx) >= 0) + # TODO (Xingyi): think about better way to implement this + # Currently it hard codes the 3x3 region + c33_reg = reg.view(n, L, 1, 4).expand(n, L, K, 4).clone() + c33_reg[:, :, [0, 3, 6], 0] -= 1 + c33_reg[:, :, [0, 3, 6], 2] += 1 + c33_reg[:, :, [2, 5, 8], 0] += 1 + c33_reg[:, :, [2, 5, 8], 2] -= 1 + c33_reg[:, :, [0, 1, 2], 1] -= 1 + c33_reg[:, :, [0, 1, 2], 3] += 1 + c33_reg[:, :, [6, 7, 8], 1] += 1 + c33_reg[:, :, [6, 7, 8], 3] -= 1 + c33_mask = c33_mask & (c33_reg.min(dim=3)[0] >= 0) # n x L x K + c33_inds.append(c33_ind) + c33_masks.append(c33_mask) + c33_regs.append(c33_reg) + + if len(level_masks) > 0: + labels = torch.cat(labels, dim=0) + level_masks = torch.cat(level_masks, dim=0) + c33_inds = torch.cat(c33_inds, dim=0).long() + c33_regs = torch.cat(c33_regs, dim=0) + c33_masks = torch.cat(c33_masks, dim=0) + else: + labels = shapes_per_level.new_zeros((0)).long() + level_masks = shapes_per_level.new_zeros((0, L)).bool() + c33_inds = shapes_per_level.new_zeros((0, L, K)).long() + c33_regs = shapes_per_level.new_zeros((0, L, K, 4)).float() + c33_masks = shapes_per_level.new_zeros((0, L, K)).bool() + return labels, level_masks, c33_inds, c33_masks, c33_regs # N x L, N x L x K diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/dense_heads/centernet_head.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/dense_heads/centernet_head.py new file mode 100644 index 0000000000000000000000000000000000000000..57e0960a57c904c097b6a717391474a4a635dd7d --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/dense_heads/centernet_head.py @@ -0,0 +1,162 @@ +import math +from typing import List +import torch +from torch import nn +from torch.nn import functional as F + +from detectron2.layers import ShapeSpec, get_norm +from detectron2.config import configurable +from ..layers.deform_conv import DFConv2d + +__all__ = ["CenterNetHead"] + +class Scale(nn.Module): + def __init__(self, init_value=1.0): + super(Scale, self).__init__() + self.scale = nn.Parameter(torch.FloatTensor([init_value])) + + def forward(self, input): + return input * self.scale + +class CenterNetHead(nn.Module): + @configurable + def __init__(self, + # input_shape: List[ShapeSpec], + in_channels, + num_levels, + *, + num_classes=80, + with_agn_hm=False, + only_proposal=False, + norm='GN', + num_cls_convs=4, + num_box_convs=4, + num_share_convs=0, + use_deformable=False, + prior_prob=0.01): + super().__init__() + self.num_classes = num_classes + self.with_agn_hm = with_agn_hm + self.only_proposal = only_proposal + self.out_kernel = 3 + + head_configs = { + "cls": (num_cls_convs if not self.only_proposal else 0, \ + use_deformable), + "bbox": (num_box_convs, use_deformable), + "share": (num_share_convs, use_deformable)} + + # in_channels = [s.channels for s in input_shape] + # assert len(set(in_channels)) == 1, \ + # "Each level must have the same channel!" + # in_channels = in_channels[0] + channels = { + 'cls': in_channels, + 'bbox': in_channels, + 'share': in_channels, + } + for head in head_configs: + tower = [] + num_convs, use_deformable = head_configs[head] + channel = channels[head] + for i in range(num_convs): + if use_deformable and i == num_convs - 1: + conv_func = DFConv2d + else: + conv_func = nn.Conv2d + tower.append(conv_func( + in_channels if i == 0 else channel, + channel, + kernel_size=3, stride=1, + padding=1, bias=True + )) + if norm == 'GN' and channel % 32 != 0: + tower.append(nn.GroupNorm(25, channel)) + elif norm != '': + tower.append(get_norm(norm, channel)) + tower.append(nn.ReLU()) + self.add_module('{}_tower'.format(head), + nn.Sequential(*tower)) + + self.bbox_pred = nn.Conv2d( + in_channels, 4, kernel_size=self.out_kernel, + stride=1, padding=self.out_kernel // 2 + ) + + self.scales = nn.ModuleList( + [Scale(init_value=1.0) for _ in range(num_levels)]) + + for modules in [ + self.cls_tower, self.bbox_tower, + self.share_tower, + self.bbox_pred, + ]: + for l in modules.modules(): + if isinstance(l, nn.Conv2d): + torch.nn.init.normal_(l.weight, std=0.01) + torch.nn.init.constant_(l.bias, 0) + + torch.nn.init.constant_(self.bbox_pred.bias, 8.) + prior_prob = prior_prob + bias_value = -math.log((1 - prior_prob) / prior_prob) + + if self.with_agn_hm: + self.agn_hm = nn.Conv2d( + in_channels, 1, kernel_size=self.out_kernel, + stride=1, padding=self.out_kernel // 2 + ) + torch.nn.init.constant_(self.agn_hm.bias, bias_value) + torch.nn.init.normal_(self.agn_hm.weight, std=0.01) + + if not self.only_proposal: + cls_kernel_size = self.out_kernel + self.cls_logits = nn.Conv2d( + in_channels, self.num_classes, + kernel_size=cls_kernel_size, + stride=1, + padding=cls_kernel_size // 2, + ) + + torch.nn.init.constant_(self.cls_logits.bias, bias_value) + torch.nn.init.normal_(self.cls_logits.weight, std=0.01) + + @classmethod + def from_config(cls, cfg, input_shape): + ret = { + # 'input_shape': input_shape, + 'in_channels': [s.channels for s in input_shape][0], + 'num_levels': len(input_shape), + 'num_classes': cfg.MODEL.CENTERNET.NUM_CLASSES, + 'with_agn_hm': cfg.MODEL.CENTERNET.WITH_AGN_HM, + 'only_proposal': cfg.MODEL.CENTERNET.ONLY_PROPOSAL, + 'norm': cfg.MODEL.CENTERNET.NORM, + 'num_cls_convs': cfg.MODEL.CENTERNET.NUM_CLS_CONVS, + 'num_box_convs': cfg.MODEL.CENTERNET.NUM_BOX_CONVS, + 'num_share_convs': cfg.MODEL.CENTERNET.NUM_SHARE_CONVS, + 'use_deformable': cfg.MODEL.CENTERNET.USE_DEFORMABLE, + 'prior_prob': cfg.MODEL.CENTERNET.PRIOR_PROB, + } + return ret + + def forward(self, x): + clss = [] + bbox_reg = [] + agn_hms = [] + for l, feature in enumerate(x): + feature = self.share_tower(feature) + cls_tower = self.cls_tower(feature) + bbox_tower = self.bbox_tower(feature) + if not self.only_proposal: + clss.append(self.cls_logits(cls_tower)) + else: + clss.append(None) + + if self.with_agn_hm: + agn_hms.append(self.agn_hm(bbox_tower)) + else: + agn_hms.append(None) + reg = self.bbox_pred(bbox_tower) + reg = self.scales[l](reg) + bbox_reg.append(F.relu(reg)) + + return clss, bbox_reg, agn_hms \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/dense_heads/utils.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/dense_heads/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c9efa287fc71315f633347023b390fe4ce57913a --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/dense_heads/utils.py @@ -0,0 +1,38 @@ +import cv2 +import torch +from torch import nn +from detectron2.utils.comm import get_world_size +from detectron2.structures import pairwise_iou, Boxes +# from .data import CenterNetCrop +import torch.nn.functional as F +import numpy as np +from detectron2.structures import Boxes, ImageList, Instances + +__all__ = ['reduce_sum', '_transpose'] + +INF = 1000000000 + +def _transpose(training_targets, num_loc_list): + ''' + This function is used to transpose image first training targets to + level first ones + :return: level first training targets + ''' + for im_i in range(len(training_targets)): + training_targets[im_i] = torch.split( + training_targets[im_i], num_loc_list, dim=0) + + targets_level_first = [] + for targets_per_level in zip(*training_targets): + targets_level_first.append( + torch.cat(targets_per_level, dim=0)) + return targets_level_first + + +def reduce_sum(tensor): + world_size = get_world_size() + if world_size < 2: + return tensor + tensor = tensor.clone() + torch.distributed.all_reduce(tensor, op=torch.distributed.ReduceOp.SUM) + return tensor \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/layers/deform_conv.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/layers/deform_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..e5650c40673882c9164ddc56fd3ee63af0be730c --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/layers/deform_conv.py @@ -0,0 +1,116 @@ +import torch +from torch import nn + +from detectron2.layers import Conv2d + + +class _NewEmptyTensorOp(torch.autograd.Function): + @staticmethod + def forward(ctx, x, new_shape): + ctx.shape = x.shape + return x.new_empty(new_shape) + + @staticmethod + def backward(ctx, grad): + shape = ctx.shape + return _NewEmptyTensorOp.apply(grad, shape), None + + +class DFConv2d(nn.Module): + """Deformable convolutional layer""" + def __init__( + self, + in_channels, + out_channels, + with_modulated_dcn=True, + kernel_size=3, + stride=1, + groups=1, + dilation=1, + deformable_groups=1, + bias=False, + padding=None + ): + super(DFConv2d, self).__init__() + if isinstance(kernel_size, (list, tuple)): + assert isinstance(stride, (list, tuple)) + assert isinstance(dilation, (list, tuple)) + assert len(kernel_size) == 2 + assert len(stride) == 2 + assert len(dilation) == 2 + padding = ( + dilation[0] * (kernel_size[0] - 1) // 2, + dilation[1] * (kernel_size[1] - 1) // 2 + ) + offset_base_channels = kernel_size[0] * kernel_size[1] + else: + padding = dilation * (kernel_size - 1) // 2 + offset_base_channels = kernel_size * kernel_size + if with_modulated_dcn: + from detectron2.layers.deform_conv import ModulatedDeformConv + offset_channels = offset_base_channels * 3 # default: 27 + conv_block = ModulatedDeformConv + else: + from detectron2.layers.deform_conv import DeformConv + offset_channels = offset_base_channels * 2 # default: 18 + conv_block = DeformConv + self.offset = Conv2d( + in_channels, + deformable_groups * offset_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + groups=1, + dilation=dilation + ) + nn.init.constant_(self.offset.weight, 0) + nn.init.constant_(self.offset.bias, 0) + ''' + for l in [self.offset, ]: + nn.init.kaiming_uniform_(l.weight, a=1) + torch.nn.init.constant_(l.bias, 0.) + ''' + self.conv = conv_block( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + deformable_groups=deformable_groups, + bias=bias + ) + self.with_modulated_dcn = with_modulated_dcn + self.kernel_size = kernel_size + self.stride = stride + self.padding = padding + self.dilation = dilation + self.offset_split = offset_base_channels * deformable_groups * 2 + + def forward(self, x, return_offset=False): + if x.numel() > 0: + if not self.with_modulated_dcn: + offset_mask = self.offset(x) + x = self.conv(x, offset_mask) + else: + offset_mask = self.offset(x) + offset = offset_mask[:, :self.offset_split, :, :] + mask = offset_mask[:, self.offset_split:, :, :].sigmoid() + x = self.conv(x, offset, mask) + if return_offset: + return x, offset_mask + return x + # get output shape + output_shape = [ + (i + 2 * p - (di * (k - 1) + 1)) // d + 1 + for i, p, di, k, d in zip( + x.shape[-2:], + self.padding, + self.dilation, + self.kernel_size, + self.stride + ) + ] + output_shape = [x.shape[0], self.conv.weight.shape[0]] + output_shape + return _NewEmptyTensorOp.apply(x, output_shape) \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/layers/heatmap_focal_loss.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/layers/heatmap_focal_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..8c0c1be13de07aaec1dc7cdb024bacc09c5a6c3b --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/layers/heatmap_focal_loss.py @@ -0,0 +1,87 @@ +import torch +from torch.nn import functional as F + +# TODO: merge these two function +def heatmap_focal_loss( + inputs, + targets, + pos_inds, + labels, + alpha: float = -1, + beta: float = 4, + gamma: float = 2, + reduction: str = 'sum', + sigmoid_clamp: float = 1e-4, + ignore_high_fp: float = -1., +): + """ + Loss used in RetinaNet for dense detection: https://arxiv.org/abs/1708.02002. + Args: + inputs: (sum_l N*Hl*Wl, C) + targets: (sum_l N*Hl*Wl, C) + pos_inds: N + labels: N + Returns: + Loss tensor with the reduction option applied. + """ + pred = torch.clamp(inputs.sigmoid_(), min=sigmoid_clamp, max=1-sigmoid_clamp) + neg_weights = torch.pow(1 - targets, beta) + pos_pred_pix = pred[pos_inds] # N x C + pos_pred = pos_pred_pix.gather(1, labels.unsqueeze(1)) + pos_loss = torch.log(pos_pred) * torch.pow(1 - pos_pred, gamma) + neg_loss = torch.log(1 - pred) * torch.pow(pred, gamma) * neg_weights + + if ignore_high_fp > 0: + not_high_fp = (pred < ignore_high_fp).float() + neg_loss = not_high_fp * neg_loss + + if reduction == "sum": + pos_loss = pos_loss.sum() + neg_loss = neg_loss.sum() + + if alpha >= 0: + pos_loss = alpha * pos_loss + neg_loss = (1 - alpha) * neg_loss + + return - pos_loss, - neg_loss + +heatmap_focal_loss_jit = torch.jit.script(heatmap_focal_loss) +# heatmap_focal_loss_jit = heatmap_focal_loss + +def binary_heatmap_focal_loss( + inputs, + targets, + pos_inds, + alpha: float = -1, + beta: float = 4, + gamma: float = 2, + sigmoid_clamp: float = 1e-4, + ignore_high_fp: float = -1., +): + """ + Args: + inputs: (sum_l N*Hl*Wl,) + targets: (sum_l N*Hl*Wl,) + pos_inds: N + Returns: + Loss tensor with the reduction option applied. + """ + pred = torch.clamp(inputs.sigmoid_(), min=sigmoid_clamp, max=1-sigmoid_clamp) + neg_weights = torch.pow(1 - targets, beta) + pos_pred = pred[pos_inds] # N + pos_loss = torch.log(pos_pred) * torch.pow(1 - pos_pred, gamma) + neg_loss = torch.log(1 - pred) * torch.pow(pred, gamma) * neg_weights + if ignore_high_fp > 0: + not_high_fp = (pred < ignore_high_fp).float() + neg_loss = not_high_fp * neg_loss + + pos_loss = - pos_loss.sum() + neg_loss = - neg_loss.sum() + + if alpha >= 0: + pos_loss = alpha * pos_loss + neg_loss = (1 - alpha) * neg_loss + + return pos_loss, neg_loss + +binary_heatmap_focal_loss_jit = torch.jit.script(binary_heatmap_focal_loss) \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/layers/iou_loss.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/layers/iou_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..6a02464651dc1a0dcec9f30285a3a4ef74209f89 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/layers/iou_loss.py @@ -0,0 +1,121 @@ +import torch +from torch import nn + + +class IOULoss(nn.Module): + def __init__(self, loc_loss_type='iou'): + super(IOULoss, self).__init__() + self.loc_loss_type = loc_loss_type + + def forward(self, pred, target, weight=None, reduction='sum'): + pred_left = pred[:, 0] + pred_top = pred[:, 1] + pred_right = pred[:, 2] + pred_bottom = pred[:, 3] + + target_left = target[:, 0] + target_top = target[:, 1] + target_right = target[:, 2] + target_bottom = target[:, 3] + + target_aera = (target_left + target_right) * \ + (target_top + target_bottom) + pred_aera = (pred_left + pred_right) * \ + (pred_top + pred_bottom) + + w_intersect = torch.min(pred_left, target_left) + \ + torch.min(pred_right, target_right) + h_intersect = torch.min(pred_bottom, target_bottom) + \ + torch.min(pred_top, target_top) + + g_w_intersect = torch.max(pred_left, target_left) + \ + torch.max(pred_right, target_right) + g_h_intersect = torch.max(pred_bottom, target_bottom) + \ + torch.max(pred_top, target_top) + ac_uion = g_w_intersect * g_h_intersect + + area_intersect = w_intersect * h_intersect + area_union = target_aera + pred_aera - area_intersect + + ious = (area_intersect + 1.0) / (area_union + 1.0) + gious = ious - (ac_uion - area_union) / ac_uion + if self.loc_loss_type == 'iou': + losses = -torch.log(ious) + elif self.loc_loss_type == 'linear_iou': + losses = 1 - ious + elif self.loc_loss_type == 'giou': + losses = 1 - gious + else: + raise NotImplementedError + + if weight is not None: + losses = losses * weight + else: + losses = losses + + if reduction == 'sum': + return losses.sum() + elif reduction == 'batch': + return losses.sum(dim=[1]) + elif reduction == 'none': + return losses + else: + raise NotImplementedError + + +def giou_loss( + boxes1: torch.Tensor, + boxes2: torch.Tensor, + reduction: str = "none", + eps: float = 1e-7, +) -> torch.Tensor: + """ + Generalized Intersection over Union Loss (Hamid Rezatofighi et. al) + https://arxiv.org/abs/1902.09630 + Gradient-friendly IoU loss with an additional penalty that is non-zero when the + boxes do not overlap and scales with the size of their smallest enclosing box. + This loss is symmetric, so the boxes1 and boxes2 arguments are interchangeable. + Args: + boxes1, boxes2 (Tensor): box locations in XYXY format, shape (N, 4) or (4,). + reduction: 'none' | 'mean' | 'sum' + 'none': No reduction will be applied to the output. + 'mean': The output will be averaged. + 'sum': The output will be summed. + eps (float): small number to prevent division by zero + """ + + x1, y1, x2, y2 = boxes1.unbind(dim=-1) + x1g, y1g, x2g, y2g = boxes2.unbind(dim=-1) + + assert (x2 >= x1).all(), "bad box: x1 larger than x2" + assert (y2 >= y1).all(), "bad box: y1 larger than y2" + + # Intersection keypoints + xkis1 = torch.max(x1, x1g) + ykis1 = torch.max(y1, y1g) + xkis2 = torch.min(x2, x2g) + ykis2 = torch.min(y2, y2g) + + intsctk = torch.zeros_like(x1) + mask = (ykis2 > ykis1) & (xkis2 > xkis1) + intsctk[mask] = (xkis2[mask] - xkis1[mask]) * (ykis2[mask] - ykis1[mask]) + unionk = (x2 - x1) * (y2 - y1) + (x2g - x1g) * (y2g - y1g) - intsctk + iouk = intsctk / (unionk + eps) + + # smallest enclosing box + xc1 = torch.min(x1, x1g) + yc1 = torch.min(y1, y1g) + xc2 = torch.max(x2, x2g) + yc2 = torch.max(y2, y2g) + + area_c = (xc2 - xc1) * (yc2 - yc1) + miouk = iouk - ((area_c - unionk) / (area_c + eps)) + + loss = 1 - miouk + + if reduction == "mean": + loss = loss.mean() + elif reduction == "sum": + loss = loss.sum() + + return loss \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/layers/ml_nms.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/layers/ml_nms.py new file mode 100644 index 0000000000000000000000000000000000000000..325d709a98422d8a355fc7c7e281179642850968 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/layers/ml_nms.py @@ -0,0 +1,31 @@ +from detectron2.layers import batched_nms + + +def ml_nms(boxlist, nms_thresh, max_proposals=-1, + score_field="scores", label_field="labels"): + """ + Performs non-maximum suppression on a boxlist, with scores specified + in a boxlist field via score_field. + Arguments: + boxlist(BoxList) + nms_thresh (float) + max_proposals (int): if > 0, then only the top max_proposals are kept + after non-maximum suppression + score_field (str) + """ + if nms_thresh <= 0: + return boxlist + if boxlist.has('pred_boxes'): + boxes = boxlist.pred_boxes.tensor + labels = boxlist.pred_classes + else: + boxes = boxlist.proposal_boxes.tensor + labels = boxlist.proposal_boxes.tensor.new_zeros( + len(boxlist.proposal_boxes.tensor)) + scores = boxlist.scores + + keep = batched_nms(boxes, scores, labels, nms_thresh) + if max_proposals > 0: + keep = keep[: max_proposals] + boxlist = boxlist[keep] + return boxlist diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/meta_arch/centernet_detector.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/meta_arch/centernet_detector.py new file mode 100644 index 0000000000000000000000000000000000000000..b7525c7b31cbbca504442e9a0dc8fb5005ea91b3 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/meta_arch/centernet_detector.py @@ -0,0 +1,69 @@ +import math +import json +import numpy as np +import torch +from torch import nn + +from detectron2.modeling.meta_arch.build import META_ARCH_REGISTRY +from detectron2.modeling import build_backbone, build_proposal_generator +from detectron2.modeling import detector_postprocess +from detectron2.structures import ImageList + +@META_ARCH_REGISTRY.register() +class CenterNetDetector(nn.Module): + def __init__(self, cfg): + super().__init__() + self.mean, self.std = cfg.MODEL.PIXEL_MEAN, cfg.MODEL.PIXEL_STD + self.register_buffer("pixel_mean", torch.Tensor(cfg.MODEL.PIXEL_MEAN).view(-1, 1, 1)) + self.register_buffer("pixel_std", torch.Tensor(cfg.MODEL.PIXEL_STD).view(-1, 1, 1)) + + self.backbone = build_backbone(cfg) + self.proposal_generator = build_proposal_generator( + cfg, self.backbone.output_shape()) # TODO: change to a more precise name + + + def forward(self, batched_inputs): + if not self.training: + return self.inference(batched_inputs) + images = self.preprocess_image(batched_inputs) + features = self.backbone(images.tensor) + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + + _, proposal_losses = self.proposal_generator( + images, features, gt_instances) + return proposal_losses + + + @property + def device(self): + return self.pixel_mean.device + + + @torch.no_grad() + def inference(self, batched_inputs, do_postprocess=True): + images = self.preprocess_image(batched_inputs) + inp = images.tensor + features = self.backbone(inp) + proposals, _ = self.proposal_generator(images, features, None) + + processed_results = [] + for results_per_image, input_per_image, image_size in zip( + proposals, batched_inputs, images.image_sizes): + if do_postprocess: + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + r = detector_postprocess(results_per_image, height, width) + processed_results.append({"instances": r}) + else: + r = results_per_image + processed_results.append(r) + return processed_results + + def preprocess_image(self, batched_inputs): + """ + Normalize, pad and batch the input images. + """ + images = [x["image"].to(self.device) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors(images, self.backbone.size_divisibility) + return images diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/roi_heads/custom_fast_rcnn.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/roi_heads/custom_fast_rcnn.py new file mode 100644 index 0000000000000000000000000000000000000000..1f0f430d0f1543b684aa0a0628c09bb1f49d7cd7 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/roi_heads/custom_fast_rcnn.py @@ -0,0 +1,170 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +# Part of the code is from https://github.com/tztztztztz/eql.detectron2/blob/master/projects/EQL/eql/fast_rcnn.py +import logging +import math +import json +from typing import Dict, Union +import torch +from fvcore.nn import giou_loss, smooth_l1_loss +from torch import nn +from torch.nn import functional as F + +from detectron2.config import configurable +from detectron2.layers import Linear, ShapeSpec, batched_nms, cat, nonzero_tuple +from detectron2.modeling.box_regression import Box2BoxTransform +from detectron2.structures import Boxes, Instances +from detectron2.utils.events import get_event_storage +from detectron2.modeling.roi_heads.fast_rcnn import FastRCNNOutputLayers +from detectron2.modeling.roi_heads.fast_rcnn import fast_rcnn_inference +from detectron2.modeling.roi_heads.fast_rcnn import _log_classification_stats +from detectron2.utils.comm import get_world_size +from .fed_loss import load_class_freq, get_fed_loss_inds + +__all__ = ["CustomFastRCNNOutputLayers"] + +class CustomFastRCNNOutputLayers(FastRCNNOutputLayers): + def __init__( + self, + cfg, + input_shape: ShapeSpec, + **kwargs + ): + super().__init__(cfg, input_shape, **kwargs) + self.use_sigmoid_ce = cfg.MODEL.ROI_BOX_HEAD.USE_SIGMOID_CE + if self.use_sigmoid_ce: + prior_prob = cfg.MODEL.ROI_BOX_HEAD.PRIOR_PROB + bias_value = -math.log((1 - prior_prob) / prior_prob) + nn.init.constant_(self.cls_score.bias, bias_value) + + self.cfg = cfg + self.use_fed_loss = cfg.MODEL.ROI_BOX_HEAD.USE_FED_LOSS + if self.use_fed_loss: + self.fed_loss_num_cat = cfg.MODEL.ROI_BOX_HEAD.FED_LOSS_NUM_CAT + self.register_buffer( + 'freq_weight', + load_class_freq( + cfg.MODEL.ROI_BOX_HEAD.CAT_FREQ_PATH, + cfg.MODEL.ROI_BOX_HEAD.FED_LOSS_FREQ_WEIGHT, + ) + ) + + def losses(self, predictions, proposals): + """ + enable advanced loss + """ + scores, proposal_deltas = predictions + gt_classes = ( + cat([p.gt_classes for p in proposals], dim=0) if len(proposals) else torch.empty(0) + ) + num_classes = self.num_classes + _log_classification_stats(scores, gt_classes) + + if len(proposals): + proposal_boxes = cat([p.proposal_boxes.tensor for p in proposals], dim=0) # Nx4 + assert not proposal_boxes.requires_grad, "Proposals should not require gradients!" + gt_boxes = cat( + [(p.gt_boxes if p.has("gt_boxes") else p.proposal_boxes).tensor for p in proposals], + dim=0, + ) + else: + proposal_boxes = gt_boxes = torch.empty((0, 4), device=proposal_deltas.device) + + if self.use_sigmoid_ce: + loss_cls = self.sigmoid_cross_entropy_loss(scores, gt_classes) + else: + loss_cls = self.softmax_cross_entropy_loss(scores, gt_classes) + return { + "loss_cls": loss_cls, + "loss_box_reg": self.box_reg_loss( + proposal_boxes, gt_boxes, proposal_deltas, gt_classes) + } + + + def sigmoid_cross_entropy_loss(self, pred_class_logits, gt_classes): + if pred_class_logits.numel() == 0: + return pred_class_logits.new_zeros([1])[0] # This is more robust than .sum() * 0. + + B = pred_class_logits.shape[0] + C = pred_class_logits.shape[1] - 1 + + target = pred_class_logits.new_zeros(B, C + 1) + target[range(len(gt_classes)), gt_classes] = 1 # B x (C + 1) + target = target[:, :C] # B x C + + weight = 1 + if self.use_fed_loss and (self.freq_weight is not None): # fedloss + appeared = get_fed_loss_inds( + gt_classes, + num_sample_cats=self.fed_loss_num_cat, + C=C, + weight=self.freq_weight) + appeared_mask = appeared.new_zeros(C + 1) + appeared_mask[appeared] = 1 # C + 1 + appeared_mask = appeared_mask[:C] + fed_w = appeared_mask.view(1, C).expand(B, C) + weight = weight * fed_w.float() + + cls_loss = F.binary_cross_entropy_with_logits( + pred_class_logits[:, :-1], target, reduction='none') # B x C + loss = torch.sum(cls_loss * weight) / B + return loss + + + def softmax_cross_entropy_loss(self, pred_class_logits, gt_classes): + """ + change _no_instance handling + """ + if pred_class_logits.numel() == 0: + return pred_class_logits.new_zeros([1])[0] + + if self.use_fed_loss and (self.freq_weight is not None): + C = pred_class_logits.shape[1] - 1 + appeared = get_fed_loss_inds( + gt_classes, + num_sample_cats=self.fed_loss_num_cat, + C=C, + weight=self.freq_weight) + appeared_mask = appeared.new_zeros(C + 1).float() + appeared_mask[appeared] = 1. # C + 1 + appeared_mask[C] = 1. + loss = F.cross_entropy( + pred_class_logits, gt_classes, + weight=appeared_mask, reduction="mean") + else: + loss = F.cross_entropy( + pred_class_logits, gt_classes, reduction="mean") + return loss + + + def inference(self, predictions, proposals): + """ + enable use proposal boxes + """ + boxes = self.predict_boxes(predictions, proposals) + scores = self.predict_probs(predictions, proposals) + if self.cfg.MODEL.ROI_BOX_HEAD.MULT_PROPOSAL_SCORE: + proposal_scores = [p.get('objectness_logits') for p in proposals] + scores = [(s * ps[:, None]) ** 0.5 \ + for s, ps in zip(scores, proposal_scores)] + image_shapes = [x.image_size for x in proposals] + return fast_rcnn_inference( + boxes, + scores, + image_shapes, + self.test_score_thresh, + self.test_nms_thresh, + self.test_topk_per_image, + ) + + + def predict_probs(self, predictions, proposals): + """ + support sigmoid + """ + scores, _ = predictions + num_inst_per_image = [len(p) for p in proposals] + if self.use_sigmoid_ce: + probs = scores.sigmoid() + else: + probs = F.softmax(scores, dim=-1) + return probs.split(num_inst_per_image, dim=0) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/roi_heads/custom_roi_heads.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/roi_heads/custom_roi_heads.py new file mode 100644 index 0000000000000000000000000000000000000000..90fadf1a9667cf836223945b22c5147b89ad98a4 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/roi_heads/custom_roi_heads.py @@ -0,0 +1,185 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import numpy as np +import json +import math +import torch +from torch import nn +from torch.autograd.function import Function +from typing import Dict, List, Optional, Tuple, Union + +from detectron2.layers import ShapeSpec +from detectron2.structures import Boxes, Instances, pairwise_iou +from detectron2.utils.events import get_event_storage + +from detectron2.modeling.box_regression import Box2BoxTransform +from detectron2.modeling.roi_heads.fast_rcnn import fast_rcnn_inference +from detectron2.modeling.roi_heads.roi_heads import ROI_HEADS_REGISTRY, StandardROIHeads +from detectron2.modeling.roi_heads.cascade_rcnn import CascadeROIHeads +from detectron2.modeling.roi_heads.box_head import build_box_head +from .custom_fast_rcnn import CustomFastRCNNOutputLayers + + +@ROI_HEADS_REGISTRY.register() +class CustomROIHeads(StandardROIHeads): + @classmethod + def _init_box_head(self, cfg, input_shape): + ret = super()._init_box_head(cfg, input_shape) + del ret['box_predictor'] + ret['box_predictor'] = CustomFastRCNNOutputLayers( + cfg, ret['box_head'].output_shape) + self.debug = cfg.DEBUG + if self.debug: + self.debug_show_name = cfg.DEBUG_SHOW_NAME + self.save_debug = cfg.SAVE_DEBUG + self.vis_thresh = cfg.VIS_THRESH + self.pixel_mean = torch.Tensor(cfg.MODEL.PIXEL_MEAN).to( + torch.device(cfg.MODEL.DEVICE)).view(3, 1, 1) + self.pixel_std = torch.Tensor(cfg.MODEL.PIXEL_STD).to( + torch.device(cfg.MODEL.DEVICE)).view(3, 1, 1) + return ret + + def forward(self, images, features, proposals, targets=None): + """ + enable debug + """ + if not self.debug: + del images + if self.training: + assert targets + proposals = self.label_and_sample_proposals(proposals, targets) + del targets + + if self.training: + losses = self._forward_box(features, proposals) + losses.update(self._forward_mask(features, proposals)) + losses.update(self._forward_keypoint(features, proposals)) + return proposals, losses + else: + pred_instances = self._forward_box(features, proposals) + pred_instances = self.forward_with_given_boxes(features, pred_instances) + if self.debug: + from ..debug import debug_second_stage + denormalizer = lambda x: x * self.pixel_std + self.pixel_mean + debug_second_stage( + [denormalizer(images[0].clone())], + pred_instances, proposals=proposals, + debug_show_name=self.debug_show_name) + return pred_instances, {} + + +@ROI_HEADS_REGISTRY.register() +class CustomCascadeROIHeads(CascadeROIHeads): + @classmethod + def _init_box_head(self, cfg, input_shape): + self.mult_proposal_score = cfg.MODEL.ROI_BOX_HEAD.MULT_PROPOSAL_SCORE + ret = super()._init_box_head(cfg, input_shape) + del ret['box_predictors'] + cascade_bbox_reg_weights = cfg.MODEL.ROI_BOX_CASCADE_HEAD.BBOX_REG_WEIGHTS + box_predictors = [] + for box_head, bbox_reg_weights in zip(ret['box_heads'], cascade_bbox_reg_weights): + box_predictors.append( + CustomFastRCNNOutputLayers( + cfg, box_head.output_shape, + box2box_transform=Box2BoxTransform(weights=bbox_reg_weights) + )) + ret['box_predictors'] = box_predictors + self.debug = cfg.DEBUG + if self.debug: + self.debug_show_name = cfg.DEBUG_SHOW_NAME + self.save_debug = cfg.SAVE_DEBUG + self.vis_thresh = cfg.VIS_THRESH + self.pixel_mean = torch.Tensor(cfg.MODEL.PIXEL_MEAN).to( + torch.device(cfg.MODEL.DEVICE)).view(3, 1, 1) + self.pixel_std = torch.Tensor(cfg.MODEL.PIXEL_STD).to( + torch.device(cfg.MODEL.DEVICE)).view(3, 1, 1) + return ret + + + def _forward_box(self, features, proposals, targets=None): + """ + Add mult proposal scores at testing + """ + if (not self.training) and self.mult_proposal_score: + if len(proposals) > 0 and proposals[0].has('scores'): + proposal_scores = [ + p.get('scores') for p in proposals] + else: + proposal_scores = [ + p.get('objectness_logits') for p in proposals] + + features = [features[f] for f in self.box_in_features] + head_outputs = [] # (predictor, predictions, proposals) + prev_pred_boxes = None + image_sizes = [x.image_size for x in proposals] + for k in range(self.num_cascade_stages): + if k > 0: + proposals = self._create_proposals_from_boxes(prev_pred_boxes, image_sizes) + if self.training: + proposals = self._match_and_label_boxes(proposals, k, targets) + predictions = self._run_stage(features, proposals, k) + prev_pred_boxes = self.box_predictor[k].predict_boxes(predictions, proposals) + head_outputs.append((self.box_predictor[k], predictions, proposals)) + + if self.training: + losses = {} + storage = get_event_storage() + for stage, (predictor, predictions, proposals) in enumerate(head_outputs): + with storage.name_scope("stage{}".format(stage)): + stage_losses = predictor.losses(predictions, proposals) + losses.update({k + "_stage{}".format(stage): v for k, v in stage_losses.items()}) + return losses + else: + # Each is a list[Tensor] of length #image. Each tensor is Ri x (K+1) + scores_per_stage = [h[0].predict_probs(h[1], h[2]) for h in head_outputs] + scores = [ + sum(list(scores_per_image)) * (1.0 / self.num_cascade_stages) + for scores_per_image in zip(*scores_per_stage) + ] + + if self.mult_proposal_score: + scores = [(s * ps[:, None]) ** 0.5 \ + for s, ps in zip(scores, proposal_scores)] + + predictor, predictions, proposals = head_outputs[-1] + boxes = predictor.predict_boxes(predictions, proposals) + pred_instances, _ = fast_rcnn_inference( + boxes, + scores, + image_sizes, + predictor.test_score_thresh, + predictor.test_nms_thresh, + predictor.test_topk_per_image, + ) + + return pred_instances + + def forward(self, images, features, proposals, targets=None): + ''' + enable debug + ''' + if not self.debug: + del images + if self.training: + proposals = self.label_and_sample_proposals(proposals, targets) + + if self.training: + losses = self._forward_box(features, proposals, targets) + losses.update(self._forward_mask(features, proposals)) + losses.update(self._forward_keypoint(features, proposals)) + return proposals, losses + else: + # import pdb; pdb.set_trace() + pred_instances = self._forward_box(features, proposals) + pred_instances = self.forward_with_given_boxes(features, pred_instances) + if self.debug: + from ..debug import debug_second_stage + denormalizer = lambda x: x * self.pixel_std + self.pixel_mean + debug_second_stage( + [denormalizer(x.clone()) for x in images], + pred_instances, proposals=proposals, + save_debug=self.save_debug, + debug_show_name=self.debug_show_name, + vis_thresh=self.vis_thresh) + return pred_instances, {} + + diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/roi_heads/fed_loss.py b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/roi_heads/fed_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..290f0f07204e78ef2c4ff918aa500b04330279e6 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/centernet/modeling/roi_heads/fed_loss.py @@ -0,0 +1,31 @@ +import torch +import json +import numpy as np +from torch.nn import functional as F + +def load_class_freq( + path='datasets/lvis/lvis_v1_train_cat_info.json', + freq_weight=0.5): + cat_info = json.load(open(path, 'r')) + cat_info = torch.tensor( + [c['image_count'] for c in sorted(cat_info, key=lambda x: x['id'])]) + freq_weight = cat_info.float() ** freq_weight + return freq_weight + +def get_fed_loss_inds( + gt_classes, num_sample_cats=50, C=1203, \ + weight=None, fed_cls_inds=-1): + appeared = torch.unique(gt_classes) # C' + prob = appeared.new_ones(C + 1).float() + prob[-1] = 0 + if len(appeared) < num_sample_cats: + if weight is not None: + prob[:C] = weight.float().clone() + prob[appeared] = 0 + if fed_cls_inds > 0: + prob[fed_cls_inds:] = 0 + more_appeared = torch.multinomial( + prob, num_sample_cats - len(appeared), + replacement=False) + appeared = torch.cat([appeared, more_appeared]) + return appeared \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/Base-CenterNet-FPN.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/Base-CenterNet-FPN.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bef3dc10dee4aaf0e7158711cc9d088f2b28c940 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/Base-CenterNet-FPN.yaml @@ -0,0 +1,28 @@ +MODEL: + META_ARCHITECTURE: "CenterNetDetector" + PROPOSAL_GENERATOR: + NAME: "CenterNet" + BACKBONE: + NAME: "build_p67_resnet_fpn_backbone" + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + OUT_FEATURES: ["res3", "res4", "res5"] + FPN: + IN_FEATURES: ["res3", "res4", "res5"] +DATASETS: + TRAIN: ("coco_2017_train",) + TEST: ("coco_2017_val",) +SOLVER: + IMS_PER_BATCH: 16 + BASE_LR: 0.01 + STEPS: (60000, 80000) + MAX_ITER: 90000 + CHECKPOINT_PERIOD: 1000000000 + WARMUP_ITERS: 4000 + WARMUP_FACTOR: 0.00025 + CLIP_GRADIENTS: + ENABLED: True +INPUT: + MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800) +OUTPUT_DIR: "./output/CenterNet2/auto" diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/Base-CenterNet2.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/Base-CenterNet2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..689372310149062acd703760d11f83800e12e74f --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/Base-CenterNet2.yaml @@ -0,0 +1,56 @@ +MODEL: + META_ARCHITECTURE: "GeneralizedRCNN" + PROPOSAL_GENERATOR: + NAME: "CenterNet" + BACKBONE: + NAME: "build_p67_resnet_fpn_backbone" + WEIGHTS: "detectron2://ImageNetPretrained/MSRA/R-50.pkl" + RESNETS: + DEPTH: 50 + OUT_FEATURES: ["res3", "res4", "res5"] + FPN: + IN_FEATURES: ["res3", "res4", "res5"] + ROI_HEADS: + NAME: CustomCascadeROIHeads + IN_FEATURES: ["p3", "p4", "p5", "p6", "p7"] + IOU_THRESHOLDS: [0.6] + NMS_THRESH_TEST: 0.7 + ROI_BOX_CASCADE_HEAD: + IOUS: [0.6, 0.7, 0.8] + ROI_BOX_HEAD: + NAME: "FastRCNNConvFCHead" + NUM_FC: 2 + POOLER_RESOLUTION: 7 + CLS_AGNOSTIC_BBOX_REG: True + MULT_PROPOSAL_SCORE: True + CENTERNET: + REG_WEIGHT: 1. + NOT_NORM_REG: True + ONLY_PROPOSAL: True + WITH_AGN_HM: True + INFERENCE_TH: 0.0001 + PRE_NMS_TOPK_TRAIN: 4000 + POST_NMS_TOPK_TRAIN: 2000 + PRE_NMS_TOPK_TEST: 1000 + POST_NMS_TOPK_TEST: 256 + NMS_TH_TRAIN: 0.9 + NMS_TH_TEST: 0.9 + POS_WEIGHT: 0.5 + NEG_WEIGHT: 0.5 + IGNORE_HIGH_FP: 0.85 +DATASETS: + TRAIN: ("coco_2017_train",) + TEST: ("coco_2017_val",) +SOLVER: + IMS_PER_BATCH: 16 + BASE_LR: 0.02 + STEPS: (60000, 80000) + MAX_ITER: 90000 + CHECKPOINT_PERIOD: 1000000000 + WARMUP_ITERS: 4000 + WARMUP_FACTOR: 0.00025 + CLIP_GRADIENTS: + ENABLED: True +INPUT: + MIN_SIZE_TRAIN: (640, 672, 704, 736, 768, 800) +OUTPUT_DIR: "./output/CenterNet2/auto" diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/Base_S4_DLA.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/Base_S4_DLA.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e01be7e5503055ebcbbe4aee7e43738f004fde0 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/Base_S4_DLA.yaml @@ -0,0 +1,40 @@ +MODEL: + META_ARCHITECTURE: "CenterNetDetector" + PROPOSAL_GENERATOR: + NAME: "CenterNet" + PIXEL_STD: [57.375, 57.120, 58.395] + BACKBONE: + NAME: "build_dla_backbone" + DLA: + NORM: "BN" + CENTERNET: + IN_FEATURES: ["dla2"] + FPN_STRIDES: [4] + SOI: [[0, 1000000]] + NUM_CLS_CONVS: 1 + NUM_BOX_CONVS: 1 + REG_WEIGHT: 1. + MORE_POS: True + HM_FOCAL_ALPHA: 0.25 +DATASETS: + TRAIN: ("coco_2017_train",) + TEST: ("coco_2017_val",) +SOLVER: + LR_SCHEDULER_NAME: "WarmupCosineLR" + MAX_ITER: 90000 + BASE_LR: 0.04 + IMS_PER_BATCH: 64 + WEIGHT_DECAY: 0.0001 + CHECKPOINT_PERIOD: 1000000 + CLIP_GRADIENTS: + ENABLED: True +INPUT: + CUSTOM_AUG: EfficientDetResizeCrop + TRAIN_SIZE: 640 + MIN_SIZE_TEST: 608 + MAX_SIZE_TEST: 900 +TEST: + EVAL_PERIOD: 7500 +DATALOADER: + NUM_WORKERS: 8 +OUTPUT_DIR: "output/CenterNet2/auto" diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet-FPN_R50_1x.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet-FPN_R50_1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ea7d9b70324d172efbff299f9cff2c60e136e93 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet-FPN_R50_1x.yaml @@ -0,0 +1,4 @@ +_BASE_: "Base-CenterNet-FPN.yaml" +MODEL: + CENTERNET: + MORE_POS: True \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet-S4_DLA_8x.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet-S4_DLA_8x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3d88be9f50b53766bd4c4b88130c9ee670a4984 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet-S4_DLA_8x.yaml @@ -0,0 +1,5 @@ +_BASE_: "Base_S4_DLA.yaml" +SOLVER: + MAX_ITER: 90000 + BASE_LR: 0.08 + IMS_PER_BATCH: 128 \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2-F_R50_1x.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2-F_R50_1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c40eecc13aaae3757dd1917ca3cfcb3cd7fc467f --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2-F_R50_1x.yaml @@ -0,0 +1,4 @@ +_BASE_: "Base-CenterNet2.yaml" +MODEL: + ROI_HEADS: + NAME: CustomROIHeads \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_DLA-BiFPN-P3_24x.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_DLA-BiFPN-P3_24x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7491447ebd7e769eec7309b533947c5577d8563 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_DLA-BiFPN-P3_24x.yaml @@ -0,0 +1,36 @@ +_BASE_: "Base-CenterNet2.yaml" +MODEL: + BACKBONE: + NAME: "build_p35_fcos_dla_bifpn_backbone" + BIFPN: + OUT_CHANNELS: 160 + NUM_LEVELS: 3 + NUM_BIFPN: 4 + DLA: + NUM_LAYERS: 34 + NORM: "SyncBN" + FPN: + IN_FEATURES: ["dla3", "dla4", "dla5"] + ROI_HEADS: + IN_FEATURES: ["p3", "p4", "p5"] + CENTERNET: + POST_NMS_TOPK_TEST: 128 + FPN_STRIDES: [8, 16, 32] + IN_FEATURES: ['p3', 'p4', 'p5'] + SOI: [[0, 64], [48, 192], [128, 1000000]] +DATASETS: + TRAIN: ("coco_2017_train",) + TEST: ("coco_2017_val",) +SOLVER: + IMS_PER_BATCH: 16 + BASE_LR: 0.02 + STEPS: (300000, 340000) + MAX_ITER: 360000 + CHECKPOINT_PERIOD: 100000 + WARMUP_ITERS: 4000 + WARMUP_FACTOR: 0.00025 +INPUT: + MIN_SIZE_TRAIN: (256, 288, 320, 352, 384, 416, 448, 480, 512, 544, 576, 608) + MAX_SIZE_TRAIN: 900 + MAX_SIZE_TEST: 736 + MIN_SIZE_TEST: 512 \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_DLA-BiFPN-P3_4x.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_DLA-BiFPN-P3_4x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7491447ebd7e769eec7309b533947c5577d8563 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_DLA-BiFPN-P3_4x.yaml @@ -0,0 +1,36 @@ +_BASE_: "Base-CenterNet2.yaml" +MODEL: + BACKBONE: + NAME: "build_p35_fcos_dla_bifpn_backbone" + BIFPN: + OUT_CHANNELS: 160 + NUM_LEVELS: 3 + NUM_BIFPN: 4 + DLA: + NUM_LAYERS: 34 + NORM: "SyncBN" + FPN: + IN_FEATURES: ["dla3", "dla4", "dla5"] + ROI_HEADS: + IN_FEATURES: ["p3", "p4", "p5"] + CENTERNET: + POST_NMS_TOPK_TEST: 128 + FPN_STRIDES: [8, 16, 32] + IN_FEATURES: ['p3', 'p4', 'p5'] + SOI: [[0, 64], [48, 192], [128, 1000000]] +DATASETS: + TRAIN: ("coco_2017_train",) + TEST: ("coco_2017_val",) +SOLVER: + IMS_PER_BATCH: 16 + BASE_LR: 0.02 + STEPS: (300000, 340000) + MAX_ITER: 360000 + CHECKPOINT_PERIOD: 100000 + WARMUP_ITERS: 4000 + WARMUP_FACTOR: 0.00025 +INPUT: + MIN_SIZE_TRAIN: (256, 288, 320, 352, 384, 416, 448, 480, 512, 544, 576, 608) + MAX_SIZE_TRAIN: 900 + MAX_SIZE_TEST: 736 + MIN_SIZE_TEST: 512 \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_DLA-BiFPN-P5_640_16x.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_DLA-BiFPN-P5_640_16x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80413a62d666a3588fec4f5adc3ca5c3af788b45 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_DLA-BiFPN-P5_640_16x.yaml @@ -0,0 +1,29 @@ +_BASE_: "Base-CenterNet2.yaml" +MODEL: + BACKBONE: + NAME: "build_p37_dla_bifpn_backbone" + BIFPN: + OUT_CHANNELS: 160 + NUM_LEVELS: 5 + NUM_BIFPN: 3 + CENTERNET: + POST_NMS_TOPK_TEST: 128 + WEIGHTS: '' + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.12, 57.375] + FPN: + IN_FEATURES: ["dla3", "dla4", "dla5"] +SOLVER: + LR_SCHEDULER_NAME: "WarmupCosineLR" + MAX_ITER: 360000 + BASE_LR: 0.08 + IMS_PER_BATCH: 64 + CHECKPOINT_PERIOD: 90000 +TEST: + EVAL_PERIOD: 7500 +INPUT: + FORMAT: RGB + CUSTOM_AUG: EfficientDetResizeCrop + TRAIN_SIZE: 640 + MIN_SIZE_TEST: 608 + MAX_SIZE_TEST: 900 diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_DLA-BiFPN-P5_640_16x_ST.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_DLA-BiFPN-P5_640_16x_ST.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8813b39c1c2cf02290e491d7efa75296d9897591 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_DLA-BiFPN-P5_640_16x_ST.yaml @@ -0,0 +1,30 @@ +_BASE_: "Base-CenterNet2.yaml" +MODEL: + BACKBONE: + NAME: "build_p37_dla_bifpn_backbone" + BIFPN: + OUT_CHANNELS: 160 + NUM_LEVELS: 5 + NUM_BIFPN: 3 + CENTERNET: + POST_NMS_TOPK_TEST: 128 + WEIGHTS: '' + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.12, 57.375] + FPN: + IN_FEATURES: ["dla3", "dla4", "dla5"] +SOLVER: + LR_SCHEDULER_NAME: "WarmupCosineLR" + MAX_ITER: 360000 + BASE_LR: 0.08 + IMS_PER_BATCH: 64 +TEST: + EVAL_PERIOD: 7500 +INPUT: + FORMAT: RGB + CUSTOM_AUG: EfficientDetResizeCrop + TRAIN_SIZE: 640 + MIN_SIZE_TEST: 608 + MAX_SIZE_TEST: 900 +DATASETS: + TRAIN: ("coco_2017_train","coco_un_yolov4_55_0.5",) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_DLA-fcosBiFPN-P5_640_16x_ST.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_DLA-fcosBiFPN-P5_640_16x_ST.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f94f1358ced6f9ea88e75db668c0afa173215111 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_DLA-fcosBiFPN-P5_640_16x_ST.yaml @@ -0,0 +1,30 @@ +_BASE_: "Base-CenterNet2.yaml" +MODEL: + BACKBONE: + NAME: "build_p37_fcos_dla_bifpn_backbone" + BIFPN: + OUT_CHANNELS: 160 + NUM_LEVELS: 5 + NUM_BIFPN: 3 + CENTERNET: + POST_NMS_TOPK_TEST: 128 + WEIGHTS: '' + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.12, 57.375] + FPN: + IN_FEATURES: ["dla3", "dla4", "dla5"] +TEST: + EVAL_PERIOD: 7500 +SOLVER: + LR_SCHEDULER_NAME: "WarmupCosineLR" + MAX_ITER: 360000 + BASE_LR: 0.08 + IMS_PER_BATCH: 64 +INPUT: + FORMAT: RGB + CUSTOM_AUG: EfficientDetResizeCrop + TRAIN_SIZE: 640 + MIN_SIZE_TEST: 608 + MAX_SIZE_TEST: 900 +DATASETS: + TRAIN: ("coco_2017_train","coco_un_yolov4_55_0.5",) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_R2-101-DCN-BiFPN_1280_4x.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_R2-101-DCN-BiFPN_1280_4x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e07574b3511a372ab9e04747e584fdeef37a9700 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_R2-101-DCN-BiFPN_1280_4x.yaml @@ -0,0 +1,32 @@ +_BASE_: "Base-CenterNet2.yaml" +MODEL: + BACKBONE: + NAME: "build_res2net_bifpn_backbone" + BIFPN: + NUM_BIFPN: 7 + OUT_CHANNELS: 288 + WEIGHTS: "output/r2_101.pkl" + RESNETS: + DEPTH: 101 + WIDTH_PER_GROUP: 26 + DEFORM_ON_PER_STAGE: [False, False, True, True] # on Res4, Res5 + DEFORM_MODULATED: True + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.12, 57.375] + CENTERNET: + USE_DEFORMABLE: True + ROI_HEADS: + IN_FEATURES: ["p3", "p4"] +INPUT: + FORMAT: RGB +TEST: + EVAL_PERIOD: 7500 +SOLVER: + MAX_ITER: 180000 + CHECKPOINT_PERIOD: 60000 + LR_SCHEDULER_NAME: "WarmupCosineLR" + BASE_LR: 0.04 + IMS_PER_BATCH: 32 +INPUT: + CUSTOM_AUG: EfficientDetResizeCrop + TRAIN_SIZE: 1280 diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_R2-101-DCN-BiFPN_4x+4x_1560_ST.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_R2-101-DCN-BiFPN_4x+4x_1560_ST.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81fcab0972a943256239705b4edd320c78312532 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_R2-101-DCN-BiFPN_4x+4x_1560_ST.yaml @@ -0,0 +1,36 @@ +_BASE_: "Base-CenterNet2.yaml" +MODEL: + BACKBONE: + NAME: "build_res2net_bifpn_backbone" + BIFPN: + NUM_BIFPN: 7 + OUT_CHANNELS: 288 + WEIGHTS: "output/r2_101.pkl" + RESNETS: + DEPTH: 101 + WIDTH_PER_GROUP: 26 + DEFORM_ON_PER_STAGE: [False, False, True, True] # on Res4, Res5 + DEFORM_MODULATED: True + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.12, 57.375] + CENTERNET: + USE_DEFORMABLE: True + ROI_HEADS: + IN_FEATURES: ["p3", "p4"] +TEST: + EVAL_PERIOD: 7500 +SOLVER: + MAX_ITER: 180000 + CHECKPOINT_PERIOD: 7500 + LR_SCHEDULER_NAME: "WarmupCosineLR" + BASE_LR: 0.04 + IMS_PER_BATCH: 32 +DATASETS: + TRAIN: "('coco_2017_train', 'coco_un_yolov4_55_0.5')" +INPUT: + FORMAT: RGB + CUSTOM_AUG: EfficientDetResizeCrop + TRAIN_SIZE: 1280 + TEST_SIZE: 1560 + TEST_INPUT_TYPE: 'square' + \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_R2-101-DCN_896_4x.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_R2-101-DCN_896_4x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd6c49ee40ca927090e1a9dcd397049e6d42e649 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_R2-101-DCN_896_4x.yaml @@ -0,0 +1,29 @@ +_BASE_: "Base-CenterNet2.yaml" +MODEL: + BACKBONE: + NAME: "build_p67_res2net_fpn_backbone" + WEIGHTS: "output/r2_101.pkl" + RESNETS: + DEPTH: 101 + WIDTH_PER_GROUP: 26 + DEFORM_ON_PER_STAGE: [False, False, True, True] # on Res4, Res5 + DEFORM_MODULATED: True + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.12, 57.375] + CENTERNET: + USE_DEFORMABLE: True + ROI_HEADS: + IN_FEATURES: ["p3", "p4"] +INPUT: + FORMAT: RGB +TEST: + EVAL_PERIOD: 7500 +SOLVER: + MAX_ITER: 180000 + CHECKPOINT_PERIOD: 600000 + LR_SCHEDULER_NAME: "WarmupCosineLR" + BASE_LR: 0.04 + IMS_PER_BATCH: 32 +INPUT: + CUSTOM_AUG: EfficientDetResizeCrop + TRAIN_SIZE: 896 \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_R50_1x.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_R50_1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9dcdf5b8b6b8c613a0d4a036dbf9fd662512558c --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_R50_1x.yaml @@ -0,0 +1 @@ +_BASE_: "Base-CenterNet2.yaml" diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_X101-DCN_2x.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_X101-DCN_2x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..009c68085bdd3340df9e9ef5325bb6ca1c003478 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/CenterNet2_X101-DCN_2x.yaml @@ -0,0 +1,22 @@ +_BASE_: "Base-CenterNet2.yaml" +MODEL: + CENTERNET: + USE_DEFORMABLE: True + WEIGHTS: "detectron2://ImageNetPretrained/FAIR/X-101-32x8d.pkl" + PIXEL_STD: [57.375, 57.120, 58.395] + RESNETS: + STRIDE_IN_1X1: False + NUM_GROUPS: 32 + WIDTH_PER_GROUP: 8 + DEPTH: 101 + DEFORM_ON_PER_STAGE: [False, False, True, True] # on Res4, Res5 + DEFORM_MODULATED: True + ROI_HEADS: + IN_FEATURES: ["p3", "p4"] +SOLVER: + STEPS: (120000, 160000) + MAX_ITER: 180000 + CHECKPOINT_PERIOD: 40000 +INPUT: + MIN_SIZE_TRAIN: (480, 960) + MIN_SIZE_TRAIN_SAMPLING: "range" diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/LVIS_CenterNet2_R50_1x.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/LVIS_CenterNet2_R50_1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..912e8925dcd72cacb1dd7e08b21c97c8acf44ca1 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/LVIS_CenterNet2_R50_1x.yaml @@ -0,0 +1,17 @@ +_BASE_: "Base-CenterNet2.yaml" +MODEL: + ROI_HEADS: + NUM_CLASSES: 1203 + SCORE_THRESH_TEST: 0.02 + NMS_THRESH_TEST: 0.5 + CENTERNET: + NUM_CLASSES: 1203 + +DATASETS: + TRAIN: ("lvis_v1_train",) + TEST: ("lvis_v1_val",) +DATALOADER: + SAMPLER_TRAIN: "RepeatFactorTrainingSampler" + REPEAT_THRESHOLD: 0.001 +TEST: + DETECTIONS_PER_IMAGE: 300 diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/LVIS_CenterNet2_R50_Fed_1x.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/LVIS_CenterNet2_R50_Fed_1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6b6c823f27f3cb1459cfac3abd34dd6166ceb55 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/LVIS_CenterNet2_R50_Fed_1x.yaml @@ -0,0 +1,19 @@ +_BASE_: "Base-CenterNet2.yaml" +MODEL: + ROI_HEADS: + NUM_CLASSES: 1203 + SCORE_THRESH_TEST: 0.02 + NMS_THRESH_TEST: 0.5 + CENTERNET: + NUM_CLASSES: 1203 + ROI_BOX_HEAD: + USE_SIGMOID_CE: True + USE_FED_LOSS: True +DATASETS: + TRAIN: ("lvis_v1_train",) + TEST: ("lvis_v1_val",) +DATALOADER: + SAMPLER_TRAIN: "RepeatFactorTrainingSampler" + REPEAT_THRESHOLD: 0.001 +TEST: + DETECTIONS_PER_IMAGE: 300 diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/O365_CenterNet2_R50_1x.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/O365_CenterNet2_R50_1x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..514e52cddca8bb42afb578f1a66be71c1e6ddbe8 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/O365_CenterNet2_R50_1x.yaml @@ -0,0 +1,13 @@ +_BASE_: "Base-CenterNet2.yaml" +MODEL: + ROI_HEADS: + NUM_CLASSES: 365 + CENTERNET: + NUM_CLASSES: 365 +DATASETS: + TRAIN: ("objects365_train",) + TEST: ("objects365_val",) +DATALOADER: + SAMPLER_TRAIN: "ClassAwareSampler" +TEST: + DETECTIONS_PER_IMAGE: 300 \ No newline at end of file diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/configs/nuImages_CenterNet2_DLA_640_8x.yaml b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/nuImages_CenterNet2_DLA_640_8x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c400e92ce787bce299306589707295d0cb1ede6f --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/configs/nuImages_CenterNet2_DLA_640_8x.yaml @@ -0,0 +1,42 @@ +_BASE_: "Base-CenterNet2.yaml" +MODEL: + MASK_ON: True + ROI_MASK_HEAD: + NAME: "MaskRCNNConvUpsampleHead" + NUM_CONV: 4 + POOLER_RESOLUTION: 14 + ROI_HEADS: + NUM_CLASSES: 10 + IN_FEATURES: ["dla2"] + BACKBONE: + NAME: "build_dla_backbone" + DLA: + NORM: "BN" + CENTERNET: + IN_FEATURES: ["dla2"] + FPN_STRIDES: [4] + SOI: [[0, 1000000]] + NUM_CLS_CONVS: 1 + NUM_BOX_CONVS: 1 + REG_WEIGHT: 1. + MORE_POS: True + HM_FOCAL_ALPHA: 0.25 + POST_NMS_TOPK_TEST: 128 + WEIGHTS: '' + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.12, 57.375] +SOLVER: + MAX_ITER: 180000 + STEPS: (120000, 160000) + BASE_LR: 0.08 + IMS_PER_BATCH: 64 +INPUT: + FORMAT: RGB + CUSTOM_AUG: EfficientDetResizeCrop + TRAIN_SIZE: 640 + MIN_SIZE_TEST: 608 + MAX_SIZE_TEST: 900 + MASK_FORMAT: bitmask +DATASETS: + TRAIN: ("nuimages_train",) + TEST: ("nuimages_val",) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/datasets/README.md b/approach/ovod/mm-ovod/third_party/CenterNet2/datasets/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0eb44cc3b23beeb1755ab8d12002d26f13434235 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/datasets/README.md @@ -0,0 +1,140 @@ +# Use Builtin Datasets + +A dataset can be used by accessing [DatasetCatalog](https://detectron2.readthedocs.io/modules/data.html#detectron2.data.DatasetCatalog) +for its data, or [MetadataCatalog](https://detectron2.readthedocs.io/modules/data.html#detectron2.data.MetadataCatalog) for its metadata (class names, etc). +This document explains how to setup the builtin datasets so they can be used by the above APIs. +[Use Custom Datasets](https://detectron2.readthedocs.io/tutorials/datasets.html) gives a deeper dive on how to use `DatasetCatalog` and `MetadataCatalog`, +and how to add new datasets to them. + +Detectron2 has builtin support for a few datasets. +The datasets are assumed to exist in a directory specified by the environment variable +`DETECTRON2_DATASETS`. +Under this directory, detectron2 will look for datasets in the structure described below, if needed. +``` +$DETECTRON2_DATASETS/ + coco/ + lvis/ + cityscapes/ + VOC20{07,12}/ +``` + +You can set the location for builtin datasets by `export DETECTRON2_DATASETS=/path/to/datasets`. +If left unset, the default is `./datasets` relative to your current working directory. + +The [model zoo](https://github.com/facebookresearch/detectron2/blob/master/MODEL_ZOO.md) +contains configs and models that use these builtin datasets. + +## Expected dataset structure for [COCO instance/keypoint detection](https://cocodataset.org/#download): + +``` +coco/ + annotations/ + instances_{train,val}2017.json + person_keypoints_{train,val}2017.json + {train,val}2017/ + # image files that are mentioned in the corresponding json +``` + +You can use the 2014 version of the dataset as well. + +Some of the builtin tests (`dev/run_*_tests.sh`) uses a tiny version of the COCO dataset, +which you can download with `./datasets/prepare_for_tests.sh`. + +## Expected dataset structure for PanopticFPN: + +Extract panoptic annotations from [COCO website](https://cocodataset.org/#download) +into the following structure: +``` +coco/ + annotations/ + panoptic_{train,val}2017.json + panoptic_{train,val}2017/ # png annotations + panoptic_stuff_{train,val}2017/ # generated by the script mentioned below +``` + +Install panopticapi by: +``` +pip install git+https://github.com/cocodataset/panopticapi.git +``` +Then, run `python datasets/prepare_panoptic_fpn.py`, to extract semantic annotations from panoptic annotations. + +## Expected dataset structure for [LVIS instance segmentation](https://www.lvisdataset.org/dataset): +``` +coco/ + {train,val,test}2017/ +lvis/ + lvis_v0.5_{train,val}.json + lvis_v0.5_image_info_test.json + lvis_v1_{train,val}.json + lvis_v1_image_info_test{,_challenge}.json +``` + +Install lvis-api by: +``` +pip install git+https://github.com/lvis-dataset/lvis-api.git +``` + +To evaluate models trained on the COCO dataset using LVIS annotations, +run `python datasets/prepare_cocofied_lvis.py` to prepare "cocofied" LVIS annotations. + +## Expected dataset structure for [cityscapes](https://www.cityscapes-dataset.com/downloads/): +``` +cityscapes/ + gtFine/ + train/ + aachen/ + color.png, instanceIds.png, labelIds.png, polygons.json, + labelTrainIds.png + ... + val/ + test/ + # below are generated Cityscapes panoptic annotation + cityscapes_panoptic_train.json + cityscapes_panoptic_train/ + cityscapes_panoptic_val.json + cityscapes_panoptic_val/ + cityscapes_panoptic_test.json + cityscapes_panoptic_test/ + leftImg8bit/ + train/ + val/ + test/ +``` +Install cityscapes scripts by: +``` +pip install git+https://github.com/mcordts/cityscapesScripts.git +``` + +Note: to create labelTrainIds.png, first prepare the above structure, then run cityscapesescript with: +``` +CITYSCAPES_DATASET=/path/to/abovementioned/cityscapes python cityscapesscripts/preparation/createTrainIdLabelImgs.py +``` +These files are not needed for instance segmentation. + +Note: to generate Cityscapes panoptic dataset, run cityscapesescript with: +``` +CITYSCAPES_DATASET=/path/to/abovementioned/cityscapes python cityscapesscripts/preparation/createPanopticImgs.py +``` +These files are not needed for semantic and instance segmentation. + +## Expected dataset structure for [Pascal VOC](http://host.robots.ox.ac.uk/pascal/VOC/index.html): +``` +VOC20{07,12}/ + Annotations/ + ImageSets/ + Main/ + trainval.txt + test.txt + # train.txt or val.txt, if you use these splits + JPEGImages/ +``` + +## Expected dataset structure for [ADE20k Scene Parsing](http://sceneparsing.csail.mit.edu/): +``` +ADEChallengeData2016/ + annotations/ + annotations_detectron2/ + images/ + objectInfo150.txt +``` +The directory `annotations_detectron2` is generated by running `python datasets/prepare_ade20k_sem_seg.py`. diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/demo.py b/approach/ovod/mm-ovod/third_party/CenterNet2/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..5213faf4d859bb109a03bcd2721a02d63d2f89ce --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/demo.py @@ -0,0 +1,185 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import argparse +import glob +import multiprocessing as mp +import os +import time +import cv2 +import tqdm + +from detectron2.config import get_cfg +from detectron2.data.detection_utils import read_image +from detectron2.utils.logger import setup_logger + +from predictor import VisualizationDemo +from centernet.config import add_centernet_config +# constants +WINDOW_NAME = "CenterNet2 detections" + +from detectron2.utils.video_visualizer import VideoVisualizer +from detectron2.utils.visualizer import ColorMode, Visualizer +from detectron2.data import MetadataCatalog + +def setup_cfg(args): + # load config from file and command-line arguments + cfg = get_cfg() + add_centernet_config(cfg) + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + # Set score_threshold for builtin models + cfg.MODEL.RETINANET.SCORE_THRESH_TEST = args.confidence_threshold + cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = args.confidence_threshold + if cfg.MODEL.META_ARCHITECTURE in ['ProposalNetwork', 'CenterNetDetector']: + cfg.MODEL.CENTERNET.INFERENCE_TH = args.confidence_threshold + cfg.MODEL.CENTERNET.NMS_TH = cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST + cfg.MODEL.PANOPTIC_FPN.COMBINE.INSTANCES_CONFIDENCE_THRESH = args.confidence_threshold + cfg.freeze() + return cfg + + +def get_parser(): + parser = argparse.ArgumentParser(description="Detectron2 demo for builtin models") + parser.add_argument( + "--config-file", + default="configs/quick_schedules/mask_rcnn_R_50_FPN_inference_acc_test.yaml", + metavar="FILE", + help="path to config file", + ) + parser.add_argument("--webcam", action="store_true", help="Take inputs from webcam.") + parser.add_argument("--video-input", help="Path to video file.") + parser.add_argument("--input", nargs="+", help="A list of space separated input images") + parser.add_argument( + "--output", + help="A file or directory to save output visualizations. " + "If not given, will show output in an OpenCV window.", + ) + + parser.add_argument( + "--confidence-threshold", + type=float, + default=0.3, + help="Minimum score for instance predictions to be shown", + ) + parser.add_argument( + "--opts", + help="Modify config options using the command-line 'KEY VALUE' pairs", + default=[], + nargs=argparse.REMAINDER, + ) + return parser + + +if __name__ == "__main__": + mp.set_start_method("spawn", force=True) + args = get_parser().parse_args() + logger = setup_logger() + logger.info("Arguments: " + str(args)) + + cfg = setup_cfg(args) + + demo = VisualizationDemo(cfg) + output_file = None + if args.input: + if len(args.input) == 1: + args.input = glob.glob(os.path.expanduser(args.input[0])) + files = os.listdir(args.input[0]) + args.input = [args.input[0] + x for x in files] + assert args.input, "The input path(s) was not found" + visualizer = VideoVisualizer( + MetadataCatalog.get( + cfg.DATASETS.TEST[0] if len(cfg.DATASETS.TEST) else "__unused" + ), + instance_mode=ColorMode.IMAGE) + for path in tqdm.tqdm(args.input, disable=not args.output): + # use PIL, to be consistent with evaluation + img = read_image(path, format="BGR") + start_time = time.time() + predictions, visualized_output = demo.run_on_image( + img, visualizer=visualizer) + if 'instances' in predictions: + logger.info( + "{}: detected {} instances in {:.2f}s".format( + path, len(predictions["instances"]), time.time() - start_time + ) + ) + else: + logger.info( + "{}: detected {} instances in {:.2f}s".format( + path, len(predictions["proposals"]), time.time() - start_time + ) + ) + + if args.output: + if os.path.isdir(args.output): + assert os.path.isdir(args.output), args.output + out_filename = os.path.join(args.output, os.path.basename(path)) + visualized_output.save(out_filename) + else: + # assert len(args.input) == 1, "Please specify a directory with args.output" + # out_filename = args.output + if output_file is None: + width = visualized_output.get_image().shape[1] + height = visualized_output.get_image().shape[0] + frames_per_second = 15 + output_file = cv2.VideoWriter( + filename=args.output, + # some installation of opencv may not support x264 (due to its license), + # you can try other format (e.g. MPEG) + fourcc=cv2.VideoWriter_fourcc(*"x264"), + fps=float(frames_per_second), + frameSize=(width, height), + isColor=True, + ) + output_file.write(visualized_output.get_image()[:, :, ::-1]) + else: + # cv2.namedWindow(WINDOW_NAME, cv2.WINDOW_NORMAL) + cv2.imshow(WINDOW_NAME, visualized_output.get_image()[:, :, ::-1]) + if cv2.waitKey(1 ) == 27: + break # esc to quit + elif args.webcam: + assert args.input is None, "Cannot have both --input and --webcam!" + cam = cv2.VideoCapture(0) + for vis in tqdm.tqdm(demo.run_on_video(cam)): + cv2.namedWindow(WINDOW_NAME, cv2.WINDOW_NORMAL) + cv2.imshow(WINDOW_NAME, vis) + if cv2.waitKey(1) == 27: + break # esc to quit + cv2.destroyAllWindows() + elif args.video_input: + video = cv2.VideoCapture(args.video_input) + width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH)) + height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT)) + frames_per_second = 15 # video.get(cv2.CAP_PROP_FPS) + num_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) + basename = os.path.basename(args.video_input) + + if args.output: + if os.path.isdir(args.output): + output_fname = os.path.join(args.output, basename) + output_fname = os.path.splitext(output_fname)[0] + ".mkv" + else: + output_fname = args.output + # assert not os.path.isfile(output_fname), output_fname + output_file = cv2.VideoWriter( + filename=output_fname, + # some installation of opencv may not support x264 (due to its license), + # you can try other format (e.g. MPEG) + fourcc=cv2.VideoWriter_fourcc(*"x264"), + fps=float(frames_per_second), + frameSize=(width, height), + isColor=True, + ) + assert os.path.isfile(args.video_input) + for vis_frame in tqdm.tqdm(demo.run_on_video(video), total=num_frames): + if args.output: + output_file.write(vis_frame) + + cv2.namedWindow(basename, cv2.WINDOW_NORMAL) + cv2.imshow(basename, vis_frame) + if cv2.waitKey(1) == 27: + break # esc to quit + video.release() + if args.output: + output_file.release() + else: + cv2.destroyAllWindows() diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/docs/MODEL_ZOO.md b/approach/ovod/mm-ovod/third_party/CenterNet2/docs/MODEL_ZOO.md new file mode 100644 index 0000000000000000000000000000000000000000..7a2a92b60d0ebf8f6444f24c3bd74b753c80c57f --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/docs/MODEL_ZOO.md @@ -0,0 +1,73 @@ +# MODEL_ZOO + +### Common settings and notes + +- Multiscale training is used by default in all models. The results are all reported using single-scale testing. +- We report runtime on our local workstation with a TitanXp GPU and a Titan RTX GPU. +- All models are trained on 8-GPU servers by default. The 1280 models are trained on 24G GPUs. Reducing the batchsize with the linear learning rate rule should be fine. +- All models can be downloaded directly from [Google drive](https://drive.google.com/drive/folders/1eae1cTX8tvIaCeof36sBgxrXEXALYlf-?usp=sharing). + + +## COCO + +### CenterNet + +| Model | val mAP | FPS (Titan Xp/ Titan RTX) | links | +|-------------------------------------------|---------|---------|-----------| +| CenterNet-S4_DLA_8x | 42.5 | 50 / 71 |[config](../configs/CenterNet-S4_DLA_8x.yaml)/[model](https://drive.google.com/file/d/1lNBhVHnZAEBRD66MFaHjm5Ij6Z4KYrJq/view?usp=sharing)| +| CenterNet-FPN_R50_1x | 40.2 | 20 / 24 |[config](../configs/CenterNet-FPN_R50_1x.yaml)/[model](https://drive.google.com/file/d/1rVG1YTthMXvutC6jr9KoE2DthT5-jhGj/view?usp=sharing)| + +#### Note + +- `CenterNet-S4_DLA_8x` is a re-implemented version of the original CenterNet (stride 4), with several changes, including + - Using top-left-right-bottom box encoding and GIoU Loss; adding regression loss to the center 3x3 region. + - Adding more positive pixels for the heatmap loss whose regression loss is small and is within the center3x3 region. + - Using more heavy crop augmentation (EfficientDet-style crop ratio 0.1-2), and removing color augmentations. + - Using standard NMS instead of max pooling. + - Using RetinaNet-style optimizer (SGD), learning rate rule (0.01 for each batch size 16), and schedule (8x12 epochs). +- `CenterNet-FPN_R50_1x` is a (new) FPN version of CenterNet. It includes the changes above, and assigns objects to FPN levels based on a fixed size range. The model is trained with standard short edge 640-800 multi-scale training with 12 epochs (1x). + + +### CenterNet2 + +| Model | val mAP | FPS (Titan Xp/ Titan RTX) | links | +|-------------------------------------------|---------|---------|-----------| +| CenterNet2-F_R50_1x | 41.7 | 22 / 27 |[config](../configs/CenterNet2-F_R50_1x.yaml)/[model](X)| +| CenterNet2_R50_1x | 42.9 | 18 / 24 |[config](../configs/CenterNet2_R50_1x.yaml)/[model](https://drive.google.com/file/d/1Osu1J_sskt_1FaGdfJKa4vd2N71TWS9W/view?usp=sharing)| +| CenterNet2_X101-DCN_2x | 49.9 | 6 / 8 |[config](../configs/CenterNet2_X101-DCN_2x.yaml)/[model](https://drive.google.com/file/d/1IHgpUHVJWpvMuFUUetgKWsw27pRNN2oK/view?usp=sharing)| +| CenterNet2_DLA-BiFPN-P3_4x | 43.8 | 40 / 50|[config](../configs/CenterNet2_DLA-BiFPN-P3_4x.yaml)/[model](https://drive.google.com/file/d/12GUNlDW9RmOs40UEMSiiUsk5QK_lpGsE/view?usp=sharing)| +| CenterNet2_DLA-BiFPN-P3_24x | 45.6 | 40 / 50 |[config](../configs/CenterNet2_DLA-BiFPN-P3_24x.yaml)/[model](https://drive.google.com/file/d/15ZES1ySxubDPzKsHPA7pYg8o_Vwmf-Mb/view?usp=sharing)| +| CenterNet2_R2-101-DCN_896_4x | 51.2 | 9 / 13 |[config](../configs/CenterNet2_R2-101-DCN_896_4x.yaml)/[model](https://drive.google.com/file/d/1S7_GE8ZDQBWuLEfKHkxzeF3KBsxsbABg/view?usp=sharing)| +| CenterNet2_R2-101-DCN-BiFPN_1280_4x | 52.9 | 6 / 8 |[config](../configs/CenterNet2_R2-101-DCN-BiFPN_1280_4x.yaml)/[model](https://drive.google.com/file/d/14EBHNMagBCNTQjOXcHoZwLYIi2lFIm7F/view?usp=sharing)| +| CenterNet2_R2-101-DCN-BiFPN_4x+4x_1560_ST | 56.1 | 3 / 5 |[config](../configs/CenterNet2_R2-101-DCN-BiFPN_4x+4x_1560_ST.yaml)/[model](https://drive.google.com/file/d/11ww9VlOi_nhpdsU_vBAecSxBU0dR_JzW/view?usp=sharing)| +| CenterNet2_DLA-BiFPN-P5_640_24x_ST | 49.2 | 33 / 38 |[config](../configs/CenterNet2_DLA-BiFPN-P5_640_24x_ST.yaml)/[model](https://drive.google.com/file/d/1qsHp2HrM1u8WrtBzF5S0oCoLMz-B40wk/view?usp=sharing)| + +#### Note + +- `CenterNet2-F_R50_1x` uses Faster RCNN as the second stage. All other CenterNet2 models use Cascade RCNN as the second stage. +- `CenterNet2_DLA-BiFPN-P3_4x` follows the same training setting as [realtime-FCOS](https://github.com/aim-uofa/AdelaiDet/blob/master/configs/FCOS-Detection/README.md). +- `CenterNet2_DLA-BiFPN-P3_24x` is trained by repeating the `4x` schedule (starting from learning rate 0.01) 6 times. +- R2 means [Res2Net](https://github.com/Res2Net/Res2Net-detectron2) backbone. To train Res2Net models, you need to download the ImageNet pre-trained weight [here](https://github.com/Res2Net/Res2Net-detectron2) and place it in `output/r2_101.pkl`. +- The last 4 models in the table are trained with the EfficientDet-style resize-and-crop augmentation, instead of the default random resizing short edge in detectron2. We found this trains faster (per-iteration) and gives better performance under a long schedule. +- `_ST` means using [self-training](https://arxiv.org/abs/2006.06882) using pseudo-labels produced by [Scaled-YOLOv4](https://github.com/WongKinYiu/ScaledYOLOv4) on COCO unlabeled images, with a hard score threshold 0.5. Our processed pseudo-labels can be downloaded [here](https://drive.google.com/file/d/1LMBjtHhLp6dYf6MjwEQmzCLWQLkmWPpw/view?usp=sharing). +- `CenterNet2_R2-101-DCN-BiFPN_4x+4x_1560_ST` finetunes from `CenterNet2_R2-101-DCN-BiFPN_1280_4x` for an additional `4x` schedule with the self-training data. It is trained under `1280x1280` but tested under `1560x1560`. + +## LVIS v1 + +| Model | val mAP box | links | +|-------------------------------------------|--------------|-----------| +| LVIS_CenterNet2_R50_1x | 26.5 |[config](../configs/LVIS_CenterNet2_R50_1x.yaml)/[model](https://drive.google.com/file/d/1gT9e-tNw8uzEBaCadQuoOOP2TEYa4kKP/view?usp=sharing)| +| LVIS_CenterNet2_R50_Fed_1x | 28.3 |[config](../configs/LVIS_CenterNet2_R50_Fed_1x.yaml)/[model](https://drive.google.com/file/d/1a9UjheMCKax0qAKEwPVpq2ZHN6vpqJv8/view?usp=sharing)| + +- The models are trained with repeat-factor sampling. +- `LVIS_CenterNet2_R50_Fed_1x` is CenterNet2 with our federated loss. Check our Appendix D of our [paper](https://arxiv.org/abs/2103.07461) or our [technical report at LVIS challenge](https://www.lvisdataset.org/assets/challenge_reports/2020/CenterNet2.pdf) for references. + +## Objects365 + +| Model | val mAP| links | +|-------------------------------------------|---------|-----------| +| O365_CenterNet2_R50_1x | 22.6 |[config](../configs/O365_CenterNet2_R50_1x.yaml)/[model](https://drive.google.com/file/d/18fG6xGchAlpNp5sx8RAtwadGkS-gdIBU/view?usp=sharing)| + +#### Note +- Objects365 dataset can be downloaded [here](https://www.objects365.org/overview.html). +- The model is trained with class-aware sampling. diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/docs/centernet2_teaser.jpg b/approach/ovod/mm-ovod/third_party/CenterNet2/docs/centernet2_teaser.jpg new file mode 100644 index 0000000000000000000000000000000000000000..7b5ab5c1fddb829782632c1d9c7120cca698a02a Binary files /dev/null and b/approach/ovod/mm-ovod/third_party/CenterNet2/docs/centernet2_teaser.jpg differ diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/predictor.py b/approach/ovod/mm-ovod/third_party/CenterNet2/predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..8a036bde3f0fffd770f9ec6fd04a3505b88b09df --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/predictor.py @@ -0,0 +1,243 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import atexit +import bisect +import multiprocessing as mp +from collections import deque +import cv2 +import torch + +from detectron2.data import MetadataCatalog +from detectron2.engine.defaults import DefaultPredictor +from detectron2.utils.video_visualizer import VideoVisualizer +from detectron2.utils.visualizer import ColorMode, Visualizer + + +class VisualizationDemo(object): + def __init__(self, cfg, instance_mode=ColorMode.IMAGE, parallel=False): + """ + Args: + cfg (CfgNode): + instance_mode (ColorMode): + parallel (bool): whether to run the model in different processes from visualization. + Useful since the visualization logic can be slow. + """ + self.metadata = MetadataCatalog.get( + cfg.DATASETS.TRAIN[0] if len(cfg.DATASETS.TRAIN) else "__unused" + ) + self.cpu_device = torch.device("cpu") + self.instance_mode = instance_mode + + self.parallel = parallel + if parallel: + num_gpu = torch.cuda.device_count() + self.predictor = AsyncPredictor(cfg, num_gpus=num_gpu) + else: + self.predictor = DefaultPredictor(cfg) + + def run_on_image(self, image, visualizer=None): + """ + Args: + image (np.ndarray): an image of shape (H, W, C) (in BGR order). + This is the format used by OpenCV. + + Returns: + predictions (dict): the output of the model. + vis_output (VisImage): the visualized image output. + """ + vis_output = None + predictions = self.predictor(image) + # Convert image from OpenCV BGR format to Matplotlib RGB format. + image = image[:, :, ::-1] + use_video_vis = True + if visualizer is None: + use_video_vis = False + visualizer = Visualizer(image, self.metadata, instance_mode=self.instance_mode) + if "panoptic_seg" in predictions: + panoptic_seg, segments_info = predictions["panoptic_seg"] + vis_output = visualizer.draw_panoptic_seg_predictions( + panoptic_seg.to(self.cpu_device), segments_info + ) + else: + if "sem_seg" in predictions: + vis_output = visualizer.draw_sem_seg( + predictions["sem_seg"].argmax(dim=0).to(self.cpu_device) + ) + if "instances" in predictions: + instances = predictions["instances"].to(self.cpu_device) + if use_video_vis: + vis_output = visualizer.draw_instance_predictions( + image, predictions=instances) + else: + vis_output = visualizer.draw_instance_predictions(predictions=instances) + elif "proposals" in predictions: + instances = predictions["proposals"].to(self.cpu_device) + instances.pred_boxes = instances.proposal_boxes + instances.scores = instances.objectness_logits + instances.pred_classes[:] = -1 + if use_video_vis: + vis_output = visualizer.draw_instance_predictions( + image, predictions=instances) + else: + vis_output = visualizer.draw_instance_predictions(predictions=instances) + + return predictions, vis_output + + def _frame_from_video(self, video): + while video.isOpened(): + success, frame = video.read() + if success: + yield frame + else: + break + + def run_on_video(self, video): + """ + Visualizes predictions on frames of the input video. + + Args: + video (cv2.VideoCapture): a :class:`VideoCapture` object, whose source can be + either a webcam or a video file. + + Yields: + ndarray: BGR visualizations of each video frame. + """ + video_visualizer = VideoVisualizer(self.metadata, self.instance_mode) + + def process_predictions(frame, predictions): + frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) + if "panoptic_seg" in predictions: + panoptic_seg, segments_info = predictions["panoptic_seg"] + vis_frame = video_visualizer.draw_panoptic_seg_predictions( + frame, panoptic_seg.to(self.cpu_device), segments_info + ) + elif "instances" in predictions: + predictions = predictions["instances"].to(self.cpu_device) + vis_frame = video_visualizer.draw_instance_predictions(frame, predictions) + elif "sem_seg" in predictions: + vis_frame = video_visualizer.draw_sem_seg( + frame, predictions["sem_seg"].argmax(dim=0).to(self.cpu_device) + ) + elif "proposals" in predictions: + predictions = predictions["proposals"].to(self.cpu_device) + predictions.pred_boxes = predictions.proposal_boxes + predictions.scores = predictions.objectness_logits + predictions.pred_classes[:] = -1 + vis_frame = video_visualizer.draw_instance_predictions(frame, predictions) + + # Converts Matplotlib RGB format to OpenCV BGR format + vis_frame = cv2.cvtColor(vis_frame.get_image(), cv2.COLOR_RGB2BGR) + return vis_frame + + frame_gen = self._frame_from_video(video) + if self.parallel: + buffer_size = self.predictor.default_buffer_size + + frame_data = deque() + + for cnt, frame in enumerate(frame_gen): + frame_data.append(frame) + self.predictor.put(frame) + + if cnt >= buffer_size: + frame = frame_data.popleft() + predictions = self.predictor.get() + yield process_predictions(frame, predictions) + + while len(frame_data): + frame = frame_data.popleft() + predictions = self.predictor.get() + yield process_predictions(frame, predictions) + else: + for frame in frame_gen: + yield process_predictions(frame, self.predictor(frame)) + + +class AsyncPredictor: + """ + A predictor that runs the model asynchronously, possibly on >1 GPUs. + Because rendering the visualization takes considerably amount of time, + this helps improve throughput when rendering videos. + """ + + class _StopToken: + pass + + class _PredictWorker(mp.Process): + def __init__(self, cfg, task_queue, result_queue): + self.cfg = cfg + self.task_queue = task_queue + self.result_queue = result_queue + super().__init__() + + def run(self): + predictor = DefaultPredictor(self.cfg) + + while True: + task = self.task_queue.get() + if isinstance(task, AsyncPredictor._StopToken): + break + idx, data = task + result = predictor(data) + self.result_queue.put((idx, result)) + + def __init__(self, cfg, num_gpus: int = 1): + """ + Args: + cfg (CfgNode): + num_gpus (int): if 0, will run on CPU + """ + num_workers = max(num_gpus, 1) + self.task_queue = mp.Queue(maxsize=num_workers * 3) + self.result_queue = mp.Queue(maxsize=num_workers * 3) + self.procs = [] + for gpuid in range(max(num_gpus, 1)): + cfg = cfg.clone() + cfg.defrost() + cfg.MODEL.DEVICE = "cuda:{}".format(gpuid) if num_gpus > 0 else "cpu" + self.procs.append( + AsyncPredictor._PredictWorker(cfg, self.task_queue, self.result_queue) + ) + + self.put_idx = 0 + self.get_idx = 0 + self.result_rank = [] + self.result_data = [] + + for p in self.procs: + p.start() + atexit.register(self.shutdown) + + def put(self, image): + self.put_idx += 1 + self.task_queue.put((self.put_idx, image)) + + def get(self): + self.get_idx += 1 # the index needed for this request + if len(self.result_rank) and self.result_rank[0] == self.get_idx: + res = self.result_data[0] + del self.result_data[0], self.result_rank[0] + return res + + while True: + # make sure the results are returned in the correct order + idx, res = self.result_queue.get() + if idx == self.get_idx: + return res + insert = bisect.bisect(self.result_rank, idx) + self.result_rank.insert(insert, idx) + self.result_data.insert(insert, res) + + def __len__(self): + return self.put_idx - self.get_idx + + def __call__(self, image): + self.put(image) + return self.get() + + def shutdown(self): + for _ in self.procs: + self.task_queue.put(AsyncPredictor._StopToken()) + + @property + def default_buffer_size(self): + return len(self.procs) * 5 diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/requirements.txt b/approach/ovod/mm-ovod/third_party/CenterNet2/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0dd006bbc37158dd5128c9b63ea0767cd68b0c28 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/requirements.txt @@ -0,0 +1 @@ +opencv-python diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/tools/README.md b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0b40d5319c0838fdaa22bc6a10ef0d88bc6578ed --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/README.md @@ -0,0 +1,49 @@ + +This directory contains a few example scripts that demonstrate features of detectron2. + + +* `train_net.py` + +An example training script that's made to train builtin models of detectron2. + +For usage, see [GETTING_STARTED.md](../GETTING_STARTED.md). + +* `plain_train_net.py` + +Similar to `train_net.py`, but implements a training loop instead of using `Trainer`. +This script includes fewer features but it may be more friendly to hackers. + +* `benchmark.py` + +Benchmark the training speed, inference speed or data loading speed of a given config. + +Usage: +``` +python benchmark.py --config-file config.yaml --task train/eval/data [optional DDP flags] +``` + +* `analyze_model.py` + +Analyze FLOPs, parameters, activations of a detectron2 model. See its `--help` for usage. + +* `visualize_json_results.py` + +Visualize the json instance detection/segmentation results dumped by `COCOEvalutor` or `LVISEvaluator` + +Usage: +``` +python visualize_json_results.py --input x.json --output dir/ --dataset coco_2017_val +``` +If not using a builtin dataset, you'll need your own script or modify this script. + +* `visualize_data.py` + +Visualize ground truth raw annotations or training data (after preprocessing/augmentations). + +Usage: +``` +python visualize_data.py --config-file config.yaml --source annotation/dataloader --output-dir dir/ [--show] +``` + +NOTE: the script does not stop by itself when using `--source dataloader` because a training +dataloader is usually infinite. diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/tools/__init__.py b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/tools/analyze_model.py b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/analyze_model.py new file mode 100644 index 0000000000000000000000000000000000000000..8e38f8b71eb3b8d1e2b670e7f01a796ec2ea4b7e --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/analyze_model.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import logging +import numpy as np +from collections import Counter +import tqdm +from fvcore.nn import flop_count_table # can also try flop_count_str + +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import CfgNode, LazyConfig, get_cfg, instantiate +from detectron2.data import build_detection_test_loader +from detectron2.engine import default_argument_parser +from detectron2.modeling import build_model +from detectron2.utils.analysis import ( + FlopCountAnalysis, + activation_count_operators, + parameter_count_table, +) +from detectron2.utils.logger import setup_logger + +logger = logging.getLogger("detectron2") + + +def setup(args): + if args.config_file.endswith(".yaml"): + cfg = get_cfg() + cfg.merge_from_file(args.config_file) + cfg.DATALOADER.NUM_WORKERS = 0 + cfg.merge_from_list(args.opts) + cfg.freeze() + else: + cfg = LazyConfig.load(args.config_file) + cfg = LazyConfig.apply_overrides(cfg, args.opts) + setup_logger(name="fvcore") + setup_logger() + return cfg + + +def do_flop(cfg): + if isinstance(cfg, CfgNode): + data_loader = build_detection_test_loader(cfg, cfg.DATASETS.TEST[0]) + model = build_model(cfg) + DetectionCheckpointer(model).load(cfg.MODEL.WEIGHTS) + else: + data_loader = instantiate(cfg.dataloader.test) + model = instantiate(cfg.model) + model.to(cfg.train.device) + DetectionCheckpointer(model).load(cfg.train.init_checkpoint) + model.eval() + + counts = Counter() + total_flops = [] + for idx, data in zip(tqdm.trange(args.num_inputs), data_loader): # noqa + flops = FlopCountAnalysis(model, data) + if idx > 0: + flops.unsupported_ops_warnings(False).uncalled_modules_warnings(False) + counts += flops.by_operator() + total_flops.append(flops.total()) + + logger.info("Flops table computed from only one input sample:\n" + flop_count_table(flops)) + logger.info( + "Average GFlops for each type of operators:\n" + + str([(k, v / (idx + 1) / 1e9) for k, v in counts.items()]) + ) + logger.info( + "Total GFlops: {:.1f}±{:.1f}".format(np.mean(total_flops) / 1e9, np.std(total_flops) / 1e9) + ) + + +def do_activation(cfg): + if isinstance(cfg, CfgNode): + data_loader = build_detection_test_loader(cfg, cfg.DATASETS.TEST[0]) + model = build_model(cfg) + DetectionCheckpointer(model).load(cfg.MODEL.WEIGHTS) + else: + data_loader = instantiate(cfg.dataloader.test) + model = instantiate(cfg.model) + model.to(cfg.train.device) + DetectionCheckpointer(model).load(cfg.train.init_checkpoint) + model.eval() + + counts = Counter() + total_activations = [] + for idx, data in zip(tqdm.trange(args.num_inputs), data_loader): # noqa + count = activation_count_operators(model, data) + counts += count + total_activations.append(sum(count.values())) + logger.info( + "(Million) Activations for Each Type of Operators:\n" + + str([(k, v / idx) for k, v in counts.items()]) + ) + logger.info( + "Total (Million) Activations: {}±{}".format( + np.mean(total_activations), np.std(total_activations) + ) + ) + + +def do_parameter(cfg): + if isinstance(cfg, CfgNode): + model = build_model(cfg) + else: + model = instantiate(cfg.model) + logger.info("Parameter Count:\n" + parameter_count_table(model, max_depth=5)) + + +def do_structure(cfg): + if isinstance(cfg, CfgNode): + model = build_model(cfg) + else: + model = instantiate(cfg.model) + logger.info("Model Structure:\n" + str(model)) + + +if __name__ == "__main__": + parser = default_argument_parser( + epilog=""" +Examples: + +To show parameters of a model: +$ ./analyze_model.py --tasks parameter \\ + --config-file ../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml + +Flops and activations are data-dependent, therefore inputs and model weights +are needed to count them: + +$ ./analyze_model.py --num-inputs 100 --tasks flop \\ + --config-file ../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \\ + MODEL.WEIGHTS /path/to/model.pkl +""" + ) + parser.add_argument( + "--tasks", + choices=["flop", "activation", "parameter", "structure"], + required=True, + nargs="+", + ) + parser.add_argument( + "-n", + "--num-inputs", + default=100, + type=int, + help="number of inputs used to compute statistics for flops/activations, " + "both are data dependent.", + ) + args = parser.parse_args() + assert not args.eval_only + assert args.num_gpus == 1 + + cfg = setup(args) + + for task in args.tasks: + { + "flop": do_flop, + "activation": do_activation, + "parameter": do_parameter, + "structure": do_structure, + }[task](cfg) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/tools/benchmark.py b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..aaac56400148f7b140b7c1356bbbc3b4293e5ce3 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/benchmark.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +""" +A script to benchmark builtin models. + +Note: this script has an extra dependency of psutil. +""" + +import itertools +import logging +import psutil +import torch +import tqdm +from fvcore.common.timer import Timer +from torch.nn.parallel import DistributedDataParallel + +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import LazyConfig, get_cfg, instantiate +from detectron2.data import ( + DatasetFromList, + build_detection_test_loader, + build_detection_train_loader, +) +from detectron2.data.benchmark import DataLoaderBenchmark +from detectron2.engine import AMPTrainer, SimpleTrainer, default_argument_parser, hooks, launch +from detectron2.modeling import build_model +from detectron2.solver import build_optimizer +from detectron2.utils import comm +from detectron2.utils.collect_env import collect_env_info +from detectron2.utils.events import CommonMetricPrinter +from detectron2.utils.logger import setup_logger + +logger = logging.getLogger("detectron2") + + +def setup(args): + if args.config_file.endswith(".yaml"): + cfg = get_cfg() + cfg.merge_from_file(args.config_file) + cfg.SOLVER.BASE_LR = 0.001 # Avoid NaNs. Not useful in this script anyway. + cfg.merge_from_list(args.opts) + cfg.freeze() + else: + cfg = LazyConfig.load(args.config_file) + cfg = LazyConfig.apply_overrides(cfg, args.opts) + setup_logger(distributed_rank=comm.get_rank()) + return cfg + + +def create_data_benchmark(cfg, args): + if args.config_file.endswith(".py"): + dl_cfg = cfg.dataloader.train + dl_cfg._target_ = DataLoaderBenchmark + return instantiate(dl_cfg) + else: + kwargs = build_detection_train_loader.from_config(cfg) + kwargs.pop("aspect_ratio_grouping", None) + kwargs["_target_"] = DataLoaderBenchmark + return instantiate(kwargs) + + +def RAM_msg(): + vram = psutil.virtual_memory() + return "RAM Usage: {:.2f}/{:.2f} GB".format( + (vram.total - vram.available) / 1024 ** 3, vram.total / 1024 ** 3 + ) + + +def benchmark_data(args): + cfg = setup(args) + logger.info("After spawning " + RAM_msg()) + + benchmark = create_data_benchmark(cfg, args) + benchmark.benchmark_distributed(250, 10) + # test for a few more rounds + for k in range(10): + logger.info(f"Iteration {k} " + RAM_msg()) + benchmark.benchmark_distributed(250, 1) + + +def benchmark_data_advanced(args): + # benchmark dataloader with more details to help analyze performance bottleneck + cfg = setup(args) + benchmark = create_data_benchmark(cfg, args) + + if comm.get_rank() == 0: + benchmark.benchmark_dataset(100) + benchmark.benchmark_mapper(100) + benchmark.benchmark_workers(100, warmup=10) + benchmark.benchmark_IPC(100, warmup=10) + if comm.get_world_size() > 1: + benchmark.benchmark_distributed(100) + logger.info("Rerun ...") + benchmark.benchmark_distributed(100) + + +def benchmark_train(args): + cfg = setup(args) + model = build_model(cfg) + logger.info("Model:\n{}".format(model)) + if comm.get_world_size() > 1: + model = DistributedDataParallel( + model, device_ids=[comm.get_local_rank()], broadcast_buffers=False + ) + optimizer = build_optimizer(cfg, model) + checkpointer = DetectionCheckpointer(model, optimizer=optimizer) + checkpointer.load(cfg.MODEL.WEIGHTS) + + cfg.defrost() + cfg.DATALOADER.NUM_WORKERS = 2 + data_loader = build_detection_train_loader(cfg) + dummy_data = list(itertools.islice(data_loader, 100)) + + def f(): + data = DatasetFromList(dummy_data, copy=False, serialize=False) + while True: + yield from data + + max_iter = 400 + trainer = (AMPTrainer if cfg.SOLVER.AMP.ENABLED else SimpleTrainer)(model, f(), optimizer) + trainer.register_hooks( + [ + hooks.IterationTimer(), + hooks.PeriodicWriter([CommonMetricPrinter(max_iter)]), + hooks.TorchProfiler( + lambda trainer: trainer.iter == max_iter - 1, cfg.OUTPUT_DIR, save_tensorboard=True + ), + ] + ) + trainer.train(1, max_iter) + + +@torch.no_grad() +def benchmark_eval(args): + cfg = setup(args) + if args.config_file.endswith(".yaml"): + model = build_model(cfg) + DetectionCheckpointer(model).load(cfg.MODEL.WEIGHTS) + + cfg.defrost() + cfg.DATALOADER.NUM_WORKERS = 0 + data_loader = build_detection_test_loader(cfg, cfg.DATASETS.TEST[0]) + else: + model = instantiate(cfg.model) + model.to(cfg.train.device) + DetectionCheckpointer(model).load(cfg.train.init_checkpoint) + + cfg.dataloader.num_workers = 0 + data_loader = instantiate(cfg.dataloader.test) + + model.eval() + logger.info("Model:\n{}".format(model)) + dummy_data = DatasetFromList(list(itertools.islice(data_loader, 100)), copy=False) + + def f(): + while True: + yield from dummy_data + + for k in range(5): # warmup + model(dummy_data[k]) + + max_iter = 300 + timer = Timer() + with tqdm.tqdm(total=max_iter) as pbar: + for idx, d in enumerate(f()): + if idx == max_iter: + break + model(d) + pbar.update() + logger.info("{} iters in {} seconds.".format(max_iter, timer.seconds())) + + +if __name__ == "__main__": + parser = default_argument_parser() + parser.add_argument("--task", choices=["train", "eval", "data", "data_advanced"], required=True) + args = parser.parse_args() + assert not args.eval_only + + logger.info("Environment info:\n" + collect_env_info()) + if "data" in args.task: + print("Initial " + RAM_msg()) + if args.task == "data": + f = benchmark_data + if args.task == "data_advanced": + f = benchmark_data_advanced + elif args.task == "train": + """ + Note: training speed may not be representative. + The training cost of a R-CNN model varies with the content of the data + and the quality of the model. + """ + f = benchmark_train + elif args.task == "eval": + f = benchmark_eval + # only benchmark single-GPU inference. + assert args.num_gpus == 1 and args.num_machines == 1 + launch(f, args.num_gpus, args.num_machines, args.machine_rank, args.dist_url, args=(args,)) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/tools/convert-torchvision-to-d2.py b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/convert-torchvision-to-d2.py new file mode 100644 index 0000000000000000000000000000000000000000..4b827d960cca69657e98bd89a9aa5623a847099d --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/convert-torchvision-to-d2.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. + +import pickle as pkl +import sys +import torch + +""" +Usage: + # download one of the ResNet{18,34,50,101,152} models from torchvision: + wget https://download.pytorch.org/models/resnet50-19c8e357.pth -O r50.pth + # run the conversion + ./convert-torchvision-to-d2.py r50.pth r50.pkl + + # Then, use r50.pkl with the following changes in config: + +MODEL: + WEIGHTS: "/path/to/r50.pkl" + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.120, 57.375] + RESNETS: + DEPTH: 50 + STRIDE_IN_1X1: False +INPUT: + FORMAT: "RGB" + + These models typically produce slightly worse results than the + pre-trained ResNets we use in official configs, which are the + original ResNet models released by MSRA. +""" + +if __name__ == "__main__": + input = sys.argv[1] + + obj = torch.load(input, map_location="cpu") + + newmodel = {} + for k in list(obj.keys()): + old_k = k + if "layer" not in k: + k = "stem." + k + for t in [1, 2, 3, 4]: + k = k.replace("layer{}".format(t), "res{}".format(t + 1)) + for t in [1, 2, 3]: + k = k.replace("bn{}".format(t), "conv{}.norm".format(t)) + k = k.replace("downsample.0", "shortcut") + k = k.replace("downsample.1", "shortcut.norm") + print(old_k, "->", k) + newmodel[k] = obj.pop(old_k).detach().numpy() + + res = {"model": newmodel, "__author__": "torchvision", "matching_heuristics": True} + + with open(sys.argv[2], "wb") as f: + pkl.dump(res, f) + if obj: + print("Unconverted keys:", obj.keys()) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/tools/deploy/CMakeLists.txt b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/deploy/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..80dae12500af4c7e7e6cfc5b7b3a5800782956c3 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/deploy/CMakeLists.txt @@ -0,0 +1,15 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# See https://pytorch.org/tutorials/advanced/cpp_frontend.html +cmake_minimum_required(VERSION 3.12 FATAL_ERROR) +project(torchscript_mask_rcnn) + +find_package(Torch REQUIRED) +find_package(OpenCV REQUIRED) +find_package(TorchVision REQUIRED) # needed by export-method=tracing/scripting + +add_executable(torchscript_mask_rcnn torchscript_mask_rcnn.cpp) +target_link_libraries( + torchscript_mask_rcnn + -Wl,--no-as-needed TorchVision::TorchVision -Wl,--as-needed + "${TORCH_LIBRARIES}" ${OpenCV_LIBS}) +set_property(TARGET torchscript_mask_rcnn PROPERTY CXX_STANDARD 14) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/tools/deploy/README.md b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/deploy/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e33cbeb54c003a5738da68c838fdaa4e0d218501 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/deploy/README.md @@ -0,0 +1,66 @@ +See [deployment tutorial](https://detectron2.readthedocs.io/tutorials/deployment.html) +for some high-level background about deployment. + +This directory contains the following examples: + +1. An example script `export_model.py` + that exports a detectron2 model for deployment using different methods and formats. + +2. A C++ example that runs inference with Mask R-CNN model in TorchScript format. + +## Build +Deployment depends on libtorch and OpenCV. Some require more dependencies: + +* Running TorchScript-format models produced by `--export-method=caffe2_tracing` requires libtorch + to be built with caffe2 enabled. +* Running TorchScript-format models produced by `--export-method=tracing/scripting` requires libtorchvision (C++ library of torchvision). + +All methods are supported in one C++ file that requires all the above dependencies. +Adjust it and remove code you don't need. +As a reference, we provide a [Dockerfile](../../docker/deploy.Dockerfile) that installs all the above dependencies and builds the C++ example. + +## Use + +We show a few example commands to export and execute a Mask R-CNN model in C++. + +* `export-method=tracing, format=torchscript`: +``` +./export_model.py --config-file ../../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml \ + --output ./output --export-method tracing --format torchscript \ + MODEL.WEIGHTS detectron2://COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/model_final_f10217.pkl \ + MODEL.DEVICE cuda + +./build/torchscript_mask_rcnn output/model.ts input.jpg tracing +``` + +* `export-method=scripting, format=torchscript`: +``` +./export_model.py --config-file ../../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml \ + --output ./output --export-method scripting --format torchscript \ + MODEL.WEIGHTS detectron2://COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/model_final_f10217.pkl \ + +./build/torchscript_mask_rcnn output/model.ts input.jpg scripting +``` + +* `export-method=caffe2_tracing, format=torchscript`: + +``` +./export_model.py --config-file ../../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml \ + --output ./output --export-method caffe2_tracing --format torchscript \ + MODEL.WEIGHTS detectron2://COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x/137849600/model_final_f10217.pkl \ + +./build/torchscript_mask_rcnn output/model.ts input.jpg caffe2_tracing +``` + + +## Notes: + +1. Tracing/Caffe2-tracing requires valid weights & sample inputs. + Therefore the above commands require pre-trained models and [COCO dataset](https://detectron2.readthedocs.io/tutorials/builtin_datasets.html). + You can modify the script to obtain sample inputs in other ways instead of from COCO. + +2. `--run-eval` is implemented only for tracing mode + to evaluate the exported model using the dataset in the config. + It's recommended to always verify the accuracy in case the conversion is not successful. + Evaluation can be slow if model is exported to CPU or dataset is too large ("coco_2017_val_100" is a small subset of COCO useful for evaluation). + `caffe2_tracing` accuracy may be slightly different (within 0.1 AP) from original model due to numerical precisions between different runtime. diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/tools/deploy/export_model.py b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/deploy/export_model.py new file mode 100644 index 0000000000000000000000000000000000000000..bb1bcee6323372e80e42e3217b055d0c3a902954 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/deploy/export_model.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +import argparse +import os +from typing import Dict, List, Tuple +import torch +from torch import Tensor, nn + +import detectron2.data.transforms as T +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import get_cfg +from detectron2.data import build_detection_test_loader, detection_utils +from detectron2.evaluation import COCOEvaluator, inference_on_dataset, print_csv_format +from detectron2.export import TracingAdapter, dump_torchscript_IR, scripting_with_instances +from detectron2.modeling import GeneralizedRCNN, RetinaNet, build_model +from detectron2.modeling.postprocessing import detector_postprocess +from detectron2.projects.point_rend import add_pointrend_config +from detectron2.structures import Boxes +from detectron2.utils.env import TORCH_VERSION +from detectron2.utils.file_io import PathManager +from detectron2.utils.logger import setup_logger + + +def setup_cfg(args): + cfg = get_cfg() + # cuda context is initialized before creating dataloader, so we don't fork anymore + cfg.DATALOADER.NUM_WORKERS = 0 + add_pointrend_config(cfg) + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + cfg.freeze() + return cfg + + +def export_caffe2_tracing(cfg, torch_model, inputs): + from detectron2.export import Caffe2Tracer + + tracer = Caffe2Tracer(cfg, torch_model, inputs) + if args.format == "caffe2": + caffe2_model = tracer.export_caffe2() + caffe2_model.save_protobuf(args.output) + # draw the caffe2 graph + caffe2_model.save_graph(os.path.join(args.output, "model.svg"), inputs=inputs) + return caffe2_model + elif args.format == "onnx": + import onnx + + onnx_model = tracer.export_onnx() + onnx.save(onnx_model, os.path.join(args.output, "model.onnx")) + elif args.format == "torchscript": + ts_model = tracer.export_torchscript() + with PathManager.open(os.path.join(args.output, "model.ts"), "wb") as f: + torch.jit.save(ts_model, f) + dump_torchscript_IR(ts_model, args.output) + + +# experimental. API not yet final +def export_scripting(torch_model): + assert TORCH_VERSION >= (1, 8) + fields = { + "proposal_boxes": Boxes, + "objectness_logits": Tensor, + "pred_boxes": Boxes, + "scores": Tensor, + "pred_classes": Tensor, + "pred_masks": Tensor, + "pred_keypoints": torch.Tensor, + "pred_keypoint_heatmaps": torch.Tensor, + } + assert args.format == "torchscript", "Scripting only supports torchscript format." + + class ScriptableAdapterBase(nn.Module): + # Use this adapter to workaround https://github.com/pytorch/pytorch/issues/46944 + # by not retuning instances but dicts. Otherwise the exported model is not deployable + def __init__(self): + super().__init__() + self.model = torch_model + self.eval() + + if isinstance(torch_model, GeneralizedRCNN): + + class ScriptableAdapter(ScriptableAdapterBase): + def forward(self, inputs: Tuple[Dict[str, torch.Tensor]]) -> List[Dict[str, Tensor]]: + instances = self.model.inference(inputs, do_postprocess=False) + return [i.get_fields() for i in instances] + + else: + + class ScriptableAdapter(ScriptableAdapterBase): + def forward(self, inputs: Tuple[Dict[str, torch.Tensor]]) -> List[Dict[str, Tensor]]: + instances = self.model(inputs) + return [i.get_fields() for i in instances] + + ts_model = scripting_with_instances(ScriptableAdapter(), fields) + with PathManager.open(os.path.join(args.output, "model.ts"), "wb") as f: + torch.jit.save(ts_model, f) + dump_torchscript_IR(ts_model, args.output) + # TODO inference in Python now missing postprocessing glue code + return None + + +# experimental. API not yet final +def export_tracing(torch_model, inputs): + assert TORCH_VERSION >= (1, 8) + image = inputs[0]["image"] + inputs = [{"image": image}] # remove other unused keys + + if isinstance(torch_model, GeneralizedRCNN): + + def inference(model, inputs): + # use do_postprocess=False so it returns ROI mask + inst = model.inference(inputs, do_postprocess=False)[0] + return [{"instances": inst}] + + else: + inference = None # assume that we just call the model directly + + traceable_model = TracingAdapter(torch_model, inputs, inference) + + if args.format == "torchscript": + ts_model = torch.jit.trace(traceable_model, (image,)) + with PathManager.open(os.path.join(args.output, "model.ts"), "wb") as f: + torch.jit.save(ts_model, f) + dump_torchscript_IR(ts_model, args.output) + elif args.format == "onnx": + with PathManager.open(os.path.join(args.output, "model.onnx"), "wb") as f: + torch.onnx.export(traceable_model, (image,), f, opset_version=11) + logger.info("Inputs schema: " + str(traceable_model.inputs_schema)) + logger.info("Outputs schema: " + str(traceable_model.outputs_schema)) + + if args.format != "torchscript": + return None + if not isinstance(torch_model, (GeneralizedRCNN, RetinaNet)): + return None + + def eval_wrapper(inputs): + """ + The exported model does not contain the final resize step, which is typically + unused in deployment but needed for evaluation. We add it manually here. + """ + input = inputs[0] + instances = traceable_model.outputs_schema(ts_model(input["image"]))[0]["instances"] + postprocessed = detector_postprocess(instances, input["height"], input["width"]) + return [{"instances": postprocessed}] + + return eval_wrapper + + +def get_sample_inputs(args): + + if args.sample_image is None: + # get a first batch from dataset + data_loader = build_detection_test_loader(cfg, cfg.DATASETS.TEST[0]) + first_batch = next(iter(data_loader)) + return first_batch + else: + # get a sample data + original_image = detection_utils.read_image(args.sample_image, format=cfg.INPUT.FORMAT) + # Do same preprocessing as DefaultPredictor + aug = T.ResizeShortestEdge( + [cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST], cfg.INPUT.MAX_SIZE_TEST + ) + height, width = original_image.shape[:2] + image = aug.get_transform(original_image).apply_image(original_image) + image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) + + inputs = {"image": image, "height": height, "width": width} + + # Sample ready + sample_inputs = [inputs] + return sample_inputs + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Export a model for deployment.") + parser.add_argument( + "--format", + choices=["caffe2", "onnx", "torchscript"], + help="output format", + default="torchscript", + ) + parser.add_argument( + "--export-method", + choices=["caffe2_tracing", "tracing", "scripting"], + help="Method to export models", + default="tracing", + ) + parser.add_argument("--config-file", default="", metavar="FILE", help="path to config file") + parser.add_argument("--sample-image", default=None, type=str, help="sample image for input") + parser.add_argument("--run-eval", action="store_true") + parser.add_argument("--output", help="output directory for the converted model") + parser.add_argument( + "opts", + help="Modify config options using the command-line", + default=None, + nargs=argparse.REMAINDER, + ) + args = parser.parse_args() + logger = setup_logger() + logger.info("Command line arguments: " + str(args)) + PathManager.mkdirs(args.output) + # Disable respecialization on new shapes. Otherwise --run-eval will be slow + torch._C._jit_set_bailout_depth(1) + + cfg = setup_cfg(args) + + # create a torch model + torch_model = build_model(cfg) + DetectionCheckpointer(torch_model).resume_or_load(cfg.MODEL.WEIGHTS) + torch_model.eval() + + # get sample data + sample_inputs = get_sample_inputs(args) + + # convert and save model + if args.export_method == "caffe2_tracing": + exported_model = export_caffe2_tracing(cfg, torch_model, sample_inputs) + elif args.export_method == "scripting": + exported_model = export_scripting(torch_model) + elif args.export_method == "tracing": + exported_model = export_tracing(torch_model, sample_inputs) + + # run evaluation with the converted model + if args.run_eval: + assert exported_model is not None, ( + "Python inference is not yet implemented for " + f"export_method={args.export_method}, format={args.format}." + ) + logger.info("Running evaluation ... this takes a long time if you export to CPU.") + dataset = cfg.DATASETS.TEST[0] + data_loader = build_detection_test_loader(cfg, dataset) + # NOTE: hard-coded evaluator. change to the evaluator for your dataset + evaluator = COCOEvaluator(dataset, output_dir=args.output) + metrics = inference_on_dataset(exported_model, data_loader, evaluator) + print_csv_format(metrics) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/tools/deploy/torchscript_mask_rcnn.cpp b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/deploy/torchscript_mask_rcnn.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b40f13b81f601788847992e6627b448d62a287e2 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/deploy/torchscript_mask_rcnn.cpp @@ -0,0 +1,187 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +// @lint-ignore-every CLANGTIDY +// This is an example code that demonstrates how to run inference +// with a torchscript format Mask R-CNN model exported by ./export_model.py +// using export method=tracing, caffe2_tracing & scripting. + +#include +#include +#include + +#include +#include +#include +#include + +// only needed for export_method=tracing +#include // @oss-only +// @fb-only: #include + +using namespace std; + +c10::IValue get_caffe2_tracing_inputs(cv::Mat& img, c10::Device device) { + const int height = img.rows; + const int width = img.cols; + // FPN models require divisibility of 32. + // Tracing mode does padding inside the graph, but caffe2_tracing does not. + assert(height % 32 == 0 && width % 32 == 0); + const int channels = 3; + + auto input = + torch::from_blob(img.data, {1, height, width, channels}, torch::kUInt8); + // NHWC to NCHW + input = input.to(device, torch::kFloat).permute({0, 3, 1, 2}).contiguous(); + + std::array im_info_data{height * 1.0f, width * 1.0f, 1.0f}; + auto im_info = + torch::from_blob(im_info_data.data(), {1, 3}).clone().to(device); + return std::make_tuple(input, im_info); +} + +c10::IValue get_tracing_inputs(cv::Mat& img, c10::Device device) { + const int height = img.rows; + const int width = img.cols; + const int channels = 3; + + auto input = + torch::from_blob(img.data, {height, width, channels}, torch::kUInt8); + // HWC to CHW + input = input.to(device, torch::kFloat).permute({2, 0, 1}).contiguous(); + return input; +} + +// create a Tuple[Dict[str, Tensor]] which is the input type of scripted model +c10::IValue get_scripting_inputs(cv::Mat& img, c10::Device device) { + const int height = img.rows; + const int width = img.cols; + const int channels = 3; + + auto img_tensor = + torch::from_blob(img.data, {height, width, channels}, torch::kUInt8); + // HWC to CHW + img_tensor = + img_tensor.to(device, torch::kFloat).permute({2, 0, 1}).contiguous(); + auto dic = c10::Dict(); + dic.insert("image", img_tensor); + return std::make_tuple(dic); +} + +c10::IValue +get_inputs(std::string export_method, cv::Mat& img, c10::Device device) { + // Given an image, create inputs in the format required by the model. + if (export_method == "tracing") + return get_tracing_inputs(img, device); + if (export_method == "caffe2_tracing") + return get_caffe2_tracing_inputs(img, device); + if (export_method == "scripting") + return get_scripting_inputs(img, device); + abort(); +} + +struct MaskRCNNOutputs { + at::Tensor pred_boxes, pred_classes, pred_masks, scores; + int num_instances() const { + return pred_boxes.sizes()[0]; + } +}; + +MaskRCNNOutputs get_outputs(std::string export_method, c10::IValue outputs) { + // Given outputs of the model, extract tensors from it to turn into a + // common MaskRCNNOutputs format. + if (export_method == "tracing") { + auto out_tuple = outputs.toTuple()->elements(); + // They are ordered alphabetically by their field name in Instances + return MaskRCNNOutputs{ + out_tuple[0].toTensor(), + out_tuple[1].toTensor(), + out_tuple[2].toTensor(), + out_tuple[3].toTensor()}; + } + if (export_method == "caffe2_tracing") { + auto out_tuple = outputs.toTuple()->elements(); + // A legacy order used by caffe2 models + return MaskRCNNOutputs{ + out_tuple[0].toTensor(), + out_tuple[2].toTensor(), + out_tuple[3].toTensor(), + out_tuple[1].toTensor()}; + } + if (export_method == "scripting") { + // With the ScriptableAdapter defined in export_model.py, the output is + // List[Dict[str, Any]]. + auto out_dict = outputs.toList().get(0).toGenericDict(); + return MaskRCNNOutputs{ + out_dict.at("pred_boxes").toTensor(), + out_dict.at("pred_classes").toTensor(), + out_dict.at("pred_masks").toTensor(), + out_dict.at("scores").toTensor()}; + } + abort(); +} + +int main(int argc, const char* argv[]) { + if (argc != 4) { + cerr << R"xx( +Usage: + ./torchscript_mask_rcnn model.ts input.jpg EXPORT_METHOD + + EXPORT_METHOD can be "tracing", "caffe2_tracing" or "scripting". +)xx"; + return 1; + } + std::string image_file = argv[2]; + std::string export_method = argv[3]; + assert( + export_method == "caffe2_tracing" || export_method == "tracing" || + export_method == "scripting"); + + torch::jit::getBailoutDepth() = 1; + torch::autograd::AutoGradMode guard(false); + auto module = torch::jit::load(argv[1]); + + assert(module.buffers().size() > 0); + // Assume that the entire model is on the same device. + // We just put input to this device. + auto device = (*begin(module.buffers())).device(); + + cv::Mat input_img = cv::imread(image_file, cv::IMREAD_COLOR); + auto inputs = get_inputs(export_method, input_img, device); + + // Run the network + auto output = module.forward({inputs}); + if (device.is_cuda()) + c10::cuda::getCurrentCUDAStream().synchronize(); + + // run 3 more times to benchmark + int N_benchmark = 3, N_warmup = 1; + auto start_time = chrono::high_resolution_clock::now(); + for (int i = 0; i < N_benchmark + N_warmup; ++i) { + if (i == N_warmup) + start_time = chrono::high_resolution_clock::now(); + output = module.forward({inputs}); + if (device.is_cuda()) + c10::cuda::getCurrentCUDAStream().synchronize(); + } + auto end_time = chrono::high_resolution_clock::now(); + auto ms = chrono::duration_cast(end_time - start_time) + .count(); + cout << "Latency (should vary with different inputs): " + << ms * 1.0 / 1e6 / N_benchmark << " seconds" << endl; + + // Parse Mask R-CNN outputs + auto rcnn_outputs = get_outputs(export_method, output); + cout << "Number of detected objects: " << rcnn_outputs.num_instances() + << endl; + + cout << "pred_boxes: " << rcnn_outputs.pred_boxes.toString() << " " + << rcnn_outputs.pred_boxes.sizes() << endl; + cout << "scores: " << rcnn_outputs.scores.toString() << " " + << rcnn_outputs.scores.sizes() << endl; + cout << "pred_classes: " << rcnn_outputs.pred_classes.toString() << " " + << rcnn_outputs.pred_classes.sizes() << endl; + cout << "pred_masks: " << rcnn_outputs.pred_masks.toString() << " " + << rcnn_outputs.pred_masks.sizes() << endl; + + cout << rcnn_outputs.pred_boxes << endl; + return 0; +} diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/tools/lazyconfig_train_net.py b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/lazyconfig_train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..bb62d36c0c171b0391453afafc2828ebab1b0da1 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/lazyconfig_train_net.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +""" +Training script using the new "LazyConfig" python config files. + +This scripts reads a given python config file and runs the training or evaluation. +It can be used to train any models or dataset as long as they can be +instantiated by the recursive construction defined in the given config file. + +Besides lazy construction of models, dataloader, etc., this scripts expects a +few common configuration parameters currently defined in "configs/common/train.py". +To add more complicated training logic, you can easily add other configs +in the config file and implement a new train_net.py to handle them. +""" +import logging + +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import LazyConfig, instantiate +from detectron2.engine import ( + AMPTrainer, + SimpleTrainer, + default_argument_parser, + default_setup, + default_writers, + hooks, + launch, +) +from detectron2.engine.defaults import create_ddp_model +from detectron2.evaluation import inference_on_dataset, print_csv_format +from detectron2.utils import comm + +logger = logging.getLogger("detectron2") + + +def do_test(cfg, model): + if "evaluator" in cfg.dataloader: + ret = inference_on_dataset( + model, instantiate(cfg.dataloader.test), instantiate(cfg.dataloader.evaluator) + ) + print_csv_format(ret) + return ret + + +def do_train(args, cfg): + """ + Args: + cfg: an object with the following attributes: + model: instantiate to a module + dataloader.{train,test}: instantiate to dataloaders + dataloader.evaluator: instantiate to evaluator for test set + optimizer: instantaite to an optimizer + lr_multiplier: instantiate to a fvcore scheduler + train: other misc config defined in `configs/common/train.py`, including: + output_dir (str) + init_checkpoint (str) + amp.enabled (bool) + max_iter (int) + eval_period, log_period (int) + device (str) + checkpointer (dict) + ddp (dict) + """ + model = instantiate(cfg.model) + logger = logging.getLogger("detectron2") + logger.info("Model:\n{}".format(model)) + model.to(cfg.train.device) + + cfg.optimizer.params.model = model + optim = instantiate(cfg.optimizer) + + train_loader = instantiate(cfg.dataloader.train) + + model = create_ddp_model(model, **cfg.train.ddp) + trainer = (AMPTrainer if cfg.train.amp.enabled else SimpleTrainer)(model, train_loader, optim) + checkpointer = DetectionCheckpointer( + model, + cfg.train.output_dir, + trainer=trainer, + ) + trainer.register_hooks( + [ + hooks.IterationTimer(), + hooks.LRScheduler(scheduler=instantiate(cfg.lr_multiplier)), + hooks.PeriodicCheckpointer(checkpointer, **cfg.train.checkpointer) + if comm.is_main_process() + else None, + hooks.EvalHook(cfg.train.eval_period, lambda: do_test(cfg, model)), + hooks.PeriodicWriter( + default_writers(cfg.train.output_dir, cfg.train.max_iter), + period=cfg.train.log_period, + ) + if comm.is_main_process() + else None, + ] + ) + + checkpointer.resume_or_load(cfg.train.init_checkpoint, resume=args.resume) + if args.resume and checkpointer.has_checkpoint(): + # The checkpoint stores the training iteration that just finished, thus we start + # at the next iteration + start_iter = trainer.iter + 1 + else: + start_iter = 0 + trainer.train(start_iter, cfg.train.max_iter) + + +def main(args): + cfg = LazyConfig.load(args.config_file) + cfg = LazyConfig.apply_overrides(cfg, args.opts) + default_setup(cfg, args) + + if args.eval_only: + model = instantiate(cfg.model) + model.to(cfg.train.device) + model = create_ddp_model(model) + DetectionCheckpointer(model).load(cfg.train.init_checkpoint) + print(do_test(cfg, model)) + else: + do_train(args, cfg) + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + launch( + main, + args.num_gpus, + num_machines=args.num_machines, + machine_rank=args.machine_rank, + dist_url=args.dist_url, + args=(args,), + ) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/tools/lightning_train_net.py b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/lightning_train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..f6734b566b6764ee54dd2af1b7310fedb34bb40d --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/lightning_train_net.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. +# Lightning Trainer should be considered beta at this point +# We have confirmed that training and validation run correctly and produce correct results +# Depending on how you launch the trainer, there are issues with processes terminating correctly +# This module is still dependent on D2 logging, but could be transferred to use Lightning logging + +import logging +import os +import time +import weakref +from collections import OrderedDict +from typing import Any, Dict, List + +import detectron2.utils.comm as comm +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import get_cfg +from detectron2.data import build_detection_test_loader, build_detection_train_loader +from detectron2.engine import ( + DefaultTrainer, + SimpleTrainer, + default_argument_parser, + default_setup, + default_writers, + hooks, +) +from detectron2.evaluation import print_csv_format +from detectron2.evaluation.testing import flatten_results_dict +from detectron2.modeling import build_model +from detectron2.solver import build_lr_scheduler, build_optimizer +from detectron2.utils.events import EventStorage +from detectron2.utils.logger import setup_logger + +import pytorch_lightning as pl # type: ignore +from pytorch_lightning import LightningDataModule, LightningModule +from train_net import build_evaluator + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("detectron2") + + +class TrainingModule(LightningModule): + def __init__(self, cfg): + super().__init__() + if not logger.isEnabledFor(logging.INFO): # setup_logger is not called for d2 + setup_logger() + self.cfg = DefaultTrainer.auto_scale_workers(cfg, comm.get_world_size()) + self.storage: EventStorage = None + self.model = build_model(self.cfg) + + self.start_iter = 0 + self.max_iter = cfg.SOLVER.MAX_ITER + + def on_save_checkpoint(self, checkpoint: Dict[str, Any]) -> None: + checkpoint["iteration"] = self.storage.iter + + def on_load_checkpoint(self, checkpointed_state: Dict[str, Any]) -> None: + self.start_iter = checkpointed_state["iteration"] + self.storage.iter = self.start_iter + + def setup(self, stage: str): + if self.cfg.MODEL.WEIGHTS: + self.checkpointer = DetectionCheckpointer( + # Assume you want to save checkpoints together with logs/statistics + self.model, + self.cfg.OUTPUT_DIR, + ) + logger.info(f"Load model weights from checkpoint: {self.cfg.MODEL.WEIGHTS}.") + # Only load weights, use lightning checkpointing if you want to resume + self.checkpointer.load(self.cfg.MODEL.WEIGHTS) + + self.iteration_timer = hooks.IterationTimer() + self.iteration_timer.before_train() + self.data_start = time.perf_counter() + self.writers = None + + def training_step(self, batch, batch_idx): + data_time = time.perf_counter() - self.data_start + # Need to manually enter/exit since trainer may launch processes + # This ideally belongs in setup, but setup seems to run before processes are spawned + if self.storage is None: + self.storage = EventStorage(0) + self.storage.__enter__() + self.iteration_timer.trainer = weakref.proxy(self) + self.iteration_timer.before_step() + self.writers = ( + default_writers(self.cfg.OUTPUT_DIR, self.max_iter) + if comm.is_main_process() + else {} + ) + + loss_dict = self.model(batch) + SimpleTrainer.write_metrics(loss_dict, data_time) + + opt = self.optimizers() + self.storage.put_scalar( + "lr", opt.param_groups[self._best_param_group_id]["lr"], smoothing_hint=False + ) + self.iteration_timer.after_step() + self.storage.step() + # A little odd to put before step here, but it's the best way to get a proper timing + self.iteration_timer.before_step() + + if self.storage.iter % 20 == 0: + for writer in self.writers: + writer.write() + return sum(loss_dict.values()) + + def training_step_end(self, training_step_outpus): + self.data_start = time.perf_counter() + return training_step_outpus + + def training_epoch_end(self, training_step_outputs): + self.iteration_timer.after_train() + if comm.is_main_process(): + self.checkpointer.save("model_final") + for writer in self.writers: + writer.write() + writer.close() + self.storage.__exit__(None, None, None) + + def _process_dataset_evaluation_results(self) -> OrderedDict: + results = OrderedDict() + for idx, dataset_name in enumerate(self.cfg.DATASETS.TEST): + results[dataset_name] = self._evaluators[idx].evaluate() + if comm.is_main_process(): + print_csv_format(results[dataset_name]) + + if len(results) == 1: + results = list(results.values())[0] + return results + + def _reset_dataset_evaluators(self): + self._evaluators = [] + for dataset_name in self.cfg.DATASETS.TEST: + evaluator = build_evaluator(self.cfg, dataset_name) + evaluator.reset() + self._evaluators.append(evaluator) + + def on_validation_epoch_start(self, _outputs): + self._reset_dataset_evaluators() + + def validation_epoch_end(self, _outputs): + results = self._process_dataset_evaluation_results(_outputs) + + flattened_results = flatten_results_dict(results) + for k, v in flattened_results.items(): + try: + v = float(v) + except Exception as e: + raise ValueError( + "[EvalHook] eval_function should return a nested dict of float. " + "Got '{}: {}' instead.".format(k, v) + ) from e + self.storage.put_scalars(**flattened_results, smoothing_hint=False) + + def validation_step(self, batch, batch_idx: int, dataloader_idx: int = 0) -> None: + if not isinstance(batch, List): + batch = [batch] + outputs = self.model(batch) + self._evaluators[dataloader_idx].process(batch, outputs) + + def configure_optimizers(self): + optimizer = build_optimizer(self.cfg, self.model) + self._best_param_group_id = hooks.LRScheduler.get_best_param_group_id(optimizer) + scheduler = build_lr_scheduler(self.cfg, optimizer) + return [optimizer], [{"scheduler": scheduler, "interval": "step"}] + + +class DataModule(LightningDataModule): + def __init__(self, cfg): + super().__init__() + self.cfg = DefaultTrainer.auto_scale_workers(cfg, comm.get_world_size()) + + def train_dataloader(self): + return build_detection_train_loader(self.cfg) + + def val_dataloader(self): + dataloaders = [] + for dataset_name in self.cfg.DATASETS.TEST: + dataloaders.append(build_detection_test_loader(self.cfg, dataset_name)) + return dataloaders + + +def main(args): + cfg = setup(args) + train(cfg, args) + + +def train(cfg, args): + trainer_params = { + # training loop is bounded by max steps, use a large max_epochs to make + # sure max_steps is met first + "max_epochs": 10 ** 8, + "max_steps": cfg.SOLVER.MAX_ITER, + "val_check_interval": cfg.TEST.EVAL_PERIOD if cfg.TEST.EVAL_PERIOD > 0 else 10 ** 8, + "num_nodes": args.num_machines, + "gpus": args.num_gpus, + "num_sanity_val_steps": 0, + } + if cfg.SOLVER.AMP.ENABLED: + trainer_params["precision"] = 16 + + last_checkpoint = os.path.join(cfg.OUTPUT_DIR, "last.ckpt") + if args.resume: + # resume training from checkpoint + trainer_params["resume_from_checkpoint"] = last_checkpoint + logger.info(f"Resuming training from checkpoint: {last_checkpoint}.") + + trainer = pl.Trainer(**trainer_params) + logger.info(f"start to train with {args.num_machines} nodes and {args.num_gpus} GPUs") + + module = TrainingModule(cfg) + data_module = DataModule(cfg) + if args.eval_only: + logger.info("Running inference") + trainer.validate(module, data_module) + else: + logger.info("Running training") + trainer.fit(module, data_module) + + +def setup(args): + """ + Create configs and perform basic setups. + """ + cfg = get_cfg() + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + cfg.freeze() + default_setup(cfg, args) + return cfg + + +if __name__ == "__main__": + parser = default_argument_parser() + args = parser.parse_args() + logger.info("Command Line Args:", args) + main(args) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/tools/plain_train_net.py b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/plain_train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..4851a8398e128bdce1986feccf0f1cca4a12f704 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/plain_train_net.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +""" +Detectron2 training script with a plain training loop. + +This script reads a given config file and runs the training or evaluation. +It is an entry point that is able to train standard models in detectron2. + +In order to let one script support training of many models, +this script contains logic that are specific to these built-in models and therefore +may not be suitable for your own project. +For example, your research project perhaps only needs a single "evaluator". + +Therefore, we recommend you to use detectron2 as a library and take +this file as an example of how to use the library. +You may want to write your own script with your datasets and other customizations. + +Compared to "train_net.py", this script supports fewer default features. +It also includes fewer abstraction, therefore is easier to add custom logic. +""" + +import logging +import os +from collections import OrderedDict +import torch +from torch.nn.parallel import DistributedDataParallel + +import detectron2.utils.comm as comm +from detectron2.checkpoint import DetectionCheckpointer, PeriodicCheckpointer +from detectron2.config import get_cfg +from detectron2.data import ( + MetadataCatalog, + build_detection_test_loader, + build_detection_train_loader, +) +from detectron2.engine import default_argument_parser, default_setup, default_writers, launch +from detectron2.evaluation import ( + CityscapesInstanceEvaluator, + CityscapesSemSegEvaluator, + COCOEvaluator, + COCOPanopticEvaluator, + DatasetEvaluators, + LVISEvaluator, + PascalVOCDetectionEvaluator, + SemSegEvaluator, + inference_on_dataset, + print_csv_format, +) +from detectron2.modeling import build_model +from detectron2.solver import build_lr_scheduler, build_optimizer +from detectron2.utils.events import EventStorage + +logger = logging.getLogger("detectron2") + + +def get_evaluator(cfg, dataset_name, output_folder=None): + """ + Create evaluator(s) for a given dataset. + This uses the special metadata "evaluator_type" associated with each builtin dataset. + For your own dataset, you can simply create an evaluator manually in your + script and do not have to worry about the hacky if-else logic here. + """ + if output_folder is None: + output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") + evaluator_list = [] + evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type + if evaluator_type in ["sem_seg", "coco_panoptic_seg"]: + evaluator_list.append( + SemSegEvaluator( + dataset_name, + distributed=True, + output_dir=output_folder, + ) + ) + if evaluator_type in ["coco", "coco_panoptic_seg"]: + evaluator_list.append(COCOEvaluator(dataset_name, output_dir=output_folder)) + if evaluator_type == "coco_panoptic_seg": + evaluator_list.append(COCOPanopticEvaluator(dataset_name, output_folder)) + if evaluator_type == "cityscapes_instance": + assert ( + torch.cuda.device_count() > comm.get_rank() + ), "CityscapesEvaluator currently do not work with multiple machines." + return CityscapesInstanceEvaluator(dataset_name) + if evaluator_type == "cityscapes_sem_seg": + assert ( + torch.cuda.device_count() > comm.get_rank() + ), "CityscapesEvaluator currently do not work with multiple machines." + return CityscapesSemSegEvaluator(dataset_name) + if evaluator_type == "pascal_voc": + return PascalVOCDetectionEvaluator(dataset_name) + if evaluator_type == "lvis": + return LVISEvaluator(dataset_name, cfg, True, output_folder) + if len(evaluator_list) == 0: + raise NotImplementedError( + "no Evaluator for the dataset {} with the type {}".format(dataset_name, evaluator_type) + ) + if len(evaluator_list) == 1: + return evaluator_list[0] + return DatasetEvaluators(evaluator_list) + + +def do_test(cfg, model): + results = OrderedDict() + for dataset_name in cfg.DATASETS.TEST: + data_loader = build_detection_test_loader(cfg, dataset_name) + evaluator = get_evaluator( + cfg, dataset_name, os.path.join(cfg.OUTPUT_DIR, "inference", dataset_name) + ) + results_i = inference_on_dataset(model, data_loader, evaluator) + results[dataset_name] = results_i + if comm.is_main_process(): + logger.info("Evaluation results for {} in csv format:".format(dataset_name)) + print_csv_format(results_i) + if len(results) == 1: + results = list(results.values())[0] + return results + + +def do_train(cfg, model, resume=False): + model.train() + optimizer = build_optimizer(cfg, model) + scheduler = build_lr_scheduler(cfg, optimizer) + + checkpointer = DetectionCheckpointer( + model, cfg.OUTPUT_DIR, optimizer=optimizer, scheduler=scheduler + ) + start_iter = ( + checkpointer.resume_or_load(cfg.MODEL.WEIGHTS, resume=resume).get("iteration", -1) + 1 + ) + max_iter = cfg.SOLVER.MAX_ITER + + periodic_checkpointer = PeriodicCheckpointer( + checkpointer, cfg.SOLVER.CHECKPOINT_PERIOD, max_iter=max_iter + ) + + writers = default_writers(cfg.OUTPUT_DIR, max_iter) if comm.is_main_process() else [] + + # compared to "train_net.py", we do not support accurate timing and + # precise BN here, because they are not trivial to implement in a small training loop + data_loader = build_detection_train_loader(cfg) + logger.info("Starting training from iteration {}".format(start_iter)) + with EventStorage(start_iter) as storage: + for data, iteration in zip(data_loader, range(start_iter, max_iter)): + storage.iter = iteration + + loss_dict = model(data) + losses = sum(loss_dict.values()) + assert torch.isfinite(losses).all(), loss_dict + + loss_dict_reduced = {k: v.item() for k, v in comm.reduce_dict(loss_dict).items()} + losses_reduced = sum(loss for loss in loss_dict_reduced.values()) + if comm.is_main_process(): + storage.put_scalars(total_loss=losses_reduced, **loss_dict_reduced) + + optimizer.zero_grad() + losses.backward() + optimizer.step() + storage.put_scalar("lr", optimizer.param_groups[0]["lr"], smoothing_hint=False) + scheduler.step() + + if ( + cfg.TEST.EVAL_PERIOD > 0 + and (iteration + 1) % cfg.TEST.EVAL_PERIOD == 0 + and iteration != max_iter - 1 + ): + do_test(cfg, model) + # Compared to "train_net.py", the test results are not dumped to EventStorage + comm.synchronize() + + if iteration - start_iter > 5 and ( + (iteration + 1) % 20 == 0 or iteration == max_iter - 1 + ): + for writer in writers: + writer.write() + periodic_checkpointer.step(iteration) + + +def setup(args): + """ + Create configs and perform basic setups. + """ + cfg = get_cfg() + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + cfg.freeze() + default_setup( + cfg, args + ) # if you don't like any of the default setup, write your own setup code + return cfg + + +def main(args): + cfg = setup(args) + + model = build_model(cfg) + logger.info("Model:\n{}".format(model)) + if args.eval_only: + DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( + cfg.MODEL.WEIGHTS, resume=args.resume + ) + return do_test(cfg, model) + + distributed = comm.get_world_size() > 1 + if distributed: + model = DistributedDataParallel( + model, device_ids=[comm.get_local_rank()], broadcast_buffers=False + ) + + do_train(cfg, model, resume=args.resume) + return do_test(cfg, model) + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + print("Command Line Args:", args) + launch( + main, + args.num_gpus, + num_machines=args.num_machines, + machine_rank=args.machine_rank, + dist_url=args.dist_url, + args=(args,), + ) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/tools/train_net.py b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..6ebf5f60a2197633fa418e83aadfedb824537b8e --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/train_net.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +""" +A main training script. + +This scripts reads a given config file and runs the training or evaluation. +It is an entry point that is made to train standard models in detectron2. + +In order to let one script support training of many models, +this script contains logic that are specific to these built-in models and therefore +may not be suitable for your own project. +For example, your research project perhaps only needs a single "evaluator". + +Therefore, we recommend you to use detectron2 as an library and take +this file as an example of how to use the library. +You may want to write your own script with your datasets and other customizations. +""" + +import logging +import os +from collections import OrderedDict +import torch + +import detectron2.utils.comm as comm +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import get_cfg +from detectron2.data import MetadataCatalog +from detectron2.engine import DefaultTrainer, default_argument_parser, default_setup, hooks, launch +from detectron2.evaluation import ( + CityscapesInstanceEvaluator, + CityscapesSemSegEvaluator, + COCOEvaluator, + COCOPanopticEvaluator, + DatasetEvaluators, + LVISEvaluator, + PascalVOCDetectionEvaluator, + SemSegEvaluator, + verify_results, +) +from detectron2.modeling import GeneralizedRCNNWithTTA + + +def build_evaluator(cfg, dataset_name, output_folder=None): + """ + Create evaluator(s) for a given dataset. + This uses the special metadata "evaluator_type" associated with each builtin dataset. + For your own dataset, you can simply create an evaluator manually in your + script and do not have to worry about the hacky if-else logic here. + """ + if output_folder is None: + output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") + evaluator_list = [] + evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type + if evaluator_type in ["sem_seg", "coco_panoptic_seg"]: + evaluator_list.append( + SemSegEvaluator( + dataset_name, + distributed=True, + output_dir=output_folder, + ) + ) + if evaluator_type in ["coco", "coco_panoptic_seg"]: + evaluator_list.append(COCOEvaluator(dataset_name, output_dir=output_folder)) + if evaluator_type == "coco_panoptic_seg": + evaluator_list.append(COCOPanopticEvaluator(dataset_name, output_folder)) + if evaluator_type == "cityscapes_instance": + assert ( + torch.cuda.device_count() > comm.get_rank() + ), "CityscapesEvaluator currently do not work with multiple machines." + return CityscapesInstanceEvaluator(dataset_name) + if evaluator_type == "cityscapes_sem_seg": + assert ( + torch.cuda.device_count() > comm.get_rank() + ), "CityscapesEvaluator currently do not work with multiple machines." + return CityscapesSemSegEvaluator(dataset_name) + elif evaluator_type == "pascal_voc": + return PascalVOCDetectionEvaluator(dataset_name) + elif evaluator_type == "lvis": + return LVISEvaluator(dataset_name, output_dir=output_folder) + if len(evaluator_list) == 0: + raise NotImplementedError( + "no Evaluator for the dataset {} with the type {}".format(dataset_name, evaluator_type) + ) + elif len(evaluator_list) == 1: + return evaluator_list[0] + return DatasetEvaluators(evaluator_list) + + +class Trainer(DefaultTrainer): + """ + We use the "DefaultTrainer" which contains pre-defined default logic for + standard training workflow. They may not work for you, especially if you + are working on a new research project. In that case you can write your + own training loop. You can use "tools/plain_train_net.py" as an example. + """ + + @classmethod + def build_evaluator(cls, cfg, dataset_name, output_folder=None): + return build_evaluator(cfg, dataset_name, output_folder) + + @classmethod + def test_with_TTA(cls, cfg, model): + logger = logging.getLogger("detectron2.trainer") + # In the end of training, run an evaluation with TTA + # Only support some R-CNN models. + logger.info("Running inference with test-time augmentation ...") + model = GeneralizedRCNNWithTTA(cfg, model) + evaluators = [ + cls.build_evaluator( + cfg, name, output_folder=os.path.join(cfg.OUTPUT_DIR, "inference_TTA") + ) + for name in cfg.DATASETS.TEST + ] + res = cls.test(cfg, model, evaluators) + res = OrderedDict({k + "_TTA": v for k, v in res.items()}) + return res + + +def setup(args): + """ + Create configs and perform basic setups. + """ + cfg = get_cfg() + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + cfg.freeze() + default_setup(cfg, args) + return cfg + + +def main(args): + cfg = setup(args) + + if args.eval_only: + model = Trainer.build_model(cfg) + DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( + cfg.MODEL.WEIGHTS, resume=args.resume + ) + res = Trainer.test(cfg, model) + if cfg.TEST.AUG.ENABLED: + res.update(Trainer.test_with_TTA(cfg, model)) + if comm.is_main_process(): + verify_results(cfg, res) + return res + + """ + If you'd like to do anything fancier than the standard training logic, + consider writing your own training loop (see plain_train_net.py) or + subclassing the trainer. + """ + trainer = Trainer(cfg) + trainer.resume_or_load(resume=args.resume) + if cfg.TEST.AUG.ENABLED: + trainer.register_hooks( + [hooks.EvalHook(0, lambda: trainer.test_with_TTA(cfg, trainer.model))] + ) + return trainer.train() + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + print("Command Line Args:", args) + launch( + main, + args.num_gpus, + num_machines=args.num_machines, + machine_rank=args.machine_rank, + dist_url=args.dist_url, + args=(args,), + ) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/tools/visualize_data.py b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/visualize_data.py new file mode 100644 index 0000000000000000000000000000000000000000..fd0ba8347bfd34fc8fac5ffef9aee10915ad1820 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/visualize_data.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. +import argparse +import os +from itertools import chain +import cv2 +import tqdm + +from detectron2.config import get_cfg +from detectron2.data import DatasetCatalog, MetadataCatalog, build_detection_train_loader +from detectron2.data import detection_utils as utils +from detectron2.data.build import filter_images_with_few_keypoints +from detectron2.utils.logger import setup_logger +from detectron2.utils.visualizer import Visualizer + + +def setup(args): + cfg = get_cfg() + if args.config_file: + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + cfg.DATALOADER.NUM_WORKERS = 0 + cfg.freeze() + return cfg + + +def parse_args(in_args=None): + parser = argparse.ArgumentParser(description="Visualize ground-truth data") + parser.add_argument( + "--source", + choices=["annotation", "dataloader"], + required=True, + help="visualize the annotations or the data loader (with pre-processing)", + ) + parser.add_argument("--config-file", metavar="FILE", help="path to config file") + parser.add_argument("--output-dir", default="./", help="path to output directory") + parser.add_argument("--show", action="store_true", help="show output in a window") + parser.add_argument( + "opts", + help="Modify config options using the command-line", + default=None, + nargs=argparse.REMAINDER, + ) + return parser.parse_args(in_args) + + +if __name__ == "__main__": + args = parse_args() + logger = setup_logger() + logger.info("Arguments: " + str(args)) + cfg = setup(args) + + dirname = args.output_dir + os.makedirs(dirname, exist_ok=True) + metadata = MetadataCatalog.get(cfg.DATASETS.TRAIN[0]) + + def output(vis, fname): + if args.show: + print(fname) + cv2.imshow("window", vis.get_image()[:, :, ::-1]) + cv2.waitKey() + else: + filepath = os.path.join(dirname, fname) + print("Saving to {} ...".format(filepath)) + vis.save(filepath) + + scale = 1.0 + if args.source == "dataloader": + train_data_loader = build_detection_train_loader(cfg) + for batch in train_data_loader: + for per_image in batch: + # Pytorch tensor is in (C, H, W) format + img = per_image["image"].permute(1, 2, 0).cpu().detach().numpy() + img = utils.convert_image_to_rgb(img, cfg.INPUT.FORMAT) + + visualizer = Visualizer(img, metadata=metadata, scale=scale) + target_fields = per_image["instances"].get_fields() + labels = [metadata.thing_classes[i] for i in target_fields["gt_classes"]] + vis = visualizer.overlay_instances( + labels=labels, + boxes=target_fields.get("gt_boxes", None), + masks=target_fields.get("gt_masks", None), + keypoints=target_fields.get("gt_keypoints", None), + ) + output(vis, str(per_image["image_id"]) + ".jpg") + else: + dicts = list(chain.from_iterable([DatasetCatalog.get(k) for k in cfg.DATASETS.TRAIN])) + if cfg.MODEL.KEYPOINT_ON: + dicts = filter_images_with_few_keypoints(dicts, 1) + for dic in tqdm.tqdm(dicts): + img = utils.read_image(dic["file_name"], "RGB") + visualizer = Visualizer(img, metadata=metadata, scale=scale) + vis = visualizer.draw_dataset_dict(dic) + output(vis, os.path.basename(dic["file_name"])) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/tools/visualize_json_results.py b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/visualize_json_results.py new file mode 100644 index 0000000000000000000000000000000000000000..472190e0b3b38b55773795915badbb5bc4599d42 --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/tools/visualize_json_results.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# Copyright (c) Facebook, Inc. and its affiliates. + +import argparse +import json +import numpy as np +import os +from collections import defaultdict +import cv2 +import tqdm + +from detectron2.data import DatasetCatalog, MetadataCatalog +from detectron2.structures import Boxes, BoxMode, Instances +from detectron2.utils.file_io import PathManager +from detectron2.utils.logger import setup_logger +from detectron2.utils.visualizer import Visualizer + + +def create_instances(predictions, image_size): + ret = Instances(image_size) + + score = np.asarray([x["score"] for x in predictions]) + chosen = (score > args.conf_threshold).nonzero()[0] + score = score[chosen] + bbox = np.asarray([predictions[i]["bbox"] for i in chosen]).reshape(-1, 4) + bbox = BoxMode.convert(bbox, BoxMode.XYWH_ABS, BoxMode.XYXY_ABS) + + labels = np.asarray([dataset_id_map(predictions[i]["category_id"]) for i in chosen]) + + ret.scores = score + ret.pred_boxes = Boxes(bbox) + ret.pred_classes = labels + + try: + ret.pred_masks = [predictions[i]["segmentation"] for i in chosen] + except KeyError: + pass + return ret + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="A script that visualizes the json predictions from COCO or LVIS dataset." + ) + parser.add_argument("--input", required=True, help="JSON file produced by the model") + parser.add_argument("--output", required=True, help="output directory") + parser.add_argument("--dataset", help="name of the dataset", default="coco_2017_val") + parser.add_argument("--conf-threshold", default=0.5, type=float, help="confidence threshold") + args = parser.parse_args() + + logger = setup_logger() + + with PathManager.open(args.input, "r") as f: + predictions = json.load(f) + + pred_by_image = defaultdict(list) + for p in predictions: + pred_by_image[p["image_id"]].append(p) + + dicts = list(DatasetCatalog.get(args.dataset)) + metadata = MetadataCatalog.get(args.dataset) + if hasattr(metadata, "thing_dataset_id_to_contiguous_id"): + + def dataset_id_map(ds_id): + return metadata.thing_dataset_id_to_contiguous_id[ds_id] + + elif "lvis" in args.dataset: + # LVIS results are in the same format as COCO results, but have a different + # mapping from dataset category id to contiguous category id in [0, #categories - 1] + def dataset_id_map(ds_id): + return ds_id - 1 + + else: + raise ValueError("Unsupported dataset: {}".format(args.dataset)) + + os.makedirs(args.output, exist_ok=True) + + for dic in tqdm.tqdm(dicts): + img = cv2.imread(dic["file_name"], cv2.IMREAD_COLOR)[:, :, ::-1] + basename = os.path.basename(dic["file_name"]) + + predictions = create_instances(pred_by_image[dic["image_id"]], img.shape[:2]) + vis = Visualizer(img, metadata) + vis_pred = vis.draw_instance_predictions(predictions).get_image() + + vis = Visualizer(img, metadata) + vis_gt = vis.draw_dataset_dict(dic).get_image() + + concat = np.concatenate((vis_pred, vis_gt), axis=1) + cv2.imwrite(os.path.join(args.output, basename), concat[:, :, ::-1]) diff --git a/approach/ovod/mm-ovod/third_party/CenterNet2/train_net.py b/approach/ovod/mm-ovod/third_party/CenterNet2/train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..d903efde074e97e1720f970ea94717ebf105d9af --- /dev/null +++ b/approach/ovod/mm-ovod/third_party/CenterNet2/train_net.py @@ -0,0 +1,228 @@ +import logging +import os +from collections import OrderedDict +import torch +from torch.nn.parallel import DistributedDataParallel +import time +import datetime +import json + +from fvcore.common.timer import Timer +import detectron2.utils.comm as comm +from detectron2.checkpoint import DetectionCheckpointer, PeriodicCheckpointer +from detectron2.config import get_cfg +from detectron2.data import ( + MetadataCatalog, + build_detection_test_loader, +) +from detectron2.engine import default_argument_parser, default_setup, launch + +from detectron2.evaluation import ( + COCOEvaluator, + LVISEvaluator, + inference_on_dataset, + print_csv_format, +) +from detectron2.modeling import build_model +from detectron2.solver import build_lr_scheduler, build_optimizer +from detectron2.utils.events import ( + CommonMetricPrinter, + EventStorage, + JSONWriter, + TensorboardXWriter, +) +from detectron2.modeling.test_time_augmentation import GeneralizedRCNNWithTTA +from detectron2.data.dataset_mapper import DatasetMapper +from detectron2.data.build import build_detection_train_loader + +from centernet.config import add_centernet_config +from centernet.data.custom_build_augmentation import build_custom_augmentation + +logger = logging.getLogger("detectron2") + +def do_test(cfg, model): + results = OrderedDict() + for dataset_name in cfg.DATASETS.TEST: + mapper = None if cfg.INPUT.TEST_INPUT_TYPE == 'default' else \ + DatasetMapper( + cfg, False, augmentations=build_custom_augmentation(cfg, False)) + data_loader = build_detection_test_loader(cfg, dataset_name, mapper=mapper) + output_folder = os.path.join( + cfg.OUTPUT_DIR, "inference_{}".format(dataset_name)) + evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type + + if evaluator_type == "lvis": + evaluator = LVISEvaluator(dataset_name, cfg, True, output_folder) + elif evaluator_type == 'coco': + evaluator = COCOEvaluator(dataset_name, cfg, True, output_folder) + else: + assert 0, evaluator_type + + results[dataset_name] = inference_on_dataset( + model, data_loader, evaluator) + if comm.is_main_process(): + logger.info("Evaluation results for {} in csv format:".format( + dataset_name)) + print_csv_format(results[dataset_name]) + if len(results) == 1: + results = list(results.values())[0] + return results + +def do_train(cfg, model, resume=False): + model.train() + optimizer = build_optimizer(cfg, model) + scheduler = build_lr_scheduler(cfg, optimizer) + + checkpointer = DetectionCheckpointer( + model, cfg.OUTPUT_DIR, optimizer=optimizer, scheduler=scheduler + ) + + start_iter = ( + checkpointer.resume_or_load( + cfg.MODEL.WEIGHTS, resume=resume, + ).get("iteration", -1) + 1 + ) + if cfg.SOLVER.RESET_ITER: + logger.info('Reset loaded iteration. Start training from iteration 0.') + start_iter = 0 + max_iter = cfg.SOLVER.MAX_ITER if cfg.SOLVER.TRAIN_ITER < 0 else cfg.SOLVER.TRAIN_ITER + + periodic_checkpointer = PeriodicCheckpointer( + checkpointer, cfg.SOLVER.CHECKPOINT_PERIOD, max_iter=max_iter + ) + + writers = ( + [ + CommonMetricPrinter(max_iter), + JSONWriter(os.path.join(cfg.OUTPUT_DIR, "metrics.json")), + TensorboardXWriter(cfg.OUTPUT_DIR), + ] + if comm.is_main_process() + else [] + ) + + + mapper = DatasetMapper(cfg, True) if cfg.INPUT.CUSTOM_AUG == '' else \ + DatasetMapper(cfg, True, augmentations=build_custom_augmentation(cfg, True)) + if cfg.DATALOADER.SAMPLER_TRAIN in ['TrainingSampler', 'RepeatFactorTrainingSampler']: + data_loader = build_detection_train_loader(cfg, mapper=mapper) + else: + from centernet.data.custom_dataset_dataloader import build_custom_train_loader + data_loader = build_custom_train_loader(cfg, mapper=mapper) + + + logger.info("Starting training from iteration {}".format(start_iter)) + with EventStorage(start_iter) as storage: + step_timer = Timer() + data_timer = Timer() + start_time = time.perf_counter() + for data, iteration in zip(data_loader, range(start_iter, max_iter)): + data_time = data_timer.seconds() + storage.put_scalars(data_time=data_time) + step_timer.reset() + iteration = iteration + 1 + storage.step() + loss_dict = model(data) + + losses = sum( + loss for k, loss in loss_dict.items()) + assert torch.isfinite(losses).all(), loss_dict + + loss_dict_reduced = {k: v.item() \ + for k, v in comm.reduce_dict(loss_dict).items()} + losses_reduced = sum(loss for loss in loss_dict_reduced.values()) + if comm.is_main_process(): + storage.put_scalars( + total_loss=losses_reduced, **loss_dict_reduced) + + optimizer.zero_grad() + losses.backward() + optimizer.step() + + storage.put_scalar( + "lr", optimizer.param_groups[0]["lr"], smoothing_hint=False) + + step_time = step_timer.seconds() + storage.put_scalars(time=step_time) + data_timer.reset() + scheduler.step() + + if ( + cfg.TEST.EVAL_PERIOD > 0 + and iteration % cfg.TEST.EVAL_PERIOD == 0 + and iteration != max_iter + ): + do_test(cfg, model) + comm.synchronize() + + if iteration - start_iter > 5 and \ + (iteration % 20 == 0 or iteration == max_iter): + for writer in writers: + writer.write() + periodic_checkpointer.step(iteration) + + total_time = time.perf_counter() - start_time + logger.info( + "Total training time: {}".format( + str(datetime.timedelta(seconds=int(total_time))))) + +def setup(args): + """ + Create configs and perform basic setups. + """ + cfg = get_cfg() + add_centernet_config(cfg) + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + if '/auto' in cfg.OUTPUT_DIR: + file_name = os.path.basename(args.config_file)[:-5] + cfg.OUTPUT_DIR = cfg.OUTPUT_DIR.replace('/auto', '/{}'.format(file_name)) + logger.info('OUTPUT_DIR: {}'.format(cfg.OUTPUT_DIR)) + cfg.freeze() + default_setup(cfg, args) + return cfg + + +def main(args): + cfg = setup(args) + + model = build_model(cfg) + logger.info("Model:\n{}".format(model)) + if args.eval_only: + DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load( + cfg.MODEL.WEIGHTS, resume=args.resume + ) + if cfg.TEST.AUG.ENABLED: + logger.info("Running inference with test-time augmentation ...") + model = GeneralizedRCNNWithTTA(cfg, model, batch_size=1) + + return do_test(cfg, model) + + distributed = comm.get_world_size() > 1 + if distributed: + model = DistributedDataParallel( + model, device_ids=[comm.get_local_rank()], broadcast_buffers=False, + find_unused_parameters=True + ) + + do_train(cfg, model, resume=args.resume) + return do_test(cfg, model) + + +if __name__ == "__main__": + args = default_argument_parser() + args.add_argument('--manual_device', default='') + args = args.parse_args() + if args.manual_device != '': + os.environ['CUDA_VISIBLE_DEVICES'] = args.manual_device + args.dist_url = 'tcp://127.0.0.1:{}'.format( + torch.randint(11111, 60000, (1,))[0].item()) + print("Command Line Args:", args) + launch( + main, + args.num_gpus, + num_machines=args.num_machines, + machine_rank=args.machine_rank, + dist_url=args.dist_url, + args=(args,), + )