diff --git a/approach/ovod/APE/ape/__init__.py b/approach/ovod/APE/ape/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6f40cb49c8625c665f98079a3b35b47778cc1032 --- /dev/null +++ b/approach/ovod/APE/ape/__init__.py @@ -0,0 +1,5 @@ +from .data import * + +# This line will be programatically read/write by setup.py. +# Leave them at the bottom of this file and don't touch them. +__version__ = "0.0" diff --git a/approach/ovod/APE/ape/checkpoint/__init__.py b/approach/ovod/APE/ape/checkpoint/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d69773dd082a8b7d8645c7d07822c2560688f3a1 --- /dev/null +++ b/approach/ovod/APE/ape/checkpoint/__init__.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- + + +from .detection_checkpoint import DetectionCheckpointer + +__all__ = ["DetectionCheckpointer"] diff --git a/approach/ovod/APE/ape/checkpoint/detection_checkpoint.py b/approach/ovod/APE/ape/checkpoint/detection_checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..a09ebe820e4b15f5a9981aefc2968da9cd404e04 --- /dev/null +++ b/approach/ovod/APE/ape/checkpoint/detection_checkpoint.py @@ -0,0 +1,45 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import os +import pickle +from collections import defaultdict +from typing import IO, Any, Dict, Iterable, List, NamedTuple, Optional, Tuple, cast + +import numpy as np +import torch + +from detectron2.checkpoint import DetectionCheckpointer as DetectionCheckpointer_d2 + + +class DetectionCheckpointer(DetectionCheckpointer_d2): + + # def __init__(self, skip_key="", **kwargs): + # super().__init__(**kwargs) + # self.skip_key = skip_key + + def _convert_ndarray_to_tensor(self, state_dict: Dict[str, Any]) -> None: + """ + In-place convert all numpy arrays in the state_dict to torch tensor. + Args: + state_dict (dict): a state-dict to be loaded to the model. + Will be modified. + """ + logger = logging.getLogger(__name__) + # model could be an OrderedDict with _metadata attribute + # (as returned by Pytorch's state_dict()). We should preserve these + # properties. + for k in list(state_dict.keys()): + + # if self.skip_key in k: + # if "model_language" in k: + # state_dict.pop(k) + # continue + + v = state_dict[k] + if not isinstance(v, np.ndarray) and not isinstance(v, torch.Tensor): + logger.warning("Unsupported type found in checkpoint! {}: {}".format(k, type(v))) + state_dict.pop(k) + continue + raise ValueError("Unsupported type found in checkpoint! {}: {}".format(k, type(v))) + if not isinstance(v, torch.Tensor): + state_dict[k] = torch.from_numpy(v) diff --git a/approach/ovod/APE/ape/data/__init__.py b/approach/ovod/APE/ape/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7ca826bdd8893b5f3fd8420f3bed8d98a903bff9 --- /dev/null +++ b/approach/ovod/APE/ape/data/__init__.py @@ -0,0 +1,20 @@ +from . import datasets +from .build_copypaste import ( + build_detection_train_loader_copypaste, + get_detection_dataset_dicts_copypaste, +) +from .build_multi_dataset import ( + build_detection_train_loader_multi_dataset, + get_detection_dataset_dicts_multi_dataset, +) +from .build_multi_dataset_copypaste import ( + build_detection_train_loader_multi_dataset_copypaste, + get_detection_dataset_dicts_multi_dataset_copypaste, +) +from .dataset_mapper import DatasetMapper_ape +from .dataset_mapper_copypaste import DatasetMapper_copypaste +from .dataset_mapper_detr_instance import DatasetMapper_detr_instance +from .dataset_mapper_detr_instance_exp import DatasetMapper_detr_instance_exp +from .dataset_mapper_detr_panoptic import DatasetMapper_detr_panoptic +from .dataset_mapper_detr_panoptic_copypaste import DatasetMapper_detr_panoptic_copypaste +from .dataset_mapper_detr_semantic import DatasetMapper_detr_semantic diff --git a/approach/ovod/APE/ape/data/build_copypaste.py b/approach/ovod/APE/ape/data/build_copypaste.py new file mode 100644 index 0000000000000000000000000000000000000000..46bf38fce94bdbd50e22c522dfc28fdbb57d4005 --- /dev/null +++ b/approach/ovod/APE/ape/data/build_copypaste.py @@ -0,0 +1,255 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import itertools +import logging + +import torch.utils.data as torchdata + +from detectron2.config import configurable +from detectron2.data.build import ( + build_batch_data_loader, + filter_images_with_few_keypoints, + filter_images_with_only_crowd_annotations, + get_detection_dataset_dicts, + load_proposals_into_dataset, + print_instances_class_histogram, +) +from detectron2.data.catalog import DatasetCatalog, MetadataCatalog +from detectron2.data.common import DatasetFromList +from detectron2.data.detection_utils import check_metadata_consistency +from detectron2.data.samplers import ( + RandomSubsetTrainingSampler, + RepeatFactorTrainingSampler, + TrainingSampler, +) +from detectron2.utils.logger import _log_api_usage + +from .common_copypaste import MapDataset_coppaste +from .dataset_mapper_copypaste import DatasetMapper_copypaste + +""" +This file contains the default logic to build a dataloader for training or testing. +""" + +__all__ = [ + "build_detection_train_loader_copypaste", +] + + +def get_detection_dataset_dicts_copypaste( + names, + filter_empty=True, + min_keypoints=0, + proposal_files=None, + check_consistency=True, + copypastes=[True], +): + """ + Load and prepare dataset dicts for instance detection/segmentation and semantic segmentation. + + Args: + names (str or list[str]): a dataset name or a list of dataset names + filter_empty (bool): whether to filter out images without instance annotations + min_keypoints (int): filter out images with fewer keypoints than + `min_keypoints`. Set to 0 to do nothing. + proposal_files (list[str]): if given, a list of object proposal files + that match each dataset in `names`. + check_consistency (bool): whether to check if datasets have consistent metadata. + + Returns: + list[dict]: a list of dicts following the standard dataset dict format. + """ + if isinstance(names, str): + names = [names] + assert len(names), names + dataset_dicts = [DatasetCatalog.get(dataset_name) for dataset_name in names] + for dataset_name, dicts in zip(names, dataset_dicts): + assert len(dicts), "Dataset '{}' is empty!".format(dataset_name) + + for copypaste, dicts in zip(copypastes, dataset_dicts): + for d in dicts: + d["copypaste"] = copypaste + + if proposal_files is not None: + assert len(names) == len(proposal_files) + # load precomputed proposals from proposal files + dataset_dicts = [ + load_proposals_into_dataset(dataset_i_dicts, proposal_file) + for dataset_i_dicts, proposal_file in zip(dataset_dicts, proposal_files) + ] + + if isinstance(dataset_dicts[0], torchdata.Dataset): + return torchdata.ConcatDataset(dataset_dicts) + + dataset_dicts = list(itertools.chain.from_iterable(dataset_dicts)) + + has_instances = "annotations" in dataset_dicts[0] + if filter_empty and has_instances: + dataset_dicts = filter_images_with_only_crowd_annotations(dataset_dicts) + if min_keypoints > 0 and has_instances: + dataset_dicts = filter_images_with_few_keypoints(dataset_dicts, min_keypoints) + + if check_consistency and has_instances: + try: + class_names = MetadataCatalog.get(names[0]).thing_classes + check_metadata_consistency("thing_classes", names) + print_instances_class_histogram(dataset_dicts, class_names) + except AttributeError: # class names are not available for this dataset + pass + + assert len(dataset_dicts), "No valid data found in {}.".format(",".join(names)) + return dataset_dicts + + +def _train_loader_from_config(cfg, mapper=None, *, dataset=None, sampler=None): + assert len(cfg.DATASETS.TRAIN) == len(cfg.DATASETS.COPYPASTE.COPYPASTE) + + if dataset is None: + dataset = get_detection_dataset_dicts_copypaste( + cfg.DATASETS.TRAIN, + filter_empty=cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS, + min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE + if cfg.MODEL.KEYPOINT_ON + else 0, + proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN if cfg.MODEL.LOAD_PROPOSALS else None, + copypastes=cfg.DATASETS.COPYPASTE.COPYPASTE, + ) + _log_api_usage("dataset." + cfg.DATASETS.TRAIN[0]) + + if True: + dataset_bg = get_detection_dataset_dicts( + cfg.DATASETS.COPYPASTE.BG, + filter_empty=cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS, + min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE + if cfg.MODEL.KEYPOINT_ON + else 0, + proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN if cfg.MODEL.LOAD_PROPOSALS else None, + ) + _log_api_usage("dataset." + cfg.DATASETS.TRAIN[0]) + + if mapper is None: + mapper = DatasetMapper_copypaste(cfg, True) + + if sampler is None: + sampler_name = cfg.DATALOADER.SAMPLER_TRAIN + logger = logging.getLogger(__name__) + logger.info("Using training sampler {}".format(sampler_name)) + if sampler_name == "TrainingSampler": + sampler = TrainingSampler(len(dataset)) + elif sampler_name == "RepeatFactorTrainingSampler": + repeat_factors = RepeatFactorTrainingSampler.repeat_factors_from_category_frequency( + dataset, cfg.DATALOADER.REPEAT_THRESHOLD + ) + sampler = RepeatFactorTrainingSampler(repeat_factors) + elif sampler_name == "RandomSubsetTrainingSampler": + sampler = RandomSubsetTrainingSampler(len(dataset), cfg.DATALOADER.RANDOM_SUBSET_RATIO) + else: + raise ValueError("Unknown training sampler: {}".format(sampler_name)) + + if True: + sampler_name = cfg.DATALOADER.COPYPASTE.SAMPLER_TRAIN + logger = logging.getLogger(__name__) + logger.info("Using training sampler {}".format(sampler_name)) + if sampler_name == "TrainingSampler": + sampler_bg = TrainingSampler(len(dataset_bg)) + elif sampler_name == "RepeatFactorTrainingSampler": + repeat_factors = RepeatFactorTrainingSampler.repeat_factors_from_category_frequency( + dataset_bg, cfg.DATALOADER.COPYPASTE.REPEAT_THRESHOLD + ) + sampler_bg = RepeatFactorTrainingSampler(repeat_factors) + elif sampler_name == "RandomSubsetTrainingSampler": + sampler_bg = RandomSubsetTrainingSampler( + len(dataset_bg), cfg.DATALOADER.COPYPASTE.RANDOM_SUBSET_RATIO + ) + else: + raise ValueError("Unknown training sampler: {}".format(sampler_name)) + + return { + "dataset": dataset, + "dataset_bg": dataset_bg, + "sampler": sampler, + "sampler_bg": sampler_bg, + "mapper": mapper, + "total_batch_size": cfg.SOLVER.IMS_PER_BATCH, + "aspect_ratio_grouping": cfg.DATALOADER.ASPECT_RATIO_GROUPING, + "num_workers": cfg.DATALOADER.NUM_WORKERS, + } + + +@configurable(from_config=_train_loader_from_config) +def build_detection_train_loader_copypaste( + dataset, + dataset_bg, + *, + mapper, + sampler=None, + sampler_bg=None, + total_batch_size, + aspect_ratio_grouping=True, + num_workers=0, + collate_fn=None, +): + """ + Build a dataloader for object detection with some default features. + This interface is experimental. + + Args: + dataset (list or torch.utils.data.Dataset): a list of dataset dicts, + or a pytorch dataset (either map-style or iterable). It can be obtained + by using :func:`DatasetCatalog.get` or :func:`get_detection_dataset_dicts`. + mapper (callable): a callable which takes a sample (dict) from dataset and + returns the format to be consumed by the model. + When using cfg, the default choice is ``DatasetMapper(cfg, is_train=True)``. + sampler (torch.utils.data.sampler.Sampler or None): a sampler that produces + indices to be applied on ``dataset``. + If ``dataset`` is map-style, the default sampler is a :class:`TrainingSampler`, + which coordinates an infinite random shuffle sequence across all workers. + Sampler must be None if ``dataset`` is iterable. + total_batch_size (int): total batch size across all workers. Batching + simply puts data into a list. + aspect_ratio_grouping (bool): whether to group images with similar + aspect ratio for efficiency. When enabled, it requires each + element in dataset be a dict with keys "width" and "height". + num_workers (int): number of parallel data loading workers + collate_fn: same as the argument of `torch.utils.data.DataLoader`. + Defaults to do no collation and return a list of data. + No collation is OK for small batch size and simple data structures. + If your batch size is large and each sample contains too many small tensors, + it's more efficient to collate them in data loader. + + Returns: + torch.utils.data.DataLoader: + a dataloader. Each output from it is a ``list[mapped_element]`` of length + ``total_batch_size / num_workers``, where ``mapped_element`` is produced + by the ``mapper``. + """ + if isinstance(dataset_bg, list): + dataset_bg = DatasetFromList(dataset_bg, copy=False) + + if isinstance(dataset_bg, torchdata.IterableDataset): + assert sampler_bg is None, "sampler must be None if dataset is IterableDataset" + else: + if sampler_bg is None: + sampler_bg = TrainingSampler(len(dataset)) + assert isinstance( + sampler_bg, torchdata.Sampler + ), f"Expect a Sampler but got {type(sampler)}" + + if isinstance(dataset, list): + dataset = DatasetFromList(dataset, copy=False) + if mapper is not None: + dataset = MapDataset_coppaste(dataset, mapper, dataset_bg, sampler_bg) + + if isinstance(dataset, torchdata.IterableDataset): + assert sampler is None, "sampler must be None if dataset is IterableDataset" + else: + if sampler is None: + sampler = TrainingSampler(len(dataset)) + assert isinstance(sampler, torchdata.Sampler), f"Expect a Sampler but got {type(sampler)}" + return build_batch_data_loader( + dataset, + sampler, + total_batch_size, + aspect_ratio_grouping=aspect_ratio_grouping, + num_workers=num_workers, + collate_fn=collate_fn, + ) diff --git a/approach/ovod/APE/ape/data/build_multi_dataset.py b/approach/ovod/APE/ape/data/build_multi_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..8dd346899e623d694253c5cee95deea616c25135 --- /dev/null +++ b/approach/ovod/APE/ape/data/build_multi_dataset.py @@ -0,0 +1,741 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import itertools +import logging +import operator +import time +from collections import defaultdict +from typing import Callable, Optional + +import numpy as np +import torch +import torch.utils.data as torchdata +from termcolor import colored +from torch.utils.data.sampler import Sampler + +from detectron2.config import configurable +from detectron2.data.build import ( + filter_images_with_few_keypoints, + filter_images_with_only_crowd_annotations, + get_detection_dataset_dicts, + load_proposals_into_dataset, + trivial_batch_collator, + worker_init_reset_seed, +) +from detectron2.data.catalog import DatasetCatalog, MetadataCatalog +from detectron2.data.common import DatasetFromList, MapDataset, ToIterableDataset +from detectron2.data.detection_utils import check_metadata_consistency +from detectron2.data.samplers import ( + RandomSubsetTrainingSampler, + RepeatFactorTrainingSampler, + TrainingSampler, +) +from detectron2.utils import comm +from detectron2.utils.comm import get_world_size +from detectron2.utils.logger import _log_api_usage, log_first_n +from tabulate import tabulate + +from .dataset_mapper import DatasetMapper_ape +from .samplers import MultiDatasetTrainingSampler + +""" +This file contains the default logic to build a dataloader for training or testing. +""" + +__all__ = [ + "build_detection_train_loader_multi_dataset", +] + + +def print_instances_class_histogram(dataset_dicts, class_names): + """ + Args: + dataset_dicts (list[dict]): list of dataset dicts. + class_names (list[str]): list of class names (zero-indexed). + """ + num_classes = len(class_names) + hist_bins = np.arange(num_classes + 1) + histogram = np.zeros((num_classes,), dtype=np.int) + total_num_out_of_class = 0 + for entry in dataset_dicts: + annos = entry["annotations"] + classes = np.asarray( + [x["category_id"] for x in annos if not x.get("iscrowd", 0)], dtype=np.int + ) + if len(classes): + assert classes.min() >= 0, f"Got an invalid category_id={classes.min()}" + # assert ( + # classes.max() < num_classes + # ), f"Got an invalid category_id={classes.max()} for a dataset of {num_classes} classes" + histogram += np.histogram(classes, bins=hist_bins)[0] + + total_num_out_of_class += sum(classes >= num_classes) + + N_COLS = min(6, len(class_names) * 2) + + def short_name(x): + # make long class names shorter. useful for lvis + if len(x) > 13: + return x[:11] + ".." + return x + + data = list( + itertools.chain(*[[short_name(class_names[i]), int(v)] for i, v in enumerate(histogram)]) + ) + total_num_instances = sum(data[1::2]) + data.extend([None] * (N_COLS - (len(data) % N_COLS))) + if num_classes > 1: + data.extend(["total", total_num_instances]) + if total_num_out_of_class > 0: + data.extend(["total out", total_num_out_of_class]) + data = itertools.zip_longest(*[data[i::N_COLS] for i in range(N_COLS)]) + table = tabulate( + data, + headers=["category", "#instances"] * (N_COLS // 2), + tablefmt="pipe", + numalign="left", + stralign="center", + ) + log_first_n( + logging.INFO, + "Distribution of instances among all {} categories:\n".format(num_classes) + + colored(table, "cyan"), + key="message", + ) + + +def DatasetCatalog_get(dataset_name, reduce_memory, reduce_memory_size): + import os, psutil + + logger = logging.getLogger(__name__) + logger.info( + "Current memory usage: {} GB".format( + psutil.Process(os.getpid()).memory_info().rss / 1024**3 + ) + ) + + dataset_dicts = DatasetCatalog.get(dataset_name) + + # logger.info( + # "Current memory usage: {} GB".format( + # psutil.Process(os.getpid()).memory_info().rss / 1024**3 + # ) + # ) + # logger.info("Reducing memory usage...") + + # for d in dataset_dicts: + # # LVIS + # if "not_exhaustive_category_ids" in d.keys(): + # del d["not_exhaustive_category_ids"] + # if "neg_category_ids" in d.keys(): + # del d["neg_category_ids"] + # if "pos_category_ids" in d.keys(): + # del d["pos_category_ids"] + + # if "annotations" not in d.keys(): + # continue + # for anno in d["annotations"]: + # if "iscrowd" in anno.keys(): + # if anno["iscrowd"] == 0: + # del anno["iscrowd"] + + logger.info( + "Current memory usage: {} GB".format( + psutil.Process(os.getpid()).memory_info().rss / 1024**3 + ) + ) + + if not reduce_memory: + return dataset_dicts + if len(dataset_dicts) < reduce_memory_size: + return dataset_dicts + + logger.info("Reducing memory usage further...") + + for d in dataset_dicts: + if "annotations" not in d.keys(): + continue + + for anno in d["annotations"]: + + if "bbox" in anno.keys(): + del anno["bbox"] + + if "bbox_mode" in anno.keys(): + del anno["bbox_mode"] + + if "segmentation" in anno.keys(): + del anno["segmentation"] + + if "phrase" in anno.keys(): + del anno["phrase"] + + logger.info( + "Current memory usage: {} GB".format( + psutil.Process(os.getpid()).memory_info().rss / 1024**3 + ) + ) + + return dataset_dicts + + +def get_detection_dataset_dicts_multi_dataset( + names, + filter_empty=True, + min_keypoints=0, + proposal_files=None, + check_consistency=True, + filter_emptys=[True], + dataloader_id=None, + reduce_memory=False, + reduce_memory_size=1e6, +): + """ + Load and prepare dataset dicts for instance detection/segmentation and semantic segmentation. + + Args: + names (str or list[str]): a dataset name or a list of dataset names + filter_empty (bool): whether to filter out images without instance annotations + min_keypoints (int): filter out images with fewer keypoints than + `min_keypoints`. Set to 0 to do nothing. + proposal_files (list[str]): if given, a list of object proposal files + that match each dataset in `names`. + check_consistency (bool): whether to check if datasets have consistent metadata. + + Returns: + list[dict]: a list of dicts following the standard dataset dict format. + """ + if isinstance(names, str): + names = [names] + assert len(names), names + # dataset_dicts = [DatasetCatalog.get(dataset_name) for dataset_name in names] + dataset_dicts = [ + DatasetCatalog_get(dataset_name, reduce_memory, reduce_memory_size) + for dataset_name in names + ] + + if isinstance(dataset_dicts[0], torchdata.Dataset): + if len(dataset_dicts) > 1: + # ConcatDataset does not work for iterable style dataset. + # We could support concat for iterable as well, but it's often + # not a good idea to concat iterables anyway. + return torchdata.ConcatDataset(dataset_dicts) + return dataset_dicts[0] + + for dataset_name, dicts in zip(names, dataset_dicts): + assert len(dicts), "Dataset '{}' is empty!".format(dataset_name) + + for dataset_id, (dataset_name, dicts) in enumerate(zip(names, dataset_dicts)): + for d in dicts: + d["dataset_id"] = dataset_id + if dataloader_id is not None: + d["dataloader_id"] = dataloader_id + + has_instances = "annotations" in dicts[0] + if not check_consistency or not has_instances: + continue + try: + class_names = MetadataCatalog.get(dataset_name).thing_classes + check_metadata_consistency("thing_classes", [dataset_name]) + print_instances_class_histogram(dicts, class_names) + except AttributeError: # class names are not available for this dataset + pass + + assert proposal_files is None + if proposal_files is not None: + assert len(names) == len(proposal_files) + # load precomputed proposals from proposal files + dataset_dicts = [ + load_proposals_into_dataset(dataset_i_dicts, proposal_file) + for dataset_i_dicts, proposal_file in zip(dataset_dicts, proposal_files) + ] + + dataset_dicts = [ + filter_images_with_only_crowd_annotations(dicts) + if flag and "annotations" in dicts[0] + else dicts + for dicts, flag in zip(dataset_dicts, filter_emptys) + ] + + dataset_dicts = list(itertools.chain.from_iterable(dataset_dicts)) + + has_instances = "annotations" in dataset_dicts[0] + if filter_empty and has_instances and False: + dataset_dicts = filter_images_with_only_crowd_annotations(dataset_dicts) + if min_keypoints > 0 and has_instances: + dataset_dicts = filter_images_with_few_keypoints(dataset_dicts, min_keypoints) + + if check_consistency and has_instances and False: + try: + class_names = MetadataCatalog.get(names[0]).thing_classes + check_metadata_consistency("thing_classes", names) + print_instances_class_histogram(dataset_dicts, class_names) + except AttributeError: # class names are not available for this dataset + pass + + assert len(dataset_dicts), "No valid data found in {}.".format(",".join(names)) + return dataset_dicts + + +def build_batch_data_loader_multi_dataset( + dataset, + sampler, + total_batch_size, + total_batch_size_list, + *, + aspect_ratio_grouping=False, + num_workers=0, + collate_fn=None, + num_datasets=1, +): + """ + Build a batched dataloader. The main differences from `torch.utils.data.DataLoader` are: + 1. support aspect ratio grouping options + 2. use no "batch collation", because this is common for detection training + + Args: + dataset (torch.utils.data.Dataset): a pytorch map-style or iterable dataset. + sampler (torch.utils.data.sampler.Sampler or None): a sampler that produces indices. + Must be provided iff. ``dataset`` is a map-style dataset. + total_batch_size, aspect_ratio_grouping, num_workers, collate_fn: see + :func:`build_detection_train_loader`. + + Returns: + iterable[list]. Length of each list is the batch size of the current + GPU. Each element in the list comes from the dataset. + """ + world_size = get_world_size() + assert ( + total_batch_size > 0 and total_batch_size % world_size == 0 + ), "Total batch size ({}) must be divisible by the number of gpus ({}).".format( + total_batch_size, world_size + ) + batch_size = total_batch_size // world_size + + if len(total_batch_size_list) < num_datasets: + total_batch_size_list += [ + total_batch_size, + ] * (num_datasets - len(total_batch_size_list)) + assert all([x > 0 for x in total_batch_size_list]) and all( + [x % world_size == 0 for x in total_batch_size_list] + ), "Total batch size ({}) must be divisible by the number of gpus ({}).".format( + total_batch_size_list, world_size + ) + batch_size = [x // world_size for x in total_batch_size_list] + + if isinstance(dataset, torchdata.IterableDataset): + assert sampler is None, "sampler must be None if dataset is IterableDataset" + else: + dataset = ToIterableDataset(dataset, sampler) + + assert aspect_ratio_grouping + if aspect_ratio_grouping: + data_loader = torchdata.DataLoader( + dataset, + num_workers=num_workers, + collate_fn=operator.itemgetter(0), # don't batch, but yield individual elements + worker_init_fn=worker_init_reset_seed, + ) # yield individual mapped dict + # data_loader = AspectRatioGroupedDataset(data_loader, batch_size) + data_loader = MultiDatasetAspectRatioGroupedDataset( + data_loader, batch_size, num_datasets=num_datasets + ) + if collate_fn is None: + return data_loader + return MapDataset(data_loader, collate_fn) + else: + return torchdata.DataLoader( + dataset, + batch_size=batch_size, + drop_last=True, + num_workers=num_workers, + collate_fn=trivial_batch_collator if collate_fn is None else collate_fn, + worker_init_fn=worker_init_reset_seed, + ) + + +def _train_loader_from_config(cfg, mapper=None, *, dataset=None, sampler=None): + assert len(cfg.DATASETS.TRAIN) == len(cfg.MULTI_DATASET.NAMES) + assert len(cfg.DATASETS.TRAIN) == len(cfg.MULTI_DATASET.ENTITIES) + assert len(cfg.DATASETS.TRAIN) == len(cfg.MULTI_DATASET.NUM_CLASSES) + assert len(cfg.DATASETS.TRAIN) == len(cfg.MULTI_DATASET.RATIOS) + assert len(cfg.DATASETS.TRAIN) == len(cfg.MULTI_DATASET.USE_CAS) + assert len(cfg.DATASETS.TRAIN) == len(cfg.MULTI_DATASET.USE_RFS) + assert len(cfg.DATASETS.TRAIN) == len(cfg.MULTI_DATASET.FILTER_EMPTY_ANNOTATIONS) + # assert len(cfg.DATASETS.TRAIN) == len(cfg.SOLVER.IMS_PER_BATCH_LIST) + # assert len(cfg.DATASETS.TRAIN) == len(cfg.SOLVER.AUGMENT_TYPE) + + seed1 = comm.shared_random_seed() + seed2 = comm.shared_random_seed() + logger = logging.getLogger(__name__) + logger.info("rank {} seed1 {} seed2 {}".format(comm.get_local_rank(), seed1, seed2)) + + # Hard-coded 2 sequent group and 1200s time wait. + wait_group = 2 + wait_time = cfg.DATALOADER.GROUP_WAIT + wait = comm.get_local_rank() % wait_group * wait_time + logger.info("rank {} _train_loader_from_config sleep {}".format(comm.get_local_rank(), wait)) + time.sleep(wait) + + if dataset is None: + dataset = get_detection_dataset_dicts_multi_dataset( + cfg.DATASETS.TRAIN, + filter_empty=cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS, + min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE + if cfg.MODEL.KEYPOINT_ON + else 0, + proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN if cfg.MODEL.LOAD_PROPOSALS else None, + filter_emptys=cfg.MULTI_DATASET.FILTER_EMPTY_ANNOTATIONS, + ) + _log_api_usage("dataset." + cfg.DATASETS.TRAIN[0]) + + if mapper is None: + mapper = DatasetMapper_ape(cfg, True) + + if sampler is None: + sampler_name = cfg.DATALOADER.SAMPLER_TRAIN + logger = logging.getLogger(__name__) + if isinstance(dataset, torchdata.IterableDataset): + logger.info("Not using any sampler since the dataset is IterableDataset.") + sampler = None + else: + logger.info("Using training sampler {}".format(sampler_name)) + if sampler_name == "TrainingSampler": + sampler = TrainingSampler(len(dataset), seed=seed1) + elif sampler_name == "RepeatFactorTrainingSampler": + repeat_factors = RepeatFactorTrainingSampler.repeat_factors_from_category_frequency( + dataset, cfg.DATALOADER.REPEAT_THRESHOLD + ) + sampler = RepeatFactorTrainingSampler(repeat_factors, seed=seed1) + elif sampler_name == "RandomSubsetTrainingSampler": + sampler = RandomSubsetTrainingSampler( + len(dataset), + cfg.DATALOADER.RANDOM_SUBSET_RATIO, + seed_shuffle=seed1, + seed_subset=seed2, + ) + elif sampler_name == "MultiDatasetSampler": + raise ValueError("Despreted training sampler: {}".format(sampler_name)) + sizes = [0 for _ in range(len(cfg.DATASETS.TRAIN))] + for d in dataset: + sizes[d["dataset_id"]] += 1 + sampler = MultiDatasetSampler(cfg, dataset, sizes, seed=seed1) + elif sampler_name == "MultiDatasetTrainingSampler": + # sampler = MultiDatasetTrainingSampler(cfg, dataset, seed=seed1) + repeat_factors = MultiDatasetTrainingSampler.get_repeat_factors( + dataset, + len(cfg.DATASETS.TRAIN), + cfg.MULTI_DATASET.RATIOS, + cfg.MULTI_DATASET.USE_RFS, + cfg.MULTI_DATASET.USE_CAS, + cfg.MULTI_DATASET.REPEAT_THRESHOLD, + cfg.MULTI_DATASET.CAS_LAMBDA, + ) + sampler = MultiDatasetTrainingSampler(repeat_factors, seed=seed1) + else: + raise ValueError("Unknown training sampler: {}".format(sampler_name)) + + return { + "dataset": dataset, + "sampler": sampler, + "mapper": mapper, + "total_batch_size": cfg.SOLVER.IMS_PER_BATCH, + "total_batch_size_list": cfg.SOLVER.IMS_PER_BATCH_LIST, + "aspect_ratio_grouping": cfg.DATALOADER.ASPECT_RATIO_GROUPING, + "num_workers": cfg.DATALOADER.NUM_WORKERS, + "num_datasets": len(cfg.DATASETS.TRAIN), + } + + +@configurable(from_config=_train_loader_from_config) +def build_detection_train_loader_multi_dataset( + dataset, + *, + mapper, + sampler=None, + total_batch_size, + total_batch_size_list, + aspect_ratio_grouping=True, + num_workers=0, + collate_fn=None, + num_datasets=1, +): + """ + Build a dataloader for object detection with some default features. + + Args: + dataset (list or torch.utils.data.Dataset): a list of dataset dicts, + or a pytorch dataset (either map-style or iterable). It can be obtained + by using :func:`DatasetCatalog.get` or :func:`get_detection_dataset_dicts`. + mapper (callable): a callable which takes a sample (dict) from dataset and + returns the format to be consumed by the model. + When using cfg, the default choice is ``DatasetMapper(cfg, is_train=True)``. + sampler (torch.utils.data.sampler.Sampler or None): a sampler that produces + indices to be applied on ``dataset``. + If ``dataset`` is map-style, the default sampler is a :class:`TrainingSampler`, + which coordinates an infinite random shuffle sequence across all workers. + Sampler must be None if ``dataset`` is iterable. + total_batch_size (int): total batch size across all workers. + aspect_ratio_grouping (bool): whether to group images with similar + aspect ratio for efficiency. When enabled, it requires each + element in dataset be a dict with keys "width" and "height". + num_workers (int): number of parallel data loading workers + collate_fn: a function that determines how to do batching, same as the argument of + `torch.utils.data.DataLoader`. Defaults to do no collation and return a list of + data. No collation is OK for small batch size and simple data structures. + If your batch size is large and each sample contains too many small tensors, + it's more efficient to collate them in data loader. + + Returns: + torch.utils.data.DataLoader: + a dataloader. Each output from it is a ``list[mapped_element]`` of length + ``total_batch_size / num_workers``, where ``mapped_element`` is produced + by the ``mapper``. + """ + # wait = round(comm.get_local_rank() * 1.0 * len(dataset) / 60000) + # logger = logging.getLogger(__name__) + # logger.info("get_detection_dataset_dicts_multi_dataset sleep {}".format(wait)) + # time.sleep(wait) + + if isinstance(sampler, Callable): + sampler = sampler(dataset) + + if isinstance(dataset, list): + dataset = DatasetFromList(dataset, copy=False) + if mapper is not None: + dataset = MapDataset(dataset, mapper) + + if isinstance(dataset, torchdata.IterableDataset): + assert sampler is None, "sampler must be None if dataset is IterableDataset" + else: + if sampler is None: + sampler = TrainingSampler(len(dataset)) + assert isinstance(sampler, torchdata.Sampler), f"Expect a Sampler but got {type(sampler)}" + return build_batch_data_loader_multi_dataset( + dataset, + sampler, + total_batch_size, + total_batch_size_list, + aspect_ratio_grouping=aspect_ratio_grouping, + num_workers=num_workers, + collate_fn=collate_fn, + num_datasets=num_datasets, + ) + + +class MultiDatasetSampler(Sampler): + def __init__(self, cfg, dataset_dicts, sizes, seed: Optional[int] = None): + self.sizes = sizes + self.sample_epoch_size = cfg.MULTI_DATASET.SAMPLE_EPOCH_SIZE + assert self.sample_epoch_size % cfg.SOLVER.IMS_PER_BATCH == 0, ( + self.sample_epoch_size % cfg.SOLVER.IMS_PER_BATCH == 0 + ) + if seed is None: + seed = comm.shared_random_seed() + self._seed = int(seed) + + self._rank = comm.get_rank() + self._world_size = comm.get_world_size() + + dataset_ratio = cfg.MULTI_DATASET.RATIOS + assert len(dataset_ratio) == len( + sizes + ), "length of dataset ratio {} should be equal to number if dataset {}".format( + len(dataset_ratio), len(sizes) + ) + dataset_weight = [ + torch.ones(s) * max(sizes) / s * r / sum(dataset_ratio) + for i, (r, s) in enumerate(zip(dataset_ratio, sizes)) + ] + st = 0 + cas_factors = [] + for i, s in enumerate(sizes): + if cfg.MULTI_DATASET.USE_CAS[i]: + cas_factor = self._get_class_balance_factor_per_dataset( + dataset_dicts[st : st + s], l=cfg.MULTI_DATASET.CAS_LAMBDA + ) + cas_factor = cas_factor * (s / cas_factor.sum()) + else: + cas_factor = torch.ones(s) + cas_factors.append(cas_factor) + st = st + s + cas_factors = torch.cat(cas_factors) + dataset_weight = torch.cat(dataset_weight) + self.weights = dataset_weight * cas_factors + + def __iter__(self): + start = self._rank + yield from itertools.islice(self._infinite_indices(), start, None, self._world_size) + + def _infinite_indices(self): + g = torch.Generator() + g.manual_seed(self._seed) + while True: + ids = torch.multinomial( + self.weights, self.sample_epoch_size, generator=g, replacement=True + ) + yield from ids + + def _get_class_balance_factor_per_dataset(self, dataset_dicts, l=1.0): + ret = [] + category_freq = defaultdict(int) + for dataset_dict in dataset_dicts: # For each image (without repeats) + cat_ids = {ann["category_id"] for ann in dataset_dict["annotations"]} + for cat_id in cat_ids: + category_freq[cat_id] += 1 + for dataset_dict in dataset_dicts: + cat_ids = {ann["category_id"] for ann in dataset_dict["annotations"]} + ret.append(sum([1.0 / (category_freq[cat_id] ** l) for cat_id in cat_ids])) + return torch.tensor(ret).float() + + +# class MultiDatasetTrainingSampler(Sampler): +# def __init__(self, cfg, dataset_dicts, *, shuffle=True, seed=None): +# sizes = [0 for _ in range(len(cfg.DATASETS.TRAIN))] +# for d in dataset_dicts: +# sizes[d["dataset_id"]] += 1 + +# dataset_ratio = cfg.MULTI_DATASET.RATIOS +# assert len(dataset_ratio) == len( +# sizes +# ), "length of dataset ratio {} should be equal to number if dataset {}".format( +# len(dataset_ratio), len(sizes) +# ) +# dataset_weight = [ +# torch.ones(s) * max(sizes) / s * r for i, (r, s) in enumerate(zip(dataset_ratio, sizes)) +# ] + +# logger = logging.getLogger(__name__) +# logger.info( +# "Training sampler dataset weight: {}".format( +# str([max(sizes) / s * r for i, (r, s) in enumerate(zip(dataset_ratio, sizes))]) +# ) +# ) + +# st = 0 +# repeat_factors = [] +# for i, s in enumerate(sizes): +# assert cfg.MULTI_DATASET.USE_RFS[i] * cfg.MULTI_DATASET.USE_CAS[i] == 0 +# if cfg.MULTI_DATASET.USE_RFS[i]: +# repeat_factor = RepeatFactorTrainingSampler.repeat_factors_from_category_frequency( +# dataset_dicts[st : st + s], cfg.MULTI_DATASET.REPEAT_THRESHOLD +# ) +# elif cfg.MULTI_DATASET.USE_CAS[i]: +# repeat_factor = MultiDatasetTrainingSampler.get_class_balance_factor_per_dataset( +# dataset_dicts[st : st + s], l=cfg.MULTI_DATASET.CAS_LAMBDA +# ) +# repeat_factor = repeat_factor * (s / repeat_factor.sum()) +# else: +# repeat_factor = torch.ones(s) +# repeat_factors.append(repeat_factor) +# st = st + s +# repeat_factors = torch.cat(repeat_factors) +# dataset_weight = torch.cat(dataset_weight) +# repeat_factors = dataset_weight * repeat_factors + +# self._shuffle = shuffle +# if seed is None: +# seed = comm.shared_random_seed() +# self._seed = int(seed) + +# self._rank = comm.get_rank() +# self._world_size = comm.get_world_size() + +# # Split into whole number (_int_part) and fractional (_frac_part) parts. +# self._int_part = torch.trunc(repeat_factors) +# self._frac_part = repeat_factors - self._int_part + +# @staticmethod +# def get_class_balance_factor_per_dataset(dataset_dicts, l=1.0): +# rep_factors = [] +# category_freq = defaultdict(int) +# for dataset_dict in dataset_dicts: # For each image (without repeats) +# cat_ids = {ann["category_id"] for ann in dataset_dict["annotations"]} +# for cat_id in cat_ids: +# category_freq[cat_id] += 1 +# for dataset_dict in dataset_dicts: +# cat_ids = {ann["category_id"] for ann in dataset_dict["annotations"]} +# rep_factor = sum([1.0 / (category_freq[cat_id] ** l) for cat_id in cat_ids]) +# rep_factors.append(rep_factor) + +# return torch.tensor(rep_factors, dtype=torch.float32) + +# def _get_epoch_indices(self, generator): +# """ +# Create a list of dataset indices (with repeats) to use for one epoch. + +# Args: +# generator (torch.Generator): pseudo random number generator used for +# stochastic rounding. + +# Returns: +# torch.Tensor: list of dataset indices to use in one epoch. Each index +# is repeated based on its calculated repeat factor. +# """ +# # Since repeat factors are fractional, we use stochastic rounding so +# # that the target repeat factor is achieved in expectation over the +# # course of training +# rands = torch.rand(len(self._frac_part), generator=generator) +# rep_factors = self._int_part + (rands < self._frac_part).float() +# # Construct a list of indices in which we repeat images as specified +# indices = [] +# for dataset_index, rep_factor in enumerate(rep_factors): +# indices.extend([dataset_index] * int(rep_factor.item())) +# return torch.tensor(indices, dtype=torch.int64) + +# def __iter__(self): +# start = self._rank +# yield from itertools.islice(self._infinite_indices(), start, None, self._world_size) + +# def _infinite_indices(self): +# g = torch.Generator() +# g.manual_seed(self._seed) +# while True: +# # Sample indices with repeats determined by stochastic rounding; each +# # "epoch" may have a slightly different size due to the rounding. +# indices = self._get_epoch_indices(g) +# if self._shuffle: +# randperm = torch.randperm(len(indices), generator=g) +# yield from indices[randperm].tolist() +# else: +# yield from indices.tolist() + + +class MultiDatasetAspectRatioGroupedDataset(torch.utils.data.IterableDataset): + """ + Batch data that have similar aspect ratio together. + In this implementation, images whose aspect ratio < (or >) 1 will + be batched together. + This improves training speed because the images then need less padding + to form a batch. + + It assumes the underlying dataset produces dicts with "width" and "height" keys. + It will then produce a list of original dicts with length = batch_size, + all with similar aspect ratios. + """ + + def __init__(self, dataset, batch_size, num_datasets): + """ + Args: + dataset: an iterable. Each element must be a dict with keys + "width" and "height", which will be used to batch data. + batch_size (int): + """ + self.dataset = dataset + self.batch_size = batch_size + self._buckets = [[] for _ in range(2 * num_datasets)] + # Hard-coded two aspect ratio groups: w > h and w < h. + # Can add support for more aspect ratio groups, but doesn't seem useful + + def __iter__(self): + for d in self.dataset: + w, h = d["width"], d["height"] + bucket_id = 0 if w > h else 1 + bucket_id = d["dataset_id"] * 2 + bucket_id + bucket = self._buckets[bucket_id] + bucket.append(d) + if len(bucket) == self.batch_size[d["dataset_id"]]: + data = bucket[:] + # Clear bucket first, because code after yield is not + # guaranteed to execute + del bucket[:] + yield data diff --git a/approach/ovod/APE/ape/data/build_multi_dataset_copypaste.py b/approach/ovod/APE/ape/data/build_multi_dataset_copypaste.py new file mode 100644 index 0000000000000000000000000000000000000000..b7dc12a6820beb61eb5b42d3c6daf83cd49eba53 --- /dev/null +++ b/approach/ovod/APE/ape/data/build_multi_dataset_copypaste.py @@ -0,0 +1,805 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import itertools +import logging +import operator +import time +from collections import defaultdict +from typing import Callable, Optional + +import numpy as np +import torch +import torch.utils.data as torchdata +from termcolor import colored +from torch.utils.data.sampler import Sampler + +from detectron2.config import configurable +from detectron2.data.build import ( + filter_images_with_few_keypoints, + filter_images_with_only_crowd_annotations, + get_detection_dataset_dicts, + load_proposals_into_dataset, + trivial_batch_collator, + worker_init_reset_seed, +) +from detectron2.data.catalog import DatasetCatalog, MetadataCatalog +from detectron2.data.common import DatasetFromList, MapDataset, ToIterableDataset +from detectron2.data.detection_utils import check_metadata_consistency +from detectron2.data.samplers import ( + RandomSubsetTrainingSampler, + RepeatFactorTrainingSampler, + TrainingSampler, +) +from detectron2.utils import comm +from detectron2.utils.comm import get_world_size +from detectron2.utils.logger import _log_api_usage, log_first_n +from tabulate import tabulate + +from .common_copypaste import MapDataset_coppaste +from .dataset_mapper_copypaste import DatasetMapper_copypaste +from .samplers import MultiDatasetTrainingSampler + +""" +This file contains the default logic to build a dataloader for training or testing. +""" + +__all__ = [ + "build_detection_train_loader_multi_dataset_copypaste", +] + + +def print_instances_class_histogram(dataset_dicts, class_names): + """ + Args: + dataset_dicts (list[dict]): list of dataset dicts. + class_names (list[str]): list of class names (zero-indexed). + """ + num_classes = len(class_names) + hist_bins = np.arange(num_classes + 1) + histogram = np.zeros((num_classes,), dtype=np.int) + total_num_out_of_class = 0 + for entry in dataset_dicts: + annos = entry["annotations"] + classes = np.asarray( + [x["category_id"] for x in annos if not x.get("iscrowd", 0)], dtype=np.int + ) + if len(classes): + assert classes.min() >= 0, f"Got an invalid category_id={classes.min()}" + # assert ( + # classes.max() < num_classes + # ), f"Got an invalid category_id={classes.max()} for a dataset of {num_classes} classes" + histogram += np.histogram(classes, bins=hist_bins)[0] + + total_num_out_of_class += sum(classes >= num_classes) + + N_COLS = min(6, len(class_names) * 2) + + def short_name(x): + # make long class names shorter. useful for lvis + if len(x) > 13: + return x[:11] + ".." + return x + + data = list( + itertools.chain(*[[short_name(class_names[i]), int(v)] for i, v in enumerate(histogram)]) + ) + total_num_instances = sum(data[1::2]) + data.extend([None] * (N_COLS - (len(data) % N_COLS))) + if num_classes > 1: + data.extend(["total", total_num_instances]) + if total_num_out_of_class > 0: + data.extend(["total out", total_num_out_of_class]) + data = itertools.zip_longest(*[data[i::N_COLS] for i in range(N_COLS)]) + table = tabulate( + data, + headers=["category", "#instances"] * (N_COLS // 2), + tablefmt="pipe", + numalign="left", + stralign="center", + ) + log_first_n( + logging.INFO, + "Distribution of instances among all {} categories:\n".format(num_classes) + + colored(table, "cyan"), + key="message", + ) + + +def DatasetCatalog_get(dataset_name, reduce_memory, reduce_memory_size): + import os, psutil + + logger = logging.getLogger(__name__) + logger.info( + "Current memory usage: {} GB".format( + psutil.Process(os.getpid()).memory_info().rss / 1024**3 + ) + ) + + dataset_dicts = DatasetCatalog.get(dataset_name) + + # logger.info( + # "Current memory usage: {} GB".format( + # psutil.Process(os.getpid()).memory_info().rss / 1024**3 + # ) + # ) + # logger.info("Reducing memory usage...") + + # for d in dataset_dicts: + # # LVIS + # if "not_exhaustive_category_ids" in d.keys(): + # del d["not_exhaustive_category_ids"] + # if "neg_category_ids" in d.keys(): + # del d["neg_category_ids"] + # if "pos_category_ids" in d.keys(): + # del d["pos_category_ids"] + + # if "annotations" not in d.keys(): + # continue + # for anno in d["annotations"]: + # if "iscrowd" in anno.keys(): + # if anno["iscrowd"] == 0: + # del anno["iscrowd"] + + logger.info( + "Current memory usage: {} GB".format( + psutil.Process(os.getpid()).memory_info().rss / 1024**3 + ) + ) + + if not reduce_memory: + return dataset_dicts + if len(dataset_dicts) < reduce_memory_size: + return dataset_dicts + + logger.info("Reducing memory usage further...") + + for d in dataset_dicts: + if "annotations" not in d.keys(): + continue + + for anno in d["annotations"]: + + if "bbox" in anno.keys(): + del anno["bbox"] + + if "bbox_mode" in anno.keys(): + del anno["bbox_mode"] + + if "segmentation" in anno.keys(): + del anno["segmentation"] + + if "phrase" in anno.keys(): + del anno["phrase"] + + logger.info( + "Current memory usage: {} GB".format( + psutil.Process(os.getpid()).memory_info().rss / 1024**3 + ) + ) + + return dataset_dicts + + +def get_detection_dataset_dicts_multi_dataset_copypaste( + names, + filter_empty=True, + min_keypoints=0, + proposal_files=None, + check_consistency=True, + filter_emptys=[True], + copypastes=[True], + dataloader_id=None, + reduce_memory=False, + reduce_memory_size=1e6, +): + """ + Load and prepare dataset dicts for instance detection/segmentation and semantic segmentation. + + Args: + names (str or list[str]): a dataset name or a list of dataset names + filter_empty (bool): whether to filter out images without instance annotations + min_keypoints (int): filter out images with fewer keypoints than + `min_keypoints`. Set to 0 to do nothing. + proposal_files (list[str]): if given, a list of object proposal files + that match each dataset in `names`. + check_consistency (bool): whether to check if datasets have consistent metadata. + + Returns: + list[dict]: a list of dicts following the standard dataset dict format. + """ + if isinstance(names, str): + names = [names] + assert len(names), names + # dataset_dicts = [DatasetCatalog.get(dataset_name) for dataset_name in names] + dataset_dicts = [ + DatasetCatalog_get(dataset_name, reduce_memory, reduce_memory_size) + for dataset_name in names + ] + + if isinstance(dataset_dicts[0], torchdata.Dataset): + if len(dataset_dicts) > 1: + # ConcatDataset does not work for iterable style dataset. + # We could support concat for iterable as well, but it's often + # not a good idea to concat iterables anyway. + return torchdata.ConcatDataset(dataset_dicts) + return dataset_dicts[0] + + for dataset_name, dicts in zip(names, dataset_dicts): + assert len(dicts), "Dataset '{}' is empty!".format(dataset_name) + + for dataset_id, (dataset_name, copypaste, dicts) in enumerate( + zip(names, copypastes, dataset_dicts) + ): + for d in dicts: + d["dataset_id"] = dataset_id + d["copypaste"] = copypaste + if dataloader_id is not None: + d["dataloader_id"] = dataloader_id + + has_instances = "annotations" in dicts[0] + if not check_consistency or not has_instances: + continue + try: + class_names = MetadataCatalog.get(dataset_name).thing_classes + check_metadata_consistency("thing_classes", [dataset_name]) + print_instances_class_histogram(dicts, class_names) + except AttributeError: # class names are not available for this dataset + pass + + assert proposal_files is None + if proposal_files is not None: + assert len(names) == len(proposal_files) + # load precomputed proposals from proposal files + dataset_dicts = [ + load_proposals_into_dataset(dataset_i_dicts, proposal_file) + for dataset_i_dicts, proposal_file in zip(dataset_dicts, proposal_files) + ] + + dataset_dicts = [ + filter_images_with_only_crowd_annotations(dicts) + if flag and "annotations" in dicts[0] + else dicts + for dicts, flag in zip(dataset_dicts, filter_emptys) + ] + + dataset_dicts = list(itertools.chain.from_iterable(dataset_dicts)) + + has_instances = "annotations" in dataset_dicts[0] + if filter_empty and has_instances and False: + dataset_dicts = filter_images_with_only_crowd_annotations(dataset_dicts) + if min_keypoints > 0 and has_instances: + dataset_dicts = filter_images_with_few_keypoints(dataset_dicts, min_keypoints) + + if check_consistency and has_instances and False: + try: + class_names = MetadataCatalog.get(names[0]).thing_classes + check_metadata_consistency("thing_classes", names) + print_instances_class_histogram(dataset_dicts, class_names) + except AttributeError: # class names are not available for this dataset + pass + + assert len(dataset_dicts), "No valid data found in {}.".format(",".join(names)) + return dataset_dicts + + +def build_batch_data_loader_multi_dataset( + dataset, + sampler, + total_batch_size, + total_batch_size_list, + *, + aspect_ratio_grouping=False, + num_workers=0, + collate_fn=None, + num_datasets=1, +): + """ + Build a batched dataloader. The main differences from `torch.utils.data.DataLoader` are: + 1. support aspect ratio grouping options + 2. use no "batch collation", because this is common for detection training + + Args: + dataset (torch.utils.data.Dataset): a pytorch map-style or iterable dataset. + sampler (torch.utils.data.sampler.Sampler or None): a sampler that produces indices. + Must be provided iff. ``dataset`` is a map-style dataset. + total_batch_size, aspect_ratio_grouping, num_workers, collate_fn: see + :func:`build_detection_train_loader`. + + Returns: + iterable[list]. Length of each list is the batch size of the current + GPU. Each element in the list comes from the dataset. + """ + world_size = get_world_size() + assert ( + total_batch_size > 0 and total_batch_size % world_size == 0 + ), "Total batch size ({}) must be divisible by the number of gpus ({}).".format( + total_batch_size, world_size + ) + batch_size = total_batch_size // world_size + + if len(total_batch_size_list) < num_datasets: + total_batch_size_list += [ + total_batch_size, + ] * (num_datasets - len(total_batch_size_list)) + assert all([x > 0 for x in total_batch_size_list]) and all( + [x % world_size == 0 for x in total_batch_size_list] + ), "Total batch size ({}) must be divisible by the number of gpus ({}).".format( + total_batch_size_list, world_size + ) + batch_size = [x // world_size for x in total_batch_size_list] + + if isinstance(dataset, torchdata.IterableDataset): + assert sampler is None, "sampler must be None if dataset is IterableDataset" + else: + dataset = ToIterableDataset(dataset, sampler) + + assert aspect_ratio_grouping + if aspect_ratio_grouping: + data_loader = torchdata.DataLoader( + dataset, + num_workers=num_workers, + collate_fn=operator.itemgetter(0), # don't batch, but yield individual elements + worker_init_fn=worker_init_reset_seed, + ) # yield individual mapped dict + # data_loader = AspectRatioGroupedDataset(data_loader, batch_size) + data_loader = MultiDatasetAspectRatioGroupedDataset( + data_loader, batch_size, num_datasets=num_datasets + ) + if collate_fn is None: + return data_loader + return MapDataset(data_loader, collate_fn) + else: + return torchdata.DataLoader( + dataset, + batch_size=batch_size, + drop_last=True, + num_workers=num_workers, + collate_fn=trivial_batch_collator if collate_fn is None else collate_fn, + worker_init_fn=worker_init_reset_seed, + ) + + +def _train_loader_from_config(cfg, mapper=None, *, dataset=None, sampler=None): + assert len(cfg.DATASETS.TRAIN) == len(cfg.MULTI_DATASET.NAMES) + assert len(cfg.DATASETS.TRAIN) == len(cfg.MULTI_DATASET.ENTITIES) + assert len(cfg.DATASETS.TRAIN) == len(cfg.MULTI_DATASET.NUM_CLASSES) + assert len(cfg.DATASETS.TRAIN) == len(cfg.MULTI_DATASET.RATIOS) + assert len(cfg.DATASETS.TRAIN) == len(cfg.MULTI_DATASET.USE_CAS) + assert len(cfg.DATASETS.TRAIN) == len(cfg.MULTI_DATASET.USE_RFS) + assert len(cfg.DATASETS.TRAIN) == len(cfg.MULTI_DATASET.FILTER_EMPTY_ANNOTATIONS) + # assert len(cfg.DATASETS.TRAIN) == len(cfg.SOLVER.IMS_PER_BATCH_LIST) + # assert len(cfg.DATASETS.TRAIN) == len(cfg.SOLVER.AUGMENT_TYPE) + assert len(cfg.DATASETS.TRAIN) == len(cfg.DATASETS.COPYPASTE.COPYPASTE) + + seed1 = comm.shared_random_seed() + seed2 = comm.shared_random_seed() + seed3 = comm.shared_random_seed() + seed4 = comm.shared_random_seed() + logger = logging.getLogger(__name__) + logger.info("rank {} seed1 {} seed2 {}".format(comm.get_local_rank(), seed1, seed2)) + logger.info("rank {} seed3 {} seed4 {}".format(comm.get_local_rank(), seed3, seed4)) + + # Hard-coded 2 sequent group and 1200s time wait. + wait_group = 2 + wait_time = cfg.DATALOADER.GROUP_WAIT + wait = comm.get_local_rank() % wait_group * wait_time + logger.info("rank {} _train_loader_from_config sleep {}".format(comm.get_local_rank(), wait)) + time.sleep(wait) + + if dataset is None: + dataset = get_detection_dataset_dicts_multi_dataset_copypaste( + cfg.DATASETS.TRAIN, + filter_empty=cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS, + min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE + if cfg.MODEL.KEYPOINT_ON + else 0, + proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN if cfg.MODEL.LOAD_PROPOSALS else None, + filter_emptys=cfg.MULTI_DATASET.FILTER_EMPTY_ANNOTATIONS, + copypastes=cfg.DATASETS.COPYPASTE.COPYPASTE, + ) + _log_api_usage("dataset." + cfg.DATASETS.TRAIN[0]) + + if True: + dataset_bg = get_detection_dataset_dicts( + cfg.DATASETS.COPYPASTE.BG, + filter_empty=cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS, + min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE + if cfg.MODEL.KEYPOINT_ON + else 0, + proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN if cfg.MODEL.LOAD_PROPOSALS else None, + ) + _log_api_usage("dataset." + cfg.DATASETS.COPYPASTE.BG[0]) + + if mapper is None: + mapper = DatasetMapper_copypaste(cfg, True) + + if sampler is None: + sampler_name = cfg.DATALOADER.SAMPLER_TRAIN + logger = logging.getLogger(__name__) + if isinstance(dataset, torchdata.IterableDataset): + logger.info("Not using any sampler since the dataset is IterableDataset.") + sampler = None + else: + logger.info("Using training sampler {}".format(sampler_name)) + if sampler_name == "TrainingSampler": + sampler = TrainingSampler(len(dataset), seed=seed1) + elif sampler_name == "RepeatFactorTrainingSampler": + repeat_factors = RepeatFactorTrainingSampler.repeat_factors_from_category_frequency( + dataset, cfg.DATALOADER.REPEAT_THRESHOLD + ) + sampler = RepeatFactorTrainingSampler(repeat_factors, seed=seed1) + elif sampler_name == "RandomSubsetTrainingSampler": + sampler = RandomSubsetTrainingSampler( + len(dataset), + cfg.DATALOADER.RANDOM_SUBSET_RATIO, + seed_shuffle=seed1, + seed_subset=seed2, + ) + elif sampler_name == "MultiDatasetSampler": + raise ValueError("Despreted training sampler: {}".format(sampler_name)) + sizes = [0 for _ in range(len(cfg.DATASETS.TRAIN))] + for d in dataset: + sizes[d["dataset_id"]] += 1 + sampler = MultiDatasetSampler(cfg, dataset, sizes, seed=seed1) + elif sampler_name == "MultiDatasetTrainingSampler": + # sampler = MultiDatasetTrainingSampler(cfg, dataset, seed=seed1) + repeat_factors = MultiDatasetTrainingSampler.get_repeat_factors( + dataset, + len(cfg.DATASETS.TRAIN), + cfg.MULTI_DATASET.RATIOS, + cfg.MULTI_DATASET.USE_RFS, + cfg.MULTI_DATASET.USE_CAS, + cfg.MULTI_DATASET.REPEAT_THRESHOLD, + cfg.MULTI_DATASET.CAS_LAMBDA, + ) + sampler = MultiDatasetTrainingSampler(repeat_factors, seed=seed1) + else: + raise ValueError("Unknown training sampler: {}".format(sampler_name)) + + if True: + sampler_name = cfg.DATALOADER.COPYPASTE.SAMPLER_TRAIN + logger = logging.getLogger(__name__) + if isinstance(dataset_bg, torchdata.IterableDataset): + logger.info("Not using any sampler since the dataset is IterableDataset.") + sampler = None + else: + logger.info("Using training sampler {}".format(sampler_name)) + if sampler_name == "TrainingSampler": + sampler_bg = TrainingSampler(len(dataset_bg), seed=seed3) + elif sampler_name == "RepeatFactorTrainingSampler": + repeat_factors = RepeatFactorTrainingSampler.repeat_factors_from_category_frequency( + dataset_bg, cfg.DATALOADER.COPYPASTE.REPEAT_THRESHOLD + ) + sampler_bg = RepeatFactorTrainingSampler(repeat_factors, seed=seed3) + elif sampler_name == "RandomSubsetTrainingSampler": + sampler_bg = RandomSubsetTrainingSampler( + len(dataset_bg), + cfg.DATALOADER.COPYPASTE.RANDOM_SUBSET_RATIO, + seed_shuffle=seed3, + seed_subset=seed4, + ) + else: + raise ValueError("Unknown training sampler: {}".format(sampler_name)) + + return { + "dataset": dataset, + "dataset_bg": dataset_bg, + "sampler": sampler, + "sampler_bg": sampler_bg, + "mapper": mapper, + "total_batch_size": cfg.SOLVER.IMS_PER_BATCH, + "total_batch_size_list": cfg.SOLVER.IMS_PER_BATCH_LIST, + "aspect_ratio_grouping": cfg.DATALOADER.ASPECT_RATIO_GROUPING, + "num_workers": cfg.DATALOADER.NUM_WORKERS, + "num_datasets": len(cfg.DATASETS.TRAIN), + } + + +@configurable(from_config=_train_loader_from_config) +def build_detection_train_loader_multi_dataset_copypaste( + dataset, + dataset_bg, + *, + mapper, + sampler=None, + sampler_bg=None, + total_batch_size, + total_batch_size_list, + aspect_ratio_grouping=True, + num_workers=0, + collate_fn=None, + num_datasets=1, +): + """ + Build a dataloader for object detection with some default features. + + Args: + dataset (list or torch.utils.data.Dataset): a list of dataset dicts, + or a pytorch dataset (either map-style or iterable). It can be obtained + by using :func:`DatasetCatalog.get` or :func:`get_detection_dataset_dicts`. + mapper (callable): a callable which takes a sample (dict) from dataset and + returns the format to be consumed by the model. + When using cfg, the default choice is ``DatasetMapper(cfg, is_train=True)``. + sampler (torch.utils.data.sampler.Sampler or None): a sampler that produces + indices to be applied on ``dataset``. + If ``dataset`` is map-style, the default sampler is a :class:`TrainingSampler`, + which coordinates an infinite random shuffle sequence across all workers. + Sampler must be None if ``dataset`` is iterable. + total_batch_size (int): total batch size across all workers. + aspect_ratio_grouping (bool): whether to group images with similar + aspect ratio for efficiency. When enabled, it requires each + element in dataset be a dict with keys "width" and "height". + num_workers (int): number of parallel data loading workers + collate_fn: a function that determines how to do batching, same as the argument of + `torch.utils.data.DataLoader`. Defaults to do no collation and return a list of + data. No collation is OK for small batch size and simple data structures. + If your batch size is large and each sample contains too many small tensors, + it's more efficient to collate them in data loader. + + Returns: + torch.utils.data.DataLoader: + a dataloader. Each output from it is a ``list[mapped_element]`` of length + ``total_batch_size / num_workers``, where ``mapped_element`` is produced + by the ``mapper``. + """ + # wait = round(comm.get_local_rank() * 1.0 * len(dataset) / 60000) + # logger = logging.getLogger(__name__) + # logger.info("get_detection_dataset_dicts_multi_dataset sleep {}".format(wait)) + # time.sleep(wait) + + if isinstance(sampler_bg, Callable): + sampler_bg = sampler_bg(dataset_bg) + if isinstance(sampler, Callable): + sampler = sampler(dataset) + + if isinstance(dataset_bg, list): + dataset_bg = DatasetFromList(dataset_bg, copy=False) + + if isinstance(dataset_bg, torchdata.IterableDataset): + assert sampler_bg is None, "sampler must be None if dataset is IterableDataset" + else: + if sampler_bg is None: + sampler_bg = TrainingSampler(len(dataset_bg)) + assert isinstance( + sampler_bg, torchdata.Sampler + ), f"Expect a Sampler but got {type(sampler)}" + + if isinstance(dataset, list): + dataset = DatasetFromList(dataset, copy=False) + if mapper is not None: + dataset = MapDataset_coppaste(dataset, mapper, dataset_bg, sampler_bg) + + if isinstance(dataset, torchdata.IterableDataset): + assert sampler is None, "sampler must be None if dataset is IterableDataset" + else: + if sampler is None: + sampler = TrainingSampler(len(dataset)) + assert isinstance(sampler, torchdata.Sampler), f"Expect a Sampler but got {type(sampler)}" + return build_batch_data_loader_multi_dataset( + dataset, + sampler, + total_batch_size, + total_batch_size_list, + aspect_ratio_grouping=aspect_ratio_grouping, + num_workers=num_workers, + collate_fn=collate_fn, + num_datasets=num_datasets, + ) + + +class MultiDatasetSampler(Sampler): + def __init__(self, cfg, dataset_dicts, sizes, seed: Optional[int] = None): + self.sizes = sizes + self.sample_epoch_size = cfg.MULTI_DATASET.SAMPLE_EPOCH_SIZE + assert self.sample_epoch_size % cfg.SOLVER.IMS_PER_BATCH == 0, ( + self.sample_epoch_size % cfg.SOLVER.IMS_PER_BATCH == 0 + ) + if seed is None: + seed = comm.shared_random_seed() + self._seed = int(seed) + + self._rank = comm.get_rank() + self._world_size = comm.get_world_size() + + dataset_ratio = cfg.MULTI_DATASET.RATIOS + assert len(dataset_ratio) == len( + sizes + ), "length of dataset ratio {} should be equal to number if dataset {}".format( + len(dataset_ratio), len(sizes) + ) + dataset_weight = [ + torch.ones(s) * max(sizes) / s * r / sum(dataset_ratio) + for i, (r, s) in enumerate(zip(dataset_ratio, sizes)) + ] + st = 0 + cas_factors = [] + for i, s in enumerate(sizes): + if cfg.MULTI_DATASET.USE_CAS[i]: + cas_factor = self._get_class_balance_factor_per_dataset( + dataset_dicts[st : st + s], l=cfg.MULTI_DATASET.CAS_LAMBDA + ) + cas_factor = cas_factor * (s / cas_factor.sum()) + else: + cas_factor = torch.ones(s) + cas_factors.append(cas_factor) + st = st + s + cas_factors = torch.cat(cas_factors) + dataset_weight = torch.cat(dataset_weight) + self.weights = dataset_weight * cas_factors + + def __iter__(self): + start = self._rank + yield from itertools.islice(self._infinite_indices(), start, None, self._world_size) + + def _infinite_indices(self): + g = torch.Generator() + g.manual_seed(self._seed) + while True: + ids = torch.multinomial( + self.weights, self.sample_epoch_size, generator=g, replacement=True + ) + yield from ids + + def _get_class_balance_factor_per_dataset(self, dataset_dicts, l=1.0): + ret = [] + category_freq = defaultdict(int) + for dataset_dict in dataset_dicts: # For each image (without repeats) + cat_ids = {ann["category_id"] for ann in dataset_dict["annotations"]} + for cat_id in cat_ids: + category_freq[cat_id] += 1 + for dataset_dict in dataset_dicts: + cat_ids = {ann["category_id"] for ann in dataset_dict["annotations"]} + ret.append(sum([1.0 / (category_freq[cat_id] ** l) for cat_id in cat_ids])) + return torch.tensor(ret).float() + + +# class MultiDatasetTrainingSampler(Sampler): +# def __init__(self, cfg, dataset_dicts, *, shuffle=True, seed=None): +# sizes = [0 for _ in range(len(cfg.DATASETS.TRAIN))] +# for d in dataset_dicts: +# sizes[d["dataset_id"]] += 1 + +# dataset_ratio = cfg.MULTI_DATASET.RATIOS +# assert len(dataset_ratio) == len( +# sizes +# ), "length of dataset ratio {} should be equal to number if dataset {}".format( +# len(dataset_ratio), len(sizes) +# ) +# dataset_weight = [ +# torch.ones(s) * max(sizes) / s * r for i, (r, s) in enumerate(zip(dataset_ratio, sizes)) +# ] + +# logger = logging.getLogger(__name__) +# logger.info( +# "Training sampler dataset weight: {}".format( +# str([max(sizes) / s * r for i, (r, s) in enumerate(zip(dataset_ratio, sizes))]) +# ) +# ) + +# st = 0 +# repeat_factors = [] +# for i, s in enumerate(sizes): +# assert cfg.MULTI_DATASET.USE_RFS[i] * cfg.MULTI_DATASET.USE_CAS[i] == 0 +# if cfg.MULTI_DATASET.USE_RFS[i]: +# repeat_factor = RepeatFactorTrainingSampler.repeat_factors_from_category_frequency( +# dataset_dicts[st : st + s], cfg.MULTI_DATASET.REPEAT_THRESHOLD +# ) +# elif cfg.MULTI_DATASET.USE_CAS[i]: +# repeat_factor = MultiDatasetTrainingSampler.get_class_balance_factor_per_dataset( +# dataset_dicts[st : st + s], l=cfg.MULTI_DATASET.CAS_LAMBDA +# ) +# repeat_factor = repeat_factor * (s / repeat_factor.sum()) +# else: +# repeat_factor = torch.ones(s) +# repeat_factors.append(repeat_factor) +# st = st + s +# repeat_factors = torch.cat(repeat_factors) +# dataset_weight = torch.cat(dataset_weight) +# repeat_factors = dataset_weight * repeat_factors + +# self._shuffle = shuffle +# if seed is None: +# seed = comm.shared_random_seed() +# self._seed = int(seed) + +# self._rank = comm.get_rank() +# self._world_size = comm.get_world_size() + +# # Split into whole number (_int_part) and fractional (_frac_part) parts. +# self._int_part = torch.trunc(repeat_factors) +# self._frac_part = repeat_factors - self._int_part + +# @staticmethod +# def get_class_balance_factor_per_dataset(dataset_dicts, l=1.0): +# rep_factors = [] +# category_freq = defaultdict(int) +# for dataset_dict in dataset_dicts: # For each image (without repeats) +# cat_ids = {ann["category_id"] for ann in dataset_dict["annotations"]} +# for cat_id in cat_ids: +# category_freq[cat_id] += 1 +# for dataset_dict in dataset_dicts: +# cat_ids = {ann["category_id"] for ann in dataset_dict["annotations"]} +# rep_factor = sum([1.0 / (category_freq[cat_id] ** l) for cat_id in cat_ids]) +# rep_factors.append(rep_factor) + +# return torch.tensor(rep_factors, dtype=torch.float32) + +# def _get_epoch_indices(self, generator): +# """ +# Create a list of dataset indices (with repeats) to use for one epoch. + +# Args: +# generator (torch.Generator): pseudo random number generator used for +# stochastic rounding. + +# Returns: +# torch.Tensor: list of dataset indices to use in one epoch. Each index +# is repeated based on its calculated repeat factor. +# """ +# # Since repeat factors are fractional, we use stochastic rounding so +# # that the target repeat factor is achieved in expectation over the +# # course of training +# rands = torch.rand(len(self._frac_part), generator=generator) +# rep_factors = self._int_part + (rands < self._frac_part).float() +# # Construct a list of indices in which we repeat images as specified +# indices = [] +# for dataset_index, rep_factor in enumerate(rep_factors): +# indices.extend([dataset_index] * int(rep_factor.item())) +# return torch.tensor(indices, dtype=torch.int64) + +# def __iter__(self): +# start = self._rank +# yield from itertools.islice(self._infinite_indices(), start, None, self._world_size) + +# def _infinite_indices(self): +# g = torch.Generator() +# g.manual_seed(self._seed) +# while True: +# # Sample indices with repeats determined by stochastic rounding; each +# # "epoch" may have a slightly different size due to the rounding. +# indices = self._get_epoch_indices(g) +# if self._shuffle: +# randperm = torch.randperm(len(indices), generator=g) +# yield from indices[randperm].tolist() +# else: +# yield from indices.tolist() + + +class MultiDatasetAspectRatioGroupedDataset(torch.utils.data.IterableDataset): + """ + Batch data that have similar aspect ratio together. + In this implementation, images whose aspect ratio < (or >) 1 will + be batched together. + This improves training speed because the images then need less padding + to form a batch. + + It assumes the underlying dataset produces dicts with "width" and "height" keys. + It will then produce a list of original dicts with length = batch_size, + all with similar aspect ratios. + """ + + def __init__(self, dataset, batch_size, num_datasets): + """ + Args: + dataset: an iterable. Each element must be a dict with keys + "width" and "height", which will be used to batch data. + batch_size (int): + """ + self.dataset = dataset + self.batch_size = batch_size + self._buckets = [[] for _ in range(2 * num_datasets)] + # Hard-coded two aspect ratio groups: w > h and w < h. + # Can add support for more aspect ratio groups, but doesn't seem useful + + def __iter__(self): + for d in self.dataset: + w, h = d["width"], d["height"] + bucket_id = 0 if w > h else 1 + bucket_id = d["dataset_id"] * 2 + bucket_id + bucket = self._buckets[bucket_id] + bucket.append(d) + if len(bucket) == self.batch_size[d["dataset_id"]]: + data = bucket[:] + # Clear bucket first, because code after yield is not + # guaranteed to execute + del bucket[:] + yield data diff --git a/approach/ovod/APE/ape/data/common_copypaste.py b/approach/ovod/APE/ape/data/common_copypaste.py new file mode 100644 index 0000000000000000000000000000000000000000..57f85bba12f8fb6e774b7c008ce1d925e9a9728d --- /dev/null +++ b/approach/ovod/APE/ape/data/common_copypaste.py @@ -0,0 +1,83 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import random + +import numpy as np +import torch.utils.data as data + +from detectron2.data.common import _MapIterableDataset +from detectron2.utils.serialize import PicklableWrapper + +__all__ = ["MapDataset_coppaste"] + + +class MapDataset_coppaste(data.Dataset): + """ + Map a function over the elements in a dataset. + """ + + def __init__(self, dataset, map_func, dataset_bg, sampler_bg): + """ + Args: + dataset: a dataset where map function is applied. Can be either + map-style or iterable dataset. When given an iterable dataset, + the returned object will also be an iterable dataset. + map_func: a callable which maps the element in dataset. map_func can + return None to skip the data (e.g. in case of errors). + How None is handled depends on the style of `dataset`. + If `dataset` is map-style, it randomly tries other elements. + If `dataset` is iterable, it skips the data and tries the next. + """ + self._dataset = dataset + self._map_func = PicklableWrapper(map_func) # wrap so that a lambda will work + + self._rng = random.Random(42) + self._fallback_candidates = set(range(len(dataset))) + + self._dataset_bg = dataset_bg + self._sampler_bg = sampler_bg + self._sampler_bg_iter = None + + def __new__(cls, dataset, map_func, dataset_bg, sampler_bg): + is_iterable = isinstance(dataset, data.IterableDataset) + if is_iterable: + assert 0 + return _MapIterableDataset(dataset, map_func) + else: + return super().__new__(cls) + + def __getnewargs__(self): + return self._dataset, self._map_func, self._dataset_bg, self._sampler_bg + + def __len__(self): + return len(self._dataset) + + def __getitem__(self, idx): + retry_count = 0 + cur_idx = int(idx) + + if self._sampler_bg_iter: + pass + else: + self._sampler_bg._seed = np.random.randint(2**31) + self._sampler_bg_iter = iter(self._sampler_bg) + + while True: + cur_idx_bg = next(self._sampler_bg_iter) + data = self._map_func(self._dataset[cur_idx], self._dataset_bg[cur_idx_bg]) + if data is not None: + self._fallback_candidates.add(cur_idx) + return data + + # _map_func fails for this idx, use a random new index from the pool + retry_count += 1 + self._fallback_candidates.discard(cur_idx) + cur_idx = self._rng.sample(self._fallback_candidates, k=1)[0] + + if retry_count >= 3: + logger = logging.getLogger(__name__) + logger.warning( + "Failed to apply `_map_func` for idx: {}, retry count: {}".format( + idx, retry_count + ) + ) diff --git a/approach/ovod/APE/ape/data/dataset_mapper.py b/approach/ovod/APE/ape/data/dataset_mapper.py new file mode 100644 index 0000000000000000000000000000000000000000..d92a5b4d262f72218c8b7a1226bd386e77699a80 --- /dev/null +++ b/approach/ovod/APE/ape/data/dataset_mapper.py @@ -0,0 +1,41 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging + +from detectron2.data import detection_utils as utils +from detectron2.data import transforms as T +from detectron2.data.dataset_mapper import DatasetMapper as DatasetMapper_d2 + +from . import detection_utils as utils_sota + +""" +This file contains the default mapping that's applied to "dataset dicts". +""" + +__all__ = ["DatasetMapper_ape"] + + +class DatasetMapper_ape(DatasetMapper_d2): + """ + A callable which takes a dataset dict in Detectron2 Dataset format, + and map it into a format used by the model. + + This is the default callable to be used to map your dataset dict into training data. + You may need to follow it to implement your own one for customized logic, + such as a different way to read or transform images. + See :doc:`/tutorials/data_loading` for details. + + The callable currently does the following: + + 1. Read the image from "file_name" + 2. Applies cropping/geometric transforms to the image and annotations + 3. Prepare data and annotations to Tensor and :class:`Instances` + """ + + def __init__(self, cfg, is_train: bool = True): + super().__init__(cfg, is_train) + augmentations = utils_sota.build_augmentation(cfg, is_train) + self.augmentations = T.AugmentationList(augmentations) + + logger = logging.getLogger(__name__) + mode = "training" if is_train else "inference" + logger.info(f"[DatasetMapper] Augmentations used in {mode}: {augmentations}") diff --git a/approach/ovod/APE/ape/data/dataset_mapper_copypaste.py b/approach/ovod/APE/ape/data/dataset_mapper_copypaste.py new file mode 100644 index 0000000000000000000000000000000000000000..4f862b9ad8d6b25c8a61771e2224e15017ecd1a0 --- /dev/null +++ b/approach/ovod/APE/ape/data/dataset_mapper_copypaste.py @@ -0,0 +1,499 @@ +import copy +import logging +import os +import random +from typing import List, Optional, Union + +import cv2 +import numpy as np +import torch + +import detectron2.utils.comm as comm +from detectron2.config import configurable +from detectron2.data import MetadataCatalog +from detectron2.data import detection_utils as utils +from detectron2.data import transforms as T +from detectron2.data.dataset_mapper import DatasetMapper as DatasetMapper_d2 +from detectron2.data.detection_utils import convert_image_to_rgb +from detectron2.structures import BitMasks, Boxes, Instances + +from . import detection_utils as utils_sota +from . import mapper_utils + +""" +This file contains the default mapping that's applied to "dataset dicts". +""" + +__all__ = ["DatasetMapper_copypaste"] + + +class DatasetMapper_copypaste(DatasetMapper_d2): + """ + A callable which takes a dataset dict in Detectron2 Dataset format, + and map it into a format used by the model. + + This is the default callable to be used to map your dataset dict into training data. + You may need to follow it to implement your own one for customized logic, + such as a different way to read or transform images. + See :doc:`/tutorials/data_loading` for details. + + The callable currently does the following: + + 1. Read the image from "file_name" + 2. Applies cropping/geometric transforms to the image and annotations + 3. Prepare data and annotations to Tensor and :class:`Instances` + """ + + @configurable + def __init__( + self, + is_train: bool, + *, + augmentations: List[Union[T.Augmentation, T.Transform]], + augmentations_d2: List[Union[T.Augmentation, T.Transform]], + augmentations_aa: List[Union[T.Augmentation, T.Transform]], + augmentations_lsj: List[Union[T.Augmentation, T.Transform]], + augmentations_type: List[str], + image_format: str, + use_instance_mask: bool = False, + use_keypoint: bool = False, + instance_mask_format: str = "polygon", + keypoint_hflip_indices: Optional[np.ndarray] = None, + precomputed_proposal_topk: Optional[int] = None, + recompute_boxes: bool = False, + copypaste_prob: float = 0.5, + output_dir: str = None, + vis_period: int = 0, + dataset_names: tuple = (), + ): + """ + NOTE: this interface is experimental. + + Args: + is_train: whether it's used in training or inference + augmentations: a list of augmentations or deterministic transforms to apply + image_format: an image format supported by :func:`detection_utils.read_image`. + use_instance_mask: whether to process instance segmentation annotations, if available + use_keypoint: whether to process keypoint annotations if available + instance_mask_format: one of "polygon" or "bitmask". Process instance segmentation + masks into this format. + keypoint_hflip_indices: see :func:`detection_utils.create_keypoint_hflip_indices` + precomputed_proposal_topk: if given, will load pre-computed + proposals from dataset_dict and keep the top k proposals for each image. + recompute_boxes: whether to overwrite bounding box annotations + by computing tight bounding boxes from instance mask annotations. + """ + if recompute_boxes: + assert use_instance_mask, "recompute_boxes requires instance masks" + # fmt: off + self.is_train = is_train + self.augmentations = T.AugmentationList(augmentations) + self.augmentations_d2 = T.AugmentationList(augmentations_d2) + self.augmentations_aa = T.AugmentationList(augmentations_aa) + self.augmentations_lsj = T.AugmentationList(augmentations_lsj) + self.augmentations_type = augmentations_type + self.image_format = image_format + self.use_instance_mask = use_instance_mask + self.instance_mask_format = instance_mask_format + self.use_keypoint = use_keypoint + self.keypoint_hflip_indices = keypoint_hflip_indices + self.proposal_topk = precomputed_proposal_topk + self.recompute_boxes = recompute_boxes + # fmt: on + logger = logging.getLogger(__name__) + mode = "training" if is_train else "inference" + logger.info(f"[DatasetMapper] Augmentations used in {mode}: {augmentations}") + logger.info(f"[DatasetMapper] D2 Augmentations D2 used in {mode}: {augmentations_d2}") + logger.info(f"[DatasetMapper] AA Augmentations used in {mode}: {augmentations_aa}") + logger.info(f"[DatasetMapper] LSJ Augmentations used in {mode}: {augmentations_lsj}") + logger.info(f"[DatasetMapper] Type Augmentations used in {mode}: {augmentations_type}") + + if output_dir is not None: + self.output_dir = os.path.join(output_dir, "vis_mapper") + os.makedirs(self.output_dir, exist_ok=True) + + self.copypaste_prob = copypaste_prob + self.vis_period = vis_period + self.iter = 0 + self.dataset_names = dataset_names + + self.metatada_list = [] + for dataset_name in self.dataset_names: + metadata = MetadataCatalog.get(dataset_name) + self.metatada_list.append(metadata) + + @classmethod + def from_config(cls, cfg, is_train: bool = True): + augs = utils_sota.build_augmentation(cfg, is_train) + augs_d2 = utils.build_augmentation(cfg, is_train) + augs_aa = utils_sota.build_augmentation_aa(cfg, is_train) + augs_lsj = utils_sota.build_augmentation_lsj(cfg, is_train) + if cfg.INPUT.CROP.ENABLED and is_train: + raise NotImplementedError("cfg.INPUT.CROP.ENABLED is not supported yet") + augs.insert(0, T.RandomCrop(cfg.INPUT.CROP.TYPE, cfg.INPUT.CROP.SIZE)) + recompute_boxes = cfg.MODEL.MASK_ON + else: + recompute_boxes = False + + if cfg.INPUT.MASK_FORMAT == "polygon": + logger = logging.getLogger(__name__) + logger.warning("Using polygon is slow, use bitmask instead") + if cfg.INPUT.MASK_FORMAT == "bitmask": + logger = logging.getLogger(__name__) + logger.warning("Using bitmask may has bug, use polygon instead") + assert ( + cfg.INPUT.SEG_PAD_VALUE == 0 + ), "PadTransform should pad bitmask with value 0. Please setting cfg.INPUT.SEG_PAD_VALUE to 0. \nNoted that cfg.INPUT.SEG_PAD_VALUE is also used to pad semantic segmentation. If semantic segmentation is used, Please set cfg.INPUT.FORMAT to polygon." + + ret = { + "is_train": is_train, + "augmentations": augs, + "augmentations_d2": augs_d2, + "augmentations_aa": augs_aa, + "augmentations_lsj": augs_lsj, + "augmentations_type": cfg.INPUT.AUGMENT_TYPE, + "image_format": cfg.INPUT.FORMAT, + "use_instance_mask": cfg.MODEL.MASK_ON, + "instance_mask_format": cfg.INPUT.MASK_FORMAT, + "use_keypoint": cfg.MODEL.KEYPOINT_ON, + "recompute_boxes": recompute_boxes, + "output_dir": cfg.OUTPUT_DIR, + "copypaste_prob": cfg.DATASETS.COPYPASTE.PROB, + "vis_period": cfg.VIS_PERIOD, + "dataset_names": cfg.DATASETS.TRAIN, + } + + if cfg.MODEL.KEYPOINT_ON: + ret["keypoint_hflip_indices"] = utils.create_keypoint_hflip_indices(cfg.DATASETS.TRAIN) + + if cfg.MODEL.LOAD_PROPOSALS: + ret["precomputed_proposal_topk"] = ( + cfg.DATASETS.PRECOMPUTED_PROPOSAL_TOPK_TRAIN + if is_train + else cfg.DATASETS.PRECOMPUTED_PROPOSAL_TOPK_TEST + ) + return ret + + def _transform_annotations(self, dataset_dict, transforms, image_shape): + # USER: Modify this if you want to keep them for some reason. + for anno in dataset_dict["annotations"]: + if not self.use_instance_mask: + anno.pop("segmentation", None) + if not self.use_keypoint: + anno.pop("keypoints", None) + + copypaste = [ + obj.get("copypaste", 0) + for obj in dataset_dict["annotations"] + if obj.get("iscrowd", 0) == 0 + ] + + phrases = [ + obj.get("phrase", "") + for obj in dataset_dict["annotations"] + if obj.get("iscrowd", 0) == 0 + ] + + # USER: Implement additional transformations if you have other types of data + annos = [ + utils.transform_instance_annotations( + obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices + ) + for obj in dataset_dict.pop("annotations") + if obj.get("iscrowd", 0) == 0 + ] + instances = utils.annotations_to_instances( + annos, image_shape, mask_format=self.instance_mask_format + ) + + instances.copypaste = torch.tensor(copypaste) + + if sum([len(x) for x in phrases]) > 0: + instances.phrase_idxs = torch.tensor(range(len(phrases))) + + # After transforms such as cropping are applied, the bounding box may no longer + # tightly bound the object. As an example, imagine a triangle object + # [(0,0), (2,0), (0,2)] cropped by a box [(1,0),(2,2)] (XYXY format). The tight + # bounding box of the cropped triangle should be [(1,0),(2,1)], which is not equal to + # the intersection of original bounding box and the cropping box. + if self.recompute_boxes and instances.has("gt_masks"): + instances.gt_boxes = instances.gt_masks.get_bounding_boxes() + dataset_dict["instances"] = utils.filter_empty_instances(instances, box_threshold=10) + + if sum([len(x) for x in phrases]) > 0: + phrases_filtered = [] + for x in dataset_dict["instances"].phrase_idxs.tolist(): + phrases_filtered.append(phrases[x]) + dataset_dict["instances"].phrases = mapper_utils.transform_phrases( + phrases_filtered, transforms + ) + dataset_dict["instances"].remove("phrase_idxs") + # dataset_dict["instances"].gt_classes = torch.tensor(range(len(phrases_filtered))) + + def __call__(self, dataset_dict, dataset_dict_bg): + """ + Args: + dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. + + Returns: + dict: a format that builtin models in detectron2 accept + """ + dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below + # USER: Write your own image loading if it's not from a file + try: + image = utils.read_image(dataset_dict["file_name"], format=self.image_format) + except Exception as e: + logger = logging.getLogger(__name__) + logger.error(f"read_image fails: {dataset_dict['file_name']}") + logger.error(f"read_image fails: {e}") + return None + utils.check_image_size(dataset_dict, image) + + # ------------------------------------------------------------------------------------ + if ( + self.is_train + and "annotations" in dataset_dict + and ( + len(dataset_dict["annotations"]) == 0 + or any(["bbox" not in anno for anno in dataset_dict["annotations"]]) + ) + ): + if "dataset_id" in dataset_dict: + dataset_id = dataset_dict["dataset_id"] + else: + dataset_id = 0 + metadata = self.metatada_list[dataset_id] + if "sa1b" in self.dataset_names[dataset_id]: + metadata = None + dataset_dict = mapper_utils.maybe_load_annotation_from_file(dataset_dict, meta=metadata) + + for anno in dataset_dict["annotations"]: + if "bbox" not in anno: + logger = logging.getLogger(__name__) + logger.warning(f"Box not found: {dataset_dict}") + return None + if "category_id" not in anno: + anno["category_id"] = 0 + # ------------------------------------------------------------------------------------ + + # ------------------------------------------------------------------------------------ + if dataset_dict["copypaste"] and self.copypaste_prob > random.uniform(0, 1): + image_cp, dataset_dict_cp = mapper_utils.copypaste( + dataset_dict, dataset_dict_bg, self.image_format, self.instance_mask_format + ) + + if dataset_dict_cp is None or image_cp is None: + pass + else: + for key in dataset_dict.keys(): + if key in dataset_dict_cp: + continue + dataset_dict_cp[key] = dataset_dict[key] + dataset_dict = dataset_dict_cp + image = image_cp + # ------------------------------------------------------------------------------------ + + # USER: Remove if you don't do semantic/panoptic segmentation. + if "sem_seg_file_name" in dataset_dict: + try: + sem_seg_gt = utils.read_image(dataset_dict.pop("sem_seg_file_name"), "L").squeeze(2) + except Exception as e: + logger = logging.getLogger(__name__) + logger.error(f"read_image fails: {e}") + logger.error(f"read_image fails: {dataset_dict}") + return None + + if "copypaste_mask" in dataset_dict: + # assume thing class is 0 + sem_seg_gt = sem_seg_gt.copy() + sem_seg_gt[dataset_dict["copypaste_mask"]] = 0 + else: + sem_seg_gt = None + + aug_input = T.AugInput(image, sem_seg=sem_seg_gt) + try: + if "dataset_id" not in dataset_dict or dataset_dict["dataset_id"] >= len( + self.augmentations_type + ): + transforms = self.augmentations(aug_input) + elif self.augmentations_type[dataset_dict["dataset_id"]] == "D2": + transforms = self.augmentations_d2(aug_input) + elif self.augmentations_type[dataset_dict["dataset_id"]] == "AA": + transforms = self.augmentations_aa(aug_input) + elif self.augmentations_type[dataset_dict["dataset_id"]] == "LSJ": + transforms = self.augmentations_lsj(aug_input) + else: + print("fall back to default augmentation") + transforms = self.augmentations(aug_input) + image, sem_seg_gt = aug_input.image, aug_input.sem_seg + except Exception as e: + logger = logging.getLogger(__name__) + logger.error(f"augment fails: {dataset_dict['file_name']}") + logger.error(f"augment fails: {e}") + return None + + image_shape = image.shape[:2] # h, w + # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, + # but not efficient on large generic data structures due to the use of pickle & mp.Queue. + # Therefore it's important to use torch.Tensor. + dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1))) + if sem_seg_gt is not None: + dataset_dict["sem_seg"] = torch.as_tensor(sem_seg_gt.astype("long")) + + # USER: Remove if you don't use pre-computed proposals. + # Most users would not need this feature. + if self.proposal_topk is not None: + utils.transform_proposals( + dataset_dict, image_shape, transforms, proposal_topk=self.proposal_topk + ) + + if not self.is_train: + # USER: Modify this if you want to keep them for some reason. + dataset_dict.pop("annotations", None) + dataset_dict.pop("sem_seg_file_name", None) + return dataset_dict + + # seperate box and region + if "annotations" in dataset_dict: + annotations = [] + annotations_phrase = [] + for ann in dataset_dict.pop("annotations"): + if ann.get("isobject", 1) == 0: + annotations_phrase.append(ann) + else: + annotations.append(ann) + if len(annotations_phrase) > 0: + dataset_dict["annotations"] = annotations_phrase + self._transform_annotations(dataset_dict, transforms, image_shape) + dataset_dict["instances_phrase"] = dataset_dict.pop("instances") + dataset_dict["annotations"] = annotations + + if "annotations" in dataset_dict: + self._transform_annotations(dataset_dict, transforms, image_shape) + + # ------------------------------------------------------------------------------------ + if self.vis_period > 0 and self.iter % self.vis_period == 0: + self.visualize_training(dataset_dict) + # ------------------------------------------------------------------------------------ + self.iter += 1 + + return dataset_dict + + def visualize_training(self, dataset_dict, prefix="", suffix=""): + if self.output_dir is None: + return + if dataset_dict is None: + return + # if "instances" not in dataset_dict: + # return + from detectron2.utils.visualizer import Visualizer + from detectron2.data import MetadataCatalog + + if "dataset_id" in dataset_dict: + dataset_id = dataset_dict["dataset_id"] + else: + dataset_id = 0 + dataset_name = self.dataset_names[dataset_id] + metadata = MetadataCatalog.get(dataset_name) + class_names = metadata.get( + "thing_classes", + [ + "thing", + ], + ) + + img = dataset_dict["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.image_format) + image_shape = img.shape[:2] # h, w + vis = Visualizer(img, metadata=metadata) + if "instances" in dataset_dict: + vis = vis.overlay_instances( + boxes=dataset_dict["instances"].gt_boxes, + masks=dataset_dict["instances"].gt_masks + if dataset_dict["instances"].has("gt_masks") + else None, + labels=[class_names[i] for i in dataset_dict["instances"].gt_classes], + ) + else: + vis = vis.overlay_instances( + boxes=None, + masks=None, + labels=None, + ) + vis_gt = vis.get_image() + + if "instances_phrase" in dataset_dict: + vis = Visualizer(img, metadata=metadata) + vis = vis.overlay_instances( + boxes=dataset_dict["instances_phrase"].gt_boxes, + masks=dataset_dict["instances_phrase"].gt_masks + if dataset_dict["instances_phrase"].has("gt_masks") + else None, + labels=dataset_dict["instances_phrase"].phrases, + ) + vis_phrase = vis.get_image() + vis_gt = np.concatenate((vis_gt, vis_phrase), axis=1) + + if "captions" in dataset_dict: + vis = Visualizer(img, metadata=metadata) + vis = vis.overlay_instances( + boxes=Boxes( + np.array( + [ + [ + 0 + i * 20, + 0 + i * 20, + image_shape[1] - 1 - i * 20, + image_shape[0] - 1 - i * 20, + ] + for i in range(len(dataset_dict["captions"])) + ] + ) + ), + masks=None, + labels=dataset_dict["captions"], + ) + vis_cap = vis.get_image() + vis_gt = np.concatenate((vis_gt, vis_cap), axis=1) + + if "sem_seg" in dataset_dict: + vis = Visualizer(img, metadata=metadata) + vis = vis.draw_sem_seg(dataset_dict["sem_seg"], area_threshold=0, alpha=0.5) + vis_sem_gt = vis.get_image() + vis_gt = np.concatenate((vis_gt, vis_sem_gt), axis=1) + + concat = np.concatenate((vis_gt, img), axis=1) + + image_name = os.path.basename(dataset_dict["file_name"]).split(".")[0] + + save_path = os.path.join( + self.output_dir, + prefix + + str(self.iter) + + "_" + + image_name + + "_g" + + str(comm.get_rank()) + + suffix + + ".png", + ) + concat = cv2.cvtColor(concat, cv2.COLOR_RGB2BGR) + cv2.imwrite(save_path, concat) + + return + + import pickle + + save_path = os.path.join( + self.output_dir, + prefix + + str(self.iter) + + "_" + + str(dataset_dict["image_id"]) + + "_g" + + str(comm.get_rank()) + + suffix + + ".pkl", + ) + with open(save_path, "wb") as save_file: + pickle.dump(dataset_dict, save_file) diff --git a/approach/ovod/APE/ape/data/dataset_mapper_detr_instance.py b/approach/ovod/APE/ape/data/dataset_mapper_detr_instance.py new file mode 100644 index 0000000000000000000000000000000000000000..0096a0c7b2b8455343c5b47c5633c34f226a34ba --- /dev/null +++ b/approach/ovod/APE/ape/data/dataset_mapper_detr_instance.py @@ -0,0 +1,288 @@ +import copy +import logging +from typing import List, Optional, Union + +import numpy as np +import torch + +from detectron2.config import configurable +from detectron2.data import MetadataCatalog +from detectron2.data import detection_utils as utils +from detectron2.data import transforms as T +from detectron2.layers import batched_nms + +from . import mapper_utils + +""" +This file contains the default mapping that's applied to "dataset dicts". +""" + +__all__ = ["DatasetMapper_detr_instance"] + + +class DatasetMapper_detr_instance: + """ + A callable which takes a dataset dict in Detectron2 Dataset format, + and map it into a format used by the model. + + This is the default callable to be used to map your dataset dict into training data. + You may need to follow it to implement your own one for customized logic, + such as a different way to read or transform images. + See :doc:`/tutorials/data_loading` for details. + + The callable currently does the following: + + 1. Read the image from "file_name" + 2. Applies cropping/geometric transforms to the image and annotations + 3. Prepare data and annotations to Tensor and :class:`Instances` + """ + + @configurable + def __init__( + self, + is_train: bool, + *, + augmentations: List[Union[T.Augmentation, T.Transform]], + augmentations_with_crop: List[Union[T.Augmentation, T.Transform]], + image_format: str, + use_instance_mask: bool = False, + use_keypoint: bool = False, + instance_mask_format: str = "polygon", + keypoint_hflip_indices: Optional[np.ndarray] = None, + precomputed_proposal_topk: Optional[int] = None, + recompute_boxes: bool = False, + dataset_names: tuple = (), + max_num_phrase: int = 0, + nms_thresh_phrase: float = 0.0, + ): + """ + NOTE: this interface is experimental. + + Args: + is_train: whether it's used in training or inference + augmentations: a list of augmentations or deterministic transforms to apply + image_format: an image format supported by :func:`detection_utils.read_image`. + use_instance_mask: whether to process instance segmentation annotations, if available + use_keypoint: whether to process keypoint annotations if available + instance_mask_format: one of "polygon" or "bitmask". Process instance segmentation + masks into this format. + keypoint_hflip_indices: see :func:`detection_utils.create_keypoint_hflip_indices` + precomputed_proposal_topk: if given, will load pre-computed + proposals from dataset_dict and keep the top k proposals for each image. + recompute_boxes: whether to overwrite bounding box annotations + by computing tight bounding boxes from instance mask annotations. + """ + if recompute_boxes: + assert use_instance_mask, "recompute_boxes requires instance masks" + # fmt: off + self.is_train = is_train + self.augmentations = T.AugmentationList(augmentations) + self.augmentations_with_crop = T.AugmentationList(augmentations_with_crop) + self.image_format = image_format + self.use_instance_mask = use_instance_mask + self.instance_mask_format = instance_mask_format + self.use_keypoint = use_keypoint + self.keypoint_hflip_indices = keypoint_hflip_indices + self.proposal_topk = precomputed_proposal_topk + self.recompute_boxes = recompute_boxes + # fmt: on + logger = logging.getLogger(__name__) + mode = "training" if is_train else "inference" + logger.info(f"[DatasetMapper] Augmentations used in {mode}: {augmentations}") + logger.info(f"[DatasetMapper] Augmentations used in {mode}: {augmentations_with_crop}") + + self.dataset_names = dataset_names + + self.metatada_list = [] + for dataset_name in self.dataset_names: + metadata = MetadataCatalog.get(dataset_name) + self.metatada_list.append(metadata) + + self.max_num_phrase = max_num_phrase + self.nms_thresh_phrase = nms_thresh_phrase + + @classmethod + def from_config(cls, cfg, is_train: bool = True): + raise NotImplementedError(self.__class__.__name__) + + def _transform_annotations(self, dataset_dict, transforms, image_shape): + # USER: Modify this if you want to keep them for some reason. + for anno in dataset_dict["annotations"]: + if not self.use_instance_mask: + anno.pop("segmentation", None) + if not self.use_keypoint: + anno.pop("keypoints", None) + + phrases = [ + obj.get("phrase", "") + for obj in dataset_dict["annotations"] + if obj.get("iscrowd", 0) == 0 + ] + + # USER: Implement additional transformations if you have other types of data + annos = [ + utils.transform_instance_annotations( + obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices + ) + for obj in dataset_dict.pop("annotations") + if obj.get("iscrowd", 0) == 0 + ] + instances = utils.annotations_to_instances( + annos, image_shape, mask_format=self.instance_mask_format + ) + + if sum([len(x) for x in phrases]) > 0: + instances.phrase_idxs = torch.tensor(range(len(phrases))) + + # After transforms such as cropping are applied, the bounding box may no longer + # tightly bound the object. As an example, imagine a triangle object + # [(0,0), (2,0), (0,2)] cropped by a box [(1,0),(2,2)] (XYXY format). The tight + # bounding box of the cropped triangle should be [(1,0),(2,1)], which is not equal to + # the intersection of original bounding box and the cropping box. + if self.recompute_boxes and instances.has("gt_masks"): + instances.gt_boxes = instances.gt_masks.get_bounding_boxes() + dataset_dict["instances"] = utils.filter_empty_instances(instances) + + if sum([len(x) for x in phrases]) > 0: + phrases_filtered = [] + for x in dataset_dict["instances"].phrase_idxs.tolist(): + phrases_filtered.append(phrases[x]) + dataset_dict["instances"].phrases = mapper_utils.transform_phrases( + phrases_filtered, transforms + ) + dataset_dict["instances"].remove("phrase_idxs") + # dataset_dict["instances"].gt_classes = torch.tensor(range(len(phrases_filtered))) + + def __call__(self, dataset_dict): + """ + Args: + dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. + + Returns: + dict: a format that builtin models in detectron2 accept + """ + dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below + # USER: Write your own image loading if it's not from a file + try: + image = utils.read_image(dataset_dict["file_name"], format=self.image_format) + dataset_dict["width"] = image.shape[1] + dataset_dict["height"] = image.shape[0] + except Exception as e: + logger = logging.getLogger(__name__) + logger.error(f"read_image fails: {dataset_dict['file_name']}") + logger.error(f"read_image fails: {e}") + return None + utils.check_image_size(dataset_dict, image) + + # ------------------------------------------------------------------------------------ + if ( + self.is_train + and "annotations" in dataset_dict + and ( + len(dataset_dict["annotations"]) == 0 + or any(["bbox" not in anno for anno in dataset_dict["annotations"]]) + ) + ): + if "dataset_id" in dataset_dict: + dataset_id = dataset_dict["dataset_id"] + else: + dataset_id = 0 + metadata = self.metatada_list[dataset_id] + if "sa1b" in self.dataset_names[dataset_id]: + metadata = None + dataset_dict = mapper_utils.maybe_load_annotation_from_file(dataset_dict, meta=metadata) + + for anno in dataset_dict["annotations"]: + if "bbox" not in anno: + logger = logging.getLogger(__name__) + logger.warning(f"Box not found: {dataset_dict}") + return None + if "category_id" not in anno: + anno["category_id"] = 0 + # ------------------------------------------------------------------------------------ + + # USER: Remove if you don't do semantic/panoptic segmentation. + if "sem_seg_file_name" in dataset_dict: + sem_seg_gt = utils.read_image(dataset_dict.pop("sem_seg_file_name"), "L").squeeze(2) + else: + sem_seg_gt = None + + # ordinal numbers + disable_crop = False + if ( + "annotations" in dataset_dict + and len(dataset_dict["annotations"]) > 0 + and "phrase" in dataset_dict["annotations"][0] + ): + disable_crop = disable_crop or mapper_utils.has_ordinal_num( + [anno["phrase"] for anno in dataset_dict["annotations"]] + ) + if "expressions" in dataset_dict: + disable_crop = disable_crop or mapper_utils.has_ordinal_num(dataset_dict["expressions"]) + + if self.augmentations_with_crop is None or disable_crop: + augmentations = self.augmentations + else: + if np.random.rand() > 0.5: + augmentations = self.augmentations + else: + augmentations = self.augmentations_with_crop + + aug_input = T.AugInput(image, sem_seg=sem_seg_gt) + # transforms = self.augmentations(aug_input) + transforms = augmentations(aug_input) + image, sem_seg_gt = aug_input.image, aug_input.sem_seg + + image_shape = image.shape[:2] # h, w + # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, + # but not efficient on large generic data structures due to the use of pickle & mp.Queue. + # Therefore it's important to use torch.Tensor. + dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1))) + if sem_seg_gt is not None: + dataset_dict["sem_seg"] = torch.as_tensor(sem_seg_gt.astype("long")) + + # USER: Remove if you don't use pre-computed proposals. + # Most users would not need this feature. + if self.proposal_topk is not None: + utils.transform_proposals( + dataset_dict, image_shape, transforms, proposal_topk=self.proposal_topk + ) + + if "expressions" in dataset_dict: + dataset_dict["expressions"] = mapper_utils.transform_expressions( + dataset_dict["expressions"], transforms + ) + + if not self.is_train: + # USER: Modify this if you want to keep them for some reason. + dataset_dict.pop("annotations", None) + dataset_dict.pop("sem_seg_file_name", None) + return dataset_dict + + if "annotations" in dataset_dict: + self._transform_annotations(dataset_dict, transforms, image_shape) + + if "instances" in dataset_dict and dataset_dict["instances"].has("phrases"): + num_instances = len(dataset_dict["instances"]) + + if self.nms_thresh_phrase > 0: + boxes = dataset_dict["instances"].gt_boxes.tensor + scores = torch.rand(num_instances) + classes = torch.zeros(num_instances) + keep = batched_nms(boxes, scores, classes, self.nms_thresh_phrase) + else: + keep = torch.randperm(num_instances) + + if self.max_num_phrase > 0: + keep = keep[: self.max_num_phrase] + + phrases = dataset_dict["instances"].phrases + phrases_filtered = [] + for x in keep: + phrases_filtered.append(phrases[x]) + + dataset_dict["instances"].remove("phrases") + dataset_dict["instances"] = dataset_dict["instances"][keep] + dataset_dict["instances"].phrases = phrases_filtered + + return dataset_dict diff --git a/approach/ovod/APE/ape/data/dataset_mapper_detr_instance_exp.py b/approach/ovod/APE/ape/data/dataset_mapper_detr_instance_exp.py new file mode 100644 index 0000000000000000000000000000000000000000..9ce0619d7ed1fc37525f7b89c7784f628585977a --- /dev/null +++ b/approach/ovod/APE/ape/data/dataset_mapper_detr_instance_exp.py @@ -0,0 +1,232 @@ +import copy +import logging +from typing import List, Optional, Union + +import numpy as np +import torch + +from detectron2.config import configurable +from detectron2.data import MetadataCatalog +from detectron2.data import detection_utils as utils +from detectron2.data import transforms as T + +from . import mapper_utils + +""" +This file contains the default mapping that's applied to "dataset dicts". +""" + +__all__ = ["DatasetMapper_detr_instance_exp"] + + +class DatasetMapper_detr_instance_exp: + """ + A callable which takes a dataset dict in Detectron2 Dataset format, + and map it into a format used by the model. + + This is the default callable to be used to map your dataset dict into training data. + You may need to follow it to implement your own one for customized logic, + such as a different way to read or transform images. + See :doc:`/tutorials/data_loading` for details. + + The callable currently does the following: + + 1. Read the image from "file_name" + 2. Applies cropping/geometric transforms to the image and annotations + 3. Prepare data and annotations to Tensor and :class:`Instances` + """ + + @configurable + def __init__( + self, + is_train: bool, + *, + augmentations: List[Union[T.Augmentation, T.Transform]], + augmentations_with_crop: List[Union[T.Augmentation, T.Transform]], + image_format: str, + use_instance_mask: bool = False, + use_keypoint: bool = False, + instance_mask_format: str = "polygon", + keypoint_hflip_indices: Optional[np.ndarray] = None, + precomputed_proposal_topk: Optional[int] = None, + recompute_boxes: bool = False, + dataset_names: tuple = (), + ): + """ + NOTE: this interface is experimental. + + Args: + is_train: whether it's used in training or inference + augmentations: a list of augmentations or deterministic transforms to apply + image_format: an image format supported by :func:`detection_utils.read_image`. + use_instance_mask: whether to process instance segmentation annotations, if available + use_keypoint: whether to process keypoint annotations if available + instance_mask_format: one of "polygon" or "bitmask". Process instance segmentation + masks into this format. + keypoint_hflip_indices: see :func:`detection_utils.create_keypoint_hflip_indices` + precomputed_proposal_topk: if given, will load pre-computed + proposals from dataset_dict and keep the top k proposals for each image. + recompute_boxes: whether to overwrite bounding box annotations + by computing tight bounding boxes from instance mask annotations. + """ + if recompute_boxes: + assert use_instance_mask, "recompute_boxes requires instance masks" + # fmt: off + self.is_train = is_train + self.augmentations = T.AugmentationList(augmentations) + self.augmentations_with_crop = T.AugmentationList(augmentations_with_crop) + self.image_format = image_format + self.use_instance_mask = use_instance_mask + self.instance_mask_format = instance_mask_format + self.use_keypoint = use_keypoint + self.keypoint_hflip_indices = keypoint_hflip_indices + self.proposal_topk = precomputed_proposal_topk + self.recompute_boxes = recompute_boxes + # fmt: on + logger = logging.getLogger(__name__) + mode = "training" if is_train else "inference" + logger.info(f"[DatasetMapper] Augmentations used in {mode}: {augmentations}") + logger.info(f"[DatasetMapper] Augmentations used in {mode}: {augmentations_with_crop}") + + self.dataset_names = dataset_names + + self.metatada_list = [] + for dataset_name in self.dataset_names: + metadata = MetadataCatalog.get(dataset_name) + self.metatada_list.append(metadata) + + @classmethod + def from_config(cls, cfg, is_train: bool = True): + raise NotImplementedError(self.__class__.__name__) + + def _transform_annotations(self, dataset_dict, transforms, image_shape): + # USER: Modify this if you want to keep them for some reason. + for anno in dataset_dict["annotations"]: + if not self.use_instance_mask: + anno.pop("segmentation", None) + if not self.use_keypoint: + anno.pop("keypoints", None) + + # USER: Implement additional transformations if you have other types of data + annos = [ + utils.transform_instance_annotations( + obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices + ) + for obj in dataset_dict.pop("annotations") + if obj.get("iscrowd", 0) == 0 + ] + instances = utils.annotations_to_instances( + annos, image_shape, mask_format=self.instance_mask_format + ) + + # After transforms such as cropping are applied, the bounding box may no longer + # tightly bound the object. As an example, imagine a triangle object + # [(0,0), (2,0), (0,2)] cropped by a box [(1,0),(2,2)] (XYXY format). The tight + # bounding box of the cropped triangle should be [(1,0),(2,1)], which is not equal to + # the intersection of original bounding box and the cropping box. + if self.recompute_boxes and instances.has("gt_masks"): + instances.gt_boxes = instances.gt_masks.get_bounding_boxes() + dataset_dict["instances"] = utils.filter_empty_instances(instances) + + def __call__(self, dataset_dict): + """ + Args: + dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. + + Returns: + dict: a format that builtin models in detectron2 accept + """ + dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below + # USER: Write your own image loading if it's not from a file + image = utils.read_image(dataset_dict["file_name"], format=self.image_format) + utils.check_image_size(dataset_dict, image) + + # ------------------------------------------------------------------------------------ + if ( + self.is_train + and "annotations" in dataset_dict + and ( + len(dataset_dict["annotations"]) == 0 + or any(["bbox" not in anno for anno in dataset_dict["annotations"]]) + ) + ): + if "dataset_id" in dataset_dict: + dataset_id = dataset_dict["dataset_id"] + else: + dataset_id = 0 + metadata = self.metatada_list[dataset_id] + if "sa1b" in self.dataset_names[dataset_id]: + metadata = None + dataset_dict = mapper_utils.maybe_load_annotation_from_file(dataset_dict, meta=metadata) + + for anno in dataset_dict["annotations"]: + if "bbox" not in anno: + logger = logging.getLogger(__name__) + logger.warning(f"Box not found: {dataset_dict}") + return None + if "category_id" not in anno: + anno["category_id"] = 0 + # ------------------------------------------------------------------------------------ + + # USER: Remove if you don't do semantic/panoptic segmentation. + if "sem_seg_file_name" in dataset_dict: + sem_seg_gt = utils.read_image(dataset_dict.pop("sem_seg_file_name"), "L").squeeze(2) + else: + sem_seg_gt = None + + # ordinal numbers + disable_crop = False + if ( + "annotations" in dataset_dict + and len(dataset_dict["annotations"]) > 0 + and "phrase" in dataset_dict["annotations"][0] + ): + disable_crop = disable_crop or mapper_utils.has_ordinal_num( + [anno["phrase"] for anno in dataset_dict["annotations"]] + ) + if "expressions" in dataset_dict: + disable_crop = disable_crop or mapper_utils.has_ordinal_num(dataset_dict["expressions"]) + + if self.augmentations_with_crop is None or disable_crop: + augmentations = self.augmentations + else: + if np.random.rand() > 0.5: + augmentations = self.augmentations + else: + augmentations = self.augmentations_with_crop + + aug_input = T.AugInput(image, sem_seg=sem_seg_gt) + # transforms = self.augmentations(aug_input) + transforms = augmentations(aug_input) + image, sem_seg_gt = aug_input.image, aug_input.sem_seg + + image_shape = image.shape[:2] # h, w + # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, + # but not efficient on large generic data structures due to the use of pickle & mp.Queue. + # Therefore it's important to use torch.Tensor. + dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1))) + if sem_seg_gt is not None: + dataset_dict["sem_seg"] = torch.as_tensor(sem_seg_gt.astype("long")) + + # USER: Remove if you don't use pre-computed proposals. + # Most users would not need this feature. + if self.proposal_topk is not None: + utils.transform_proposals( + dataset_dict, image_shape, transforms, proposal_topk=self.proposal_topk + ) + + if "expressions" in dataset_dict: + dataset_dict["expressions"] = mapper_utils.transform_expressions( + dataset_dict["expressions"], transforms + ) + + if not self.is_train: + # USER: Modify this if you want to keep them for some reason. + dataset_dict.pop("annotations", None) + dataset_dict.pop("sem_seg_file_name", None) + return dataset_dict + + if "annotations" in dataset_dict: + self._transform_annotations(dataset_dict, transforms, image_shape) + + return dataset_dict diff --git a/approach/ovod/APE/ape/data/dataset_mapper_detr_panoptic.py b/approach/ovod/APE/ape/data/dataset_mapper_detr_panoptic.py new file mode 100644 index 0000000000000000000000000000000000000000..b03b91de90ce320edc0a6bc3f61c7d8e0b0b15c5 --- /dev/null +++ b/approach/ovod/APE/ape/data/dataset_mapper_detr_panoptic.py @@ -0,0 +1,443 @@ +import copy +import logging +import re +from typing import List, Optional, Union + +import numpy as np +import torch + +from detectron2.config import configurable +from detectron2.data import MetadataCatalog +from detectron2.data import detection_utils as utils +from detectron2.data import transforms as T +from detectron2.structures import BitMasks, Boxes, Instances, PolygonMasks + +from . import mapper_utils + +""" +This file contains the default mapping that's applied to "dataset dicts". +""" + +__all__ = ["DatasetMapper_detr_panoptic"] + + +class DatasetMapper_detr_panoptic: + """ + A callable which takes a dataset dict in Detectron2 Dataset format, + and map it into a format used by the model. + + This is the default callable to be used to map your dataset dict into training data. + You may need to follow it to implement your own one for customized logic, + such as a different way to read or transform images. + See :doc:`/tutorials/data_loading` for details. + + The callable currently does the following: + + 1. Read the image from "file_name" + 2. Applies cropping/geometric transforms to the image and annotations + 3. Prepare data and annotations to Tensor and :class:`Instances` + """ + + @configurable + def __init__( + self, + is_train: bool, + *, + augmentations: List[Union[T.Augmentation, T.Transform]], + augmentations_with_crop: List[Union[T.Augmentation, T.Transform]], + image_format: str, + use_instance_mask: bool = False, + use_keypoint: bool = False, + instance_mask_format: str = "polygon", + keypoint_hflip_indices: Optional[np.ndarray] = None, + precomputed_proposal_topk: Optional[int] = None, + recompute_boxes: bool = False, + ignore_label: int = 255, + stuff_classes_offset: int = 80, + stuff_classes_decomposition: bool = False, + dataset_names: tuple = (), + ): + """ + NOTE: this interface is experimental. + + Args: + is_train: whether it's used in training or inference + augmentations: a list of augmentations or deterministic transforms to apply + image_format: an image format supported by :func:`detection_utils.read_image`. + use_instance_mask: whether to process instance segmentation annotations, if available + use_keypoint: whether to process keypoint annotations if available + instance_mask_format: one of "polygon" or "bitmask". Process instance segmentation + masks into this format. + keypoint_hflip_indices: see :func:`detection_utils.create_keypoint_hflip_indices` + precomputed_proposal_topk: if given, will load pre-computed + proposals from dataset_dict and keep the top k proposals for each image. + recompute_boxes: whether to overwrite bounding box annotations + by computing tight bounding boxes from instance mask annotations. + """ + if recompute_boxes: + assert use_instance_mask, "recompute_boxes requires instance masks" + # fmt: off + self.is_train = is_train + self.augmentations = T.AugmentationList(augmentations) + self.augmentations_with_crop = T.AugmentationList(augmentations_with_crop) + self.image_format = image_format + self.use_instance_mask = use_instance_mask + self.instance_mask_format = instance_mask_format + self.use_keypoint = use_keypoint + self.keypoint_hflip_indices = keypoint_hflip_indices + self.proposal_topk = precomputed_proposal_topk + self.recompute_boxes = recompute_boxes + self.ignore_label = ignore_label + self.stuff_classes_offset = stuff_classes_offset + self.stuff_classes_decomposition = stuff_classes_decomposition + # fmt: on + logger = logging.getLogger(__name__) + mode = "training" if is_train else "inference" + logger.info(f"[DatasetMapper] Augmentations used in {mode}: {augmentations}") + logger.info(f"[DatasetMapper] Augmentations used in {mode}: {augmentations_with_crop}") + + self.dataset_names = dataset_names + + self.metatada_list = [] + for dataset_name in self.dataset_names: + metadata = MetadataCatalog.get(dataset_name) + self.metatada_list.append(metadata) + + @classmethod + def from_config(cls, cfg, is_train: bool = True): + raise NotImplementedError(self.__class__.__name__) + + def _transform_annotations(self, dataset_dict, transforms, image_shape): + # USER: Modify this if you want to keep them for some reason. + for anno in dataset_dict["annotations"]: + if not self.use_instance_mask: + anno.pop("segmentation", None) + if not self.use_keypoint: + anno.pop("keypoints", None) + + # USER: Implement additional transformations if you have other types of data + annos = [ + utils.transform_instance_annotations( + obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices + ) + for obj in dataset_dict.pop("annotations") + if obj.get("iscrowd", 0) == 0 + ] + instances = utils.annotations_to_instances( + annos, image_shape, mask_format=self.instance_mask_format + ) + + # After transforms such as cropping are applied, the bounding box may no longer + # tightly bound the object. As an example, imagine a triangle object + # [(0,0), (2,0), (0,2)] cropped by a box [(1,0),(2,2)] (XYXY format). The tight + # bounding box of the cropped triangle should be [(1,0),(2,1)], which is not equal to + # the intersection of original bounding box and the cropping box. + if self.recompute_boxes and instances.has("gt_masks"): + instances.gt_boxes = instances.gt_masks.get_bounding_boxes() + dataset_dict["instances"] = utils.filter_empty_instances(instances) + + def __call__(self, dataset_dict): + """ + Args: + dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. + + Returns: + dict: a format that builtin models in detectron2 accept + """ + dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below + # USER: Write your own image loading if it's not from a file + image = utils.read_image(dataset_dict["file_name"], format=self.image_format) + utils.check_image_size(dataset_dict, image) + + # ------------------------------------------------------------------------------------ + if "dataset_id" in dataset_dict: + dataset_id = dataset_dict["dataset_id"] + else: + dataset_id = 0 + metadata = self.metatada_list[dataset_id] + if "sa1b" in self.dataset_names[dataset_id]: + metadata = None + if ( + self.is_train + and "annotations" in dataset_dict + and ( + len(dataset_dict["annotations"]) == 0 + or any(["bbox" not in anno for anno in dataset_dict["annotations"]]) + ) + ): + dataset_dict = mapper_utils.maybe_load_annotation_from_file(dataset_dict, meta=metadata) + + for anno in dataset_dict["annotations"]: + if "bbox" not in anno: + logger = logging.getLogger(__name__) + logger.warning(f"Box not found: {dataset_dict}") + return None + if "category_id" not in anno: + anno["category_id"] = 0 + # ------------------------------------------------------------------------------------ + + # USER: Remove if you don't do semantic/panoptic segmentation. + if "sem_seg_file_name" in dataset_dict: + sem_seg_gt = utils.read_image(dataset_dict.pop("sem_seg_file_name"), "L").squeeze(2) + else: + sem_seg_gt = None + + # ordinal numbers + disable_crop = False + if ( + "annotations" in dataset_dict + and len(dataset_dict["annotations"]) > 0 + and "phrase" in dataset_dict["annotations"][0] + ): + disable_crop = disable_crop or mapper_utils.has_ordinal_num( + [anno["phrase"] for anno in dataset_dict["annotations"]] + ) + if "expressions" in dataset_dict: + disable_crop = disable_crop or mapper_utils.has_ordinal_num(dataset_dict["expressions"]) + + if self.augmentations_with_crop is None or disable_crop: + augmentations = self.augmentations + else: + if np.random.rand() > 0.5: + augmentations = self.augmentations + else: + augmentations = self.augmentations_with_crop + + aug_input = T.AugInput(image, sem_seg=sem_seg_gt) + # transforms = self.augmentations(aug_input) + transforms = augmentations(aug_input) + image, sem_seg_gt = aug_input.image, aug_input.sem_seg + + image_shape = image.shape[:2] # h, w + # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, + # but not efficient on large generic data structures due to the use of pickle & mp.Queue. + # Therefore it's important to use torch.Tensor. + dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1))) + if sem_seg_gt is not None: + dataset_dict["sem_seg"] = torch.as_tensor(sem_seg_gt.astype("long")) + + # USER: Remove if you don't use pre-computed proposals. + # Most users would not need this feature. + if self.proposal_topk is not None: + utils.transform_proposals( + dataset_dict, image_shape, transforms, proposal_topk=self.proposal_topk + ) + + if "expressions" in dataset_dict: + dataset_dict["expressions"] = mapper_utils.transform_expressions( + dataset_dict["expressions"], transforms + ) + + if not self.is_train: + # USER: Modify this if you want to keep them for some reason. + dataset_dict.pop("annotations", None) + dataset_dict.pop("sem_seg_file_name", None) + dataset_dict.pop("pan_seg_file_name", None) + dataset_dict.pop("segments_info", None) + return dataset_dict + + if "annotations" in dataset_dict: + self._transform_annotations(dataset_dict, transforms, image_shape) + + dataset_dict["instances"].is_thing = torch.tensor( + [True for _ in range(len(dataset_dict["instances"]))], dtype=torch.bool + ) + + # Prepare per-category binary masks + if sem_seg_gt is not None and not self.stuff_classes_decomposition: + instances = Instances(image_shape) + classes = np.unique(sem_seg_gt).astype(np.int64) + # remove ignored region + classes = classes[classes != self.ignore_label] + + if self.stuff_classes_offset > 0: + classes = classes[classes != 0] + instances.gt_classes = torch.tensor( + classes + self.stuff_classes_offset - 1, dtype=torch.int64 + ) + else: + instances.gt_classes = torch.tensor(classes, dtype=torch.int64) + + masks = [] + for class_id in classes: + masks.append(sem_seg_gt == class_id) + + if len(masks) == 0: + # # Some image does not have annotation (all ignored) + # instances.gt_masks = torch.zeros((0, sem_seg_gt.shape[-2], sem_seg_gt.shape[-1])) + masks = BitMasks(torch.zeros((0, sem_seg_gt.shape[-2], sem_seg_gt.shape[-1]))) + else: + masks = BitMasks( + torch.stack([torch.from_numpy(np.ascontiguousarray(x.copy())) for x in masks]) + ) + + instances.gt_masks = masks + instances.gt_boxes = masks.get_bounding_boxes() + + instances.is_thing = torch.tensor( + [False for _ in range(len(instances))], dtype=torch.bool + ) + + if "instances" in dataset_dict and dataset_dict["instances"].has("copypaste"): + instances.copypaste = torch.tensor([False for _ in range(len(instances))]) + + if len(instances) > 0: + if "instances" in dataset_dict and len(dataset_dict["instances"]) > 0: + dataset_dict["instances"] = Instances.cat( + [dataset_dict["instances"], instances] + ) + else: + dataset_dict["instances"] = instances + + # Prepare per-category binary masks + if sem_seg_gt is not None and self.stuff_classes_decomposition: + classes = np.unique(sem_seg_gt) + # remove ignored region + classes = classes[classes != self.ignore_label] + + if self.stuff_classes_offset > 0: + classes = classes[classes != 0] + + gt_masks = [] + gt_classes = [] + for class_id in classes: + bitmask = sem_seg_gt == class_id + pygmask, _ = mapper_utils.mask_to_polygons_2(bitmask) + for mask in pygmask: + gt_masks.append([mask]) + gt_classes.append(class_id) + + # if len(gt_masks) == 0: + # return None + + instances = Instances(image_shape) + instances.gt_classes = torch.tensor(gt_classes, dtype=torch.int64) + if self.stuff_classes_offset > 0: + instances.gt_classes += self.stuff_classes_offset - 1 + if self.instance_mask_format == "polygon": + instances.gt_masks = PolygonMasks(gt_masks) + else: + assert self.instance_mask_format == "bitmask", self.instance_mask_format + instances.gt_masks = BitMasks.from_polygon_masks( + gt_masks, image_shape[0], image_shape[1] + ) + instances.gt_boxes = instances.gt_masks.get_bounding_boxes() + + if self.instance_mask_format == "polygon": + area = instances.gt_masks.area() + else: + assert self.instance_mask_format == "bitmask", self.instance_mask_format + area = instances.gt_masks.tensor.sum((1, 2)) + instances = instances[area > 8 * 8] + + instances.is_thing = torch.tensor( + [False for _ in range(len(instances))], dtype=torch.bool + ) + + if "instances" in dataset_dict and dataset_dict["instances"].has("copypaste"): + instances.copypaste = torch.tensor([False for _ in range(len(instances))]) + + if len(instances) > 0: + if "instances" in dataset_dict and len(dataset_dict["instances"]) > 0: + dataset_dict["instances"] = Instances.cat( + [dataset_dict["instances"], instances] + ) + else: + dataset_dict["instances"] = instances + + if "pan_seg_file_name" in dataset_dict and not self.stuff_classes_decomposition: + pan_seg_gt = utils.read_image(dataset_dict.pop("pan_seg_file_name"), "RGB") + segments_info = dataset_dict["segments_info"] + + # apply the same transformation to panoptic segmentation + pan_seg_gt = transforms.apply_segmentation(pan_seg_gt) + + from panopticapi.utils import rgb2id + + pan_seg_gt = rgb2id(pan_seg_gt) + + instances = Instances(image_shape) + classes = [] + masks = [] + for segment_info in segments_info: + class_id = segment_info["category_id"] + if not segment_info["iscrowd"]: + classes.append(class_id) + masks.append(pan_seg_gt == segment_info["id"]) + + classes = np.array(classes) + instances.gt_classes = torch.tensor(classes, dtype=torch.int64) + if len(masks) == 0: + # Some image does not have annotation (all ignored) + instances.gt_masks = torch.zeros((0, pan_seg_gt.shape[-2], pan_seg_gt.shape[-1])) + instances.gt_boxes = Boxes(torch.zeros((0, 4))) + else: + masks = BitMasks( + torch.stack([torch.from_numpy(np.ascontiguousarray(x.copy())) for x in masks]) + ) + instances.gt_masks = masks.tensor + instances.gt_boxes = masks.get_bounding_boxes() + + if "instances" in dataset_dict and dataset_dict["instances"].has("copypaste"): + instances.copypaste = torch.tensor([False for _ in range(len(instances))]) + + dataset_dict["instances"] = instances + + if "pan_seg_file_name" in dataset_dict and self.stuff_classes_decomposition: + pan_seg_gt = utils.read_image(dataset_dict.pop("pan_seg_file_name"), "RGB") + segments_info = dataset_dict["segments_info"] + + # apply the same transformation to panoptic segmentation + pan_seg_gt = transforms.apply_segmentation(pan_seg_gt) + + from panopticapi.utils import rgb2id + + pan_seg_gt = rgb2id(pan_seg_gt) + + instances = Instances(image_shape) + classes = [] + masks = [] + for segment_info in segments_info: + class_id = segment_info["category_id"] + if not segment_info["iscrowd"]: + if class_id in metadata.thing_dataset_id_to_contiguous_id.values(): + classes.append(class_id) + masks.append(pan_seg_gt == segment_info["id"]) + else: + bitmask = pan_seg_gt == segment_info["id"] + pygmask, _ = mapper_utils.mask_to_polygons_2(bitmask) + for mask in pygmask: + mask = ( + BitMasks.from_polygon_masks( + [[mask]], image_shape[0], image_shape[1] + ) + .tensor[0, ...] + .numpy() + ) + classes.append(class_id) + masks.append(mask) + + classes = np.array(classes) + instances.gt_classes = torch.tensor(classes, dtype=torch.int64) + if len(masks) == 0: + # Some image does not have annotation (all ignored) + instances.gt_masks = torch.zeros((0, pan_seg_gt.shape[-2], pan_seg_gt.shape[-1])) + instances.gt_boxes = Boxes(torch.zeros((0, 4))) + else: + masks = BitMasks( + torch.stack([torch.from_numpy(np.ascontiguousarray(x.copy())) for x in masks]) + ) + instances.gt_masks = masks.tensor + instances.gt_boxes = masks.get_bounding_boxes() + + if "instances" in dataset_dict and dataset_dict["instances"].has("copypaste"): + instances.copypaste = torch.tensor([False for _ in range(len(instances))]) + + dataset_dict["instances"] = instances + + if "instances" in dataset_dict and len(dataset_dict["instances"]) > 0: + pass + else: + return None + + return dataset_dict diff --git a/approach/ovod/APE/ape/data/dataset_mapper_detr_panoptic_copypaste.py b/approach/ovod/APE/ape/data/dataset_mapper_detr_panoptic_copypaste.py new file mode 100644 index 0000000000000000000000000000000000000000..61d6b98c60492c38d41f0d8130369bbd4877f45d --- /dev/null +++ b/approach/ovod/APE/ape/data/dataset_mapper_detr_panoptic_copypaste.py @@ -0,0 +1,676 @@ +import copy +import logging +import os +import random +from typing import List, Optional, Union + +import cv2 +import numpy as np +import torch + +import detectron2.utils.comm as comm +from detectron2.config import configurable +from detectron2.data import MetadataCatalog +from detectron2.data import detection_utils as utils +from detectron2.data import transforms as T +from detectron2.data.detection_utils import convert_image_to_rgb +from detectron2.layers import batched_nms +from detectron2.structures import BitMasks, Boxes, Instances, PolygonMasks + +from . import mapper_utils + +""" +This file contains the default mapping that's applied to "dataset dicts". +""" + +__all__ = ["DatasetMapper_detr_panoptic_copypaste"] + + +class DatasetMapper_detr_panoptic_copypaste: + """ + A callable which takes a dataset dict in Detectron2 Dataset format, + and map it into a format used by the model. + + This is the default callable to be used to map your dataset dict into training data. + You may need to follow it to implement your own one for customized logic, + such as a different way to read or transform images. + See :doc:`/tutorials/data_loading` for details. + + The callable currently does the following: + + 1. Read the image from "file_name" + 2. Applies cropping/geometric transforms to the image and annotations + 3. Prepare data and annotations to Tensor and :class:`Instances` + """ + + @configurable + def __init__( + self, + is_train: bool, + *, + augmentations: List[Union[T.Augmentation, T.Transform]], + augmentations_with_crop: List[Union[T.Augmentation, T.Transform]], + image_format: str, + use_instance_mask: bool = False, + use_keypoint: bool = False, + instance_mask_format: str = "polygon", + keypoint_hflip_indices: Optional[np.ndarray] = None, + precomputed_proposal_topk: Optional[int] = None, + recompute_boxes: bool = False, + ignore_label: int = 255, + stuff_classes_offset: int = 80, + stuff_classes_decomposition: bool = False, + copypaste_prob: float = 0.5, + output_dir: str = None, + vis_period: int = 0, + dataset_names: tuple = (), + max_num_phrase: int = 0, + nms_thresh_phrase: float = 0.0, + ): + """ + NOTE: this interface is experimental. + + Args: + is_train: whether it's used in training or inference + augmentations: a list of augmentations or deterministic transforms to apply + image_format: an image format supported by :func:`detection_utils.read_image`. + use_instance_mask: whether to process instance segmentation annotations, if available + use_keypoint: whether to process keypoint annotations if available + instance_mask_format: one of "polygon" or "bitmask". Process instance segmentation + masks into this format. + keypoint_hflip_indices: see :func:`detection_utils.create_keypoint_hflip_indices` + precomputed_proposal_topk: if given, will load pre-computed + proposals from dataset_dict and keep the top k proposals for each image. + recompute_boxes: whether to overwrite bounding box annotations + by computing tight bounding boxes from instance mask annotations. + """ + if recompute_boxes: + assert use_instance_mask, "recompute_boxes requires instance masks" + # fmt: off + self.is_train = is_train + self.augmentations = T.AugmentationList(augmentations) + self.augmentations_with_crop = T.AugmentationList(augmentations_with_crop) + self.image_format = image_format + self.use_instance_mask = use_instance_mask + self.instance_mask_format = instance_mask_format + self.use_keypoint = use_keypoint + self.keypoint_hflip_indices = keypoint_hflip_indices + self.proposal_topk = precomputed_proposal_topk + self.recompute_boxes = recompute_boxes + self.ignore_label = ignore_label + self.stuff_classes_offset = stuff_classes_offset + self.stuff_classes_decomposition = stuff_classes_decomposition + # fmt: on + logger = logging.getLogger(__name__) + mode = "training" if is_train else "inference" + logger.info(f"[DatasetMapper] Augmentations used in {mode}: {augmentations}") + logger.info(f"[DatasetMapper] Augmentations used in {mode}: {augmentations_with_crop}") + + if output_dir is not None: + self.output_dir = os.path.join(output_dir, "vis_mapper") + os.makedirs(self.output_dir, exist_ok=True) + + self.copypaste_prob = copypaste_prob + self.vis_period = vis_period + self.iter = 0 + self.dataset_names = dataset_names + + self.metatada_list = [] + for dataset_name in self.dataset_names: + metadata = MetadataCatalog.get(dataset_name) + self.metatada_list.append(metadata) + + self.max_num_phrase = max_num_phrase + self.nms_thresh_phrase = nms_thresh_phrase + + @classmethod + def from_config(cls, cfg, is_train: bool = True): + raise NotImplementedError(self.__class__.__name__) + + def _transform_annotations(self, dataset_dict, transforms, image_shape): + # USER: Modify this if you want to keep them for some reason. + for anno in dataset_dict["annotations"]: + if not self.use_instance_mask: + anno.pop("segmentation", None) + if not self.use_keypoint: + anno.pop("keypoints", None) + + copypaste = [ + obj.get("copypaste", 0) + for obj in dataset_dict["annotations"] + if obj.get("iscrowd", 0) == 0 + ] + + phrases = [ + obj.get("phrase", "") + for obj in dataset_dict["annotations"] + if obj.get("iscrowd", 0) == 0 + ] + + # USER: Implement additional transformations if you have other types of data + annos = [ + utils.transform_instance_annotations( + obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices + ) + for obj in dataset_dict.pop("annotations") + if obj.get("iscrowd", 0) == 0 + ] + instances = utils.annotations_to_instances( + annos, image_shape, mask_format=self.instance_mask_format + ) + + instances.copypaste = torch.tensor(copypaste) + + if sum([len(x) for x in phrases]) > 0: + instances.phrase_idxs = torch.tensor(range(len(phrases))) + + # After transforms such as cropping are applied, the bounding box may no longer + # tightly bound the object. As an example, imagine a triangle object + # [(0,0), (2,0), (0,2)] cropped by a box [(1,0),(2,2)] (XYXY format). The tight + # bounding box of the cropped triangle should be [(1,0),(2,1)], which is not equal to + # the intersection of original bounding box and the cropping box. + if self.recompute_boxes and instances.has("gt_masks"): + instances.gt_boxes = instances.gt_masks.get_bounding_boxes() + dataset_dict["instances"] = utils.filter_empty_instances(instances) + + if sum([len(x) for x in phrases]) > 0: + phrases_filtered = [] + for x in dataset_dict["instances"].phrase_idxs.tolist(): + phrases_filtered.append(phrases[x]) + dataset_dict["instances"].phrases = mapper_utils.transform_phrases( + phrases_filtered, transforms + ) + dataset_dict["instances"].remove("phrase_idxs") + # dataset_dict["instances"].gt_classes = torch.tensor(range(len(phrases_filtered))) + + def __call__(self, dataset_dict, dataset_dict_bg): + """ + Args: + dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. + + Returns: + dict: a format that builtin models in detectron2 accept + """ + dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below + # USER: Write your own image loading if it's not from a file + try: + image = utils.read_image(dataset_dict["file_name"], format=self.image_format) + except Exception as e: + logger = logging.getLogger(__name__) + logger.error(f"read_image fails: {dataset_dict['file_name']}") + logger.error(f"read_image fails: {e}") + return None + utils.check_image_size(dataset_dict, image) + + # ------------------------------------------------------------------------------------ + if "dataset_id" in dataset_dict: + dataset_id = dataset_dict["dataset_id"] + else: + dataset_id = 0 + metadata = self.metatada_list[dataset_id] + if "sa1b" in self.dataset_names[dataset_id]: + metadata = None + if ( + self.is_train + and "annotations" in dataset_dict + and ( + len(dataset_dict["annotations"]) == 0 + or any(["bbox" not in anno for anno in dataset_dict["annotations"]]) + ) + ): + dataset_dict = mapper_utils.maybe_load_annotation_from_file(dataset_dict, meta=metadata) + + for anno in dataset_dict["annotations"]: + if "bbox" not in anno: + logger = logging.getLogger(__name__) + logger.warning(f"Box not found: {dataset_dict}") + return None + if "category_id" not in anno: + anno["category_id"] = 0 + # ------------------------------------------------------------------------------------ + + # ------------------------------------------------------------------------------------ + if dataset_dict["copypaste"] and self.copypaste_prob > random.uniform(0, 1): + image_cp, dataset_dict_cp = mapper_utils.copypaste( + dataset_dict, dataset_dict_bg, self.image_format, self.instance_mask_format + ) + + if dataset_dict_cp is None or image_cp is None: + pass + else: + for key in dataset_dict.keys(): + if key in dataset_dict_cp: + continue + dataset_dict_cp[key] = dataset_dict[key] + dataset_dict = dataset_dict_cp + image = image_cp + # ------------------------------------------------------------------------------------ + + # USER: Remove if you don't do semantic/panoptic segmentation. + if "sem_seg_file_name" in dataset_dict: + try: + sem_seg_gt = utils.read_image(dataset_dict.pop("sem_seg_file_name"), "L").squeeze(2) + except Exception as e: + logger = logging.getLogger(__name__) + logger.error(f"read_image fails: {e}") + logger.error(f"read_image fails: {dataset_dict}") + return None + + if "copypaste_mask" in dataset_dict: + # assume thing class is 0 + sem_seg_gt = sem_seg_gt.copy() + sem_seg_gt[dataset_dict["copypaste_mask"]] = 0 + else: + sem_seg_gt = None + + # ordinal numbers + disable_crop = False + if ( + "annotations" in dataset_dict + and len(dataset_dict["annotations"]) > 0 + and "phrase" in dataset_dict["annotations"][0] + ): + disable_crop = disable_crop or mapper_utils.has_ordinal_num( + [anno["phrase"] for anno in dataset_dict["annotations"]] + ) + if "expressions" in dataset_dict: + disable_crop = disable_crop or mapper_utils.has_ordinal_num(dataset_dict["expressions"]) + + if self.augmentations_with_crop is None or disable_crop: + augmentations = self.augmentations + else: + if np.random.rand() > 0.5: + augmentations = self.augmentations + else: + augmentations = self.augmentations_with_crop + + aug_input = T.AugInput(image, sem_seg=sem_seg_gt) + # transforms = self.augmentations(aug_input) + transforms = augmentations(aug_input) + image, sem_seg_gt = aug_input.image, aug_input.sem_seg + + image_shape = image.shape[:2] # h, w + # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, + # but not efficient on large generic data structures due to the use of pickle & mp.Queue. + # Therefore it's important to use torch.Tensor. + dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1))) + if sem_seg_gt is not None: + dataset_dict["sem_seg"] = torch.as_tensor(sem_seg_gt.astype("long")) + + # USER: Remove if you don't use pre-computed proposals. + # Most users would not need this feature. + if self.proposal_topk is not None: + utils.transform_proposals( + dataset_dict, image_shape, transforms, proposal_topk=self.proposal_topk + ) + + if "expressions" in dataset_dict: + dataset_dict["expressions"] = mapper_utils.transform_expressions( + dataset_dict["expressions"], transforms + ) + + if not self.is_train: + # USER: Modify this if you want to keep them for some reason. + dataset_dict.pop("annotations", None) + dataset_dict.pop("sem_seg_file_name", None) + dataset_dict.pop("pan_seg_file_name", None) + dataset_dict.pop("segments_info", None) + return dataset_dict + + if "annotations" in dataset_dict: + self._transform_annotations(dataset_dict, transforms, image_shape) + + dataset_dict["instances"].is_thing = torch.tensor( + [True for _ in range(len(dataset_dict["instances"]))], dtype=torch.bool + ) + + if "instances" in dataset_dict and dataset_dict["instances"].has("phrases"): + num_instances = len(dataset_dict["instances"]) + + if self.nms_thresh_phrase > 0: + boxes = dataset_dict["instances"].gt_boxes.tensor + scores = torch.rand(num_instances) + classes = torch.zeros(num_instances) + keep = batched_nms(boxes, scores, classes, self.nms_thresh_phrase) + else: + keep = torch.randperm(num_instances) + + if self.max_num_phrase > 0: + keep = keep[: self.max_num_phrase] + + phrases = dataset_dict["instances"].phrases + phrases_filtered = [] + for x in keep: + phrases_filtered.append(phrases[x]) + + dataset_dict["instances"].remove("phrases") + dataset_dict["instances"] = dataset_dict["instances"][keep] + dataset_dict["instances"].phrases = phrases_filtered + + # Prepare per-category binary masks + if sem_seg_gt is not None and not self.stuff_classes_decomposition: + instances = Instances(image_shape) + classes = np.unique(sem_seg_gt).astype(np.int64) + # remove ignored region + classes = classes[classes != self.ignore_label] + + if self.stuff_classes_offset > 0: + classes = classes[classes != 0] + instances.gt_classes = torch.tensor( + classes + self.stuff_classes_offset - 1, dtype=torch.int64 + ) + else: + instances.gt_classes = torch.tensor(classes, dtype=torch.int64) + + masks = [] + for class_id in classes: + masks.append(sem_seg_gt == class_id) + + if len(masks) == 0: + # # Some image does not have annotation (all ignored) + # instances.gt_masks = torch.zeros((0, sem_seg_gt.shape[-2], sem_seg_gt.shape[-1])) + masks = BitMasks(torch.zeros((0, sem_seg_gt.shape[-2], sem_seg_gt.shape[-1]))) + else: + masks = BitMasks( + torch.stack([torch.from_numpy(np.ascontiguousarray(x.copy())) for x in masks]) + ) + + instances.gt_masks = masks + instances.gt_boxes = masks.get_bounding_boxes() + + instances.is_thing = torch.tensor( + [False for _ in range(len(instances))], dtype=torch.bool + ) + + if "instances" in dataset_dict and dataset_dict["instances"].has("copypaste"): + instances.copypaste = torch.tensor([False for _ in range(len(instances))]) + + if len(instances) > 0: + if "instances" in dataset_dict and len(dataset_dict["instances"]) > 0: + dataset_dict["instances"] = Instances.cat( + [dataset_dict["instances"], instances] + ) + else: + dataset_dict["instances"] = instances + + # Prepare per-category binary masks + if sem_seg_gt is not None and self.stuff_classes_decomposition: + classes = np.unique(sem_seg_gt) + # remove ignored region + classes = classes[classes != self.ignore_label] + + if self.stuff_classes_offset > 0: + classes = classes[classes != 0] + + gt_masks = [] + gt_classes = [] + for class_id in classes: + bitmask = sem_seg_gt == class_id + pygmask, _ = mapper_utils.mask_to_polygons_2(bitmask) + for mask in pygmask: + gt_masks.append([mask]) + gt_classes.append(class_id) + + # if len(gt_masks) == 0: + # return None + + instances = Instances(image_shape) + instances.gt_classes = torch.tensor(gt_classes, dtype=torch.int64) + if self.stuff_classes_offset > 0: + instances.gt_classes += self.stuff_classes_offset - 1 + if self.instance_mask_format == "polygon": + instances.gt_masks = PolygonMasks(gt_masks) + else: + assert self.instance_mask_format == "bitmask", self.instance_mask_format + instances.gt_masks = BitMasks.from_polygon_masks( + gt_masks, image_shape[0], image_shape[1] + ) + instances.gt_boxes = instances.gt_masks.get_bounding_boxes() + + if self.instance_mask_format == "polygon": + area = instances.gt_masks.area() + else: + assert self.instance_mask_format == "bitmask", self.instance_mask_format + area = instances.gt_masks.tensor.sum((1, 2)) + instances = instances[area > 8 * 8] + + instances.is_thing = torch.tensor( + [False for _ in range(len(instances))], dtype=torch.bool + ) + + if "instances" in dataset_dict and dataset_dict["instances"].has("copypaste"): + instances.copypaste = torch.tensor([False for _ in range(len(instances))]) + + if len(instances) > 0: + if "instances" in dataset_dict and len(dataset_dict["instances"]) > 0: + dataset_dict["instances"] = Instances.cat( + [dataset_dict["instances"], instances] + ) + else: + dataset_dict["instances"] = instances + + if "pan_seg_file_name" in dataset_dict and not self.stuff_classes_decomposition: + pan_seg_gt = utils.read_image(dataset_dict.pop("pan_seg_file_name"), "RGB") + segments_info = dataset_dict["segments_info"] + + # apply the same transformation to panoptic segmentation + pan_seg_gt = transforms.apply_segmentation(pan_seg_gt) + + from panopticapi.utils import rgb2id + + pan_seg_gt = rgb2id(pan_seg_gt) + + instances = Instances(image_shape) + classes = [] + masks = [] + for segment_info in segments_info: + class_id = segment_info["category_id"] + if not segment_info["iscrowd"]: + classes.append(class_id) + masks.append(pan_seg_gt == segment_info["id"]) + + classes = np.array(classes) + instances.gt_classes = torch.tensor(classes, dtype=torch.int64) + if len(masks) == 0: + # Some image does not have annotation (all ignored) + instances.gt_masks = torch.zeros((0, pan_seg_gt.shape[-2], pan_seg_gt.shape[-1])) + instances.gt_boxes = Boxes(torch.zeros((0, 4))) + else: + masks = BitMasks( + torch.stack([torch.from_numpy(np.ascontiguousarray(x.copy())) for x in masks]) + ) + instances.gt_masks = masks.tensor + instances.gt_boxes = masks.get_bounding_boxes() + + if "instances" in dataset_dict and dataset_dict["instances"].has("copypaste"): + instances.copypaste = torch.tensor([False for _ in range(len(instances))]) + + dataset_dict["instances"] = instances + + if "pan_seg_file_name" in dataset_dict and self.stuff_classes_decomposition: + pan_seg_gt = utils.read_image(dataset_dict.pop("pan_seg_file_name"), "RGB") + segments_info = dataset_dict["segments_info"] + + # apply the same transformation to panoptic segmentation + pan_seg_gt = transforms.apply_segmentation(pan_seg_gt) + + from panopticapi.utils import rgb2id + + pan_seg_gt = rgb2id(pan_seg_gt) + + instances = Instances(image_shape) + classes = [] + masks = [] + for segment_info in segments_info: + class_id = segment_info["category_id"] + if not segment_info["iscrowd"]: + if class_id in metadata.thing_dataset_id_to_contiguous_id.values(): + classes.append(class_id) + masks.append(pan_seg_gt == segment_info["id"]) + else: + bitmask = pan_seg_gt == segment_info["id"] + pygmask, _ = mapper_utils.mask_to_polygons_2(bitmask) + for mask in pygmask: + mask = ( + BitMasks.from_polygon_masks( + [[mask]], image_shape[0], image_shape[1] + ) + .tensor[0, ...] + .numpy() + ) + classes.append(class_id) + masks.append(mask) + + classes = np.array(classes) + instances.gt_classes = torch.tensor(classes, dtype=torch.int64) + if len(masks) == 0: + # Some image does not have annotation (all ignored) + instances.gt_masks = torch.zeros((0, pan_seg_gt.shape[-2], pan_seg_gt.shape[-1])) + instances.gt_boxes = Boxes(torch.zeros((0, 4))) + else: + masks = BitMasks( + torch.stack([torch.from_numpy(np.ascontiguousarray(x.copy())) for x in masks]) + ) + instances.gt_masks = masks.tensor + instances.gt_boxes = masks.get_bounding_boxes() + + if "instances" in dataset_dict and dataset_dict["instances"].has("copypaste"): + instances.copypaste = torch.tensor([False for _ in range(len(instances))]) + + dataset_dict["instances"] = instances + + if "instances" in dataset_dict and len(dataset_dict["instances"]) > 0: + pass + else: + return None + + # ------------------------------------------------------------------------------------ + if self.vis_period > 0 and self.iter % self.vis_period == 0: + self.visualize_training(dataset_dict) + # ------------------------------------------------------------------------------------ + self.iter += 1 + + return dataset_dict + + def visualize_training(self, dataset_dict, prefix="", suffix=""): + if self.output_dir is None: + return + if dataset_dict is None: + return + # if "instances" not in dataset_dict: + # return + from detectron2.utils.visualizer import Visualizer + from detectron2.data import MetadataCatalog + + if "dataset_id" in dataset_dict: + dataset_id = dataset_dict["dataset_id"] + else: + dataset_id = 0 + dataset_name = self.dataset_names[dataset_id] + metadata = MetadataCatalog.get(dataset_name) + class_names = ( + metadata.get("thing_classes", []) + metadata.get("stuff_classes", ["thing"])[1:] + ) + + if "instances" in dataset_dict and dataset_dict["instances"].has("phrases"): + labels = dataset_dict["instances"].phrases + elif "expressions" in dataset_dict: + labels = [dataset_dict["expressions"]] + else: + labels = [class_names[i] for i in dataset_dict["instances"].gt_classes] + + img = dataset_dict["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.image_format) + image_shape = img.shape[:2] # h, w + vis = Visualizer(img, metadata=metadata) + if "instances" in dataset_dict: + vis = vis.overlay_instances( + boxes=dataset_dict["instances"].gt_boxes, + masks=dataset_dict["instances"].gt_masks + if dataset_dict["instances"].has("gt_masks") + else None, + labels=labels, + ) + else: + vis = vis.overlay_instances( + boxes=None, + masks=None, + labels=None, + ) + vis_gt = vis.get_image() + + if "instances_phrase" in dataset_dict: + vis = Visualizer(img, metadata=metadata) + vis = vis.overlay_instances( + boxes=dataset_dict["instances_phrase"].gt_boxes, + masks=dataset_dict["instances_phrase"].gt_masks + if dataset_dict["instances_phrase"].has("gt_masks") + else None, + labels=dataset_dict["instances_phrase"].phrases, + ) + vis_phrase = vis.get_image() + vis_gt = np.concatenate((vis_gt, vis_phrase), axis=1) + + if "captions" in dataset_dict: + vis = Visualizer(img, metadata=metadata) + vis = vis.overlay_instances( + boxes=Boxes( + np.array( + [ + [ + 0 + i * 20, + 0 + i * 20, + image_shape[1] - 1 - i * 20, + image_shape[0] - 1 - i * 20, + ] + for i in range(len(dataset_dict["captions"])) + ] + ) + ), + masks=None, + labels=dataset_dict["captions"], + ) + vis_cap = vis.get_image() + vis_gt = np.concatenate((vis_gt, vis_cap), axis=1) + + if "sem_seg" in dataset_dict: + vis = Visualizer(img, metadata=metadata) + vis = vis.draw_sem_seg(dataset_dict["sem_seg"], area_threshold=0, alpha=0.5) + vis_sem_gt = vis.get_image() + vis_gt = np.concatenate((vis_gt, vis_sem_gt), axis=1) + + concat = np.concatenate((vis_gt, img), axis=1) + + image_name = os.path.basename(dataset_dict["file_name"]).split(".")[0] + + save_path = os.path.join( + self.output_dir, + prefix + + str(self.iter) + + "_" + + image_name + + "_g" + + str(comm.get_rank()) + + suffix + + ".png", + ) + concat = cv2.cvtColor(concat, cv2.COLOR_RGB2BGR) + cv2.imwrite(save_path, concat) + + return + + import pickle + + save_path = os.path.join( + self.output_dir, + prefix + + str(self.iter) + + "_" + + str(dataset_dict["image_id"]) + + "_g" + + str(comm.get_rank()) + + suffix + + ".pkl", + ) + with open(save_path, "wb") as save_file: + pickle.dump(dataset_dict, save_file) diff --git a/approach/ovod/APE/ape/data/dataset_mapper_detr_semantic.py b/approach/ovod/APE/ape/data/dataset_mapper_detr_semantic.py new file mode 100644 index 0000000000000000000000000000000000000000..604d5494d9576c39f1113fd4bf7bd4af5e95aa70 --- /dev/null +++ b/approach/ovod/APE/ape/data/dataset_mapper_detr_semantic.py @@ -0,0 +1,244 @@ +import copy +import logging +from typing import List, Optional, Union + +import cv2 +import numpy as np +import torch + +from detectron2.config import configurable +from detectron2.data import detection_utils as utils +from detectron2.data import transforms as T +from detectron2.projects.point_rend import ColorAugSSDTransform +from detectron2.structures import BitMasks, Instances, PolygonMasks + +from . import mapper_utils + +""" +This file contains the default mapping that's applied to "dataset dicts". +""" + +__all__ = ["DatasetMapper_detr_semantic"] + + +class DatasetMapper_detr_semantic: + """ + A callable which takes a dataset dict in Detectron2 Dataset format, + and map it into a format used by the model. + + This is the default callable to be used to map your dataset dict into training data. + You may need to follow it to implement your own one for customized logic, + such as a different way to read or transform images. + See :doc:`/tutorials/data_loading` for details. + + The callable currently does the following: + + 1. Read the image from "file_name" + 2. Applies cropping/geometric transforms to the image and annotations + 3. Prepare data and annotations to Tensor and :class:`Instances` + """ + + @configurable + def __init__( + self, + is_train: bool, + *, + augmentations: List[Union[T.Augmentation, T.Transform]], + augmentations_with_crop: List[Union[T.Augmentation, T.Transform]], + image_format: str, + use_instance_mask: bool = False, + use_keypoint: bool = False, + instance_mask_format: str = "polygon", + keypoint_hflip_indices: Optional[np.ndarray] = None, + precomputed_proposal_topk: Optional[int] = None, + recompute_boxes: bool = False, + ignore_label: int = 255, + stuff_classes_decomposition: bool = False, + ): + """ + NOTE: this interface is experimental. + + Args: + is_train: whether it's used in training or inference + augmentations: a list of augmentations or deterministic transforms to apply + image_format: an image format supported by :func:`detection_utils.read_image`. + use_instance_mask: whether to process instance segmentation annotations, if available + use_keypoint: whether to process keypoint annotations if available + instance_mask_format: one of "polygon" or "bitmask". Process instance segmentation + masks into this format. + keypoint_hflip_indices: see :func:`detection_utils.create_keypoint_hflip_indices` + precomputed_proposal_topk: if given, will load pre-computed + proposals from dataset_dict and keep the top k proposals for each image. + recompute_boxes: whether to overwrite bounding box annotations + by computing tight bounding boxes from instance mask annotations. + """ + if recompute_boxes: + assert use_instance_mask, "recompute_boxes requires instance masks" + # fmt: off + self.is_train = is_train + self.augmentations = T.AugmentationList(augmentations) + self.augmentations_with_crop = T.AugmentationList(augmentations_with_crop) + self.image_format = image_format + self.use_instance_mask = use_instance_mask + self.instance_mask_format = instance_mask_format + self.use_keypoint = use_keypoint + self.keypoint_hflip_indices = keypoint_hflip_indices + self.proposal_topk = precomputed_proposal_topk + self.recompute_boxes = recompute_boxes + self.ignore_label = ignore_label + self.stuff_classes_decomposition = stuff_classes_decomposition + # fmt: on + logger = logging.getLogger(__name__) + mode = "training" if is_train else "inference" + logger.info(f"[DatasetMapper] Augmentations used in {mode}: {augmentations}") + logger.info(f"[DatasetMapper] Augmentations used in {mode}: {augmentations_with_crop}") + + @classmethod + def from_config(cls, cfg, is_train: bool = True): + raise NotImplementedError(self.__class__.__name__) + + def _transform_annotations(self, dataset_dict, transforms, image_shape): + # USER: Modify this if you want to keep them for some reason. + for anno in dataset_dict["annotations"]: + if not self.use_instance_mask: + anno.pop("segmentation", None) + if not self.use_keypoint: + anno.pop("keypoints", None) + + # USER: Implement additional transformations if you have other types of data + annos = [ + utils.transform_instance_annotations( + obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices + ) + for obj in dataset_dict.pop("annotations") + if obj.get("iscrowd", 0) == 0 + ] + instances = utils.annotations_to_instances( + annos, image_shape, mask_format=self.instance_mask_format + ) + + # After transforms such as cropping are applied, the bounding box may no longer + # tightly bound the object. As an example, imagine a triangle object + # [(0,0), (2,0), (0,2)] cropped by a box [(1,0),(2,2)] (XYXY format). The tight + # bounding box of the cropped triangle should be [(1,0),(2,1)], which is not equal to + # the intersection of original bounding box and the cropping box. + if self.recompute_boxes: + instances.gt_boxes = instances.gt_masks.get_bounding_boxes() + dataset_dict["instances"] = utils.filter_empty_instances(instances) + + def __call__(self, dataset_dict): + """ + Args: + dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. + + Returns: + dict: a format that builtin models in detectron2 accept + """ + dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below + # USER: Write your own image loading if it's not from a file + image = utils.read_image(dataset_dict["file_name"], format=self.image_format) + utils.check_image_size(dataset_dict, image) + + # USER: Remove if you don't do semantic/panoptic segmentation. + if "sem_seg_file_name" in dataset_dict: + # sem_seg_gt = utils.read_image(dataset_dict.pop("sem_seg_file_name")).astype("double") + sem_seg_gt = utils.read_image(dataset_dict.pop("sem_seg_file_name"), "L").squeeze(2) + else: + sem_seg_gt = None + + if self.augmentations_with_crop is None: + augmentations = self.augmentations + else: + if np.random.rand() > 0.5: + augmentations = self.augmentations + else: + augmentations = self.augmentations_with_crop + + aug_input = T.AugInput(image, sem_seg=sem_seg_gt) + # transforms = self.augmentations(aug_input) + transforms = augmentations(aug_input) + image, sem_seg_gt = aug_input.image, aug_input.sem_seg + + image_shape = image.shape[:2] # h, w + # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, + # but not efficient on large generic data structures due to the use of pickle & mp.Queue. + # Therefore it's important to use torch.Tensor. + dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1))) + if sem_seg_gt is not None: + dataset_dict["sem_seg"] = torch.as_tensor(sem_seg_gt.astype("long")) + + # USER: Remove if you don't use pre-computed proposals. + # Most users would not need this feature. + if self.proposal_topk is not None: + utils.transform_proposals( + dataset_dict, image_shape, transforms, proposal_topk=self.proposal_topk + ) + + if not self.is_train: + # USER: Modify this if you want to keep them for some reason. + dataset_dict.pop("annotations", None) + dataset_dict.pop("sem_seg_file_name", None) + return dataset_dict + + if "annotations" in dataset_dict: + self._transform_annotations(dataset_dict, transforms, image_shape) + + # Prepare per-category binary masks + if sem_seg_gt is not None and not self.stuff_classes_decomposition: + instances = Instances(image_shape) + classes = np.unique(sem_seg_gt) + # remove ignored region + classes = classes[classes != self.ignore_label] + instances.gt_classes = torch.tensor(classes, dtype=torch.int64) + + masks = [] + for class_id in classes: + masks.append(sem_seg_gt == class_id) + + if len(masks) == 0: + # # Some image does not have annotation (all ignored) + # instances.gt_masks = torch.zeros((0, sem_seg_gt.shape[-2], sem_seg_gt.shape[-1])) + masks = BitMasks(torch.zeros((0, sem_seg_gt.shape[-2], sem_seg_gt.shape[-1]))) + else: + masks = BitMasks( + torch.stack([torch.from_numpy(np.ascontiguousarray(x.copy())) for x in masks]) + ) + + instances.gt_masks = masks + instances.gt_boxes = masks.get_bounding_boxes() + dataset_dict["instances"] = instances + + # Prepare per-category binary masks + if sem_seg_gt is not None and self.stuff_classes_decomposition: + classes = np.unique(sem_seg_gt) + # remove ignored region + classes = classes[classes != self.ignore_label] + + gt_masks = [] + gt_classes = [] + for class_id in classes: + bitmask = sem_seg_gt == class_id + pygmask, _ = mapper_utils.mask_to_polygons_2(bitmask) + for mask in pygmask: + gt_masks.append([mask]) + gt_classes.append(class_id) + + # if len(gt_masks) == 0: + # return None + + instances = Instances(image_shape) + instances.gt_classes = torch.tensor(gt_classes, dtype=torch.int64) + instances.gt_masks = PolygonMasks(gt_masks) + instances.gt_boxes = instances.gt_masks.get_bounding_boxes() + + area = instances.gt_masks.area() + instances = instances[area > 8 * 8] + + dataset_dict["instances"] = instances + + if "instances" in dataset_dict and len(dataset_dict["instances"]) > 0: + pass + else: + return None + + return dataset_dict diff --git a/approach/ovod/APE/ape/data/detection_utils.py b/approach/ovod/APE/ape/data/detection_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ad4c5f16f72507fb306b626f2915dc0b1cb2ad16 --- /dev/null +++ b/approach/ovod/APE/ape/data/detection_utils.py @@ -0,0 +1,233 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +""" +Common data processing utilities that are used in a +typical object detection data pipeline. +""" +import json +import logging +import os +from typing import List, Union + +import numpy as np +import pycocotools.mask as mask_util +import torch + +from detectron2.data import transforms as T +from detectron2.data.catalog import MetadataCatalog +from detectron2.data.detection_utils import build_augmentation as build_augmentation_d2 +from detectron2.data.detection_utils import check_metadata_consistency + +from .transforms import AutoAugment, LargeScaleJitter + +__all__ = [ + "build_augmentation", +] + + +def load_fed_loss_cls_weights(class_freq_path: str, freq_weight_power=1.0): + logger = logging.getLogger(__name__) + logger.info("Loading " + class_freq_path) + assert os.path.exists(class_freq_path) + + class_info = json.load(open(class_freq_path, "r")) + class_freq = torch.tensor([c["image_count"] for c in sorted(class_info, key=lambda x: x["id"])]) + + class_freq_weight = class_freq.float() ** freq_weight_power + return class_freq_weight + + +def get_fed_loss_cls_weights(dataset_names: Union[str, List[str]], freq_weight_power=1.0): + """ + Get frequency weight for each class sorted by class id. + We now calcualte freqency weight using image_count to the power freq_weight_power. + + Args: + dataset_names: list of dataset names + freq_weight_power: power value + """ + if isinstance(dataset_names, str): + dataset_names = [dataset_names] + + logger = logging.getLogger(__name__) + class_freq_path = MetadataCatalog.get(dataset_names[0]).json_file[:-5] + "_cat_info.json" + if os.path.exists(class_freq_path): + logger.info( + "Search outside metadata 'image_count' for dataset '{}' from '{}'".format( + dataset_names[0], class_freq_path + ) + ) + return load_fed_loss_cls_weights(class_freq_path, freq_weight_power) + logger.info("Using builtin metadata 'image_count' for dataset '{}'".format(dataset_names)) + + check_metadata_consistency("class_image_count", dataset_names) + + meta = MetadataCatalog.get(dataset_names[0]) + class_freq_meta = meta.class_image_count + class_freq = torch.tensor( + [c["image_count"] for c in sorted(class_freq_meta, key=lambda x: x["id"])] + ) + class_freq_weight = class_freq.float() ** freq_weight_power + return class_freq_weight + + +def get_fed_loss_cls_weights_v2(dataset_names: Union[str, List[str]], freq_weight_power=1.0): + """ + Get frequency weight for each class sorted by class id. + We now calcualte freqency weight using image_count to the power freq_weight_power. + + Args: + dataset_names: list of dataset names + freq_weight_power: power value + """ + if isinstance(dataset_names, str): + dataset_names = [dataset_names] + + logger = logging.getLogger(__name__) + + class_freq_weight_list = [] + for dataset_name in dataset_names: + if MetadataCatalog.get(dataset_name).get("json_file") is None: + continue + class_freq_path = MetadataCatalog.get(dataset_name).json_file[:-5] + "_cat_info.json" + if os.path.exists(class_freq_path): + logger.info( + "Search outside metadata 'image_count' for dataset '{}' from '{}'".format( + dataset_name, class_freq_path + ) + ) + # return load_fed_loss_cls_weights(class_freq_path, freq_weight_power) + class_freq_weight_list.append( + load_fed_loss_cls_weights(class_freq_path, freq_weight_power) + ) + continue + else: + logger.info( + "Nofind outside metadata 'image_count' for dataset '{}' from '{}'".format( + dataset_name, class_freq_path + ) + ) + + logger.info("Using builtin metadata 'image_count' for dataset '{}'".format(dataset_name)) + + # check_metadata_consistency("class_image_count", dataset_names) + + meta = MetadataCatalog.get(dataset_name) + class_freq_meta = meta.class_image_count + class_freq = torch.tensor( + [c["image_count"] for c in sorted(class_freq_meta, key=lambda x: x["id"])] + ) + class_freq_weight = class_freq.float() ** freq_weight_power + # return class_freq_weight + class_freq_weight_list.append(class_freq_weight) + + return class_freq_weight_list[0] if len(class_freq_weight_list) == 1 else class_freq_weight_list + + +def build_augmentation(cfg, is_train): + """ + Create a list of default :class:`Augmentation` from config. + Now it includes resizing and flipping. + + Returns: + list[Augmentation] + """ + assert not (cfg.INPUT.AUTOAUGMENT.ENABLED and cfg.INPUT.LSJ.ENABLED) + + augmentation = [] + if is_train and cfg.INPUT.AUTOAUGMENT.ENABLED: + augmentation.append(AutoAugment(cfg)) + + if cfg.INPUT.RANDOM_FLIP != "none": + augmentation.append( + T.RandomFlip( + horizontal=cfg.INPUT.RANDOM_FLIP == "horizontal", + vertical=cfg.INPUT.RANDOM_FLIP == "vertical", + ) + ) + if cfg.INPUT.RANDOM_COLOR.ENABLED: + augmentation.append(T.RandomBrightness(0.5, 1.5)) + augmentation.append(T.RandomContrast(0.5, 1.5)) + augmentation.append(T.RandomSaturation(0.0, 2.0)) + return augmentation + + if is_train and cfg.INPUT.LSJ.ENABLED: + augmentation.append(LargeScaleJitter(cfg)) + + if cfg.INPUT.RANDOM_FLIP != "none": + augmentation.append( + T.RandomFlip( + horizontal=cfg.INPUT.RANDOM_FLIP == "horizontal", + vertical=cfg.INPUT.RANDOM_FLIP == "vertical", + ) + ) + if cfg.INPUT.RANDOM_COLOR.ENABLED: + augmentation.append(T.RandomBrightness(0.5, 1.5)) + augmentation.append(T.RandomContrast(0.5, 1.5)) + augmentation.append(T.RandomSaturation(0.0, 2.0)) + return augmentation + + return build_augmentation_d2(cfg, is_train) + + +def build_augmentation_lsj(cfg, is_train): + """ + Create a list of default :class:`Augmentation` from config. + Now it includes resizing and flipping. + + Returns: + list[Augmentation] + """ + augmentation = [] + if is_train: + augmentation.append(LargeScaleJitter(cfg)) + + if cfg.INPUT.RANDOM_FLIP != "none": + augmentation.append( + T.RandomFlip( + horizontal=cfg.INPUT.RANDOM_FLIP == "horizontal", + vertical=cfg.INPUT.RANDOM_FLIP == "vertical", + ) + ) + if cfg.INPUT.RANDOM_COLOR.ENABLED: + augmentation.append(T.RandomBrightness(0.5, 1.5)) + augmentation.append(T.RandomContrast(0.5, 1.5)) + augmentation.append(T.RandomSaturation(0.0, 2.0)) + return augmentation + + return build_augmentation_d2(cfg, is_train) + + +def build_augmentation_aa(cfg, is_train): + """ + Create a list of default :class:`Augmentation` from config. + Now it includes resizing and flipping. + + Returns: + list[Augmentation] + """ + augmentation = [] + if is_train: + augmentation.append(AutoAugment(cfg)) + + if cfg.INPUT.RANDOM_FLIP != "none": + augmentation.append( + T.RandomFlip( + horizontal=cfg.INPUT.RANDOM_FLIP == "horizontal", + vertical=cfg.INPUT.RANDOM_FLIP == "vertical", + ) + ) + if cfg.INPUT.RANDOM_COLOR.ENABLED: + augmentation.append(T.RandomBrightness(0.5, 1.5)) + augmentation.append(T.RandomContrast(0.5, 1.5)) + augmentation.append(T.RandomSaturation(0.0, 2.0)) + return augmentation + + return build_augmentation_d2(cfg, is_train) + + +build_transform_gen = build_augmentation +""" +Alias for backward-compatibility. +""" diff --git a/approach/ovod/APE/ape/data/mapper_utils.py b/approach/ovod/APE/ape/data/mapper_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b75ed3554f0dfa494fbe459890f062b516aabd0b --- /dev/null +++ b/approach/ovod/APE/ape/data/mapper_utils.py @@ -0,0 +1,488 @@ +# -*- coding: utf-8 -*- +import copy +import json +import logging +import os +import random +import re + +import cv2 +import numpy as np +import pycocotools.mask as mask_util +import torch +from scipy.ndimage import gaussian_filter + +from detectron2.data import detection_utils as utils +from detectron2.structures import ( + BitMasks, + Boxes, + BoxMode, + Instances, + PolygonMasks, + polygons_to_bitmask, +) +from fvcore.transforms.transform import HFlipTransform + +__all__ = [ + "copypaste", + "maybe_load_annotation_from_file", +] + + +def clean_string(phrase): + # return re.sub(r"([.,'!?\"()*#:;])", "", phrase.lower()).replace("-", " ").replace("/", " ") + + phrase = re.sub(r"([.,'!?\"()*#:;])", "", phrase.lower()).replace("-", " ").replace("/", " ") + phrase = phrase.strip("\n").strip("\r").strip().lstrip(" ").rstrip(" ") + phrase = re.sub(" +", " ", phrase) + + replacements = { + "½": "half", + "—": "-", + "™": "", + "¢": "cent", + "ç": "c", + "û": "u", + "é": "e", + "°": " degree", + "è": "e", + "…": "", + } + for k, v in replacements.items(): + phrase = phrase.replace(k, v) + + return phrase + + +def transform_phrases(phrases, transforms): + # clean + phrases = [clean_string(phrase) for phrase in phrases] + # hflip + for x in transforms: + if isinstance(x, HFlipTransform): + phrases = [ + phrase.replace("left", "@").replace("right", "left").replace("@", "right") + for phrase in phrases + ] + return phrases + + +def transform_expressions(expressions, transforms): + # pick one expression if there are multiple expressions + expression = expressions[np.random.choice(len(expressions))] + expression = clean_string(expression) + # deal with hflip for expression + for x in transforms: + if isinstance(x, HFlipTransform): + expression = ( + expression.replace("left", "@").replace("right", "left").replace("@", "right") + ) + return expression + + +def has_ordinal_num(phrases): + # oridinal numbers + ordinal_nums = [ + "first", + "second", + "third", + "fourth", + "fifth", + "sixth", + "seventh", + "eighth", + "ninth", + "tenth", + ] + + flag = False + for phrase in phrases: + phrase_low = phrase.lower() + for word in ordinal_nums: + if word in phrase_low: + flag = True + break + if flag == True: + break + return flag + + +# from detectron2/utils/visualizer.py +def mask_to_polygons_2(mask): + # cv2.RETR_CCOMP flag retrieves all the contours and arranges them to a 2-level + # hierarchy. External contours (boundary) of the object are placed in hierarchy-1. + # Internal contours (holes) are placed in hierarchy-2. + # cv2.CHAIN_APPROX_NONE flag gets vertices of polygons from contours. + mask = np.ascontiguousarray(mask) # some versions of cv2 does not support incontiguous arr + res = cv2.findContours(mask.astype("uint8"), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + hierarchy = res[-1] + if hierarchy is None: # empty mask + return [], False + has_holes = (hierarchy.reshape(-1, 4)[:, 3] >= 0).sum() > 0 + res = res[-2] + res = [x.flatten() for x in res] + # These coordinates from OpenCV are integers in range [0, W-1 or H-1]. + # We add 0.5 to turn them into real-value coordinate space. A better solution + # would be to first +0.5 and then dilate the returned polygon by 0.5. + res = [x + 0.5 for x in res if len(x) >= 6] + return res, has_holes + + +# from detectron2/utils/visualizer.py +def mask_to_polygons(mask): + # cv2.RETR_CCOMP flag retrieves all the contours and arranges them to a 2-level + # hierarchy. External contours (boundary) of the object are placed in hierarchy-1. + # Internal contours (holes) are placed in hierarchy-2. + # cv2.CHAIN_APPROX_NONE flag gets vertices of polygons from contours. + mask = np.ascontiguousarray(mask) # some versions of cv2 does not support incontiguous arr + res = cv2.findContours(mask.astype("uint8"), cv2.RETR_CCOMP, cv2.CHAIN_APPROX_NONE) + hierarchy = res[-1] + if hierarchy is None: # empty mask + return [], False + has_holes = (hierarchy.reshape(-1, 4)[:, 3] >= 0).sum() > 0 + res = res[-2] + res = [x.flatten() for x in res] + # These coordinates from OpenCV are integers in range [0, W-1 or H-1]. + # We add 0.5 to turn them into real-value coordinate space. A better solution + # would be to first +0.5 and then dilate the returned polygon by 0.5. + res = [x + 0.5 for x in res if len(x) >= 6] + return res, has_holes + + +def close_contour(contour): + if not np.array_equal(contour[0], contour[-1]): + contour = np.vstack((contour, contour[0])) + return contour + + +# from pycococreatortools/pycococreatortools.py +def binary_mask_to_polygon(binary_mask, tolerance=0): + """Converts a binary mask to COCO polygon representation + Args: + binary_mask: a 2D binary numpy array where '1's represent the object + tolerance: Maximum distance from original points of polygon to approximated + polygonal chain. If tolerance is 0, the original coordinate array is returned. + """ "" + polygons = [] + # pad mask to close contours of shapes which start and end at an edge + padded_binary_mask = np.pad(binary_mask, pad_width=1, mode="constant", constant_values=0) + contours = measure.find_contours(padded_binary_mask, 0.5) + contours = np.subtract(contours, 1) + for contour in contours: + contour = close_contour(contour) + contour = measure.approximate_polygon(contour, tolerance) + if len(contour) < 3: + continue + contour = np.flip(contour, axis=1) + segmentation = contour.ravel().tolist() + # after padding and subtracting 1 we may get -0.5 points in our segmentation + segmentation = [0 if i < 0 else i for i in segmentation] + polygons.append(segmentation) + + return polygons + + +def instances_to_annotations(instances, img_id, bbox_mode, instance_mask_format): + num_instance = len(instances) + if num_instance == 0: + return [] + + boxes = instances.gt_boxes.tensor.numpy() + boxes = BoxMode.convert(boxes, BoxMode.XYXY_ABS, bbox_mode) + boxes = boxes.tolist() + classes = instances.gt_classes.tolist() + + if instance_mask_format == "polygon": + segms = [[p.reshape(-1) for p in mask] for mask in instances.gt_masks] + + elif instance_mask_format == "bitmask" and False: + masks = [np.array(mask, dtype=np.uint8) for mask in instances.gt_masks] + + else: + rles = [ + mask_util.encode(np.array(mask[:, :, None], order="F", dtype="uint8"))[0] + for mask in instances.gt_masks + ] + for rle in rles: + # "counts" is an array encoded by mask_util as a byte-stream. Python3's + # json writer which always produces strings cannot serialize a bytestream + # unless you decode it. Thankfully, utf-8 works out (which is also what + # the pycocotools/_mask.pyx does). + rle["counts"] = rle["counts"].decode("utf-8") + + annotations = [] + for k in range(num_instance): + anno = { + "image_id": img_id, + "category_id": classes[k], + "bbox": boxes[k], + "bbox_mode": bbox_mode, + } + if instance_mask_format == "polygon": + anno["segmentation"] = segms[k] + elif instance_mask_format == "bitmask" and False: + anno["segmentation"] = masks[k] + else: + anno["segmentation"] = rles[k] + annotations.append(anno) + + return annotations + + +def copypaste(dataset_dict, dataset_dict_bg, image_format, instance_mask_format): + dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below + # USER: Write your own image loading if it's not from a file + image = utils.read_image(dataset_dict["file_name"], format=image_format) + utils.check_image_size(dataset_dict, image) + + dataset_dict_bg = copy.deepcopy(dataset_dict_bg) # it will be modified by code below + # USER: Write your own image loading if it's not from a file + image_bg = utils.read_image(dataset_dict_bg["file_name"], format=image_format) + utils.check_image_size(dataset_dict_bg, image_bg) + + image_bg = image_bg.copy() + + image_size = image_shape = image.shape[:2] # h, w + image_size_bg = image_shape_bg = image_bg.shape[:2] # h, w + + instances = utils.annotations_to_instances( + # dataset_dict["annotations"], + [obj for obj in dataset_dict["annotations"] if obj.get("iscrowd", 0) == 0], + image_shape, + mask_format=instance_mask_format, + ) + if "annotations" in dataset_dict_bg: + instances_bg = utils.annotations_to_instances( + # dataset_dict_bg["annotations"], + [obj for obj in dataset_dict_bg["annotations"] if obj.get("iscrowd", 0) == 0], + image_shape_bg, + mask_format=instance_mask_format, + ) + else: + instances_bg = None + + if instances_bg is None or len(instances_bg) == 0: + bitmasks_bg = torch.zeros((1, image_size_bg[0], image_size_bg[1])).to(torch.bool) + elif instance_mask_format == "polygon": + bitmasks_bg = [ + polygons_to_bitmask(polygon, *image_size_bg) for polygon in instances_bg.gt_masks + ] + bitmasks_bg = torch.tensor(np.array(bitmasks_bg)) + else: + bitmasks_bg = instances_bg.gt_masks.tensor + + if instance_mask_format == "polygon": + bitmasks = [polygons_to_bitmask(polygon, *image_size) for polygon in instances.gt_masks] + bitmasks = torch.tensor(np.array(bitmasks)) + else: + bitmasks = instances.gt_masks.tensor + + assert bitmasks_bg.dtype == torch.bool, bitmasks_bg.dtype + # foreground_mask = torch.sum(bitmasks_bg, dim=0) + foreground_mask = torch.max(bitmasks_bg, dim=0)[0] + copypaste_mask = torch.zeros_like(foreground_mask) + + if instance_mask_format == "polygon": + mask_areas = instances.gt_masks.area().numpy() + else: + mask_areas = instances.gt_masks.tensor.sum(dim=1).sum(dim=1).numpy() + + instance_list = [] + for i in mask_areas.argsort(): + i = int(i) + + box = instances.gt_boxes[i].tensor.numpy()[0] + x1 = int(box[0]) + y1 = int(box[1]) + x2 = int(box[2]) + y2 = int(box[3]) + + if x1 + 1 > x2 or y1 + 1 > y2: + continue + + image_p = image[y1:y2, x1:x2, :] + bitmasks_p = bitmasks[i, y1:y2, x1:x2] + + h, w = bitmasks_p.shape + + trial = 10 + for _ in range(trial): + if w + 10 >= image_size_bg[1] or h + 10 >= image_size_bg[0]: + break + + x1 = random.randint(0, image_size_bg[1] - w) + y1 = random.randint(0, image_size_bg[0] - h) + x2 = x1 + w + y2 = y1 + h + + bitmask = torch.zeros_like(foreground_mask) + bitmask[y1:y2, x1:x2] = bitmasks_p + + # bitmask = bitmask * (1 - foreground_mask) + bitmask = bitmask & (~foreground_mask) + + if bitmask.sum() < 100: + continue + + instance = Instances(image_size_bg) + instance.gt_classes = instances[i].gt_classes + + # if bitmask.sum() < bitmasks_p.sum(): + bitmasks_p = bitmask[y1:y2, x1:x2] + + if instance_mask_format == "polygon": + mask = [mask_to_polygons(bitmask)[0]] + instance.gt_masks = PolygonMasks(mask) + else: + instance.gt_masks = BitMasks(bitmask.unsqueeze(0)) + + bitmasks_p = bitmasks_p.numpy() + if bitmask.sum() > 128 * 64: + bitmasks_p = gaussian_filter(bitmasks_p.astype(float), sigma=5, truncate=1) + + image_bg_p = image_bg[y1:y2, x1:x2, :] + image_fgbg_p = image_p * bitmasks_p[..., np.newaxis] + image_bg_p * ( + 1 - bitmasks_p[..., np.newaxis] + ) + + image_bg[y1:y2, x1:x2, :] = image_fgbg_p + + foreground_mask = foreground_mask | bitmask + copypaste_mask = copypaste_mask | bitmask + + instance_list.append(instance) + break + + if len(instance_list) > 0: + instances = Instances.cat(instance_list) + instances.gt_boxes = instances.gt_masks.get_bounding_boxes() + + image_id = dataset_dict["image_id"] + bbox_mode = dataset_dict["annotations"][0]["bbox_mode"] + annotations = instances_to_annotations(instances, image_id, bbox_mode, instance_mask_format) + + for annotation in annotations: + annotation["copypaste"] = 1 + + if "annotations" in dataset_dict_bg: + dataset_dict_bg["annotations"] += annotations + else: + dataset_dict_bg["annotations"] = annotations + + dataset_dict_bg["image_id"] = ( + str(dataset_dict["image_id"]) + "_" + str(dataset_dict_bg["image_id"]) + ) + + dataset_dict_bg["copypaste_mask"] = copypaste_mask.numpy() + + return image_bg, dataset_dict_bg + else: + return None, None + + +# from SotA-T/ape/data/datasets/coco.py +def maybe_load_annotation_from_file(record, meta=None, extra_annotation_keys=None): + + file_name = record["file_name"] + image_ext = file_name.split(".")[-1] + file_name = file_name[: -len(image_ext)] + "json" + + if not os.path.isfile(file_name): + return record + + try: + with open(file_name, "r") as f: + json_data = json.load(f) + except Exception as e: + logger = logging.getLogger(__name__) + logger.warning(f"json.load fails: {file_name}") + logger.warning(f"json.load fails: {e}") + return record + if "image" not in json_data or "annotations" not in json_data: + return record + + image_id = record["image_id"] + if "image_id" in json_data["image"]: + assert json_data["image"]["image_id"] == image_id + if "id" in json_data["image"]: + assert json_data["image"]["id"] == image_id + + id_map = meta.thing_dataset_id_to_contiguous_id if meta is not None else None + ann_keys = ["iscrowd", "bbox", "keypoints", "category_id"] + (extra_annotation_keys or []) + + ann_keys += ["phrase", "isobject"] + + num_instances_without_valid_segmentation = 0 + + if True: + anno_dict_list = json_data["annotations"] + + objs = [] + for anno in anno_dict_list: + if "image_id" not in anno: + anno["image_id"] = image_id + # Check that the image_id in this annotation is the same as + # the image_id we're looking at. + # This fails only when the data parsing logic or the annotation file is buggy. + + # The original COCO valminusminival2014 & minival2014 annotation files + # actually contains bugs that, together with certain ways of using COCO API, + # can trigger this assertion. + assert anno["image_id"] == image_id + + assert anno.get("ignore", 0) == 0, '"ignore" in COCO json file is not supported.' + + obj = {key: anno[key] for key in ann_keys if key in anno} + if "bbox" in obj and len(obj["bbox"]) == 0: + raise ValueError( + f"One annotation of image {image_id} contains empty 'bbox' value! " + "This json does not have valid COCO format." + ) + + segm = anno.get("segmentation", None) + if segm: # either list[list[float]] or dict(RLE) + if isinstance(segm, dict): + if isinstance(segm["counts"], list): + # convert to compressed RLE + segm = mask_util.frPyObjects(segm, *segm["size"]) + else: + # filter out invalid polygons (< 3 points) + segm = [poly for poly in segm if len(poly) % 2 == 0 and len(poly) >= 6] + if len(segm) == 0: + num_instances_without_valid_segmentation += 1 + continue # ignore this instance + obj["segmentation"] = segm + + keypts = anno.get("keypoints", None) + if keypts: # list[int] + for idx, v in enumerate(keypts): + if idx % 3 != 2: + # COCO's segmentation coordinates are floating points in [0, H or W], + # but keypoint coordinates are integers in [0, H-1 or W-1] + # Therefore we assume the coordinates are "pixel indices" and + # add 0.5 to convert to floating point coordinates. + keypts[idx] = v + 0.5 + obj["keypoints"] = keypts + + # phrase = anno.get("phrase", None) + # if phrase: + # obj["phrase"] = phrase + + # isobject = anno.get("isobject", None) + # if isobject: + # obj["isobject"] = isobject + + obj["bbox_mode"] = BoxMode.XYWH_ABS + if id_map: + annotation_category_id = obj["category_id"] + try: + obj["category_id"] = id_map[annotation_category_id] + except KeyError as e: + raise KeyError( + f"Encountered category_id={annotation_category_id} " + "but this id does not exist in 'categories' of the json file." + ) from e + objs.append(obj) + record["annotations"] = objs + + return record diff --git a/approach/ovod/APE/ape/engine/__init__.py b/approach/ovod/APE/ape/engine/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ee63d6e0b4590644d92d12384f49cf27e64f9978 --- /dev/null +++ b/approach/ovod/APE/ape/engine/__init__.py @@ -0,0 +1,4 @@ +from .defaults import * +from .train_loop import * + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/approach/ovod/APE/ape/engine/defaults.py b/approach/ovod/APE/ape/engine/defaults.py new file mode 100644 index 0000000000000000000000000000000000000000..de9ac4223eb9cb72208c11953ca3eb108e73dee8 --- /dev/null +++ b/approach/ovod/APE/ape/engine/defaults.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +""" +This file contains components with some default boilerplate logic user may need +in training / testing. They will not work for everyone, but many users may find them useful. + +The behavior of functions/classes in this file is subject to change, +since they are meant to represent the "common default behavior" people need in their projects. +""" + +import copy +import os +import sys + +import torch + +from ape.checkpoint import DetectionCheckpointer +from detectron2.config import instantiate + +__all__ = [ + "DefaultPredictor", +] + + +class DefaultPredictor: + """ + Create a simple end-to-end predictor with the given config that runs on + single device for a single input image. + + Compared to using the model directly, this class does the following additions: + + 1. Load checkpoint from `cfg.MODEL.WEIGHTS`. + 2. Always take BGR image as the input and apply conversion defined by `cfg.INPUT.FORMAT`. + 3. Apply resizing defined by `cfg.INPUT.{MIN,MAX}_SIZE_TEST`. + 4. Take one input image and produce a single output, instead of a batch. + + This is meant for simple demo purposes, so it does the above steps automatically. + This is not meant for benchmarks or running complicated inference logic. + If you'd like to do anything more complicated, please refer to its source code as + examples to build and use the model manually. + + Attributes: + metadata (Metadata): the metadata of the underlying dataset, obtained from + cfg.DATASETS.TEST. + + Examples: + :: + pred = DefaultPredictor(cfg) + inputs = cv2.imread("input.jpg") + outputs = pred(inputs) + """ + + def __init__(self, cfg): + self.cfg = copy.deepcopy(cfg) # cfg can be modified by model + self.model = instantiate(cfg.model) + self.model.to(cfg.train.device) + self.model.eval() + + checkpointer = DetectionCheckpointer(self.model) + checkpointer.load(cfg.train.init_checkpoint) + + self.aug = instantiate(cfg.dataloader.test.mapper.augmentations[0]) + if "model_vision" in cfg.model: + self.input_format = cfg.model.model_vision.input_format + else: + self.input_format = cfg.model.input_format + assert self.input_format in ["RGB", "BGR"], self.input_format + + def __call__(self, original_image, text_prompt=None, mask_prompt=None): + """ + Args: + original_image (np.ndarray): an image of shape (H, W, C) (in BGR order). + + Returns: + predictions (dict): + the output of the model for one image only. + See :doc:`/tutorials/models` for details about the format. + """ + with torch.no_grad(): # https://github.com/sphinx-doc/sphinx/issues/4258 + # Apply pre-processing to image. + if self.input_format == "RGB": + # whether the model expects BGR inputs or RGB + original_image = original_image[:, :, ::-1] + height, width = original_image.shape[:2] + image = self.aug.get_transform(original_image).apply_image(original_image) + image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) + + inputs = {"image": image, "height": height, "width": width} + if text_prompt is not None: + inputs["prompt"] = "text" + inputs["text_prompt"] = text_prompt + if mask_prompt is not None: + mask_prompt = self.aug.get_transform(mask_prompt).apply_image(mask_prompt) + inputs["mask_prompt"] = torch.as_tensor(mask_prompt.astype("float32")) + predictions = self.model([inputs])[0] + return predictions diff --git a/approach/ovod/APE/ape/engine/train_loop.py b/approach/ovod/APE/ape/engine/train_loop.py new file mode 100644 index 0000000000000000000000000000000000000000..380b3e79ad9f61faec05ccca26394d619fe29811 --- /dev/null +++ b/approach/ovod/APE/ape/engine/train_loop.py @@ -0,0 +1,415 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. +import concurrent.futures +import logging +import time +import weakref +from typing import List, Mapping, Optional + +import numpy as np +import torch +from torch.nn.parallel import DataParallel, DistributedDataParallel + +import detectron2.utils.comm as comm +from detectron2.engine.train_loop import HookBase, TrainerBase +from detectron2.utils.events import EventStorage, get_event_storage +from detectron2.utils.logger import _log_api_usage + +__all__ = ["SimpleTrainer", "AMPTrainer"] + + +class SimpleTrainer(TrainerBase): + """ + A simple trainer for the most common type of task: + single-cost single-optimizer single-data-source iterative optimization, + optionally using data-parallelism. + It assumes that every step, you: + + 1. Compute the loss with a data from the data_loader. + 2. Compute the gradients with the above loss. + 3. Update the model with the optimizer. + + All other tasks during training (checkpointing, logging, evaluation, LR schedule) + are maintained by hooks, which can be registered by :meth:`TrainerBase.register_hooks`. + + If you want to do anything fancier than this, + either subclass TrainerBase and implement your own `run_step`, + or write your own training loop. + """ + + def __init__( + self, + model, + data_loader, + optimizer, + gather_metric_period=1, + zero_grad_before_forward=False, + async_write_metrics=False, + ): + """ + Args: + model: a torch Module. Takes a data from data_loader and returns a + dict of losses. + data_loader: an iterable. Contains data to be used to call model. + optimizer: a torch optimizer. + gather_metric_period: an int. Every gather_metric_period iterations + the metrics are gathered from all the ranks to rank 0 and logged. + zero_grad_before_forward: whether to zero the gradients before the forward. + async_write_metrics: bool. If True, then write metrics asynchronously to improve + training speed + """ + super().__init__() + + """ + We set the model to training mode in the trainer. + However it's valid to train a model that's in eval mode. + If you want your model (or a submodule of it) to behave + like evaluation during training, you can overwrite its train() method. + """ + model.train() + + self.model = model + self.data_loader = data_loader + # to access the data loader iterator, call `self._data_loader_iter` + self._data_loader_iter_obj = None + self.optimizer = optimizer + self.gather_metric_period = gather_metric_period + self.zero_grad_before_forward = zero_grad_before_forward + self.async_write_metrics = async_write_metrics + # create a thread pool that can execute non critical logic in run_step asynchronically + # use only 1 worker so tasks will be executred in order of submitting. + self.concurrent_executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) + + def run_step(self): + """ + Implement the standard training logic described above. + """ + assert self.model.training, "[SimpleTrainer] model was changed to eval mode!" + start = time.perf_counter() + """ + If you want to do something with the data, you can wrap the dataloader. + """ + data = next(self._data_loader_iter) + data_time = time.perf_counter() - start + + # ------------------------------------------------------------------ + for d in data: + self.dataset_image_counts[self.dataset_names[d.get("dataset_id", 0)]] += 1 + self.dataset_object_counts[self.dataset_names[d.get("dataset_id", 0)]] += len( + d.get("instances", []) + ) + dataset_image_counts = {f"count_image/{k}": v for k, v in self.dataset_image_counts.items()} + dataset_object_counts = { + f"count_object/{k}": v for k, v in self.dataset_object_counts.items() + } + if self.async_write_metrics: + # write metrics asynchronically + self.concurrent_executor.submit( + self._write_metrics_common, dataset_image_counts, iter=self.iter + ) + self.concurrent_executor.submit( + self._write_metrics_common, dataset_object_counts, iter=self.iter + ) + else: + self._write_metrics_common(dataset_image_counts) + self._write_metrics_common(dataset_object_counts) + # ------------------------------------------------------------------ + + if self.zero_grad_before_forward: + """ + If you need to accumulate gradients or do something similar, you can + wrap the optimizer with your custom `zero_grad()` method. + """ + self.optimizer.zero_grad() + + """ + If you want to do something with the losses, you can wrap the model. + """ + loss_dict = self.model(data) + if isinstance(loss_dict, torch.Tensor): + losses = loss_dict + loss_dict = {"total_loss": loss_dict} + else: + losses = sum(loss_dict.values()) + if not self.zero_grad_before_forward: + """ + If you need to accumulate gradients or do something similar, you can + wrap the optimizer with your custom `zero_grad()` method. + """ + self.optimizer.zero_grad() + losses.backward() + + self.after_backward() + + if self.async_write_metrics: + # write metrics asynchronically + self.concurrent_executor.submit( + self._write_metrics, loss_dict, data_time, iter=self.iter + ) + else: + self._write_metrics(loss_dict, data_time) + + """ + If you need gradient clipping/scaling or other processing, you can + wrap the optimizer with your custom `step()` method. But it is + suboptimal as explained in https://arxiv.org/abs/2006.15704 Sec 3.2.4 + """ + self.optimizer.step() + + @property + def _data_loader_iter(self): + # only create the data loader iterator when it is used + if self._data_loader_iter_obj is None: + self._data_loader_iter_obj = iter(self.data_loader) + return self._data_loader_iter_obj + + def reset_data_loader(self, data_loader_builder): + """ + Delete and replace the current data loader with a new one, which will be created + by calling `data_loader_builder` (without argument). + """ + del self.data_loader + data_loader = data_loader_builder() + self.data_loader = data_loader + self._data_loader_iter_obj = None + + def _write_metrics( + self, + loss_dict: Mapping[str, torch.Tensor], + data_time: float, + prefix: str = "", + iter: Optional[int] = None, + ) -> None: + logger = logging.getLogger(__name__) + + iter = self.iter if iter is None else iter + if (iter + 1) % self.gather_metric_period == 0: + try: + SimpleTrainer.write_metrics(loss_dict, data_time, iter, prefix) + except Exception: + logger.exception("Exception in writing metrics: ") + raise + + @staticmethod + def write_metrics( + loss_dict: Mapping[str, torch.Tensor], + data_time: float, + cur_iter: int, + prefix: str = "", + ) -> None: + """ + Args: + loss_dict (dict): dict of scalar losses + data_time (float): time taken by the dataloader iteration + prefix (str): prefix for logging keys + """ + metrics_dict = {k: v.detach().cpu().item() for k, v in loss_dict.items()} + metrics_dict["data_time"] = data_time + + # Gather metrics among all workers for logging + # This assumes we do DDP-style training, which is currently the only + # supported method in detectron2. + all_metrics_dict = comm.gather(metrics_dict) + + if comm.is_main_process(): + storage = get_event_storage() + + # data_time among workers can have high variance. The actual latency + # caused by data_time is the maximum among workers. + data_time = np.max([x.pop("data_time") for x in all_metrics_dict]) + storage.put_scalar("data_time", data_time, cur_iter=cur_iter) + + # average the rest metrics + all_metrics_key = [] + for metrics_dict in all_metrics_dict: + for key in metrics_dict.keys(): + if key not in all_metrics_key: + all_metrics_key.append(key) + metrics_dict = { + k: np.mean([x[k] for x in all_metrics_dict if k in x]) for k in all_metrics_key + } + total_losses_reduced = sum(metrics_dict.values()) + if not np.isfinite(total_losses_reduced): + raise FloatingPointError( + f"Loss became infinite or NaN at iteration={cur_iter}!\n" + f"loss_dict = {metrics_dict}" + ) + + storage.put_scalar( + "{}total_loss".format(prefix), total_losses_reduced, cur_iter=cur_iter + ) + if len(metrics_dict) > 1: + storage.put_scalars(cur_iter=cur_iter, **metrics_dict) + + def state_dict(self): + ret = super().state_dict() + ret["optimizer"] = self.optimizer.state_dict() + return ret + + def load_state_dict(self, state_dict): + super().load_state_dict(state_dict) + self.optimizer.load_state_dict(state_dict["optimizer"]) + + def after_train(self): + super().after_train() + self.concurrent_executor.shutdown(wait=True) + + def _write_metrics_common( + self, + metrics_dict: Mapping[str, torch.Tensor], + prefix: str = "", + iter: Optional[int] = None, + ) -> None: + logger = logging.getLogger(__name__) + + iter = self.iter if iter is None else iter + if (iter + 1) % self.gather_metric_period == 0: + try: + SimpleTrainer.write_metrics_common(metrics_dict, iter, prefix) + except Exception: + logger.exception("Exception in writing metrics: ") + raise + + @staticmethod + def write_metrics_common( + metrics_dict: Mapping[str, torch.Tensor], + cur_iter: int, + prefix: str = "", + ) -> None: + """ + Args: + metrics_dict (dict): dict of scalar losses + prefix (str): prefix for logging keys + """ + metrics_dict = {k: v.detach().cpu().item() for k, v in metrics_dict.items()} + all_metrics_dict = comm.gather(metrics_dict) + if comm.is_main_process(): + storage = get_event_storage() + + metrics_dict = { + k: np.sum([x[k] for x in all_metrics_dict]) for k in all_metrics_dict[0].keys() + } + + if len(metrics_dict) > 1: + storage.put_scalars(cur_iter=cur_iter, **metrics_dict) + + +class AMPTrainer(SimpleTrainer): + """ + Like :class:`SimpleTrainer`, but uses PyTorch's native automatic mixed precision + in the training loop. + """ + + def __init__( + self, + model, + data_loader, + optimizer, + gather_metric_period=1, + zero_grad_before_forward=False, + grad_scaler=None, + precision: torch.dtype = torch.float16, + log_grad_scaler: bool = False, + async_write_metrics=False, + ): + """ + Args: + model, data_loader, optimizer, gather_metric_period, zero_grad_before_forward, + async_write_metrics: same as in :class:`SimpleTrainer`. + grad_scaler: torch GradScaler to automatically scale gradients. + precision: torch.dtype as the target precision to cast to in computations + """ + unsupported = "AMPTrainer does not support single-process multi-device training!" + if isinstance(model, DistributedDataParallel): + assert not (model.device_ids and len(model.device_ids) > 1), unsupported + assert not isinstance(model, DataParallel), unsupported + + super().__init__( + model, data_loader, optimizer, gather_metric_period, zero_grad_before_forward + ) + + if grad_scaler is None: + from torch.cuda.amp import GradScaler + + grad_scaler = GradScaler() + self.grad_scaler = grad_scaler + self.precision = precision + self.log_grad_scaler = log_grad_scaler + + def run_step(self): + """ + Implement the AMP training logic. + """ + assert self.model.training, "[AMPTrainer] model was changed to eval mode!" + assert torch.cuda.is_available(), "[AMPTrainer] CUDA is required for AMP training!" + from torch.cuda.amp import autocast + + start = time.perf_counter() + data = next(self._data_loader_iter) + data_time = time.perf_counter() - start + + # ------------------------------------------------------------------ + for d in data: + self.dataset_image_counts[self.dataset_names[d.get("dataset_id", 0)]] += 1 + self.dataset_object_counts[self.dataset_names[d.get("dataset_id", 0)]] += len( + d.get("instances", []) + ) + dataset_image_counts = { + f"count_image/{k}": v for k, v in self.dataset_image_counts.items() + } + dataset_object_counts = { + f"count_object/{k}": v for k, v in self.dataset_object_counts.items() + } + if self.async_write_metrics: + # write metrics asynchronically + self.concurrent_executor.submit( + self._write_metrics_common, dataset_image_counts, iter=self.iter + ) + self.concurrent_executor.submit( + self._write_metrics_common, dataset_object_counts, iter=self.iter + ) + else: + self._write_metrics_common(dataset_image_counts) + self._write_metrics_common(dataset_object_counts) + # ------------------------------------------------------------------ + + if self.zero_grad_before_forward: + self.optimizer.zero_grad() + with autocast(dtype=self.precision): + loss_dict = self.model(data) + if isinstance(loss_dict, torch.Tensor): + losses = loss_dict + loss_dict = {"total_loss": loss_dict} + else: + losses = sum(loss_dict.values()) + + if not self.zero_grad_before_forward: + self.optimizer.zero_grad() + + self.grad_scaler.scale(losses).backward() + + if self.log_grad_scaler: + storage = get_event_storage() + storage.put_scalar("[metric] grad_scaler", self.grad_scaler.get_scale()) + + self.after_backward() + + if self.async_write_metrics: + # write metrics asynchronically + self.concurrent_executor.submit( + self._write_metrics, loss_dict, data_time, iter=self.iter + ) + else: + self._write_metrics(loss_dict, data_time) + + self.grad_scaler.step(self.optimizer) + self.grad_scaler.update() + + def state_dict(self): + ret = super().state_dict() + ret["grad_scaler"] = self.grad_scaler.state_dict() + return ret + + def load_state_dict(self, state_dict): + super().load_state_dict(state_dict) + self.grad_scaler.load_state_dict(state_dict["grad_scaler"]) diff --git a/approach/ovod/APE/ape/evaluation/__init__.py b/approach/ovod/APE/ape/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..150dcc393d5ee6cacc77c500165641a919b5f18b --- /dev/null +++ b/approach/ovod/APE/ape/evaluation/__init__.py @@ -0,0 +1,8 @@ +from .d3_evaluation import D3Evaluator +from .evaluator import inference_on_dataset +from .instance_evaluation import InstanceSegEvaluator +from .lvis_evaluation import LVISEvaluator +from .oideval import OIDEvaluator +from .refcoco_evaluation import RefCOCOEvaluator + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/approach/ovod/APE/ape/evaluation/d3_evaluation.py b/approach/ovod/APE/ape/evaluation/d3_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..0f4db0430b59aa03ecb8f767bb113dc40b71bdfd --- /dev/null +++ b/approach/ovod/APE/ape/evaluation/d3_evaluation.py @@ -0,0 +1,771 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import contextlib +import copy +import io +import itertools +import json +import logging +import os +import pickle +from collections import OrderedDict + +import numpy as np +import pycocotools.mask as mask_util +import torch +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval + +import detectron2.utils.comm as comm +from detectron2.config import CfgNode +from detectron2.data import MetadataCatalog +from detectron2.data.datasets.coco import convert_to_coco_json +from detectron2.evaluation import DatasetEvaluator +from detectron2.structures import Boxes, BoxMode, pairwise_iou +from detectron2.utils.file_io import PathManager +from detectron2.utils.logger import create_small_table +from tabulate import tabulate + +try: + from detectron2.evaluation.fast_eval_api import COCOeval_opt +except ImportError: + COCOeval_opt = COCOeval + + +class D3Evaluator(DatasetEvaluator): + """ + Evaluate AR for object proposals, AP for instance detection/segmentation, AP + for keypoint detection outputs using COCO's metrics. + See http://cocodataset.org/#detection-eval and + http://cocodataset.org/#keypoints-eval to understand its metrics. + The metrics range from 0 to 100 (instead of 0 to 1), where a -1 or NaN means + the metric cannot be computed (e.g. due to no predictions made). + + In addition to COCO, this evaluator is able to support any bounding box detection, + instance segmentation, or keypoint detection dataset. + """ + + def __init__( + self, + dataset_name, + tasks=None, + distributed=True, + output_dir=None, + *, + max_dets_per_image=None, + use_fast_impl=True, + kpt_oks_sigmas=(), + allow_cached_coco=True, + mode="FULL", # FULL, PRES, ABS + ): + """ + Args: + dataset_name (str): name of the dataset to be evaluated. + It must have either the following corresponding metadata: + + "json_file": the path to the COCO format annotation + + Or it must be in detectron2's standard dataset format + so it can be converted to COCO format automatically. + tasks (tuple[str]): tasks that can be evaluated under the given + configuration. A task is one of "bbox", "segm", "keypoints". + By default, will infer this automatically from predictions. + distributed (True): if True, will collect results from all ranks and run evaluation + in the main process. + Otherwise, will only evaluate the results in the current process. + output_dir (str): optional, an output directory to dump all + results predicted on the dataset. The dump contains two files: + + 1. "instances_predictions.pth" a file that can be loaded with `torch.load` and + contains all the results in the format they are produced by the model. + 2. "coco_instances_results.json" a json file in COCO's result format. + max_dets_per_image (int): limit on the maximum number of detections per image. + By default in COCO, this limit is to 100, but this can be customized + to be greater, as is needed in evaluation metrics AP fixed and AP pool + (see https://arxiv.org/pdf/2102.01066.pdf) + This doesn't affect keypoint evaluation. + use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP. + Although the results should be very close to the official implementation in COCO + API, it is still recommended to compute results with the official API for use in + papers. The faster implementation also uses more RAM. + kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS. + See http://cocodataset.org/#keypoints-eval + When empty, it will use the defaults in COCO. + Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS. + allow_cached_coco (bool): Whether to use cached coco json from previous validation + runs. You should set this to False if you need to use different validation data. + Defaults to True. + """ + self._logger = logging.getLogger(__name__) + self._distributed = distributed + self._output_dir = output_dir + + if use_fast_impl and (COCOeval_opt is COCOeval): + self._logger.info("Fast COCO eval is not built. Falling back to official COCO eval.") + use_fast_impl = False + self._use_fast_impl = use_fast_impl + + # COCOeval requires the limit on the number of detections per image (maxDets) to be a list + # with at least 3 elements. The default maxDets in COCOeval is [1, 10, 100], in which the + # 3rd element (100) is used as the limit on the number of detections per image when + # evaluating AP. COCOEvaluator expects an integer for max_dets_per_image, so for COCOeval, + # we reformat max_dets_per_image into [1, 10, max_dets_per_image], based on the defaults. + if max_dets_per_image is None: + max_dets_per_image = [1, 10, 100] + else: + max_dets_per_image = [1, 10, max_dets_per_image] + self._max_dets_per_image = max_dets_per_image + + if tasks is not None and isinstance(tasks, CfgNode): + kpt_oks_sigmas = ( + tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas + ) + self._logger.warn( + "COCO Evaluator instantiated using config, this is deprecated behavior." + " Please pass in explicit arguments instead." + ) + self._tasks = None # Infering it from predictions should be better + else: + self._tasks = tasks + + self._cpu_device = torch.device("cpu") + + self._metadata = MetadataCatalog.get(dataset_name) + if not hasattr(self._metadata, "json_file"): + if output_dir is None: + raise ValueError( + "output_dir must be provided to COCOEvaluator " + "for datasets not in COCO format." + ) + self._logger.info(f"Trying to convert '{dataset_name}' to COCO format ...") + + cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") + self._metadata.json_file = {} + self._metadata.json_file[mode] = cache_path + convert_to_coco_json(dataset_name, cache_path, allow_cached=allow_cached_coco) + + json_file = PathManager.get_local_path(self._metadata.json_file[mode]) + with contextlib.redirect_stdout(io.StringIO()): + self._coco_api = COCO(json_file) + + # Test set json files do not contain annotations (evaluation must be + # performed using the COCO evaluation server). + self._do_evaluation = "annotations" in self._coco_api.dataset + if self._do_evaluation: + self._kpt_oks_sigmas = kpt_oks_sigmas + + self.mode = mode + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a COCO model (e.g., GeneralizedRCNN). + It is a list of dict. Each dict corresponds to an image and + contains keys like "height", "width", "file_name", "image_id". + outputs: the outputs of a COCO model. It is a list of dicts with key + "instances" that contains :class:`Instances`. + """ + for input, output in zip(inputs, outputs): + prediction = {"image_id": input["image_id"]} + + if "instances" in output: + instances = output["instances"].to(self._cpu_device) + + if self._metadata.group == "intra": + instances = instances[instances.pred_classes < len(input["sent_ids"])] + instances.pred_classes = torch.as_tensor( + [ + input["sent_ids"][pred_class] + for pred_class in instances.pred_classes.tolist() + ] + ) + elif self._metadata.group == "inter": + pass + else: + assert False + + prediction["instances"] = instances_to_coco_json(instances, input["image_id"]) + if "proposals" in output: + prediction["proposals"] = output["proposals"].to(self._cpu_device) + if len(prediction) > 1: + self._predictions.append(prediction) + + def evaluate(self, img_ids=None): + """ + Args: + img_ids: a list of image IDs to evaluate on. Default to None for the whole dataset + """ + if self._distributed: + comm.synchronize() + predictions = comm.gather(self._predictions, dst=0) + predictions = list(itertools.chain(*predictions)) + + if not comm.is_main_process(): + return {} + else: + predictions = self._predictions + + if len(predictions) == 0: + self._logger.warning("[COCOEvaluator] Did not receive valid predictions.") + return {} + + if self._output_dir: + PathManager.mkdirs(self._output_dir) + file_path = os.path.join(self._output_dir, f"instances_predictions_{self.mode}.pth") + with PathManager.open(file_path, "wb") as f: + torch.save(predictions, f) + + self._results = OrderedDict() + if "proposals" in predictions[0]: + self._eval_box_proposals(predictions) + if "instances" in predictions[0]: + self._eval_predictions(predictions, img_ids=img_ids) + # Copy so the caller can do whatever with results + self._results = {f"{k}_{self.mode}": v for k, v in self._results.items()} + return copy.deepcopy(self._results) + + def _tasks_from_predictions(self, predictions): + """ + Get COCO API "tasks" (i.e. iou_type) from COCO-format predictions. + """ + tasks = {"bbox"} + for pred in predictions: + if "segmentation" in pred: + tasks.add("segm") + if "keypoints" in pred: + tasks.add("keypoints") + return sorted(tasks) + + def _eval_predictions(self, predictions, img_ids=None): + """ + Evaluate predictions. Fill self._results with the metrics of the tasks. + """ + self._logger.info("Preparing results for COCO format ...") + coco_results = list(itertools.chain(*[x["instances"] for x in predictions])) + tasks = self._tasks or self._tasks_from_predictions(coco_results) + + # unmap the category ids for COCO + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"): + dataset_id_to_contiguous_id = self._metadata.thing_dataset_id_to_contiguous_id + all_contiguous_ids = list(dataset_id_to_contiguous_id.values()) + num_classes = len(all_contiguous_ids) + assert min(all_contiguous_ids) == 0 and max(all_contiguous_ids) == num_classes - 1 + + reverse_id_mapping = {v: k for k, v in dataset_id_to_contiguous_id.items()} + for result in coco_results: + category_id = result["category_id"] + assert category_id < num_classes, ( + f"A prediction has class={category_id}, " + f"but the dataset only has {num_classes} classes and " + f"predicted class id should be in [0, {num_classes - 1}]." + ) + result["category_id"] = reverse_id_mapping[category_id] + + if self._output_dir: + file_path = os.path.join(self._output_dir, f"coco_instances_results_{self.mode}.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(coco_results)) + f.flush() + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info( + "Evaluating predictions with {} COCO API...".format( + "unofficial" if self._use_fast_impl else "official" + ) + ) + for task in sorted(tasks): + assert task in {"bbox", "segm", "keypoints"}, f"Got unknown task: {task}!" + coco_eval = ( + _evaluate_predictions_on_coco( + self._coco_api, + coco_results, + task, + kpt_oks_sigmas=self._kpt_oks_sigmas, + cocoeval_fn=COCOeval_opt if self._use_fast_impl else COCOeval, + img_ids=img_ids, + max_dets_per_image=self._max_dets_per_image, + ) + if len(coco_results) > 0 + else None # cocoapi does not handle empty results very well + ) + + res = self._derive_coco_results( + coco_eval, task, class_names=self._metadata.get("thing_classes") + ) + self._results[task] = res + + def _eval_box_proposals(self, predictions): + """ + Evaluate the box proposals in predictions. + Fill self._results with the metrics for "box_proposals" task. + """ + if self._output_dir: + # Saving generated box proposals to file. + # Predicted box_proposals are in XYXY_ABS mode. + bbox_mode = BoxMode.XYXY_ABS.value + ids, boxes, objectness_logits = [], [], [] + for prediction in predictions: + ids.append(prediction["image_id"]) + boxes.append(prediction["proposals"].proposal_boxes.tensor.numpy()) + objectness_logits.append(prediction["proposals"].objectness_logits.numpy()) + + proposal_data = { + "boxes": boxes, + "objectness_logits": objectness_logits, + "ids": ids, + "bbox_mode": bbox_mode, + } + with PathManager.open(os.path.join(self._output_dir, "box_proposals.pkl"), "wb") as f: + pickle.dump(proposal_data, f) + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info("Evaluating bbox proposals ...") + res = {} + areas = {"all": "", "small": "s", "medium": "m", "large": "l"} + for limit in [100, 1000]: + for area, suffix in areas.items(): + stats = _evaluate_box_proposals(predictions, self._coco_api, area=area, limit=limit) + key = "AR{}@{:d}".format(suffix, limit) + res[key] = float(stats["ar"].item() * 100) + self._logger.info("Proposal metrics: \n" + create_small_table(res)) + self._results["box_proposals"] = res + + def _derive_coco_results(self, coco_eval, iou_type, class_names=None): + """ + Derive the desired score numbers from summarized COCOeval. + + Args: + coco_eval (None or COCOEval): None represents no predictions from model. + iou_type (str): + class_names (None or list[str]): if provided, will use it to predict + per-category AP. + + Returns: + a dict of {metric name: score} + """ + + metrics = { + "bbox": [ + "AP", + "AP50", + "AP75", + "APs", + "APm", + "APl", + "AR@1", + "AR@10", + "AR@100", + "ARs", + "ARm", + "ARl", + ], + "segm": [ + "AP", + "AP50", + "AP75", + "APs", + "APm", + "APl", + "AR@1", + "AR@10", + "AR@100", + "ARs", + "ARm", + "ARl", + ], + "keypoints": ["AP", "AP50", "AP75", "APm", "APl"], + }[iou_type] + + if coco_eval is None: + self._logger.warn("No predictions from the model!") + return {metric: float("nan") for metric in metrics} + + # the standard metrics + results = { + metric: float(coco_eval.stats[idx] * 100 if coco_eval.stats[idx] >= 0 else "nan") + for idx, metric in enumerate(metrics) + } + self._logger.info( + "Evaluation results for {}: \n".format(iou_type) + create_small_table(results) + ) + if not np.isfinite(sum(results.values())): + self._logger.info("Some metrics cannot be computed and is shown as NaN.") + + if class_names is None or len(class_names) <= 1: + return results + # Compute per-category AP + # from https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L222-L252 # noqa + precisions = coco_eval.eval["precision"] + # precision has dims (iou, recall, cls, area range, max dets) + + if len(class_names) > precisions.shape[2]: + class_names = [category["name"] for category in self._coco_api.dataset["categories"]] + + assert len(class_names) == precisions.shape[2] + + results_per_category = [] + for idx, name in enumerate(class_names): + # area range index 0: all area ranges + # max dets index -1: typically 100 per image + precision = precisions[:, :, idx, 0, -1] + precision = precision[precision > -1] + ap = np.mean(precision) if precision.size else float("nan") + results_per_category.append(("{}".format(name), float(ap * 100))) + + # tabulate it + N_COLS = min(6, len(results_per_category) * 2) + results_flatten = list(itertools.chain(*results_per_category)) + results_2d = itertools.zip_longest(*[results_flatten[i::N_COLS] for i in range(N_COLS)]) + table = tabulate( + results_2d, + tablefmt="pipe", + floatfmt=".3f", + headers=["category", "AP"] * (N_COLS // 2), + numalign="left", + ) + self._logger.info("Per-category {} AP: \n".format(iou_type) + table) + + results.update({"AP-" + name: ap for name, ap in results_per_category}) + return results + + +def instances_to_coco_json(instances, img_id): + """ + Dump an "Instances" object to a COCO-format json that's used for evaluation. + + Args: + instances (Instances): + img_id (int): the image id + + Returns: + list[dict]: list of json annotations in COCO format. + """ + num_instance = len(instances) + if num_instance == 0: + return [] + + boxes = instances.pred_boxes.tensor.numpy() + boxes = BoxMode.convert(boxes, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS) + boxes = boxes.tolist() + scores = instances.scores.tolist() + classes = instances.pred_classes.tolist() + + has_mask = instances.has("pred_masks") + if has_mask: + # use RLE to encode the masks, because they are too large and takes memory + # since this evaluator stores outputs of the entire dataset + rles = [ + mask_util.encode(np.array(mask[:, :, None], order="F", dtype="uint8"))[0] + for mask in instances.pred_masks + ] + for rle in rles: + # "counts" is an array encoded by mask_util as a byte-stream. Python3's + # json writer which always produces strings cannot serialize a bytestream + # unless you decode it. Thankfully, utf-8 works out (which is also what + # the pycocotools/_mask.pyx does). + rle["counts"] = rle["counts"].decode("utf-8") + + has_keypoints = instances.has("pred_keypoints") + if has_keypoints: + keypoints = instances.pred_keypoints + + results = [] + for k in range(num_instance): + result = { + "image_id": img_id, + "category_id": classes[k], + "bbox": boxes[k], + "score": scores[k], + } + if has_mask: + result["segmentation"] = rles[k] + if has_keypoints: + # In COCO annotations, + # keypoints coordinates are pixel indices. + # However our predictions are floating point coordinates. + # Therefore we subtract 0.5 to be consistent with the annotation format. + # This is the inverse of data loading logic in `datasets/coco.py`. + keypoints[k][:, :2] -= 0.5 + result["keypoints"] = keypoints[k].flatten().tolist() + results.append(result) + return results + + +# inspired from Detectron: +# https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L255 # noqa +def _evaluate_box_proposals(dataset_predictions, coco_api, thresholds=None, area="all", limit=None): + """ + Evaluate detection proposal recall metrics. This function is a much + faster alternative to the official COCO API recall evaluation code. However, + it produces slightly different results. + """ + # Record max overlap value for each gt box + # Return vector of overlap values + areas = { + "all": 0, + "small": 1, + "medium": 2, + "large": 3, + "96-128": 4, + "128-256": 5, + "256-512": 6, + "512-inf": 7, + } + area_ranges = [ + [0**2, 1e5**2], # all + [0**2, 32**2], # small + [32**2, 96**2], # medium + [96**2, 1e5**2], # large + [96**2, 128**2], # 96-128 + [128**2, 256**2], # 128-256 + [256**2, 512**2], # 256-512 + [512**2, 1e5**2], + ] # 512-inf + assert area in areas, "Unknown area range: {}".format(area) + area_range = area_ranges[areas[area]] + gt_overlaps = [] + num_pos = 0 + + for prediction_dict in dataset_predictions: + predictions = prediction_dict["proposals"] + + # sort predictions in descending order + # TODO maybe remove this and make it explicit in the documentation + inds = predictions.objectness_logits.sort(descending=True)[1] + predictions = predictions[inds] + + ann_ids = coco_api.getAnnIds(imgIds=prediction_dict["image_id"]) + anno = coco_api.loadAnns(ann_ids) + gt_boxes = [ + BoxMode.convert(obj["bbox"], BoxMode.XYWH_ABS, BoxMode.XYXY_ABS) + for obj in anno + if obj["iscrowd"] == 0 + ] + gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4) # guard against no boxes + gt_boxes = Boxes(gt_boxes) + gt_areas = torch.as_tensor([obj["area"] for obj in anno if obj["iscrowd"] == 0]) + + if len(gt_boxes) == 0 or len(predictions) == 0: + continue + + valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1]) + gt_boxes = gt_boxes[valid_gt_inds] + + num_pos += len(gt_boxes) + + if len(gt_boxes) == 0: + continue + + if limit is not None and len(predictions) > limit: + predictions = predictions[:limit] + + overlaps = pairwise_iou(predictions.proposal_boxes, gt_boxes) + + _gt_overlaps = torch.zeros(len(gt_boxes)) + for j in range(min(len(predictions), len(gt_boxes))): + # find which proposal box maximally covers each gt box + # and get the iou amount of coverage for each gt box + max_overlaps, argmax_overlaps = overlaps.max(dim=0) + + # find which gt box is 'best' covered (i.e. 'best' = most iou) + gt_ovr, gt_ind = max_overlaps.max(dim=0) + assert gt_ovr >= 0 + # find the proposal box that covers the best covered gt box + box_ind = argmax_overlaps[gt_ind] + # record the iou coverage of this gt box + _gt_overlaps[j] = overlaps[box_ind, gt_ind] + assert _gt_overlaps[j] == gt_ovr + # mark the proposal box and the gt box as used + overlaps[box_ind, :] = -1 + overlaps[:, gt_ind] = -1 + + # append recorded iou coverage level + gt_overlaps.append(_gt_overlaps) + gt_overlaps = ( + torch.cat(gt_overlaps, dim=0) if len(gt_overlaps) else torch.zeros(0, dtype=torch.float32) + ) + gt_overlaps, _ = torch.sort(gt_overlaps) + + if thresholds is None: + step = 0.05 + thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32) + recalls = torch.zeros_like(thresholds) + # compute recall for each iou threshold + for i, t in enumerate(thresholds): + recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos) + # ar = 2 * np.trapz(recalls, thresholds) + ar = recalls.mean() + return { + "ar": ar, + "recalls": recalls, + "thresholds": thresholds, + "gt_overlaps": gt_overlaps, + "num_pos": num_pos, + } + + +def _evaluate_predictions_on_coco( + coco_gt, + coco_results, + iou_type, + kpt_oks_sigmas=None, + cocoeval_fn=COCOeval_opt, + img_ids=None, + max_dets_per_image=None, +): + """ + Evaluate the coco results using COCOEval API. + """ + assert len(coco_results) > 0 + + if iou_type == "segm": + coco_results = copy.deepcopy(coco_results) + # When evaluating mask AP, if the results contain bbox, cocoapi will + # use the box area as the area of the instance, instead of the mask area. + # This leads to a different definition of small/medium/large. + # We remove the bbox field to let mask AP use mask area. + for c in coco_results: + c.pop("bbox", None) + + coco_dt = coco_gt.loadRes(coco_results) + coco_eval = cocoeval_fn(coco_gt, coco_dt, iou_type) + # For COCO, the default max_dets_per_image is [1, 10, 100]. + if max_dets_per_image is None: + max_dets_per_image = [1, 10, 100] # Default from COCOEval + else: + assert ( + len(max_dets_per_image) >= 3 + ), "COCOeval requires maxDets (and max_dets_per_image) to have length at least 3" + # In the case that user supplies a custom input for max_dets_per_image, + # apply COCOevalMaxDets to evaluate AP with the custom input. + if max_dets_per_image[2] != 100: + coco_eval = COCOevalMaxDets(coco_gt, coco_dt, iou_type) + if iou_type != "keypoints": + coco_eval.params.maxDets = max_dets_per_image + + if img_ids is not None: + coco_eval.params.imgIds = img_ids + + if iou_type == "keypoints": + # Use the COCO default keypoint OKS sigmas unless overrides are specified + if kpt_oks_sigmas: + assert hasattr(coco_eval.params, "kpt_oks_sigmas"), "pycocotools is too old!" + coco_eval.params.kpt_oks_sigmas = np.array(kpt_oks_sigmas) + # COCOAPI requires every detection and every gt to have keypoints, so + # we just take the first entry from both + num_keypoints_dt = len(coco_results[0]["keypoints"]) // 3 + num_keypoints_gt = len(next(iter(coco_gt.anns.values()))["keypoints"]) // 3 + num_keypoints_oks = len(coco_eval.params.kpt_oks_sigmas) + assert num_keypoints_oks == num_keypoints_dt == num_keypoints_gt, ( + f"[COCOEvaluator] Prediction contain {num_keypoints_dt} keypoints. " + f"Ground truth contains {num_keypoints_gt} keypoints. " + f"The length of cfg.TEST.KEYPOINT_OKS_SIGMAS is {num_keypoints_oks}. " + "They have to agree with each other. For meaning of OKS, please refer to " + "http://cocodataset.org/#keypoints-eval." + ) + + coco_eval.evaluate() + coco_eval.accumulate() + coco_eval.summarize() + + return coco_eval + + +class COCOevalMaxDets(COCOeval): + """ + Modified version of COCOeval for evaluating AP with a custom + maxDets (by default for COCO, maxDets is 100) + """ + + def summarize(self): + """ + Compute and display summary metrics for evaluation results given + a custom value for max_dets_per_image + """ + + def _summarize(ap=1, iouThr=None, areaRng="all", maxDets=100): + p = self.params + iStr = " {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}" + titleStr = "Average Precision" if ap == 1 else "Average Recall" + typeStr = "(AP)" if ap == 1 else "(AR)" + iouStr = ( + "{:0.2f}:{:0.2f}".format(p.iouThrs[0], p.iouThrs[-1]) + if iouThr is None + else "{:0.2f}".format(iouThr) + ) + + aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng] + mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets] + if ap == 1: + # dimension of precision: [TxRxKxAxM] + s = self.eval["precision"] + # IoU + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] + s = s[:, :, :, aind, mind] + else: + # dimension of recall: [TxKxAxM] + s = self.eval["recall"] + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] + s = s[:, :, aind, mind] + if len(s[s > -1]) == 0: + mean_s = -1 + else: + mean_s = np.mean(s[s > -1]) + print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s)) + return mean_s + + def _summarizeDets(): + stats = np.zeros((12,)) + # Evaluate AP using the custom limit on maximum detections per image + stats[0] = _summarize(1, maxDets=self.params.maxDets[2]) + stats[1] = _summarize(1, iouThr=0.5, maxDets=self.params.maxDets[2]) + stats[2] = _summarize(1, iouThr=0.75, maxDets=self.params.maxDets[2]) + stats[3] = _summarize(1, areaRng="small", maxDets=self.params.maxDets[2]) + stats[4] = _summarize(1, areaRng="medium", maxDets=self.params.maxDets[2]) + stats[5] = _summarize(1, areaRng="large", maxDets=self.params.maxDets[2]) + stats[6] = _summarize(0, maxDets=self.params.maxDets[0]) + stats[7] = _summarize(0, maxDets=self.params.maxDets[1]) + stats[8] = _summarize(0, maxDets=self.params.maxDets[2]) + stats[9] = _summarize(0, areaRng="small", maxDets=self.params.maxDets[2]) + stats[10] = _summarize(0, areaRng="medium", maxDets=self.params.maxDets[2]) + stats[11] = _summarize(0, areaRng="large", maxDets=self.params.maxDets[2]) + return stats + + def _summarizeKps(): + stats = np.zeros((10,)) + stats[0] = _summarize(1, maxDets=20) + stats[1] = _summarize(1, maxDets=20, iouThr=0.5) + stats[2] = _summarize(1, maxDets=20, iouThr=0.75) + stats[3] = _summarize(1, maxDets=20, areaRng="medium") + stats[4] = _summarize(1, maxDets=20, areaRng="large") + stats[5] = _summarize(0, maxDets=20) + stats[6] = _summarize(0, maxDets=20, iouThr=0.5) + stats[7] = _summarize(0, maxDets=20, iouThr=0.75) + stats[8] = _summarize(0, maxDets=20, areaRng="medium") + stats[9] = _summarize(0, maxDets=20, areaRng="large") + return stats + + if not self.eval: + raise Exception("Please run accumulate() first") + iouType = self.params.iouType + if iouType == "segm" or iouType == "bbox": + summarize = _summarizeDets + elif iouType == "keypoints": + summarize = _summarizeKps + self.stats = summarize() + + def __str__(self): + self.summarize() diff --git a/approach/ovod/APE/ape/evaluation/evaluator.py b/approach/ovod/APE/ape/evaluation/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..0fb5b89daa4ea5e66cf5a95b3ad2769d28fffe0b --- /dev/null +++ b/approach/ovod/APE/ape/evaluation/evaluator.py @@ -0,0 +1,177 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import datetime +import logging +import time +from collections import abc +from contextlib import ExitStack +from typing import List, Union + +import torch +from torch import nn + +from detectron2.evaluation import DatasetEvaluator, DatasetEvaluators, inference_context +from detectron2.utils.comm import get_world_size +from detectron2.utils.logger import log_every_n_seconds + + +def inference_on_dataset( + model, data_loader, evaluator: Union[DatasetEvaluator, List[DatasetEvaluator], None] +): + """ + Run model on the data_loader and evaluate the metrics with evaluator. + Also benchmark the inference speed of `model.__call__` accurately. + The model will be used in eval mode. + + Args: + model (callable): a callable which takes an object from + `data_loader` and returns some outputs. + + If it's an nn.Module, it will be temporarily set to `eval` mode. + If you wish to evaluate a model in `training` mode instead, you can + wrap the given model and override its behavior of `.eval()` and `.train()`. + data_loader: an iterable object with a length. + The elements it generates will be the inputs to the model. + evaluator: the evaluator(s) to run. Use `None` if you only want to benchmark, + but don't want to do any evaluation. + + Returns: + The return value of `evaluator.evaluate()` + """ + num_devices = get_world_size() + logger = logging.getLogger(__name__) + logger.info("Start inference on {} batches".format(len(data_loader))) + + total = len(data_loader) # inference data loader must have a fixed length + if evaluator is None: + # create a no-op evaluator + evaluator = DatasetEvaluators([]) + if isinstance(evaluator, abc.MutableSequence): + evaluator = DatasetEvaluators(evaluator) + evaluator.reset() + + num_warmup = min(5, total - 1) + start_time = time.perf_counter() + total_data_time = 0 + total_compute_time = 0 + total_eval_time = 0 + + total_preprocess_time = 0 + total_backbone_time = 0 + total_transformer_time = 0 + total_postprocess_time = 0 + + with ExitStack() as stack: + if isinstance(model, nn.Module): + stack.enter_context(inference_context(model)) + stack.enter_context(torch.no_grad()) + + start_data_time = time.perf_counter() + for idx, inputs in enumerate(data_loader): + total_data_time += time.perf_counter() - start_data_time + if idx == num_warmup: + start_time = time.perf_counter() + total_data_time = 0 + total_compute_time = 0 + total_eval_time = 0 + + total_preprocess_time = 0 + total_backbone_time = 0 + total_transformer_time = 0 + total_postprocess_time = 0 + + start_compute_time = time.perf_counter() + outputs = model(inputs) + if torch.cuda.is_available(): + torch.cuda.synchronize() + total_compute_time += time.perf_counter() - start_compute_time + + start_eval_time = time.perf_counter() + evaluator.process(inputs, outputs) + total_eval_time += time.perf_counter() - start_eval_time + + if hasattr(model.module, "preprocess_time"): + total_preprocess_time += model.module.preprocess_time + if hasattr(model.module, "model_vision") and hasattr( + model.module.model_vision, "preprocess_time" + ): + total_preprocess_time += model.module.model_vision.preprocess_time + if hasattr(model.module, "backbone_time"): + total_backbone_time += model.module.backbone_time + if hasattr(model.module, "model_vision") and hasattr( + model.module.model_vision, "backbone_time" + ): + total_backbone_time += model.module.model_vision.backbone_time + if hasattr(model.module, "transformer_time"): + total_transformer_time += model.module.transformer_time + if hasattr(model.module, "model_vision") and hasattr( + model.module.model_vision, "transformer_time" + ): + total_transformer_time += model.module.model_vision.transformer_time + if hasattr(model.module, "postprocess_time"): + total_postprocess_time += model.module.postprocess_time + if hasattr(model.module, "model_vision") and hasattr( + model.module.model_vision, "postprocess_time" + ): + total_postprocess_time += model.module.model_vision.postprocess_time + + iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup) + data_seconds_per_iter = total_data_time / iters_after_start + compute_seconds_per_iter = total_compute_time / iters_after_start + eval_seconds_per_iter = total_eval_time / iters_after_start + total_seconds_per_iter = (time.perf_counter() - start_time) / iters_after_start + + preprocess_seconds_per_iter = total_preprocess_time / iters_after_start + backbone_seconds_per_iter = total_backbone_time / iters_after_start + transformer_seconds_per_iter = total_transformer_time / iters_after_start + postprocess_seconds_per_iter = total_postprocess_time / iters_after_start + + if idx >= num_warmup * 2 or compute_seconds_per_iter > 5: + eta = datetime.timedelta(seconds=int(total_seconds_per_iter * (total - idx - 1))) + if torch.cuda.is_available(): + max_mem_mb = torch.cuda.max_memory_allocated() / 1024.0 / 1024.0 + else: + max_mem_mb = 0 + log_every_n_seconds( + logging.INFO, + ( + f"Inference done {idx + 1}/{total}. " + f"Dataloading: {data_seconds_per_iter:.4f} s/iter. " + f"Inference: {compute_seconds_per_iter:.4f} s/iter. " + f"Eval: {eval_seconds_per_iter:.4f} s/iter. " + f"Total: {total_seconds_per_iter:.4f} s/iter. " + f"ETA={eta}" + f". " + f"preprocess: {preprocess_seconds_per_iter:.4f} s/iter. " + f"backbone: {backbone_seconds_per_iter:.4f} s/iter. " + f"transformer: {transformer_seconds_per_iter:.4f} s/iter. " + f"postprocess: {postprocess_seconds_per_iter:.4f} s/iter. " + f"max_mem: {max_mem_mb:.0f}M. " + ), + n=5, + ) + if idx < num_warmup * 2: + torch.cuda.reset_peak_memory_stats() + start_data_time = time.perf_counter() + + # Measure the time only for this worker (before the synchronization barrier) + total_time = time.perf_counter() - start_time + total_time_str = str(datetime.timedelta(seconds=total_time)) + # NOTE this format is parsed by grep + logger.info( + "Total inference time: {} ({:.6f} s / iter per device, on {} devices)".format( + total_time_str, total_time / (total - num_warmup), num_devices + ) + ) + total_compute_time_str = str(datetime.timedelta(seconds=int(total_compute_time))) + logger.info( + "Total inference pure compute time: {} ({:.6f} s / iter per device, on {} devices)".format( + total_compute_time_str, total_compute_time / (total - num_warmup), num_devices + ) + ) + + results = evaluator.evaluate() + # An evaluator may return None when not in main process. + # Replace it by an empty dict instead to make it easier for downstream code to handle + if results is None: + results = {} + return results diff --git a/approach/ovod/APE/ape/evaluation/instance_evaluation.py b/approach/ovod/APE/ape/evaluation/instance_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..7ed95db049fa4994d6f2021cf0d969a26c84b503 --- /dev/null +++ b/approach/ovod/APE/ape/evaluation/instance_evaluation.py @@ -0,0 +1,112 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import contextlib +import copy +import io +import itertools +import json +import logging +import os +import pickle +from collections import OrderedDict + +import numpy as np +import pycocotools.mask as mask_util +import torch +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval + +import detectron2.utils.comm as comm +from detectron2.config import CfgNode +from detectron2.data import MetadataCatalog +from detectron2.data.datasets.coco import convert_to_coco_json +from detectron2.evaluation.coco_evaluation import COCOEvaluator, _evaluate_predictions_on_coco +from detectron2.structures import Boxes, BoxMode, pairwise_iou +from detectron2.utils.file_io import PathManager +from detectron2.utils.logger import create_small_table +from tabulate import tabulate + +try: + from detectron2.evaluation.fast_eval_api import COCOeval_opt +except ImportError: + COCOeval_opt = COCOeval + + +# modified from COCOEvaluator for instance segmetnat +class InstanceSegEvaluator(COCOEvaluator): + """ + Evaluate AR for object proposals, AP for instance detection/segmentation, AP + for keypoint detection outputs using COCO's metrics. + See http://cocodataset.org/#detection-eval and + http://cocodataset.org/#keypoints-eval to understand its metrics. + The metrics range from 0 to 100 (instead of 0 to 1), where a -1 or NaN means + the metric cannot be computed (e.g. due to no predictions made). + + In addition to COCO, this evaluator is able to support any bounding box detection, + instance segmentation, or keypoint detection dataset. + """ + + def _eval_predictions(self, predictions, img_ids=None): + """ + Evaluate predictions. Fill self._results with the metrics of the tasks. + """ + self._logger.info("Preparing results for COCO format ...") + coco_results = list(itertools.chain(*[x["instances"] for x in predictions])) + tasks = self._tasks or self._tasks_from_predictions(coco_results) + + # unmap the category ids for COCO + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"): + dataset_id_to_contiguous_id = self._metadata.thing_dataset_id_to_contiguous_id + # all_contiguous_ids = list(dataset_id_to_contiguous_id.values()) + # num_classes = len(all_contiguous_ids) + # assert min(all_contiguous_ids) == 0 and max(all_contiguous_ids) == num_classes - 1 + + reverse_id_mapping = {v: k for k, v in dataset_id_to_contiguous_id.items()} + for result in coco_results: + category_id = result["category_id"] + # assert category_id < num_classes, ( + # f"A prediction has class={category_id}, " + # f"but the dataset only has {num_classes} classes and " + # f"predicted class id should be in [0, {num_classes - 1}]." + # ) + assert category_id in reverse_id_mapping, ( + f"A prediction has class={category_id}, " + f"but the dataset only has class ids in {dataset_id_to_contiguous_id}." + ) + result["category_id"] = reverse_id_mapping[category_id] + + if self._output_dir: + file_path = os.path.join(self._output_dir, "coco_instances_results.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(coco_results)) + f.flush() + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info( + "Evaluating predictions with {} COCO API...".format( + "unofficial" if self._use_fast_impl else "official" + ) + ) + for task in sorted(tasks): + assert task in {"bbox", "segm", "keypoints"}, f"Got unknown task: {task}!" + coco_eval = ( + _evaluate_predictions_on_coco( + self._coco_api, + coco_results, + task, + kpt_oks_sigmas=self._kpt_oks_sigmas, + cocoeval_fn=COCOeval_opt if self._use_fast_impl else COCOeval, + img_ids=img_ids, + max_dets_per_image=self._max_dets_per_image, + ) + if len(coco_results) > 0 + else None # cocoapi does not handle empty results very well + ) + + res = self._derive_coco_results( + coco_eval, task, class_names=self._metadata.get("thing_classes") + ) + self._results[task] = res diff --git a/approach/ovod/APE/ape/evaluation/lvis_evaluation.py b/approach/ovod/APE/ape/evaluation/lvis_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..89db2a06602d5642a822f1d1c7537c1ef74603a3 --- /dev/null +++ b/approach/ovod/APE/ape/evaluation/lvis_evaluation.py @@ -0,0 +1,453 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import itertools +import json +import logging +import os +import pickle +from collections import OrderedDict + +import numpy as np +import torch + +import detectron2.utils.comm as comm +from detectron2.config import CfgNode +from detectron2.data import MetadataCatalog +from detectron2.evaluation.coco_evaluation import instances_to_coco_json +from detectron2.evaluation.evaluator import DatasetEvaluator +from detectron2.structures import Boxes, BoxMode, pairwise_iou +from detectron2.utils.file_io import PathManager +from detectron2.utils.logger import create_small_table +from tabulate import tabulate + + +class LVISEvaluator(DatasetEvaluator): + """ + Evaluate object proposal and instance detection/segmentation outputs using + LVIS's metrics and evaluation API. + """ + + def __init__( + self, + dataset_name, + tasks=None, + distributed=True, + output_dir=None, + *, + max_dets_per_image=None, + ): + """ + Args: + dataset_name (str): name of the dataset to be evaluated. + It must have the following corresponding metadata: + "json_file": the path to the LVIS format annotation + tasks (tuple[str]): tasks that can be evaluated under the given + configuration. A task is one of "bbox", "segm". + By default, will infer this automatically from predictions. + distributed (True): if True, will collect results from all ranks for evaluation. + Otherwise, will evaluate the results in the current process. + output_dir (str): optional, an output directory to dump results. + max_dets_per_image (None or int): limit on maximum detections per image in evaluating AP + This limit, by default of the LVIS dataset, is 300. + """ + from lvis import LVIS + + self._logger = logging.getLogger(__name__) + + if tasks is not None and isinstance(tasks, CfgNode): + self._logger.warn( + "COCO Evaluator instantiated using config, this is deprecated behavior." + " Please pass in explicit arguments instead." + ) + self._tasks = None # Infering it from predictions should be better + else: + self._tasks = tasks + + self._distributed = distributed + self._output_dir = output_dir + self._max_dets_per_image = max_dets_per_image + + self._cpu_device = torch.device("cpu") + + self._metadata = MetadataCatalog.get(dataset_name) + json_file = PathManager.get_local_path(self._metadata.json_file) + self._lvis_api = LVIS(json_file) + # Test set json files do not contain annotations (evaluation must be + # performed using the LVIS evaluation server). + self._do_evaluation = len(self._lvis_api.get_ann_ids()) > 0 + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a LVIS model (e.g., GeneralizedRCNN). + It is a list of dict. Each dict corresponds to an image and + contains keys like "height", "width", "file_name", "image_id". + outputs: the outputs of a LVIS model. It is a list of dicts with key + "instances" that contains :class:`Instances`. + """ + for input, output in zip(inputs, outputs): + prediction = {"image_id": input["image_id"]} + + if "instances" in output: + instances = output["instances"].to(self._cpu_device) + prediction["instances"] = instances_to_coco_json(instances, input["image_id"]) + if "proposals" in output: + prediction["proposals"] = output["proposals"].to(self._cpu_device) + self._predictions.append(prediction) + + def evaluate(self): + if self._distributed: + comm.synchronize() + predictions = comm.gather(self._predictions, dst=0) + predictions = list(itertools.chain(*predictions)) + + if not comm.is_main_process(): + return + else: + predictions = self._predictions + + if len(predictions) == 0: + self._logger.warning("[LVISEvaluator] Did not receive valid predictions.") + return {} + + if self._output_dir: + PathManager.mkdirs(self._output_dir) + file_path = os.path.join(self._output_dir, "instances_predictions.pth") + with PathManager.open(file_path, "wb") as f: + torch.save(predictions, f) + + self._results = OrderedDict() + if "proposals" in predictions[0]: + self._eval_box_proposals(predictions) + if "instances" in predictions[0]: + self._eval_predictions(predictions) + # Copy so the caller can do whatever with results + return copy.deepcopy(self._results) + + def _tasks_from_predictions(self, predictions): + for pred in predictions: + if "segmentation" in pred: + return ("bbox", "segm") + return ("bbox",) + + def _eval_predictions(self, predictions): + """ + Evaluate predictions. Fill self._results with the metrics of the tasks. + + Args: + predictions (list[dict]): list of outputs from the model + """ + self._logger.info("Preparing results in the LVIS format ...") + lvis_results = list(itertools.chain(*[x["instances"] for x in predictions])) + tasks = self._tasks or self._tasks_from_predictions(lvis_results) + + # LVIS evaluator can be used to evaluate results for COCO dataset categories. + # In this case `_metadata` variable will have a field with COCO-specific category mapping. + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"): + reverse_id_mapping = { + v: k for k, v in self._metadata.thing_dataset_id_to_contiguous_id.items() + } + for result in lvis_results: + result["category_id"] = reverse_id_mapping[result["category_id"]] + else: + # unmap the category ids for LVIS (from 0-indexed to 1-indexed) + for result in lvis_results: + result["category_id"] += 1 + + if self._output_dir: + file_path = os.path.join(self._output_dir, "lvis_instances_results.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(lvis_results)) + f.flush() + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info("Evaluating predictions ...") + for task in sorted(tasks): + lvis_eval = _evaluate_predictions_on_lvis( + self._lvis_api, + lvis_results, + task, + max_dets_per_image=self._max_dets_per_image, + class_names=self._metadata.get("thing_classes"), + ) + + res = self._derive_lvis_results( + lvis_eval, task, class_names=self._metadata.get("thing_classes") + ) + self._results[task] = res + + def _eval_box_proposals(self, predictions): + """ + Evaluate the box proposals in predictions. + Fill self._results with the metrics for "box_proposals" task. + """ + if self._output_dir: + # Saving generated box proposals to file. + # Predicted box_proposals are in XYXY_ABS mode. + bbox_mode = BoxMode.XYXY_ABS.value + ids, boxes, objectness_logits = [], [], [] + for prediction in predictions: + ids.append(prediction["image_id"]) + boxes.append(prediction["proposals"].proposal_boxes.tensor.numpy()) + objectness_logits.append(prediction["proposals"].objectness_logits.numpy()) + + proposal_data = { + "boxes": boxes, + "objectness_logits": objectness_logits, + "ids": ids, + "bbox_mode": bbox_mode, + } + with PathManager.open(os.path.join(self._output_dir, "box_proposals.pkl"), "wb") as f: + pickle.dump(proposal_data, f) + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info("Evaluating bbox proposals ...") + res = {} + areas = {"all": "", "small": "s", "medium": "m", "large": "l"} + for limit in [100, 1000]: + for area, suffix in areas.items(): + stats = _evaluate_box_proposals(predictions, self._lvis_api, area=area, limit=limit) + key = "AR{}@{:d}".format(suffix, limit) + res[key] = float(stats["ar"].item() * 100) + self._logger.info("Proposal metrics: \n" + create_small_table(res)) + self._results["box_proposals"] = res + + def _derive_lvis_results(self, lvis_eval, iou_type, class_names=None): + """ + Derive the desired score numbers from summarized COCOeval. + + Args: + lvis_eval (None or LVISEval): None represents no predictions from model. + iou_type (str): + class_names (None or list[str]): if provided, will use it to predict + per-category AP. + + Returns: + a dict of {metric name: score} + """ + + metrics = { + "bbox": ["AP", "AP50", "AP75", "APs", "APm", "APl", "APr", "APc", "APf"], + "segm": ["AP", "AP50", "AP75", "APs", "APm", "APl", "APr", "APc", "APf"], + }[iou_type] + + if lvis_eval is None: + self._logger.warn("No predictions from the model!") + return {metric: float("nan") for metric in metrics} + + # the standard metrics + # Pull the standard metrics from the LVIS results + results = lvis_eval.get_results() + results = {metric: float(results[metric] * 100) for metric in metrics} + self._logger.info( + "Evaluation results for {}: \n".format(iou_type) + create_small_table(results) + ) + if not np.isfinite(sum(results.values())): + self._logger.info("Some metrics cannot be computed and is shown as NaN.") + + if class_names is None or len(class_names) <= 1: + return results + # Compute per-category AP + # from https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L222-L252 # noqa + precisions = lvis_eval.eval["precision"] + # precision has dims (iou, recall, cls, area range, max dets) + assert len(class_names) == precisions.shape[2] + + results_per_category = [] + for idx, name in enumerate(class_names): + # area range index 0: all area ranges + precision = precisions[:, :, idx, 0] + precision = precision[precision > -1] + ap = np.mean(precision) if precision.size else float("nan") + results_per_category.append(("{}".format(name), float(ap * 100))) + + # tabulate it + N_COLS = min(6, len(results_per_category) * 2) + results_flatten = list(itertools.chain(*results_per_category)) + results_2d = itertools.zip_longest(*[results_flatten[i::N_COLS] for i in range(N_COLS)]) + table = tabulate( + results_2d, + tablefmt="pipe", + floatfmt=".3f", + headers=["category", "AP"] * (N_COLS // 2), + numalign="left", + ) + self._logger.info("Per-category {} AP: \n".format(iou_type) + table) + + results.update({"AP-" + name: ap for name, ap in results_per_category}) + return results + + +# inspired from Detectron: +# https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L255 # noqa +def _evaluate_box_proposals(dataset_predictions, lvis_api, thresholds=None, area="all", limit=None): + """ + Evaluate detection proposal recall metrics. This function is a much + faster alternative to the official LVIS API recall evaluation code. However, + it produces slightly different results. + """ + # Record max overlap value for each gt box + # Return vector of overlap values + areas = { + "all": 0, + "small": 1, + "medium": 2, + "large": 3, + "96-128": 4, + "128-256": 5, + "256-512": 6, + "512-inf": 7, + } + area_ranges = [ + [0**2, 1e5**2], # all + [0**2, 32**2], # small + [32**2, 96**2], # medium + [96**2, 1e5**2], # large + [96**2, 128**2], # 96-128 + [128**2, 256**2], # 128-256 + [256**2, 512**2], # 256-512 + [512**2, 1e5**2], + ] # 512-inf + assert area in areas, "Unknown area range: {}".format(area) + area_range = area_ranges[areas[area]] + gt_overlaps = [] + num_pos = 0 + + for prediction_dict in dataset_predictions: + predictions = prediction_dict["proposals"] + + # sort predictions in descending order + # TODO maybe remove this and make it explicit in the documentation + inds = predictions.objectness_logits.sort(descending=True)[1] + predictions = predictions[inds] + + ann_ids = lvis_api.get_ann_ids(img_ids=[prediction_dict["image_id"]]) + anno = lvis_api.load_anns(ann_ids) + gt_boxes = [ + BoxMode.convert(obj["bbox"], BoxMode.XYWH_ABS, BoxMode.XYXY_ABS) for obj in anno + ] + gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4) # guard against no boxes + gt_boxes = Boxes(gt_boxes) + gt_areas = torch.as_tensor([obj["area"] for obj in anno]) + + if len(gt_boxes) == 0 or len(predictions) == 0: + continue + + valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1]) + gt_boxes = gt_boxes[valid_gt_inds] + + num_pos += len(gt_boxes) + + if len(gt_boxes) == 0: + continue + + if limit is not None and len(predictions) > limit: + predictions = predictions[:limit] + + overlaps = pairwise_iou(predictions.proposal_boxes, gt_boxes) + + _gt_overlaps = torch.zeros(len(gt_boxes)) + for j in range(min(len(predictions), len(gt_boxes))): + # find which proposal box maximally covers each gt box + # and get the iou amount of coverage for each gt box + max_overlaps, argmax_overlaps = overlaps.max(dim=0) + + # find which gt box is 'best' covered (i.e. 'best' = most iou) + gt_ovr, gt_ind = max_overlaps.max(dim=0) + assert gt_ovr >= 0 + # find the proposal box that covers the best covered gt box + box_ind = argmax_overlaps[gt_ind] + # record the iou coverage of this gt box + _gt_overlaps[j] = overlaps[box_ind, gt_ind] + assert _gt_overlaps[j] == gt_ovr + # mark the proposal box and the gt box as used + overlaps[box_ind, :] = -1 + overlaps[:, gt_ind] = -1 + + # append recorded iou coverage level + gt_overlaps.append(_gt_overlaps) + gt_overlaps = ( + torch.cat(gt_overlaps, dim=0) if len(gt_overlaps) else torch.zeros(0, dtype=torch.float32) + ) + gt_overlaps, _ = torch.sort(gt_overlaps) + + if thresholds is None: + step = 0.05 + thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32) + recalls = torch.zeros_like(thresholds) + # compute recall for each iou threshold + for i, t in enumerate(thresholds): + recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos) + # ar = 2 * np.trapz(recalls, thresholds) + ar = recalls.mean() + return { + "ar": ar, + "recalls": recalls, + "thresholds": thresholds, + "gt_overlaps": gt_overlaps, + "num_pos": num_pos, + } + + +def _evaluate_predictions_on_lvis( + lvis_gt, lvis_results, iou_type, max_dets_per_image=None, class_names=None +): + """ + Args: + iou_type (str): + max_dets_per_image (None or int): limit on maximum detections per image in evaluating AP + This limit, by default of the LVIS dataset, is 300. + class_names (None or list[str]): if provided, will use it to predict + per-category AP. + + Returns: + a dict of {metric name: score} + """ + metrics = { + "bbox": ["AP", "AP50", "AP75", "APs", "APm", "APl", "APr", "APc", "APf"], + "segm": ["AP", "AP50", "AP75", "APs", "APm", "APl", "APr", "APc", "APf"], + }[iou_type] + + logger = logging.getLogger(__name__) + + if len(lvis_results) == 0: # TODO: check if needed + logger.warn("No predictions from the model!") + return None + return {metric: float("nan") for metric in metrics} + + if iou_type == "segm": + lvis_results = copy.deepcopy(lvis_results) + # When evaluating mask AP, if the results contain bbox, LVIS API will + # use the box area as the area of the instance, instead of the mask area. + # This leads to a different definition of small/medium/large. + # We remove the bbox field to let mask AP use mask area. + for c in lvis_results: + c.pop("bbox", None) + + if max_dets_per_image is None: + max_dets_per_image = 300 # Default for LVIS dataset + + from lvis import LVISEval, LVISResults + + logger.info(f"Evaluating with max detections per image = {max_dets_per_image}") + lvis_results = LVISResults(lvis_gt, lvis_results, max_dets=max_dets_per_image) + lvis_eval = LVISEval(lvis_gt, lvis_results, iou_type) + lvis_eval.run() + lvis_eval.print_results() + + # Pull the standard metrics from the LVIS results + results = lvis_eval.get_results() + results = {metric: float(results[metric] * 100) for metric in metrics} + logger.info("Evaluation results for {}: \n".format(iou_type) + create_small_table(results)) + return lvis_eval + return results diff --git a/approach/ovod/APE/ape/evaluation/multi_dataset_evaluator.py b/approach/ovod/APE/ape/evaluation/multi_dataset_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..9948db1b5fcd43a9a60b90bc6c7944485a0fe451 --- /dev/null +++ b/approach/ovod/APE/ape/evaluation/multi_dataset_evaluator.py @@ -0,0 +1,382 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +# Modified by Xingyi Zhou +import copy +import glob +import itertools +import json +import os +from collections import OrderedDict + +import pycocotools.mask as mask_util +from PIL import Image + +import detectron2.utils.comm as comm +from detectron2.evaluation.coco_evaluation import ( + COCOEvaluator, + _evaluate_predictions_on_coco, + instances_to_coco_json, +) +from fvcore.common.file_io import PathManager + +from .oideval import OIDEvaluator, _evaluate_predictions_on_oid + + +def get_unified_evaluator(evaluator_type, dataset_name, cfg, distributed, output_dir): + unified_label_file = cfg.MULTI_DATASET.UNIFIED_LABEL_FILE + if evaluator_type == "coco": + evaluator = UnifiedCOCOEvaluator( + unified_label_file, dataset_name, cfg, distributed, output_dir + ) + elif evaluator_type == "oid": + evaluator = UnifiedOIDEvaluator( + unified_label_file, dataset_name, cfg, distributed, output_dir + ) + elif evaluator_type == "cityscapes_instance": + evaluator = UnifiedCityscapesEvaluator( + unified_label_file, dataset_name, cfg, distributed, output_dir + ) + else: + assert 0, evaluator_type + return evaluator + + +def map_back_unified_id(results, map_back, reverse_id_mapping=None): + ret = [] + for result in results: + if result["category_id"] in map_back: + result["category_id"] = map_back[result["category_id"]] + if reverse_id_mapping is not None: + result["category_id"] = reverse_id_mapping[result["category_id"]] + ret.append(result) + return ret + + +def map_back_unified_id_novel_classes(results, map_back, reverse_id_mapping=None): + ret = [] + for result in results: + if result["category_id"] in map_back: + original_id_list = map_back[result["category_id"]] + for original_id in original_id_list: + result_copy = copy.deepcopy(result) + result_copy["category_id"] = original_id + if reverse_id_mapping is not None: + result_copy["category_id"] = reverse_id_mapping[result_copy["category_id"]] + ret.append(result_copy) + return ret + + +class UnifiedCOCOEvaluator(COCOEvaluator): + def _eval_predictions(self, tasks, predictions): + """ + Evaluate predictions. Fill self._results with the metrics of the tasks. + """ + self._logger.info("Preparing results for COCO format ...") + coco_results = list(itertools.chain(*[x["instances"] for x in predictions])) + tasks = self._tasks or self._tasks_from_predictions(coco_results) + + # unmap the category ids for COCO + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id") and False: + dataset_id_to_contiguous_id = self._metadata.thing_dataset_id_to_contiguous_id + all_contiguous_ids = list(dataset_id_to_contiguous_id.values()) + num_classes = len(all_contiguous_ids) + assert min(all_contiguous_ids) == 0 and max(all_contiguous_ids) == num_classes - 1 + + reverse_id_mapping = {v: k for k, v in dataset_id_to_contiguous_id.items()} + for result in coco_results: + category_id = result["category_id"] + assert category_id < num_classes, ( + f"A prediction has class={category_id}, " + f"but the dataset only has {num_classes} classes and " + f"predicted class id should be in [0, {num_classes - 1}]." + ) + result["category_id"] = reverse_id_mapping[category_id] + + if self._output_dir: + file_path = os.path.join(self._output_dir, "coco_instances_results.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(coco_results)) + f.flush() + + if not self._do_evaluation and False: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info( + "Evaluating predictions with {} COCO API...".format( + "unofficial" if self._use_fast_impl else "official" + ) + ) + for task in sorted(tasks): + assert task in {"bbox", "segm", "keypoints"}, f"Got unknown task: {task}!" + coco_eval = ( + _evaluate_predictions_on_coco( + self._coco_api, + coco_results, + task, + kpt_oks_sigmas=self._kpt_oks_sigmas, + use_fast_impl=self._use_fast_impl, + img_ids=img_ids, + max_dets_per_image=self._max_dets_per_image, + ) + if len(coco_results) > 0 + else None # cocoapi does not handle empty results very well + ) + + res = self._derive_coco_results( + coco_eval, task, class_names=self._metadata.get("thing_classes") + ) + self._results[task] = res + + +class UnifiedCityscapesEvaluator(COCOEvaluator): + def __init__(self, unified_label_file, dataset_name, cfg, distributed, output_dir=None): + super().__init__(dataset_name, cfg, distributed, output_dir=output_dir) + meta_dataset_name = dataset_name[: dataset_name.find("_")] + print("meta_dataset_name", meta_dataset_name) + + self.unified_novel_classes_eval = cfg.MULTI_DATASET.UNIFIED_NOVEL_CLASSES_EVAL + if self.unified_novel_classes_eval: + match_novel_classes_file = cfg.MULTI_DATASET.MATCH_NOVEL_CLASSES_FILE + print("Loading map back from", match_novel_classes_file) + novel_classes_map = json.load(open(match_novel_classes_file, "r"))[meta_dataset_name] + self.map_back = {} + for c, match in enumerate(novel_classes_map): + for m in match: + self.map_back[m] = c + else: + unified_label_data = json.load(open(unified_label_file, "r")) + label_map = unified_label_data["label_map"] + label_map = label_map[meta_dataset_name] + self.map_back = {int(v): i for i, v in enumerate(label_map)} + + self._logger.info("saving outputs to {}".format(self._output_dir)) + self._temp_dir = self._output_dir + "/cityscapes_style_eval_tmp/" + self._logger.info( + "Writing cityscapes results to temporary directory {} ...".format(self._temp_dir) + ) + PathManager.mkdirs(self._temp_dir) + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a COCO model (e.g., GeneralizedRCNN). + It is a list of dict. Each dict corresponds to an image and + contains keys like "height", "width", "file_name", "image_id". + outputs: the outputs of a COCO model. It is a list of dicts with key + "instances" that contains :class:`Instances`. + """ + for input, output in zip(inputs, outputs): + prediction = {"image_id": input["image_id"], "file_name": input["file_name"]} + + instances = output["instances"].to(self._cpu_device) + prediction["instances"] = instances_to_coco_json(instances, input["image_id"]) + for x in prediction["instances"]: + x["file_name"] = input["file_name"] + # if len(prediction['instances']) == 0: + # self._logger.info("No prediction for {}".format(x['file_name'])) + # prediction['instances'] = [ + # {'file_name': input['file_name'], + # ''}] + self._predictions.append(prediction) + + def _eval_predictions(self, tasks, predictions): + self._logger.info("Preparing results for COCO format ...") + _unified_results = list(itertools.chain(*[x["instances"] for x in predictions])) + all_file_names = [x["file_name"] for x in predictions] + file_path = os.path.join(self._output_dir, "unified_instances_results.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(_unified_results)) + f.flush() + + mapped = False + thing_classes = None + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"): + self._logger.info( + "Evaluating COCO-stype cityscapes! " + "Using buildin meta to mapback IDs." + ) + reverse_id_mapping = { + v: k for k, v in self._metadata.thing_dataset_id_to_contiguous_id.items() + } + mapped = True + thing_classes = { + k: self._metadata.thing_classes[v] + for k, v in self._metadata.thing_dataset_id_to_contiguous_id.items() + } + else: + self._logger.info("Evaluating cityscapes! " + "Using eval script to map back IDs.") + reverse_id_mapping = None + thing_classes = self._metadata.thing_classes + + if self.unified_novel_classes_eval: + coco_results = map_back_unified_id_novel_classes( + _unified_results, self.map_back, reverse_id_mapping=reverse_id_mapping + ) + else: + coco_results = map_back_unified_id( + _unified_results, self.map_back, reverse_id_mapping=reverse_id_mapping + ) + + self.write_as_cityscapes( + coco_results, + all_file_names, + temp_dir=self._temp_dir, + mapped=mapped, + thing_classes=thing_classes, + ) + + os.environ["CITYSCAPES_DATASET"] = os.path.abspath( + os.path.join(self._metadata.gt_dir, "..", "..") + ) + # Load the Cityscapes eval script *after* setting the required env var, + # since the script reads CITYSCAPES_DATASET into global variables at load time. + import cityscapesscripts.evaluation.evalInstanceLevelSemanticLabeling as cityscapes_eval + + self._logger.info("Evaluating results under {} ...".format(self._temp_dir)) + # set some global states in cityscapes evaluation API, before evaluating + cityscapes_eval.args.predictionPath = os.path.abspath(self._temp_dir) + cityscapes_eval.args.predictionWalk = None + cityscapes_eval.args.JSONOutput = False + cityscapes_eval.args.colorized = False + cityscapes_eval.args.gtInstancesFile = os.path.join(self._temp_dir, "gtInstances.json") + + # These lines are adopted from + # https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/evaluation/evalInstanceLevelSemanticLabeling.py # noqa + groundTruthImgList = glob.glob(cityscapes_eval.args.groundTruthSearch) + assert len( + groundTruthImgList + ), "Cannot find any ground truth images to use for evaluation. Searched for: {}".format( + cityscapes_eval.args.groundTruthSearch + ) + predictionImgList = [] + for gt in groundTruthImgList: + predictionImgList.append(cityscapes_eval.getPrediction(gt, cityscapes_eval.args)) + results = cityscapes_eval.evaluateImgLists( + predictionImgList, groundTruthImgList, cityscapes_eval.args + )["averages"] + + ret = OrderedDict() + ret["segm"] = {"AP": results["allAp"] * 100, "AP50": results["allAp50%"] * 100} + return ret + + @staticmethod + def write_as_cityscapes( + coco_results, + all_file_names, + temp_dir, + mapped=False, + thing_classes=None, + ext="_pred.txt", + subfolder="", + ): + from cityscapesscripts.helpers.labels import name2label + + results_per_image = {x: [] for x in all_file_names} + for x in coco_results: + results_per_image[x["file_name"]].append(x) + if subfolder != "": + PathManager.mkdirs(temp_dir + "/" + subfolder) + N = len(results_per_image) + for i, (file_name, coco_list) in enumerate(results_per_image.items()): + if i % (N // 10) == 0: + print("{}%".format(i // (N // 10) * 10), end=",", flush=True) + basename = os.path.splitext(os.path.basename(file_name))[0] + pred_txt = os.path.join(temp_dir, basename + ext) + + num_instances = len(coco_list) + with open(pred_txt, "w") as fout: + for i in range(num_instances): + if not mapped: + pred_class = coco_list[i]["category_id"] + classes = thing_classes[pred_class] + class_id = name2label[classes].id + else: + class_id = coco_list[i]["category_id"] + classes = thing_classes[class_id] + score = coco_list[i]["score"] + mask = mask_util.decode(coco_list[i]["segmentation"])[:, :].astype("uint8") + # mask = output.pred_masks[i].numpy().astype("uint8") + if subfolder != "": + png_filename = os.path.join( + temp_dir, + subfolder, + basename + "_{}_{}.png".format(i, classes.replace(" ", "_")), + ) + Image.fromarray(mask * 255).save(png_filename) + fout.write( + "{} {} {}\n".format( + subfolder + "/" + os.path.basename(png_filename), class_id, score + ) + ) + + else: + png_filename = os.path.join( + temp_dir, basename + "_{}_{}.png".format(i, classes.replace(" ", "_")) + ) + + Image.fromarray(mask * 255).save(png_filename) + fout.write( + "{} {} {}\n".format(os.path.basename(png_filename), class_id, score) + ) + + +class UnifiedOIDEvaluator(OIDEvaluator): + def __init__(self, unified_label_file, dataset_name, cfg, distributed, output_dir=None): + super().__init__(dataset_name, cfg, distributed, output_dir=output_dir) + meta_dataset_name = dataset_name[: dataset_name.find("_")] + print("meta_dataset_name", meta_dataset_name) + unified_label_data = json.load(open(unified_label_file, "r")) + label_map = unified_label_data["label_map"] + label_map = label_map[meta_dataset_name] + self.map_back = {int(v): i for i, v in enumerate(label_map)} + self._logger.info("saving outputs to {}".format(self._output_dir)) + + def evaluate(self): + if self._distributed: + comm.synchronize() + self._predictions = comm.gather(self._predictions, dst=0) + self._predictions = list(itertools.chain(*self._predictions)) + + if not comm.is_main_process(): + return + + if len(self._predictions) == 0: + self._logger.warning("[LVISEvaluator] Did not receive valid predictions.") + return {} + + self._logger.info("Preparing results in the OID format ...") + _unified_results = list(itertools.chain(*[x["instances"] for x in self._predictions])) + + if self._output_dir: + PathManager.mkdirs(self._output_dir) + + file_path = os.path.join(self._output_dir, "unified_instances_results.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(_unified_results)) + f.flush() + + self._oid_results = map_back_unified_id(_unified_results, self.map_back) + + # unmap the category ids for LVIS (from 0-indexed to 1-indexed) + for result in self._oid_results: + result["category_id"] += 1 + + PathManager.mkdirs(self._output_dir) + file_path = os.path.join(self._output_dir, "oid_instances_results.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(self._oid_results)) + f.flush() + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info("Evaluating predictions ...") + self._results = OrderedDict() + res = _evaluate_predictions_on_oid(self._oid_api, file_path, eval_seg=self._mask_on) + self._results["bbox"] = res + + return copy.deepcopy(self._results) diff --git a/approach/ovod/APE/ape/evaluation/oideval.py b/approach/ovod/APE/ape/evaluation/oideval.py new file mode 100644 index 0000000000000000000000000000000000000000..d88afbcea07e7e061019b9e6c0a6d1d6a0cb5335 --- /dev/null +++ b/approach/ovod/APE/ape/evaluation/oideval.py @@ -0,0 +1,905 @@ +# Part of the code is from https://github.com/tensorflow/models/blob/master/research/object_detection/metrics/oid_challenge_evaluation.py +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# The original code is under Apache License, Version 2.0 (the "License"); +# Part of the code is from https://github.com/lvis-dataset/lvis-api/blob/master/lvis/eval.py +# Copyright (c) 2019, Agrim Gupta and Ross Girshick +# Modified by Xingyi Zhou +# This script re-implement OpenImages evaluation in detectron2 +import copy +import datetime +import itertools +import json +import logging +import os +from collections import OrderedDict, defaultdict + +import numpy as np +import pycocotools.mask as mask_utils +import torch +from lvis.lvis import LVIS +from lvis.results import LVISResults + +import detectron2.utils.comm as comm +from detectron2.data import MetadataCatalog +from detectron2.evaluation import DatasetEvaluator +from detectron2.evaluation.coco_evaluation import instances_to_coco_json +from detectron2.utils.logger import create_small_table +from fvcore.common.file_io import PathManager +from tabulate import tabulate + + +def compute_average_precision(precision, recall): + """Compute Average Precision according to the definition in VOCdevkit. + + Precision is modified to ensure that it does not decrease as recall + decrease. + + Args: + precision: A float [N, 1] numpy array of precisions + recall: A float [N, 1] numpy array of recalls + + Raises: + ValueError: if the input is not of the correct format + + Returns: + average_precison: The area under the precision recall curve. NaN if + precision and recall are None. + + """ + if precision is None: + if recall is not None: + raise ValueError("If precision is None, recall must also be None") + return np.NAN + + if not isinstance(precision, np.ndarray) or not isinstance(recall, np.ndarray): + raise ValueError("precision and recall must be numpy array") + if precision.dtype != np.float or recall.dtype != np.float: + raise ValueError("input must be float numpy array.") + if len(precision) != len(recall): + raise ValueError("precision and recall must be of the same size.") + if not precision.size: + return 0.0 + if np.amin(precision) < 0 or np.amax(precision) > 1: + raise ValueError("Precision must be in the range of [0, 1].") + if np.amin(recall) < 0 or np.amax(recall) > 1: + raise ValueError("recall must be in the range of [0, 1].") + if not all(recall[i] <= recall[i + 1] for i in range(len(recall) - 1)): + raise ValueError("recall must be a non-decreasing array") + + recall = np.concatenate([[0], recall, [1]]) + precision = np.concatenate([[0], precision, [0]]) + + for i in range(len(precision) - 2, -1, -1): + precision[i] = np.maximum(precision[i], precision[i + 1]) + indices = np.where(recall[1:] != recall[:-1])[0] + 1 + average_precision = np.sum((recall[indices] - recall[indices - 1]) * precision[indices]) + return average_precision + + +class OIDEval: + def __init__( + self, + lvis_gt, + lvis_dt, + iou_type="bbox", + expand_pred_label=False, + oid_hierarchy_path="./datasets/openimages/annotations/challenge-2019-label500-hierarchy.json", + ): + """Constructor for OIDEval. + Args: + lvis_gt (LVIS class instance, or str containing path of annotation file) + lvis_dt (LVISResult class instance, or str containing path of result file, + or list of dict) + iou_type (str): segm or bbox evaluation + """ + self.logger = logging.getLogger(__name__) + + if iou_type not in ["bbox", "segm"]: + raise ValueError("iou_type: {} is not supported.".format(iou_type)) + + if isinstance(lvis_gt, LVIS): + self.lvis_gt = lvis_gt + elif isinstance(lvis_gt, str): + self.lvis_gt = LVIS(lvis_gt) + else: + raise TypeError("Unsupported type {} of lvis_gt.".format(lvis_gt)) + + if isinstance(lvis_dt, LVISResults): + self.lvis_dt = lvis_dt + elif isinstance(lvis_dt, (str, list)): + self.lvis_dt = LVISResults(self.lvis_gt, lvis_dt, max_dets=-1) + else: + raise TypeError("Unsupported type {} of lvis_dt.".format(lvis_dt)) + + if expand_pred_label: + oid_hierarchy = json.load(open(oid_hierarchy_path, "r")) + cat_info = self.lvis_gt.dataset["categories"] + freebase2id = {x["freebase_id"]: x["id"] for x in cat_info} + id2freebase = {x["id"]: x["freebase_id"] for x in cat_info} + id2name = {x["id"]: x["name"] for x in cat_info} + + fas = defaultdict(set) + + def dfs(hierarchy, cur_id): + all_childs = set() + all_keyed_child = {} + if "Subcategory" in hierarchy: + for x in hierarchy["Subcategory"]: + childs = dfs(x, freebase2id[x["LabelName"]]) + all_childs.update(childs) + if cur_id != -1: + for c in all_childs: + fas[c].add(cur_id) + all_childs.add(cur_id) + return all_childs + + dfs(oid_hierarchy, -1) + + expanded_pred = [] + id_count = 0 + for d in self.lvis_dt.dataset["annotations"]: + cur_id = d["category_id"] + ids = [cur_id] + [x for x in fas[cur_id]] + for cat_id in ids: + new_box = copy.deepcopy(d) + id_count = id_count + 1 + new_box["id"] = id_count + new_box["category_id"] = cat_id + expanded_pred.append(new_box) + + self.logger.info( + "Expanding original {} preds to {} preds".format( + len(self.lvis_dt.dataset["annotations"]), len(expanded_pred) + ) + ) + self.lvis_dt.dataset["annotations"] = expanded_pred + self.lvis_dt._create_index() + + # per-image per-category evaluation results + self.eval_imgs = defaultdict(list) + self.eval = {} # accumulated evaluation results + self._gts = defaultdict(list) # gt for evaluation + self._dts = defaultdict(list) # dt for evaluation + self.params = Params(iou_type=iou_type) # parameters + self.results = OrderedDict() + self.ious = {} # ious between all gts and dts + + self.params.img_ids = sorted(self.lvis_gt.get_img_ids()) + self.params.cat_ids = sorted(self.lvis_gt.get_cat_ids()) + + def _to_mask(self, anns, lvis): + for ann in anns: + rle = lvis.ann_to_rle(ann) + ann["segmentation"] = rle + + def _prepare(self): + """Prepare self._gts and self._dts for evaluation based on params.""" + + cat_ids = self.params.cat_ids if self.params.cat_ids else None + + gts = self.lvis_gt.load_anns( + self.lvis_gt.get_ann_ids(img_ids=self.params.img_ids, cat_ids=cat_ids) + ) + dts = self.lvis_dt.load_anns( + self.lvis_dt.get_ann_ids(img_ids=self.params.img_ids, cat_ids=cat_ids) + ) + # convert ground truth to mask if iou_type == 'segm' + if self.params.iou_type == "segm": + self._to_mask(gts, self.lvis_gt) + self._to_mask(dts, self.lvis_dt) + + for gt in gts: + self._gts[gt["image_id"], gt["category_id"]].append(gt) + + # For federated dataset evaluation we will filter out all dt for an + # image which belong to categories not present in gt and not present in + # the negative list for an image. In other words detector is not penalized + # for categories about which we don't have gt information about their + # presence or absence in an image. + img_data = self.lvis_gt.load_imgs(ids=self.params.img_ids) + # per image map of categories not present in image + img_nl = {d["id"]: d["neg_category_ids"] for d in img_data} + # per image list of categories present in image + img_pl = {d["id"]: d["pos_category_ids"] for d in img_data} + # img_pl = defaultdict(set) + for ann in gts: + # img_pl[ann["image_id"]].add(ann["category_id"]) + assert ann["category_id"] in img_pl[ann["image_id"]] + + for dt in dts: + img_id, cat_id = dt["image_id"], dt["category_id"] + if cat_id not in img_nl[img_id] and cat_id not in img_pl[img_id]: + continue + self._dts[img_id, cat_id].append(dt) + + self.freq_groups = self._prepare_freq_group() + + def _prepare_freq_group(self): + freq_groups = [[] for _ in self.params.img_count_lbl] + cat_data = self.lvis_gt.load_cats(self.params.cat_ids) + for idx, _cat_data in enumerate(cat_data): + if "frequency" in _cat_data: + frequency = _cat_data["frequency"] + else: + frequency = "f" + freq_groups[self.params.img_count_lbl.index(frequency)].append(idx) + return freq_groups + + def evaluate(self): + """ + Run per image evaluation on given images and store results + (a list of dict) in self.eval_imgs. + """ + self.logger.info("Running per image evaluation.") + self.logger.info("Evaluate annotation type *{}*".format(self.params.iou_type)) + + self.params.img_ids = list(np.unique(self.params.img_ids)) + + if self.params.use_cats: + cat_ids = self.params.cat_ids + else: + cat_ids = [-1] + + self._prepare() + + self.ious = { + (img_id, cat_id): self.compute_iou(img_id, cat_id) + for img_id in self.params.img_ids + for cat_id in cat_ids + } + + # loop through images, area range, max detection number + self.eval_imgs = [ + self.evaluate_img_google(img_id, cat_id, area_rng) + for cat_id in cat_ids + for area_rng in self.params.area_rng + for img_id in self.params.img_ids + ] + + def _get_gt_dt(self, img_id, cat_id): + """Create gt, dt which are list of anns/dets. If use_cats is true + only anns/dets corresponding to tuple (img_id, cat_id) will be + used. Else, all anns/dets in image are used and cat_id is not used. + """ + if self.params.use_cats: + gt = self._gts[img_id, cat_id] + dt = self._dts[img_id, cat_id] + else: + gt = [_ann for _cat_id in self.params.cat_ids for _ann in self._gts[img_id, cat_id]] + dt = [_ann for _cat_id in self.params.cat_ids for _ann in self._dts[img_id, cat_id]] + return gt, dt + + def compute_iou(self, img_id, cat_id): + gt, dt = self._get_gt_dt(img_id, cat_id) + + if len(gt) == 0 and len(dt) == 0: + return [] + + # Sort detections in decreasing order of score. + idx = np.argsort([-d["score"] for d in dt], kind="mergesort") + dt = [dt[i] for i in idx] + + # iscrowd = [int(False)] * len(gt) + iscrowd = [int("iscrowd" in g and g["iscrowd"] > 0) for g in gt] + + if self.params.iou_type == "segm": + ann_type = "segmentation" + elif self.params.iou_type == "bbox": + ann_type = "bbox" + else: + raise ValueError("Unknown iou_type for iou computation.") + gt = [g[ann_type] for g in gt] + dt = [d[ann_type] for d in dt] + + # compute iou between each dt and gt region + # will return array of shape len(dt), len(gt) + ious = mask_utils.iou(dt, gt, iscrowd) + return ious + + def evaluate_img_google(self, img_id, cat_id, area_rng): + """Perform evaluation for single category and image.""" + gt, dt = self._get_gt_dt(img_id, cat_id) + + if len(gt) == 0 and len(dt) == 0: + return None + + if len(dt) == 0: + return { + "image_id": img_id, + "category_id": cat_id, + "area_rng": area_rng, + "dt_ids": [], + "dt_matches": np.array([], dtype=np.int32).reshape(1, -1), + "dt_scores": [], + "dt_ignore": np.array([], dtype=np.int32).reshape(1, -1), + "num_gt": len(gt), + } + + no_crowd_inds = [i for i, g in enumerate(gt) if ("iscrowd" not in g) or g["iscrowd"] == 0] + crowd_inds = [i for i, g in enumerate(gt) if "iscrowd" in g and g["iscrowd"] == 1] + dt_idx = np.argsort([-d["score"] for d in dt], kind="mergesort") + + if len(self.ious[img_id, cat_id]) > 0: + ious = self.ious[img_id, cat_id] + iou = ious[:, no_crowd_inds] + iou = iou[dt_idx] + ioa = ious[:, crowd_inds] + ioa = ioa[dt_idx] + else: + iou = np.zeros((len(dt_idx), 0)) + ioa = np.zeros((len(dt_idx), 0)) + scores = np.array([dt[i]["score"] for i in dt_idx]) + + num_detected_boxes = len(dt) + tp_fp_labels = np.zeros(num_detected_boxes, dtype=bool) + is_matched_to_group_of = np.zeros(num_detected_boxes, dtype=bool) + + def compute_match_iou(iou): + max_overlap_gt_ids = np.argmax(iou, axis=1) + is_gt_detected = np.zeros(iou.shape[1], dtype=bool) + for i in range(num_detected_boxes): + gt_id = max_overlap_gt_ids[i] + is_evaluatable = ( + not tp_fp_labels[i] and iou[i, gt_id] >= 0.5 and not is_matched_to_group_of[i] + ) + if is_evaluatable: + if not is_gt_detected[gt_id]: + tp_fp_labels[i] = True + is_gt_detected[gt_id] = True + + def compute_match_ioa(ioa): + scores_group_of = np.zeros(ioa.shape[1], dtype=float) + tp_fp_labels_group_of = np.ones(ioa.shape[1], dtype=float) + max_overlap_group_of_gt_ids = np.argmax(ioa, axis=1) + for i in range(num_detected_boxes): + gt_id = max_overlap_group_of_gt_ids[i] + is_evaluatable = ( + not tp_fp_labels[i] and ioa[i, gt_id] >= 0.5 and not is_matched_to_group_of[i] + ) + if is_evaluatable: + is_matched_to_group_of[i] = True + scores_group_of[gt_id] = max(scores_group_of[gt_id], scores[i]) + selector = np.where((scores_group_of > 0) & (tp_fp_labels_group_of > 0)) + scores_group_of = scores_group_of[selector] + tp_fp_labels_group_of = tp_fp_labels_group_of[selector] + + return scores_group_of, tp_fp_labels_group_of + + if iou.shape[1] > 0: + compute_match_iou(iou) + + scores_box_group_of = np.ndarray([0], dtype=float) + tp_fp_labels_box_group_of = np.ndarray([0], dtype=float) + + if ioa.shape[1] > 0: + scores_box_group_of, tp_fp_labels_box_group_of = compute_match_ioa(ioa) + + valid_entries = ~is_matched_to_group_of + + scores = np.concatenate((scores[valid_entries], scores_box_group_of)) + tp_fps = np.concatenate( + (tp_fp_labels[valid_entries].astype(float), tp_fp_labels_box_group_of) + ) + + return { + "image_id": img_id, + "category_id": cat_id, + "area_rng": area_rng, + "dt_matches": np.array([1 if x > 0 else 0 for x in tp_fps], dtype=np.int32).reshape( + 1, -1 + ), + "dt_scores": [x for x in scores], + "dt_ignore": np.array([0 for x in scores], dtype=np.int32).reshape(1, -1), + "num_gt": len(gt), + } + + def accumulate(self): + """Accumulate per image evaluation results and store the result in + self.eval. + """ + self.logger.info("Accumulating evaluation results.") + + if not self.eval_imgs: + self.logger.warn("Please run evaluate first.") + + if self.params.use_cats: + cat_ids = self.params.cat_ids + else: + cat_ids = [-1] + + num_thrs = len(self.params.iou_thrs) + num_recalls = len(self.params.rec_thrs) + num_cats = len(cat_ids) + num_area_rngs = len(self.params.area_rng) + num_imgs = len(self.params.img_ids) + + # -1 for absent categories + precision = -np.ones((num_thrs, num_recalls, num_cats, num_area_rngs)) + recall = -np.ones((num_thrs, num_cats, num_area_rngs)) + + # Initialize dt_pointers + dt_pointers = {} + for cat_idx in range(num_cats): + dt_pointers[cat_idx] = {} + for area_idx in range(num_area_rngs): + dt_pointers[cat_idx][area_idx] = {} + + # Per category evaluation + for cat_idx in range(num_cats): + Nk = cat_idx * num_area_rngs * num_imgs + for area_idx in range(num_area_rngs): + Na = area_idx * num_imgs + E = [self.eval_imgs[Nk + Na + img_idx] for img_idx in range(num_imgs)] + # Remove elements which are None + E = [e for e in E if not e is None] + if len(E) == 0: + continue + + dt_scores = np.concatenate([e["dt_scores"] for e in E], axis=0) + dt_idx = np.argsort(-dt_scores, kind="mergesort") + dt_scores = dt_scores[dt_idx] + dt_m = np.concatenate([e["dt_matches"] for e in E], axis=1)[:, dt_idx] + dt_ig = np.concatenate([e["dt_ignore"] for e in E], axis=1)[:, dt_idx] + + num_gt = sum([e["num_gt"] for e in E]) + if num_gt == 0: + continue + + tps = np.logical_and(dt_m, np.logical_not(dt_ig)) + fps = np.logical_and(np.logical_not(dt_m), np.logical_not(dt_ig)) + + tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float) + fp_sum = np.cumsum(fps, axis=1).astype(dtype=np.float) + + dt_pointers[cat_idx][area_idx] = { + "tps": tps, + "fps": fps, + } + + for iou_thr_idx, (tp, fp) in enumerate(zip(tp_sum, fp_sum)): + tp = np.array(tp) + fp = np.array(fp) + num_tp = len(tp) + rc = tp / num_gt + if num_tp: + recall[iou_thr_idx, cat_idx, area_idx] = rc[-1] + else: + recall[iou_thr_idx, cat_idx, area_idx] = 0 + + # np.spacing(1) ~= eps + pr = tp / (fp + tp + np.spacing(1)) + pr = pr.tolist() + + # Replace each precision value with the maximum precision + # value to the right of that recall level. This ensures + # that the calculated AP value will be less suspectable + # to small variations in the ranking. + for i in range(num_tp - 1, 0, -1): + if pr[i] > pr[i - 1]: + pr[i - 1] = pr[i] + + mAP = compute_average_precision( + np.array(pr, np.float).reshape(-1), np.array(rc, np.float).reshape(-1) + ) + precision[iou_thr_idx, :, cat_idx, area_idx] = mAP + + self.eval = { + "params": self.params, + "counts": [num_thrs, num_recalls, num_cats, num_area_rngs], + "date": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "precision": precision, + "recall": recall, + "dt_pointers": dt_pointers, + } + + def _summarize(self, summary_type, iou_thr=None, area_rng="all", freq_group_idx=None): + aidx = [ + idx for idx, _area_rng in enumerate(self.params.area_rng_lbl) if _area_rng == area_rng + ] + + if summary_type == "ap": + s = self.eval["precision"] + if iou_thr is not None: + tidx = np.where(iou_thr == self.params.iou_thrs)[0] + s = s[tidx] + if freq_group_idx is not None: + s = s[:, :, self.freq_groups[freq_group_idx], aidx] + else: + s = s[:, :, :, aidx] + else: + s = self.eval["recall"] + if iou_thr is not None: + tidx = np.where(iou_thr == self.params.iou_thrs)[0] + s = s[tidx] + s = s[:, :, aidx] + + if len(s[s > -1]) == 0: + mean_s = -1 + else: + mean_s = np.mean(s[s > -1]) + return mean_s + + def summarize(self): + """Compute and display summary metrics for evaluation results.""" + if not self.eval: + raise RuntimeError("Please run accumulate() first.") + + max_dets = self.params.max_dets + + self.results["AP"] = self._summarize("ap") + self.results["AP50"] = self._summarize("ap", iou_thr=0.50) + self.results["AP75"] = self._summarize("ap", iou_thr=0.75) + self.results["APs"] = self._summarize("ap", area_rng="small") + self.results["APm"] = self._summarize("ap", area_rng="medium") + self.results["APl"] = self._summarize("ap", area_rng="large") + self.results["APr"] = self._summarize("ap", freq_group_idx=0) + self.results["APc"] = self._summarize("ap", freq_group_idx=1) + self.results["APf"] = self._summarize("ap", freq_group_idx=2) + + key = "AR@{}".format(max_dets) + self.results[key] = self._summarize("ar") + + for area_rng in ["small", "medium", "large"]: + key = "AR{}@{}".format(area_rng[0], max_dets) + self.results[key] = self._summarize("ar", area_rng=area_rng) + + def run(self): + """Wrapper function which calculates the results.""" + self.evaluate() + self.accumulate() + self.summarize() + + def print_results(self): + template = " {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} catIds={:>3s}] = {:0.3f}" + + for key, value in self.results.items(): + max_dets = self.params.max_dets + if "AP" in key: + title = "Average Precision" + _type = "(AP)" + else: + title = "Average Recall" + _type = "(AR)" + + if len(key) > 2 and key[2].isdigit(): + iou_thr = float(key[2:]) / 100 + iou = "{:0.2f}".format(iou_thr) + else: + iou = "{:0.2f}:{:0.2f}".format(self.params.iou_thrs[0], self.params.iou_thrs[-1]) + + if len(key) > 2 and key[2] in ["r", "c", "f"]: + cat_group_name = key[2] + else: + cat_group_name = "all" + + if len(key) > 2 and key[2] in ["s", "m", "l"]: + area_rng = key[2] + else: + area_rng = "all" + + self.logger.info( + template.format(title, _type, iou, area_rng, max_dets, cat_group_name, value) + ) + + def get_results(self): + if not self.results: + self.logger.warn("results is empty. Call run().") + return self.results + + +class Params: + def __init__(self, iou_type): + """Params for LVIS evaluation API.""" + self.img_ids = [] + self.cat_ids = [] + # np.arange causes trouble. the data point on arange is slightly + # larger than the true value + self.iou_thrs = np.linspace( + 0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True + ) + self.rec_thrs = np.linspace( + 0.0, 1.00, int(np.round((1.00 - 0.0) / 0.01)) + 1, endpoint=True + ) + self.max_dets = 1000 + self.google_style = True + + self.area_rng = [ + [0**2, 1e5**2], + [0**2, 32**2], + [32**2, 96**2], + [96**2, 1e5**2], + ] + self.area_rng_lbl = ["all", "small", "medium", "large"] + self.use_cats = 1 + # We bin categories in three bins based how many images of the training + # set the category is present in. + # r: Rare : < 10 + # c: Common : >= 10 and < 100 + # f: Frequent: >= 100 + self.img_count_lbl = ["r", "c", "f"] + self.iou_type = iou_type + + +class OIDEvaluator(DatasetEvaluator): + def __init__( + self, + dataset_name, + tasks=None, + distributed=True, + output_dir=None, + *, + max_dets_per_image=None, + ): + """ + Args: + dataset_name (str): name of the dataset to be evaluated. + It must have the following corresponding metadata: + "json_file": the path to the LVIS format annotation + tasks (tuple[str]): tasks that can be evaluated under the given + configuration. A task is one of "bbox", "segm". + By default, will infer this automatically from predictions. + distributed (True): if True, will collect results from all ranks for evaluation. + Otherwise, will evaluate the results in the current process. + output_dir (str): optional, an output directory to dump results. + max_dets_per_image (None or int): limit on maximum detections per image in evaluating AP + This limit, by default of the LVIS dataset, is 300. + """ + from lvis import LVIS + + self._logger = logging.getLogger(__name__) + + if tasks is not None and isinstance(tasks, CfgNode): + self._logger.warn( + "COCO Evaluator instantiated using config, this is deprecated behavior." + " Please pass in explicit arguments instead." + ) + self._tasks = None # Infering it from predictions should be better + else: + self._tasks = tasks + + self._distributed = distributed + self._output_dir = output_dir + self._max_dets_per_image = max_dets_per_image + + self._cpu_device = torch.device("cpu") + + self._metadata = MetadataCatalog.get(dataset_name) + json_file = PathManager.get_local_path(self._metadata.json_file) + self._oid_api = LVIS(json_file) + # Test set json files do not contain annotations (evaluation must be + # performed using the LVIS evaluation server). + self._do_evaluation = len(self._oid_api.get_ann_ids()) > 0 + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a LVIS model (e.g., GeneralizedRCNN). + It is a list of dict. Each dict corresponds to an image and + contains keys like "height", "width", "file_name", "image_id". + outputs: the outputs of a LVIS model. It is a list of dicts with key + "instances" that contains :class:`Instances`. + """ + for input, output in zip(inputs, outputs): + prediction = {"image_id": input["image_id"]} + + if "instances" in output: + instances = output["instances"].to(self._cpu_device) + prediction["instances"] = instances_to_coco_json(instances, input["image_id"]) + if "proposals" in output: + prediction["proposals"] = output["proposals"].to(self._cpu_device) + self._predictions.append(prediction) + + def evaluate(self): + if self._distributed: + comm.synchronize() + predictions = comm.gather(self._predictions, dst=0) + predictions = list(itertools.chain(*predictions)) + + if not comm.is_main_process(): + return + else: + predictions = self._predictions + + if len(predictions) == 0: + self._logger.warning("[LVISEvaluator] Did not receive valid predictions.") + return {} + + if self._output_dir: + PathManager.mkdirs(self._output_dir) + file_path = os.path.join(self._output_dir, "instances_predictions.pth") + with PathManager.open(file_path, "wb") as f: + torch.save(predictions, f) + + self._results = OrderedDict() + if "proposals" in predictions[0]: + self._eval_box_proposals(predictions) + if "instances" in predictions[0]: + self._eval_predictions(predictions) + # Copy so the caller can do whatever with results + return copy.deepcopy(self._results) + + def _tasks_from_predictions(self, predictions): + return ("bbox", "bbox_expand") + for pred in predictions: + if "segmentation" in pred: + return ("bbox", "bbox_expand", "segm") + return ("bbox", "bbox_expand") + + def _eval_predictions(self, predictions): + """ + Evaluate predictions. Fill self._results with the metrics of the tasks. + + Args: + predictions (list[dict]): list of outputs from the model + """ + self._logger.info("Preparing results in the OID format ...") + oid_results = list(itertools.chain(*[x["instances"] for x in predictions])) + tasks = self._tasks or self._tasks_from_predictions(oid_results) + + # LVIS evaluator can be used to evaluate results for COCO dataset categories. + # In this case `_metadata` variable will have a field with COCO-specific category mapping. + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"): + reverse_id_mapping = { + v: k for k, v in self._metadata.thing_dataset_id_to_contiguous_id.items() + } + for result in oid_results: + result["category_id"] = reverse_id_mapping[result["category_id"]] + else: + # unmap the category ids for LVIS (from 0-indexed to 1-indexed) + for result in oid_results: + result["category_id"] += 1 + + if self._output_dir: + file_path = os.path.join(self._output_dir, "oid_instances_results.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(oid_results)) + f.flush() + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info("Evaluating predictions ...") + for task in sorted(tasks): + oid_eval = _evaluate_predictions_on_oid( + self._oid_api, + oid_results, + task, + max_dets_per_image=self._max_dets_per_image, + ) + + res = self._derive_oid_results( + oid_eval, task, class_names=self._metadata.get("thing_classes") + ) + self._results[task] = res + + def _derive_oid_results(self, oid_eval, iou_type, class_names=None): + """ + Derive the desired score numbers from summarized COCOeval. + + Args: + lvis_eval (None or LVISEval): None represents no predictions from model. + iou_type (str): + class_names (None or list[str]): if provided, will use it to predict + per-category AP. + + Returns: + a dict of {metric name: score} + """ + + metrics = { + "bbox": ["AP", "AP50", "AP75", "APs", "APm", "APl", "APr", "APc", "APf"], + "bbox_expand": ["AP", "AP50", "AP75", "APs", "APm", "APl", "APr", "APc", "APf"], + "segm": ["AP", "AP50", "AP75", "APs", "APm", "APl", "APr", "APc", "APf"], + }[iou_type] + + if oid_eval is None: + self._logger.warn("No predictions from the model!") + return {metric: float("nan") for metric in metrics} + + # the standard metrics + # Pull the standard metrics from the LVIS results + results = oid_eval.get_results() + results = {metric: float(results[metric] * 100) for metric in metrics} + self._logger.info( + "Evaluation results for {}: \n".format(iou_type) + create_small_table(results) + ) + if not np.isfinite(sum(results.values())): + self._logger.info("Some metrics cannot be computed and is shown as NaN.") + + if class_names is None or len(class_names) <= 1: + return results + # Compute per-category AP + # from https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L222-L252 # noqa + precisions = oid_eval.eval["precision"] + # precision has dims (iou, recall, cls, area range, max dets) + assert len(class_names) == precisions.shape[2] + + results_per_category = [] + for idx, name in enumerate(class_names): + # area range index 0: all area ranges + precision = precisions[:, :, idx, 0] + precision = precision[precision > -1] + ap = np.mean(precision) if precision.size else float("nan") + results_per_category.append(("{}".format(name), float(ap * 100))) + + # tabulate it + N_COLS = min(6, len(results_per_category) * 2) + results_flatten = list(itertools.chain(*results_per_category)) + results_2d = itertools.zip_longest(*[results_flatten[i::N_COLS] for i in range(N_COLS)]) + table = tabulate( + results_2d, + tablefmt="pipe", + floatfmt=".3f", + headers=["category", "AP"] * (N_COLS // 2), + numalign="left", + ) + self._logger.info("Per-category {} AP: \n".format(iou_type) + table) + + results.update({"AP-" + name: ap for name, ap in results_per_category}) + return results + + +def _evaluate_predictions_on_oid( + oid_gt, + oid_results, + iou_type, + max_dets_per_image=None, +): + """ + Args: + iou_type (str): + max_dets_per_image (None or int): limit on maximum detections per image in evaluating AP + This limit, by default of the LVIS dataset, is 300. + class_names (None or list[str]): if provided, will use it to predict + per-category AP. + + Returns: + a dict of {metric name: score} + """ + metrics = { + "bbox": ["AP", "AP50", "AP75", "APs", "APm", "APl", "APr", "APc", "APf"], + "bbox_expand": ["AP", "AP50", "AP75", "APs", "APm", "APl", "APr", "APc", "APf"], + "segm": ["AP", "AP50", "AP75", "APs", "APm", "APl", "APr", "APc", "APf"], + }[iou_type] + + logger = logging.getLogger(__name__) + + if len(oid_results) == 0: # TODO: check if needed + logger.warn("No predictions from the model!") + return {metric: float("nan") for metric in metrics} + + if max_dets_per_image is None: + max_dets_per_image = 1000 # Default for OID dataset + + from lvis import LVISEval, LVISResults + + logger.info(f"Evaluating with max detections per image = {max_dets_per_image}") + oid_results = LVISResults(oid_gt, oid_results, max_dets=max_dets_per_image) + + if "segm" in iou_type: + oid_eval = OIDEval(oid_gt, oid_results, "segm", expand_pred_label=False) + oid_eval.run() + oid_eval.print_results() + elif "bbox_expand" in iou_type: + oid_eval = OIDEval(oid_gt, oid_results, "bbox", expand_pred_label=True) + oid_eval.run() + oid_eval.print_results() + elif "bbox" in iou_type: + oid_eval = OIDEval(oid_gt, oid_results, "bbox", expand_pred_label=False) + oid_eval.run() + oid_eval.print_results() + else: + return None + return {metric: float("nan") for metric in metrics} + + # Pull the standard metrics from the LVIS results + results = oid_eval.get_results() + results = {metric: float(results[metric] * 100) for metric in metrics} + logger.info("Evaluation results for {}: \n".format(iou_type) + create_small_table(results)) + return oid_eval + return results diff --git a/approach/ovod/APE/ape/evaluation/refcoco_evaluation.py b/approach/ovod/APE/ape/evaluation/refcoco_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..532c63769e959774108cd1bcb0f14aacff4f96ff --- /dev/null +++ b/approach/ovod/APE/ape/evaluation/refcoco_evaluation.py @@ -0,0 +1,753 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import contextlib +import copy +import io +import itertools +import json +import logging +import os +import pickle +from collections import OrderedDict + +import numpy as np +import pycocotools.mask as mask_util +import torch +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval + +import detectron2.utils.comm as comm +from detectron2.config import CfgNode +from detectron2.data import MetadataCatalog +from detectron2.data.datasets.coco import convert_to_coco_json +from detectron2.structures import Boxes, BoxMode, pairwise_iou +from detectron2.utils.file_io import PathManager +from detectron2.utils.logger import create_small_table +from tabulate import tabulate + +from .evaluator import DatasetEvaluator +from .refcocoeval import RefCOCOeval + + +class RefCOCOEvaluator(DatasetEvaluator): + """ + Evaluate AR for object proposals, AP for instance detection/segmentation, AP + for keypoint detection outputs using COCO's metrics. + See http://cocodataset.org/#detection-eval and + http://cocodataset.org/#keypoints-eval to understand its metrics. + The metrics range from 0 to 100 (instead of 0 to 1), where a -1 or NaN means + the metric cannot be computed (e.g. due to no predictions made). + + In addition to COCO, this evaluator is able to support any bounding box detection, + instance segmentation, or keypoint detection dataset. + """ + + def __init__( + self, + dataset_name, + tasks=None, + distributed=True, + output_dir=None, + *, + max_dets_per_image=None, + kpt_oks_sigmas=(), + allow_cached_coco=True, + force_tasks=None, + ): + """ + Args: + dataset_name (str): name of the dataset to be evaluated. + It must have either the following corresponding metadata: + + "json_file": the path to the COCO format annotation + + Or it must be in detectron2's standard dataset format + so it can be converted to COCO format automatically. + tasks (tuple[str]): tasks that can be evaluated under the given + configuration. A task is one of "bbox", "segm", "keypoints". + By default, will infer this automatically from predictions. + distributed (True): if True, will collect results from all ranks and run evaluation + in the main process. + Otherwise, will only evaluate the results in the current process. + output_dir (str): optional, an output directory to dump all + results predicted on the dataset. The dump contains two files: + + 1. "instances_predictions.pth" a file that can be loaded with `torch.load` and + contains all the results in the format they are produced by the model. + 2. "coco_instances_results.json" a json file in COCO's result format. + max_dets_per_image (int): limit on the maximum number of detections per image. + By default in COCO, this limit is to 100, but this can be customized + to be greater, as is needed in evaluation metrics AP fixed and AP pool + (see https://arxiv.org/pdf/2102.01066.pdf) + This doesn't affect keypoint evaluation. + kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS. + See http://cocodataset.org/#keypoints-eval + When empty, it will use the defaults in COCO. + Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS. + allow_cached_coco (bool): Whether to use cached coco json from previous validation + runs. You should set this to False if you need to use different validation data. + Defaults to True. + """ + self.dataset_name = dataset_name + self._logger = logging.getLogger(__name__) + self._distributed = distributed + self._output_dir = output_dir + self.force_tasks = force_tasks + + # COCOeval requires the limit on the number of detections per image (maxDets) to be a list + # with at least 3 elements. The default maxDets in COCOeval is [1, 10, 100], in which the + # 3rd element (100) is used as the limit on the number of detections per image when + # evaluating AP. COCOEvaluator expects an integer for max_dets_per_image, so for COCOeval, + # we reformat max_dets_per_image into [1, 10, max_dets_per_image], based on the defaults. + if max_dets_per_image is None: + max_dets_per_image = [1, 10, 100] + else: + max_dets_per_image = [1, 10, max_dets_per_image] + self._max_dets_per_image = max_dets_per_image + + if tasks is not None and isinstance(tasks, CfgNode): + kpt_oks_sigmas = ( + tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas + ) + self._logger.warn( + "COCO Evaluator instantiated using config, this is deprecated behavior." + " Please pass in explicit arguments instead." + ) + self._tasks = None # Infering it from predictions should be better + else: + self._tasks = tasks + + self._cpu_device = torch.device("cpu") + + self._metadata = MetadataCatalog.get(dataset_name) + if not hasattr(self._metadata, "json_file"): + if output_dir is None: + raise ValueError( + "output_dir must be provided to COCOEvaluator " + "for datasets not in COCO format." + ) + self._logger.info(f"Trying to convert '{dataset_name}' to COCO format ...") + + cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") + self._metadata.json_file = cache_path + convert_to_coco_json(dataset_name, cache_path, allow_cached=allow_cached_coco) + + json_file = PathManager.get_local_path(self._metadata.json_file) + with contextlib.redirect_stdout(io.StringIO()): + self._coco_api = COCO(json_file) + + # Test set json files do not contain annotations (evaluation must be + # performed using the COCO evaluation server). + self._do_evaluation = "annotations" in self._coco_api.dataset + if self._do_evaluation: + self._kpt_oks_sigmas = kpt_oks_sigmas + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a COCO model (e.g., GeneralizedRCNN). + It is a list of dict. Each dict corresponds to an image and + contains keys like "height", "width", "file_name", "image_id". + outputs: the outputs of a COCO model. It is a list of dicts with key + "instances" that contains :class:`Instances`. + """ + for input, output in zip(inputs, outputs): + prediction = {"image_id": input["image_id"]} + + if "instances" in output: + instances = output["instances"].to(self._cpu_device) + prediction["instances"] = instances_to_coco_json(instances, input["image_id"]) + if "proposals" in output: + prediction["proposals"] = output["proposals"].to(self._cpu_device) + if len(prediction) > 1: + self._predictions.append(prediction) + + def evaluate(self, img_ids=None): + """ + Args: + img_ids: a list of image IDs to evaluate on. Default to None for the whole dataset + """ + if self._distributed: + comm.synchronize() + predictions = comm.gather(self._predictions, dst=0) + predictions = list(itertools.chain(*predictions)) + + if not comm.is_main_process(): + return {} + else: + predictions = self._predictions + + if len(predictions) == 0: + self._logger.warning("[COCOEvaluator] Did not receive valid predictions.") + return {} + + if self._output_dir: + PathManager.mkdirs(self._output_dir) + file_path = os.path.join(self._output_dir, "instances_predictions.pth") + with PathManager.open(file_path, "wb") as f: + torch.save(predictions, f) + + self._results = OrderedDict() + if "proposals" in predictions[0]: + self._eval_box_proposals(predictions) + if "instances" in predictions[0]: + self._eval_predictions(predictions, img_ids=img_ids) + # Copy so the caller can do whatever with results + return copy.deepcopy(self._results) + + def _tasks_from_predictions(self, predictions): + """ + Get COCO API "tasks" (i.e. iou_type) from COCO-format predictions. + """ + tasks = {"bbox"} + for pred in predictions: + if "segmentation" in pred: + tasks.add("segm") + if "keypoints" in pred: + tasks.add("keypoints") + return sorted(tasks) + + def _eval_predictions(self, predictions, img_ids=None): + """ + Evaluate predictions. Fill self._results with the metrics of the tasks. + """ + self._logger.info("Preparing results for COCO format ...") + coco_results = list(itertools.chain(*[x["instances"] for x in predictions])) + tasks = self._tasks or self._tasks_from_predictions(coco_results) + if self.force_tasks is not None: + tasks = self.force_tasks + # unmap the category ids for COCO + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"): + dataset_id_to_contiguous_id = self._metadata.thing_dataset_id_to_contiguous_id + all_contiguous_ids = list(dataset_id_to_contiguous_id.values()) + num_classes = len(all_contiguous_ids) + assert min(all_contiguous_ids) == 0 and max(all_contiguous_ids) == num_classes - 1 + + reverse_id_mapping = {v: k for k, v in dataset_id_to_contiguous_id.items()} + for result in coco_results: + category_id = result["category_id"] + assert category_id < num_classes, ( + f"A prediction has class={category_id}, " + f"but the dataset only has {num_classes} classes and " + f"predicted class id should be in [0, {num_classes - 1}]." + ) + result["category_id"] = reverse_id_mapping[category_id] + + if self._output_dir: + file_path = os.path.join( + self._output_dir, "{}_instances_results.json".format(self.dataset_name) + ) + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(coco_results)) + f.flush() + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info("Evaluating predictions with RefCOCO API...") + for task in sorted(tasks): + assert task in {"bbox", "segm", "keypoints"}, f"Got unknown task: {task}!" + coco_eval = ( + _evaluate_predictions_on_coco( + self._coco_api, + coco_results, + task, + kpt_oks_sigmas=self._kpt_oks_sigmas, + cocoeval_fn=RefCOCOeval, + img_ids=img_ids, + max_dets_per_image=self._max_dets_per_image, + ) + if len(coco_results) > 0 + else None # cocoapi does not handle empty results very well + ) + res = self._derive_refcoco_results(coco_eval, task) + self._results[task] = res + + def _eval_box_proposals(self, predictions): + """ + Evaluate the box proposals in predictions. + Fill self._results with the metrics for "box_proposals" task. + """ + if self._output_dir: + # Saving generated box proposals to file. + # Predicted box_proposals are in XYXY_ABS mode. + bbox_mode = BoxMode.XYXY_ABS.value + ids, boxes, objectness_logits = [], [], [] + for prediction in predictions: + ids.append(prediction["image_id"]) + boxes.append(prediction["proposals"].proposal_boxes.tensor.numpy()) + objectness_logits.append(prediction["proposals"].objectness_logits.numpy()) + + proposal_data = { + "boxes": boxes, + "objectness_logits": objectness_logits, + "ids": ids, + "bbox_mode": bbox_mode, + } + with PathManager.open(os.path.join(self._output_dir, "box_proposals.pkl"), "wb") as f: + pickle.dump(proposal_data, f) + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info("Evaluating bbox proposals ...") + res = {} + areas = {"all": "", "small": "s", "medium": "m", "large": "l"} + for limit in [100, 1000]: + for area, suffix in areas.items(): + stats = _evaluate_box_proposals(predictions, self._coco_api, area=area, limit=limit) + key = "AR{}@{:d}".format(suffix, limit) + res[key] = float(stats["ar"].item() * 100) + self._logger.info("Proposal metrics: \n" + create_small_table(res)) + self._results["box_proposals"] = res + + def _derive_coco_results(self, coco_eval, iou_type, class_names=None): + """ + Derive the desired score numbers from summarized COCOeval. + + Args: + coco_eval (None or COCOEval): None represents no predictions from model. + iou_type (str): + class_names (None or list[str]): if provided, will use it to predict + per-category AP. + + Returns: + a dict of {metric name: score} + """ + + metrics = { + "bbox": ["AP", "AP50", "AP75", "APs", "APm", "APl"], + "segm": ["AP", "AP50", "AP75", "APs", "APm", "APl"], + "keypoints": ["AP", "AP50", "AP75", "APm", "APl"], + }[iou_type] + + if coco_eval is None: + self._logger.warn("No predictions from the model!") + return {metric: float("nan") for metric in metrics} + + # the standard metrics + results = { + metric: float(coco_eval.stats[idx] * 100 if coco_eval.stats[idx] >= 0 else "nan") + for idx, metric in enumerate(metrics) + } + self._logger.info( + "Evaluation results for {}: \n".format(iou_type) + create_small_table(results) + ) + if not np.isfinite(sum(results.values())): + self._logger.info("Some metrics cannot be computed and is shown as NaN.") + + if class_names is None or len(class_names) <= 1: + return results + # Compute per-category AP + # from https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L222-L252 # noqa + precisions = coco_eval.eval["precision"] + # precision has dims (iou, recall, cls, area range, max dets) + assert len(class_names) == precisions.shape[2] + + results_per_category = [] + for idx, name in enumerate(class_names): + # area range index 0: all area ranges + # max dets index -1: typically 100 per image + precision = precisions[:, :, idx, 0, -1] + precision = precision[precision > -1] + ap = np.mean(precision) if precision.size else float("nan") + results_per_category.append(("{}".format(name), float(ap * 100))) + + # tabulate it + N_COLS = min(6, len(results_per_category) * 2) + results_flatten = list(itertools.chain(*results_per_category)) + results_2d = itertools.zip_longest(*[results_flatten[i::N_COLS] for i in range(N_COLS)]) + table = tabulate( + results_2d, + tablefmt="pipe", + floatfmt=".3f", + headers=["category", "AP"] * (N_COLS // 2), + numalign="left", + ) + self._logger.info("Per-category {} AP: \n".format(iou_type) + table) + + results.update({"AP-" + name: ap for name, ap in results_per_category}) + return results + + def _derive_refcoco_results(self, coco_eval, iou_type): + """ + Derive the desired score numbers from summarized COCOeval. + + Args: + coco_eval (None or COCOEval): None represents no predictions from model. + iou_type (str): + class_names (None or list[str]): if provided, will use it to predict + per-category AP. + + Returns: + a dict of {metric name: score} + """ + + metrics = {"bbox": ["P@0.5", "P@0.6", "P@0.7", "P@0.8", "P@0.9"], "segm": ["oIoU", "mIoU"]}[ + iou_type + ] + + if coco_eval is None: + self._logger.warn("No predictions from the model!") + return {metric: float("nan") for metric in metrics} + + # the standard metrics + results = {metric: float("nan") for idx, metric in enumerate(metrics)} + ious = np.array([v for (k, v) in coco_eval.ious.items()]) + total_intersection_area = coco_eval.total_intersection_area + total_union_area = coco_eval.total_union_area + iou_list = coco_eval.iou_list + # compute metrics + if iou_type == "bbox": + results["P@0.5"] = np.sum(ious > 0.5) / len(ious) * 100 + results["P@0.6"] = np.sum(ious > 0.6) / len(ious) * 100 + results["P@0.7"] = np.sum(ious > 0.7) / len(ious) * 100 + results["P@0.8"] = np.sum(ious > 0.8) / len(ious) * 100 + results["P@0.9"] = np.sum(ious > 0.9) / len(ious) * 100 + elif iou_type == "segm": + results["oIoU"] = total_intersection_area / total_union_area * 100 + results["mIoU"] = np.mean(ious) * 100 + else: + raise ValueError("Unsupported iou_type!") + self._logger.info( + "Evaluation results for {}: \n".format(iou_type) + create_small_table(results) + ) + + # results.update({"AP-" + name: ap for name, ap in results_per_category}) + return results + + +def instances_to_coco_json(instances, img_id): + """ + Dump an "Instances" object to a COCO-format json that's used for evaluation. + + Args: + instances (Instances): + img_id (int): the image id + + Returns: + list[dict]: list of json annotations in COCO format. + """ + num_instance = len(instances) + if num_instance == 0: + return [] + + boxes = instances.pred_boxes.tensor.numpy() + boxes = BoxMode.convert(boxes, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS) + boxes = boxes.tolist() + scores = instances.scores.tolist() + classes = instances.pred_classes.tolist() + + has_mask = instances.has("pred_masks") + if has_mask: + # use RLE to encode the masks, because they are too large and takes memory + # since this evaluator stores outputs of the entire dataset + rles = [ + mask_util.encode(np.array(mask[:, :, None], order="F", dtype="uint8"))[0] + for mask in instances.pred_masks + ] + for rle in rles: + # "counts" is an array encoded by mask_util as a byte-stream. Python3's + # json writer which always produces strings cannot serialize a bytestream + # unless you decode it. Thankfully, utf-8 works out (which is also what + # the pycocotools/_mask.pyx does). + rle["counts"] = rle["counts"].decode("utf-8") + + has_keypoints = instances.has("pred_keypoints") + if has_keypoints: + keypoints = instances.pred_keypoints + + results = [] + for k in range(num_instance): + result = { + "image_id": img_id, + "category_id": classes[k], + "bbox": boxes[k], + "score": scores[k], + } + if has_mask: + result["segmentation"] = rles[k] + if has_keypoints: + # In COCO annotations, + # keypoints coordinates are pixel indices. + # However our predictions are floating point coordinates. + # Therefore we subtract 0.5 to be consistent with the annotation format. + # This is the inverse of data loading logic in `datasets/coco.py`. + keypoints[k][:, :2] -= 0.5 + result["keypoints"] = keypoints[k].flatten().tolist() + results.append(result) + return results + + +# inspired from Detectron: +# https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L255 # noqa +def _evaluate_box_proposals(dataset_predictions, coco_api, thresholds=None, area="all", limit=None): + """ + Evaluate detection proposal recall metrics. This function is a much + faster alternative to the official COCO API recall evaluation code. However, + it produces slightly different results. + """ + # Record max overlap value for each gt box + # Return vector of overlap values + areas = { + "all": 0, + "small": 1, + "medium": 2, + "large": 3, + "96-128": 4, + "128-256": 5, + "256-512": 6, + "512-inf": 7, + } + area_ranges = [ + [0**2, 1e5**2], # all + [0**2, 32**2], # small + [32**2, 96**2], # medium + [96**2, 1e5**2], # large + [96**2, 128**2], # 96-128 + [128**2, 256**2], # 128-256 + [256**2, 512**2], # 256-512 + [512**2, 1e5**2], + ] # 512-inf + assert area in areas, "Unknown area range: {}".format(area) + area_range = area_ranges[areas[area]] + gt_overlaps = [] + num_pos = 0 + + for prediction_dict in dataset_predictions: + predictions = prediction_dict["proposals"] + + # sort predictions in descending order + # TODO maybe remove this and make it explicit in the documentation + inds = predictions.objectness_logits.sort(descending=True)[1] + predictions = predictions[inds] + + ann_ids = coco_api.getAnnIds(imgIds=prediction_dict["image_id"]) + anno = coco_api.loadAnns(ann_ids) + gt_boxes = [ + BoxMode.convert(obj["bbox"], BoxMode.XYWH_ABS, BoxMode.XYXY_ABS) + for obj in anno + if obj["iscrowd"] == 0 + ] + gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4) # guard against no boxes + gt_boxes = Boxes(gt_boxes) + gt_areas = torch.as_tensor([obj["area"] for obj in anno if obj["iscrowd"] == 0]) + + if len(gt_boxes) == 0 or len(predictions) == 0: + continue + + valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1]) + gt_boxes = gt_boxes[valid_gt_inds] + + num_pos += len(gt_boxes) + + if len(gt_boxes) == 0: + continue + + if limit is not None and len(predictions) > limit: + predictions = predictions[:limit] + + overlaps = pairwise_iou(predictions.proposal_boxes, gt_boxes) + + _gt_overlaps = torch.zeros(len(gt_boxes)) + for j in range(min(len(predictions), len(gt_boxes))): + # find which proposal box maximally covers each gt box + # and get the iou amount of coverage for each gt box + max_overlaps, argmax_overlaps = overlaps.max(dim=0) + + # find which gt box is 'best' covered (i.e. 'best' = most iou) + gt_ovr, gt_ind = max_overlaps.max(dim=0) + assert gt_ovr >= 0 + # find the proposal box that covers the best covered gt box + box_ind = argmax_overlaps[gt_ind] + # record the iou coverage of this gt box + _gt_overlaps[j] = overlaps[box_ind, gt_ind] + assert _gt_overlaps[j] == gt_ovr + # mark the proposal box and the gt box as used + overlaps[box_ind, :] = -1 + overlaps[:, gt_ind] = -1 + + # append recorded iou coverage level + gt_overlaps.append(_gt_overlaps) + gt_overlaps = ( + torch.cat(gt_overlaps, dim=0) if len(gt_overlaps) else torch.zeros(0, dtype=torch.float32) + ) + gt_overlaps, _ = torch.sort(gt_overlaps) + + if thresholds is None: + step = 0.05 + thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32) + recalls = torch.zeros_like(thresholds) + # compute recall for each iou threshold + for i, t in enumerate(thresholds): + recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos) + # ar = 2 * np.trapz(recalls, thresholds) + ar = recalls.mean() + return { + "ar": ar, + "recalls": recalls, + "thresholds": thresholds, + "gt_overlaps": gt_overlaps, + "num_pos": num_pos, + } + + +def _evaluate_predictions_on_coco( + coco_gt, + coco_results, + iou_type, + kpt_oks_sigmas=None, + cocoeval_fn=RefCOCOeval, + img_ids=None, + max_dets_per_image=None, +): + """ + Evaluate the coco results using COCOEval API. + """ + assert len(coco_results) > 0 + + if iou_type == "segm": + coco_results = copy.deepcopy(coco_results) + # When evaluating mask AP, if the results contain bbox, cocoapi will + # use the box area as the area of the instance, instead of the mask area. + # This leads to a different definition of small/medium/large. + # We remove the bbox field to let mask AP use mask area. + for c in coco_results: + c.pop("bbox", None) + + coco_dt = coco_gt.loadRes(coco_results) + coco_eval = cocoeval_fn(coco_gt, coco_dt, iou_type) + # For COCO, the default max_dets_per_image is [1, 10, 100]. + if max_dets_per_image is None: + max_dets_per_image = [1, 10, 100] # Default from COCOEval + else: + assert ( + len(max_dets_per_image) >= 3 + ), "COCOeval requires maxDets (and max_dets_per_image) to have length at least 3" + # In the case that user supplies a custom input for max_dets_per_image, + # apply COCOevalMaxDets to evaluate AP with the custom input. + if max_dets_per_image[2] != 100: + coco_eval = COCOevalMaxDets(coco_gt, coco_dt, iou_type) + if iou_type != "keypoints": + coco_eval.params.maxDets = max_dets_per_image + + if img_ids is not None: + coco_eval.params.imgIds = img_ids + + if iou_type == "keypoints": + # Use the COCO default keypoint OKS sigmas unless overrides are specified + if kpt_oks_sigmas: + assert hasattr(coco_eval.params, "kpt_oks_sigmas"), "pycocotools is too old!" + coco_eval.params.kpt_oks_sigmas = np.array(kpt_oks_sigmas) + # COCOAPI requires every detection and every gt to have keypoints, so + # we just take the first entry from both + num_keypoints_dt = len(coco_results[0]["keypoints"]) // 3 + num_keypoints_gt = len(next(iter(coco_gt.anns.values()))["keypoints"]) // 3 + num_keypoints_oks = len(coco_eval.params.kpt_oks_sigmas) + assert num_keypoints_oks == num_keypoints_dt == num_keypoints_gt, ( + f"[COCOEvaluator] Prediction contain {num_keypoints_dt} keypoints. " + f"Ground truth contains {num_keypoints_gt} keypoints. " + f"The length of cfg.TEST.KEYPOINT_OKS_SIGMAS is {num_keypoints_oks}. " + "They have to agree with each other. For meaning of OKS, please refer to " + "http://cocodataset.org/#keypoints-eval." + ) + + coco_eval.evaluate() + + return coco_eval + + +class COCOevalMaxDets(COCOeval): + """ + Modified version of COCOeval for evaluating AP with a custom + maxDets (by default for COCO, maxDets is 100) + """ + + def summarize(self): + """ + Compute and display summary metrics for evaluation results given + a custom value for max_dets_per_image + """ + + def _summarize(ap=1, iouThr=None, areaRng="all", maxDets=100): + p = self.params + iStr = " {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}" + titleStr = "Average Precision" if ap == 1 else "Average Recall" + typeStr = "(AP)" if ap == 1 else "(AR)" + iouStr = ( + "{:0.2f}:{:0.2f}".format(p.iouThrs[0], p.iouThrs[-1]) + if iouThr is None + else "{:0.2f}".format(iouThr) + ) + + aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng] + mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets] + if ap == 1: + # dimension of precision: [TxRxKxAxM] + s = self.eval["precision"] + # IoU + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] + s = s[:, :, :, aind, mind] + else: + # dimension of recall: [TxKxAxM] + s = self.eval["recall"] + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] + s = s[:, :, aind, mind] + if len(s[s > -1]) == 0: + mean_s = -1 + else: + mean_s = np.mean(s[s > -1]) + print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s)) + return mean_s + + def _summarizeDets(): + stats = np.zeros((12,)) + # Evaluate AP using the custom limit on maximum detections per image + stats[0] = _summarize(1, maxDets=self.params.maxDets[2]) + stats[1] = _summarize(1, iouThr=0.5, maxDets=self.params.maxDets[2]) + stats[2] = _summarize(1, iouThr=0.75, maxDets=self.params.maxDets[2]) + stats[3] = _summarize(1, areaRng="small", maxDets=self.params.maxDets[2]) + stats[4] = _summarize(1, areaRng="medium", maxDets=self.params.maxDets[2]) + stats[5] = _summarize(1, areaRng="large", maxDets=self.params.maxDets[2]) + stats[6] = _summarize(0, maxDets=self.params.maxDets[0]) + stats[7] = _summarize(0, maxDets=self.params.maxDets[1]) + stats[8] = _summarize(0, maxDets=self.params.maxDets[2]) + stats[9] = _summarize(0, areaRng="small", maxDets=self.params.maxDets[2]) + stats[10] = _summarize(0, areaRng="medium", maxDets=self.params.maxDets[2]) + stats[11] = _summarize(0, areaRng="large", maxDets=self.params.maxDets[2]) + return stats + + def _summarizeKps(): + stats = np.zeros((10,)) + stats[0] = _summarize(1, maxDets=20) + stats[1] = _summarize(1, maxDets=20, iouThr=0.5) + stats[2] = _summarize(1, maxDets=20, iouThr=0.75) + stats[3] = _summarize(1, maxDets=20, areaRng="medium") + stats[4] = _summarize(1, maxDets=20, areaRng="large") + stats[5] = _summarize(0, maxDets=20) + stats[6] = _summarize(0, maxDets=20, iouThr=0.5) + stats[7] = _summarize(0, maxDets=20, iouThr=0.75) + stats[8] = _summarize(0, maxDets=20, areaRng="medium") + stats[9] = _summarize(0, maxDets=20, areaRng="large") + return stats + + if not self.eval: + raise Exception("Please run accumulate() first") + iouType = self.params.iouType + if iouType == "segm" or iouType == "bbox": + summarize = _summarizeDets + elif iouType == "keypoints": + summarize = _summarizeKps + self.stats = summarize() + + def __str__(self): + self.summarize() diff --git a/approach/ovod/APE/ape/evaluation/refcocoeval.py b/approach/ovod/APE/ape/evaluation/refcocoeval.py new file mode 100644 index 0000000000000000000000000000000000000000..b2396dfd5528c02453bef73ce9a98d652187f84c --- /dev/null +++ b/approach/ovod/APE/ape/evaluation/refcocoeval.py @@ -0,0 +1,593 @@ +__author__ = "tsungyi" + +import copy +import datetime +import time +from collections import defaultdict + +import numpy as np +import torch +from pycocotools import mask as maskUtils +from pycocotools.mask import decode +from torch._C import InterfaceType + +from torchvision.ops.boxes import box_area + + +def compute_bbox_iou(boxes1: torch.Tensor, boxes2: torch.Tensor): + # both boxes: xyxy + area1 = box_area(boxes1) + area2 = box_area(boxes2) + + lt = torch.max(boxes1[:, None, :2], boxes2[:, :2]) # [N,M,2] + rb = torch.min(boxes1[:, None, 2:], boxes2[:, 2:]) # [N,M,2] + + wh = (rb - lt).clamp(min=0) # [N,M,2] + inter = wh[:, :, 0] * wh[:, :, 1] # [N,M] + + union = area1[:, None] + area2 - inter + + iou = (inter + 1e-6) / (union + 1e-6) + return iou, inter, union + + +def compute_mask_iou(outputs: torch.Tensor, labels: torch.Tensor, EPS=1e-6): + outputs = outputs.int() + intersection = (outputs & labels).float().sum((1, 2)) # Will be zero if Truth=0 or Prediction=0 + union = (outputs | labels).float().sum((1, 2)) # Will be zero if both are 0 + iou = (intersection + EPS) / (union + EPS) # EPS is used to avoid division by zero + return iou, intersection, union + + +class RefCOCOeval: + # Interface for evaluating detection on the Microsoft COCO dataset. + # + # The usage for CocoEval is as follows: + # cocoGt=..., cocoDt=... # load dataset and results + # E = CocoEval(cocoGt,cocoDt); # initialize CocoEval object + # E.params.recThrs = ...; # set parameters as desired + # E.evaluate(); # run per image evaluation + # E.accumulate(); # accumulate per image results + # E.summarize(); # display summary metrics of results + # For example usage see evalDemo.m and http://mscoco.org/. + # + # The evaluation parameters are as follows (defaults in brackets): + # imgIds - [all] N img ids to use for evaluation + # catIds - [all] K cat ids to use for evaluation + # iouThrs - [.5:.05:.95] T=10 IoU thresholds for evaluation + # recThrs - [0:.01:1] R=101 recall thresholds for evaluation + # areaRng - [...] A=4 object area ranges for evaluation + # maxDets - [1 10 100] M=3 thresholds on max detections per image + # iouType - ['segm'] set iouType to 'segm', 'bbox' or 'keypoints' + # iouType replaced the now DEPRECATED useSegm parameter. + # useCats - [1] if true use category labels for evaluation + # Note: if useCats=0 category labels are ignored as in proposal scoring. + # Note: multiple areaRngs [Ax2] and maxDets [Mx1] can be specified. + # + # evaluate(): evaluates detections on every image and every category and + # concats the results into the "evalImgs" with fields: + # dtIds - [1xD] id for each of the D detections (dt) + # gtIds - [1xG] id for each of the G ground truths (gt) + # dtMatches - [TxD] matching gt id at each IoU or 0 + # gtMatches - [TxG] matching dt id at each IoU or 0 + # dtScores - [1xD] confidence of each dt + # gtIgnore - [1xG] ignore flag for each gt + # dtIgnore - [TxD] ignore flag for each dt at each IoU + # + # accumulate(): accumulates the per-image, per-category evaluation + # results in "evalImgs" into the dictionary "eval" with fields: + # params - parameters used for evaluation + # date - date evaluation was performed + # counts - [T,R,K,A,M] parameter dimensions (see above) + # precision - [TxRxKxAxM] precision for every evaluation setting + # recall - [TxKxAxM] max recall for every evaluation setting + # Note: precision and recall==-1 for settings with no gt objects. + # + # See also coco, mask, pycocoDemo, pycocoEvalDemo + # + # Microsoft COCO Toolbox. version 2.0 + # Data, paper, and tutorials available at: http://mscoco.org/ + # Code written by Piotr Dollar and Tsung-Yi Lin, 2015. + # Licensed under the Simplified BSD License [see coco/license.txt] + def __init__(self, cocoGt=None, cocoDt=None, iouType="segm"): + """ + Initialize CocoEval using coco APIs for gt and dt + :param cocoGt: coco object with ground truth annotations + :param cocoDt: coco object with detection results + :return: None + """ + if not iouType: + print("iouType not specified. use default iouType segm") + self.cocoGt = cocoGt # ground truth COCO API + self.cocoDt = cocoDt # detections COCO API + self.evalImgs = defaultdict( + list + ) # per-image per-category evaluation results [KxAxI] elements + self.eval = {} # accumulated evaluation results + self._gts = defaultdict(list) # gt for evaluation + self._dts = defaultdict(list) # dt for evaluation + self.params = Params(iouType=iouType) # parameters + self._paramsEval = {} # parameters for evaluation + self.stats = [] # result summarization + self.ious = {} # ious between all gts and dts + # for computing overall iou + self.total_intersection_area = 0 + self.total_union_area = 0 + self.iou_list = [] + if not cocoGt is None: + self.params.imgIds = sorted(cocoGt.getImgIds()) + self.params.catIds = sorted(cocoGt.getCatIds()) + + def _prepare(self): + """ + Prepare ._gts and ._dts for evaluation based on params + :return: None + """ + + def _toMask(anns, coco): + # modify ann['segmentation'] by reference + for ann in anns: + rle = coco.annToRLE(ann) + ann["segmentation"] = rle + + p = self.params + if p.useCats: + gts = self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds)) + dts = self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds)) + else: + gts = self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds)) + dts = self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds)) + + # convert ground truth to mask if iouType == 'segm' + if p.iouType == "segm": + _toMask(gts, self.cocoGt) + _toMask(dts, self.cocoDt) + # set ignore flag + for gt in gts: + gt["ignore"] = gt["ignore"] if "ignore" in gt else 0 + gt["ignore"] = "iscrowd" in gt and gt["iscrowd"] + if p.iouType == "keypoints": + gt["ignore"] = (gt["num_keypoints"] == 0) or gt["ignore"] + self._gts = defaultdict(list) # gt for evaluation + self._dts = defaultdict(list) # dt for evaluation + for gt in gts: + self._gts[gt["image_id"], gt["category_id"]].append(gt) + for dt in dts: + self._dts[dt["image_id"], dt["category_id"]].append(dt) + self.evalImgs = defaultdict(list) # per-image per-category evaluation results + self.eval = {} # accumulated evaluation results + + def evaluate(self): + """ + Run per image evaluation on given images and store results (a list of dict) in self.evalImgs + :return: None + """ + tic = time.time() + print("Running per image evaluation...") + p = self.params + # add backward compatibility if useSegm is specified in params + if not p.useSegm is None: + p.iouType = "segm" if p.useSegm == 1 else "bbox" + print("useSegm (deprecated) is not None. Running {} evaluation".format(p.iouType)) + print("Evaluate annotation type *{}*".format(p.iouType)) + p.imgIds = list(np.unique(p.imgIds)) + if p.useCats: + p.catIds = list(np.unique(p.catIds)) + p.maxDets = sorted(p.maxDets) + self.params = p + + self._prepare() + # loop through images, area range, max detection number + catIds = p.catIds if p.useCats else [-1] + + if p.iouType == "segm" or p.iouType == "bbox": + computeIoU = self.computeIoU + elif p.iouType == "keypoints": + computeIoU = self.computeOks + self.ious = { + (imgId, catId): computeIoU(imgId, catId) for imgId in p.imgIds for catId in catIds + } + # evaluateImg = self.evaluateImg + # maxDet = p.maxDets[-1] + # self.evalImgs = [evaluateImg(imgId, catId, areaRng, maxDet) + # for catId in catIds + # for areaRng in p.areaRng + # for imgId in p.imgIds + # ] + # self._paramsEval = copy.deepcopy(self.params) + toc = time.time() + print("DONE (t={:0.2f}s).".format(toc - tic)) + + def computeIoU(self, imgId, catId): + p = self.params + if p.useCats: + gt = self._gts[imgId, catId] + dt = self._dts[imgId, catId] + else: + gt = [_ for cId in p.catIds for _ in self._gts[imgId, cId]] + dt = [_ for cId in p.catIds for _ in self._dts[imgId, cId]] + if len(gt) == 0 and len(dt) == 0: + return [] + inds = np.argsort([-d["score"] for d in dt], kind="mergesort") + dt = [dt[i] for i in inds] + if len(dt) > p.maxDets[-1]: + dt = dt[0 : p.maxDets[-1]] + + if p.iouType == "segm": + g = [g["segmentation"] for g in gt] + d = [d["segmentation"] for d in dt] + elif p.iouType == "bbox": + g = [g["bbox"] for g in gt] + d = [d["bbox"] for d in dt] + else: + raise Exception("unknown iouType for iou computation") + + # compute iou between each dt and gt region + iscrowd = [int(o["iscrowd"]) for o in gt] + ious = maskUtils.iou(d, g, iscrowd) + + # for computing overall iou + # there is only one bbox and segm + if p.iouType == "bbox": + g, d = g[0], d[0] + g_bbox = [g[0], g[1], g[2] + g[0], g[3] + g[1]] # x1y1wh -> x1y1x2y2 + d_bbox = [d[0], d[1], d[2] + d[0], d[3] + d[1]] # x1y1wh -> x1y1x2y2 + g_bbox = torch.tensor(g_bbox).unsqueeze(0) + d_bbox = torch.tensor(d_bbox).unsqueeze(0) + iou, intersection, union = compute_bbox_iou(d_bbox, g_bbox) + elif p.iouType == "segm": + g_segm = decode(g[0]) + d_segm = decode(d[0]) + g_segm = torch.tensor(g_segm).unsqueeze(0) + d_segm = torch.tensor(d_segm).unsqueeze(0) + iou, intersection, union = compute_mask_iou(d_segm, g_segm) + else: + raise Exception("unknown iouType for iou computation") + iou, intersection, union = iou.item(), intersection.item(), union.item() + self.total_intersection_area += intersection + self.total_union_area += union + self.iou_list.append(iou) + return ious + + def evaluateImg(self, imgId, catId, aRng, maxDet): + """ + perform evaluation for single category and image + :return: dict (single image results) + """ + p = self.params + if p.useCats: + gt = self._gts[imgId, catId] + dt = self._dts[imgId, catId] + else: + gt = [_ for cId in p.catIds for _ in self._gts[imgId, cId]] + dt = [_ for cId in p.catIds for _ in self._dts[imgId, cId]] + if len(gt) == 0 and len(dt) == 0: + return None + + for g in gt: + if g["ignore"] or (g["area"] < aRng[0] or g["area"] > aRng[1]): + g["_ignore"] = 1 + else: + g["_ignore"] = 0 + + # sort dt highest score first, sort gt ignore last + gtind = np.argsort([g["_ignore"] for g in gt], kind="mergesort") + gt = [gt[i] for i in gtind] + dtind = np.argsort([-d["score"] for d in dt], kind="mergesort") + dt = [dt[i] for i in dtind[0:maxDet]] + iscrowd = [int(o["iscrowd"]) for o in gt] + # load computed ious + ious = ( + self.ious[imgId, catId][:, gtind] + if len(self.ious[imgId, catId]) > 0 + else self.ious[imgId, catId] + ) + + T = len(p.iouThrs) + G = len(gt) + D = len(dt) + gtm = np.zeros((T, G)) + dtm = np.zeros((T, D)) + gtIg = np.array([g["_ignore"] for g in gt]) + dtIg = np.zeros((T, D)) + if not len(ious) == 0: + for tind, t in enumerate(p.iouThrs): + for dind, d in enumerate(dt): + # information about best match so far (m=-1 -> unmatched) + iou = min([t, 1 - 1e-10]) + m = -1 + for gind, g in enumerate(gt): + # if this gt already matched, and not a crowd, continue + if gtm[tind, gind] > 0 and not iscrowd[gind]: + continue + # if dt matched to reg gt, and on ignore gt, stop + if m > -1 and gtIg[m] == 0 and gtIg[gind] == 1: + break + # continue to next gt unless better match made + if ious[dind, gind] < iou: + continue + # if match successful and best so far, store appropriately + iou = ious[dind, gind] + m = gind + # if match made store id of match for both dt and gt + if m == -1: + continue + dtIg[tind, dind] = gtIg[m] + dtm[tind, dind] = gt[m]["id"] + gtm[tind, m] = d["id"] + # set unmatched detections outside of area range to ignore + a = np.array([d["area"] < aRng[0] or d["area"] > aRng[1] for d in dt]).reshape((1, len(dt))) + dtIg = np.logical_or(dtIg, np.logical_and(dtm == 0, np.repeat(a, T, 0))) + # store results for given image and category + return { + "image_id": imgId, + "category_id": catId, + "aRng": aRng, + "maxDet": maxDet, + "dtIds": [d["id"] for d in dt], + "gtIds": [g["id"] for g in gt], + "dtMatches": dtm, + "gtMatches": gtm, + "dtScores": [d["score"] for d in dt], + "gtIgnore": gtIg, + "dtIgnore": dtIg, + } + + def accumulate(self, p=None): + """ + Accumulate per image evaluation results and store the result in self.eval + :param p: input params for evaluation + :return: None + """ + print("Accumulating evaluation results...") + tic = time.time() + if not self.evalImgs: + print("Please run evaluate() first") + # allows input customized parameters + if p is None: + p = self.params + p.catIds = p.catIds if p.useCats == 1 else [-1] + T = len(p.iouThrs) + R = len(p.recThrs) + K = len(p.catIds) if p.useCats else 1 + A = len(p.areaRng) + M = len(p.maxDets) + precision = -np.ones((T, R, K, A, M)) # -1 for the precision of absent categories + recall = -np.ones((T, K, A, M)) + scores = -np.ones((T, R, K, A, M)) + + # create dictionary for future indexing + _pe = self._paramsEval + catIds = _pe.catIds if _pe.useCats else [-1] + setK = set(catIds) + setA = set(map(tuple, _pe.areaRng)) + setM = set(_pe.maxDets) + setI = set(_pe.imgIds) + # get inds to evaluate + k_list = [n for n, k in enumerate(p.catIds) if k in setK] + m_list = [m for n, m in enumerate(p.maxDets) if m in setM] + a_list = [n for n, a in enumerate(map(lambda x: tuple(x), p.areaRng)) if a in setA] + i_list = [n for n, i in enumerate(p.imgIds) if i in setI] + I0 = len(_pe.imgIds) + A0 = len(_pe.areaRng) + # retrieve E at each category, area range, and max number of detections + for k, k0 in enumerate(k_list): + Nk = k0 * A0 * I0 + for a, a0 in enumerate(a_list): + Na = a0 * I0 + for m, maxDet in enumerate(m_list): + E = [self.evalImgs[Nk + Na + i] for i in i_list] + E = [e for e in E if not e is None] + if len(E) == 0: + continue + dtScores = np.concatenate([e["dtScores"][0:maxDet] for e in E]) + + # different sorting method generates slightly different results. + # mergesort is used to be consistent as Matlab implementation. + inds = np.argsort(-dtScores, kind="mergesort") + dtScoresSorted = dtScores[inds] + + dtm = np.concatenate([e["dtMatches"][:, 0:maxDet] for e in E], axis=1)[:, inds] + dtIg = np.concatenate([e["dtIgnore"][:, 0:maxDet] for e in E], axis=1)[:, inds] + gtIg = np.concatenate([e["gtIgnore"] for e in E]) + npig = np.count_nonzero(gtIg == 0) + if npig == 0: + continue + tps = np.logical_and(dtm, np.logical_not(dtIg)) + fps = np.logical_and(np.logical_not(dtm), np.logical_not(dtIg)) + + tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float) + fp_sum = np.cumsum(fps, axis=1).astype(dtype=np.float) + for t, (tp, fp) in enumerate(zip(tp_sum, fp_sum)): + tp = np.array(tp) + fp = np.array(fp) + nd = len(tp) + rc = tp / npig + pr = tp / (fp + tp + np.spacing(1)) + q = np.zeros((R,)) + ss = np.zeros((R,)) + + if nd: + recall[t, k, a, m] = rc[-1] + else: + recall[t, k, a, m] = 0 + + # numpy is slow without cython optimization for accessing elements + # use python array gets significant speed improvement + pr = pr.tolist() + q = q.tolist() + + for i in range(nd - 1, 0, -1): + if pr[i] > pr[i - 1]: + pr[i - 1] = pr[i] + + inds = np.searchsorted(rc, p.recThrs, side="left") + try: + for ri, pi in enumerate(inds): + q[ri] = pr[pi] + ss[ri] = dtScoresSorted[pi] + except: + pass + precision[t, :, k, a, m] = np.array(q) + scores[t, :, k, a, m] = np.array(ss) + self.eval = { + "params": p, + "counts": [T, R, K, A, M], + "date": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "precision": precision, + "recall": recall, + "scores": scores, + } + toc = time.time() + print("DONE (t={:0.2f}s).".format(toc - tic)) + + def summarize(self): + """ + Compute and display summary metrics for evaluation results. + Note this functin can *only* be applied on the default parameter setting + """ + + def _summarize(ap=1, iouThr=None, areaRng="all", maxDets=100): + p = self.params + iStr = " {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}" + titleStr = "Average Precision" if ap == 1 else "Average Recall" + typeStr = "(AP)" if ap == 1 else "(AR)" + iouStr = ( + "{:0.2f}:{:0.2f}".format(p.iouThrs[0], p.iouThrs[-1]) + if iouThr is None + else "{:0.2f}".format(iouThr) + ) + + aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng] + mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets] + if ap == 1: + # dimension of precision: [TxRxKxAxM] + s = self.eval["precision"] + # IoU + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] + s = s[:, :, :, aind, mind] + else: + # dimension of recall: [TxKxAxM] + s = self.eval["recall"] + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] + s = s[:, :, aind, mind] + if len(s[s > -1]) == 0: + mean_s = -1 + else: + mean_s = np.mean(s[s > -1]) + print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s)) + return mean_s + + def _summarizeDets(): + stats = np.zeros((12,)) + stats[0] = _summarize(1) + stats[1] = _summarize(1, iouThr=0.5, maxDets=self.params.maxDets[2]) + stats[2] = _summarize(1, iouThr=0.75, maxDets=self.params.maxDets[2]) + stats[3] = _summarize(1, areaRng="small", maxDets=self.params.maxDets[2]) + stats[4] = _summarize(1, areaRng="medium", maxDets=self.params.maxDets[2]) + stats[5] = _summarize(1, areaRng="large", maxDets=self.params.maxDets[2]) + stats[6] = _summarize(0, maxDets=self.params.maxDets[0]) + stats[7] = _summarize(0, maxDets=self.params.maxDets[1]) + stats[8] = _summarize(0, maxDets=self.params.maxDets[2]) + stats[9] = _summarize(0, areaRng="small", maxDets=self.params.maxDets[2]) + stats[10] = _summarize(0, areaRng="medium", maxDets=self.params.maxDets[2]) + stats[11] = _summarize(0, areaRng="large", maxDets=self.params.maxDets[2]) + return stats + + def _summarizeKps(): + stats = np.zeros((10,)) + stats[0] = _summarize(1, maxDets=20) + stats[1] = _summarize(1, maxDets=20, iouThr=0.5) + stats[2] = _summarize(1, maxDets=20, iouThr=0.75) + stats[3] = _summarize(1, maxDets=20, areaRng="medium") + stats[4] = _summarize(1, maxDets=20, areaRng="large") + stats[5] = _summarize(0, maxDets=20) + stats[6] = _summarize(0, maxDets=20, iouThr=0.5) + stats[7] = _summarize(0, maxDets=20, iouThr=0.75) + stats[8] = _summarize(0, maxDets=20, areaRng="medium") + stats[9] = _summarize(0, maxDets=20, areaRng="large") + return stats + + if not self.eval: + raise Exception("Please run accumulate() first") + iouType = self.params.iouType + if iouType == "segm" or iouType == "bbox": + summarize = _summarizeDets + elif iouType == "keypoints": + summarize = _summarizeKps + self.stats = summarize() + + def __str__(self): + self.summarize() + + +class Params: + """ + Params for coco evaluation api + """ + + def setDetParams(self): + self.imgIds = [] + self.catIds = [] + # np.arange causes trouble. the data point on arange is slightly larger than the true value + self.iouThrs = np.linspace(0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True) + self.recThrs = np.linspace(0.0, 1.00, int(np.round((1.00 - 0.0) / 0.01)) + 1, endpoint=True) + self.maxDets = [1, 10, 100] + self.areaRng = [ + [0**2, 1e5**2], + [0**2, 32**2], + [32**2, 96**2], + [96**2, 1e5**2], + ] + self.areaRngLbl = ["all", "small", "medium", "large"] + self.useCats = 1 + + def setKpParams(self): + self.imgIds = [] + self.catIds = [] + # np.arange causes trouble. the data point on arange is slightly larger than the true value + self.iouThrs = np.linspace(0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True) + self.recThrs = np.linspace(0.0, 1.00, int(np.round((1.00 - 0.0) / 0.01)) + 1, endpoint=True) + self.maxDets = [20] + self.areaRng = [[0**2, 1e5**2], [32**2, 96**2], [96**2, 1e5**2]] + self.areaRngLbl = ["all", "medium", "large"] + self.useCats = 1 + self.kpt_oks_sigmas = ( + np.array( + [ + 0.26, + 0.25, + 0.25, + 0.35, + 0.35, + 0.79, + 0.79, + 0.72, + 0.72, + 0.62, + 0.62, + 1.07, + 1.07, + 0.87, + 0.87, + 0.89, + 0.89, + ] + ) + / 10.0 + ) + + def __init__(self, iouType="segm"): + if iouType == "segm" or iouType == "bbox": + self.setDetParams() + elif iouType == "keypoints": + self.setKpParams() + else: + raise Exception("iouType not supported") + self.iouType = iouType + # useSegm is deprecated + self.useSegm = None diff --git a/approach/ovod/APE/ape/layers/__init__.py b/approach/ovod/APE/ape/layers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c6ac2191296240bb2495626ba1510b967de439f3 --- /dev/null +++ b/approach/ovod/APE/ape/layers/__init__.py @@ -0,0 +1,8 @@ +from .fuse_helper import BiAttentionBlock, BiMultiHeadAttention +from .multi_scale_deform_attn import ( + MultiScaleDeformableAttention, + multi_scale_deformable_attn_pytorch, +) +from .vision_language_align import StillClassifier, VisionLanguageAlign +from .vision_language_fusion import VisionLanguageFusion +from .zero_shot_fc import ZeroShotFC diff --git a/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_attn.h b/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_attn.h new file mode 100644 index 0000000000000000000000000000000000000000..7e5913b595033d487cc30feeaf9b2572d3c6b3ea --- /dev/null +++ b/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_attn.h @@ -0,0 +1,64 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once + +#include "ms_deform_attn_cpu.h" + +#ifdef WITH_CUDA +#include "ms_deform_attn_cuda.h" +#endif + +namespace ape { + +at::Tensor +ms_deform_attn_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int64_t im2col_step) +{ + if (value.type().is_cuda()) + { +#ifdef WITH_CUDA + return ms_deform_attn_cuda_forward( + value, spatial_shapes, level_start_index, sampling_loc, attn_weight, im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +std::vector +ms_deform_attn_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int64_t im2col_step) +{ + if (value.type().is_cuda()) + { +#ifdef WITH_CUDA + return ms_deform_attn_cuda_backward( + value, spatial_shapes, level_start_index, sampling_loc, attn_weight, grad_output, im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +} // namespace ape diff --git a/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_attn_cpu.cpp b/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_attn_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0b9db0b76923a8fbf398b862a26af07f95768f26 --- /dev/null +++ b/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_attn_cpu.cpp @@ -0,0 +1,42 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include + +#include + +namespace ape { + +at::Tensor +ms_deform_attn_cpu_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + +std::vector +ms_deform_attn_cpu_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + +} // namespace ape diff --git a/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_attn_cpu.h b/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_attn_cpu.h new file mode 100644 index 0000000000000000000000000000000000000000..47709dd42e5caca6ee80f4ef247a89793299f68e --- /dev/null +++ b/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_attn_cpu.h @@ -0,0 +1,35 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once +#include + +namespace ape { + +at::Tensor +ms_deform_attn_cpu_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step); + +std::vector +ms_deform_attn_cpu_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step); + +} // namespace ape diff --git a/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_attn_cuda.cu b/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_attn_cuda.cu new file mode 100644 index 0000000000000000000000000000000000000000..51bb2220d602c3b9528d53cd628016f1e5251d42 --- /dev/null +++ b/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_attn_cuda.cu @@ -0,0 +1,156 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#include +#include "ms_deform_im2col_cuda.cuh" + +#include +#include +#include +#include + +namespace ape { + +at::Tensor ms_deform_attn_cuda_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step) +{ + AT_ASSERTM(value.is_contiguous(), "value tensor has to be contiguous"); + AT_ASSERTM(spatial_shapes.is_contiguous(), "spatial_shapes tensor has to be contiguous"); + AT_ASSERTM(level_start_index.is_contiguous(), "level_start_index tensor has to be contiguous"); + AT_ASSERTM(sampling_loc.is_contiguous(), "sampling_loc tensor has to be contiguous"); + AT_ASSERTM(attn_weight.is_contiguous(), "attn_weight tensor has to be contiguous"); + + AT_ASSERTM(value.type().is_cuda(), "value must be a CUDA tensor"); + AT_ASSERTM(spatial_shapes.type().is_cuda(), "spatial_shapes must be a CUDA tensor"); + AT_ASSERTM(level_start_index.type().is_cuda(), "level_start_index must be a CUDA tensor"); + AT_ASSERTM(sampling_loc.type().is_cuda(), "sampling_loc must be a CUDA tensor"); + AT_ASSERTM(attn_weight.type().is_cuda(), "attn_weight must be a CUDA tensor"); + + const int batch = value.size(0); + const int spatial_size = value.size(1); + const int num_heads = value.size(2); + const int channels = value.size(3); + + const int num_levels = spatial_shapes.size(0); + + const int num_query = sampling_loc.size(1); + const int num_point = sampling_loc.size(4); + + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + auto output = at::zeros({batch, num_query, num_heads, channels}, value.options()); + + const int batch_n = im2col_step_; + auto output_n = output.view({batch/im2col_step_, batch_n, num_query, num_heads, channels}); + auto per_value_size = spatial_size * num_heads * channels; + auto per_sample_loc_size = num_query * num_heads * num_levels * num_point * 2; + auto per_attn_weight_size = num_query * num_heads * num_levels * num_point; + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto columns = output_n.select(0, n); + AT_DISPATCH_FLOATING_TYPES_AND_HALF(value.scalar_type(), "ms_deform_attn_forward_cuda", ([&] { + ms_deformable_im2col_cuda(at::cuda::getCurrentCUDAStream(), + value.data() + n * im2col_step_ * per_value_size, + spatial_shapes.data(), + level_start_index.data(), + sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, + attn_weight.data() + n * im2col_step_ * per_attn_weight_size, + batch_n, spatial_size, num_heads, channels, num_levels, num_query, num_point, + columns.data()); + + })); + } + + output = output.view({batch, num_query, num_heads*channels}); + + return output; +} + + +std::vector ms_deform_attn_cuda_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step) +{ + + AT_ASSERTM(value.is_contiguous(), "value tensor has to be contiguous"); + AT_ASSERTM(spatial_shapes.is_contiguous(), "spatial_shapes tensor has to be contiguous"); + AT_ASSERTM(level_start_index.is_contiguous(), "level_start_index tensor has to be contiguous"); + AT_ASSERTM(sampling_loc.is_contiguous(), "sampling_loc tensor has to be contiguous"); + AT_ASSERTM(attn_weight.is_contiguous(), "attn_weight tensor has to be contiguous"); + AT_ASSERTM(grad_output.is_contiguous(), "grad_output tensor has to be contiguous"); + + AT_ASSERTM(value.type().is_cuda(), "value must be a CUDA tensor"); + AT_ASSERTM(spatial_shapes.type().is_cuda(), "spatial_shapes must be a CUDA tensor"); + AT_ASSERTM(level_start_index.type().is_cuda(), "level_start_index must be a CUDA tensor"); + AT_ASSERTM(sampling_loc.type().is_cuda(), "sampling_loc must be a CUDA tensor"); + AT_ASSERTM(attn_weight.type().is_cuda(), "attn_weight must be a CUDA tensor"); + AT_ASSERTM(grad_output.type().is_cuda(), "grad_output must be a CUDA tensor"); + + const int batch = value.size(0); + const int spatial_size = value.size(1); + const int num_heads = value.size(2); + const int channels = value.size(3); + + const int num_levels = spatial_shapes.size(0); + + const int num_query = sampling_loc.size(1); + const int num_point = sampling_loc.size(4); + + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + auto grad_value = at::zeros_like(value); + auto grad_sampling_loc = at::zeros_like(sampling_loc); + auto grad_attn_weight = at::zeros_like(attn_weight); + + const int batch_n = im2col_step_; + auto per_value_size = spatial_size * num_heads * channels; + auto per_sample_loc_size = num_query * num_heads * num_levels * num_point * 2; + auto per_attn_weight_size = num_query * num_heads * num_levels * num_point; + auto grad_output_n = grad_output.view({batch/im2col_step_, batch_n, num_query, num_heads, channels}); + + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto grad_output_g = grad_output_n.select(0, n); + AT_DISPATCH_FLOATING_TYPES_AND_HALF(value.type(), "ms_deform_attn_backward_cuda", ([&] { + ms_deformable_col2im_cuda(at::cuda::getCurrentCUDAStream(), + grad_output_g.data(), + value.data() + n * im2col_step_ * per_value_size, + spatial_shapes.data(), + level_start_index.data(), + sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, + attn_weight.data() + n * im2col_step_ * per_attn_weight_size, + batch_n, spatial_size, num_heads, channels, num_levels, num_query, num_point, + grad_value.data() + n * im2col_step_ * per_value_size, + grad_sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, + grad_attn_weight.data() + n * im2col_step_ * per_attn_weight_size); + + })); + } + + return { + grad_value, grad_sampling_loc, grad_attn_weight + }; +} + +} // namespace ape diff --git a/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_attn_cuda.h b/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_attn_cuda.h new file mode 100644 index 0000000000000000000000000000000000000000..05a1eae47c6107927fa28ccd57293aad14a22bff --- /dev/null +++ b/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_attn_cuda.h @@ -0,0 +1,33 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +#pragma once +#include + +namespace ape { + +at::Tensor ms_deform_attn_cuda_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step); + +std::vector ms_deform_attn_cuda_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step); + +} // namespace ape \ No newline at end of file diff --git a/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_im2col_cuda.cuh b/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_im2col_cuda.cuh new file mode 100644 index 0000000000000000000000000000000000000000..6bc2acb7aea0eab2e9e91e769a16861e1652c284 --- /dev/null +++ b/approach/ovod/APE/ape/layers/csrc/MsDeformAttn/ms_deform_im2col_cuda.cuh @@ -0,0 +1,1327 @@ +/*! +************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************** +* Modified from DCN (https://github.com/msracver/Deformable-ConvNets) +* Copyright (c) 2018 Microsoft +************************************************************************** +*/ + +#include +#include +#include + +#include +#include + +#include + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ + i < (n); \ + i += blockDim.x * gridDim.x) + +const int CUDA_NUM_THREADS = 1024; +inline int GET_BLOCKS(const int N, const int num_threads) +{ + return (N + num_threads - 1) / num_threads; +} + + +template +__device__ scalar_t ms_deform_attn_im2col_bilinear(const scalar_t* &bottom_data, + const int &height, const int &width, const int &nheads, const int &channels, + const scalar_t &h, const scalar_t &w, const int &m, const int &c) +{ + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const scalar_t lh = h - h_low; + const scalar_t lw = w - w_low; + const scalar_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * channels + c; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + } + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + } + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + } + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + } + + const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + + +template +__device__ void ms_deform_attn_col2im_bilinear(const scalar_t* &bottom_data, + const int &height, const int &width, const int &nheads, const int &channels, + const scalar_t &h, const scalar_t &w, const int &m, const int &c, + const scalar_t &top_grad, + const scalar_t &attn_weight, + scalar_t* &grad_value, + scalar_t* grad_sampling_loc, + scalar_t* grad_attn_weight) +{ + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const scalar_t lh = h - h_low; + const scalar_t lw = w - w_low; + const scalar_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * channels + c; + + const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + const scalar_t top_grad_value = top_grad * attn_weight; + scalar_t grad_h_weight = 0, grad_w_weight = 0; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + grad_h_weight -= hw * v1; + grad_w_weight -= hh * v1; + atomicAdd(grad_value+ptr1, w1*top_grad_value); + } + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + grad_h_weight -= lw * v2; + grad_w_weight += hh * v2; + atomicAdd(grad_value+ptr2, w2*top_grad_value); + } + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + grad_h_weight += hw * v3; + grad_w_weight -= lh * v3; + atomicAdd(grad_value+ptr3, w3*top_grad_value); + } + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + grad_h_weight += lw * v4; + grad_w_weight += lh * v4; + atomicAdd(grad_value+ptr4, w4*top_grad_value); + } + + const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + *grad_attn_weight = top_grad * val; + *grad_sampling_loc = width * grad_w_weight * top_grad_value; + *(grad_sampling_loc + 1) = height * grad_h_weight * top_grad_value; +} + + +template +__device__ void ms_deform_attn_col2im_bilinear_gm(const scalar_t* &bottom_data, + const int &height, const int &width, const int &nheads, const int &channels, + const scalar_t &h, const scalar_t &w, const int &m, const int &c, + const scalar_t &top_grad, + const scalar_t &attn_weight, + scalar_t* &grad_value, + scalar_t* grad_sampling_loc, + scalar_t* grad_attn_weight) +{ + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const scalar_t lh = h - h_low; + const scalar_t lw = w - w_low; + const scalar_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * channels + c; + + const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + const scalar_t top_grad_value = top_grad * attn_weight; + scalar_t grad_h_weight = 0, grad_w_weight = 0; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + grad_h_weight -= hw * v1; + grad_w_weight -= hh * v1; + atomicAdd(grad_value+ptr1, w1*top_grad_value); + } + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + grad_h_weight -= lw * v2; + grad_w_weight += hh * v2; + atomicAdd(grad_value+ptr2, w2*top_grad_value); + } + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + grad_h_weight += hw * v3; + grad_w_weight -= lh * v3; + atomicAdd(grad_value+ptr3, w3*top_grad_value); + } + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + grad_h_weight += lw * v4; + grad_w_weight += lh * v4; + atomicAdd(grad_value+ptr4, w4*top_grad_value); + } + + const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + atomicAdd(grad_attn_weight, top_grad * val); + atomicAdd(grad_sampling_loc, width * grad_w_weight * top_grad_value); + atomicAdd(grad_sampling_loc + 1, height * grad_h_weight * top_grad_value); +} + + +template +__global__ void ms_deformable_im2col_gpu_kernel(const int n, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *data_col) +{ + CUDA_KERNEL_LOOP(index, n) + { + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + scalar_t *data_col_ptr = data_col + index; + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + scalar_t col = 0; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const scalar_t *data_value_ptr = data_value + (data_value_ptr_init_offset + level_start_id * qid_stride); + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + col += ms_deform_attn_im2col_bilinear(data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col) * weight; + } + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + } + } + *data_col_ptr = col; + } +} + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + __shared__ scalar_t cache_grad_sampling_loc[blockSize * 2]; + __shared__ scalar_t cache_grad_attn_weight[blockSize]; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + if (tid == 0) + { + scalar_t _grad_w=cache_grad_sampling_loc[0], _grad_h=cache_grad_sampling_loc[1], _grad_a=cache_grad_attn_weight[0]; + int sid=2; + for (unsigned int tid = 1; tid < blockSize; ++tid) + { + _grad_w += cache_grad_sampling_loc[sid]; + _grad_h += cache_grad_sampling_loc[sid + 1]; + _grad_a += cache_grad_attn_weight[tid]; + sid += 2; + } + + + *grad_sampling_loc = _grad_w; + *(grad_sampling_loc + 1) = _grad_h; + *grad_attn_weight = _grad_a; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + __shared__ scalar_t cache_grad_sampling_loc[blockSize * 2]; + __shared__ scalar_t cache_grad_attn_weight[blockSize]; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s=blockSize/2; s>0; s>>=1) + { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; + } + __syncthreads(); + } + + if (tid == 0) + { + *grad_sampling_loc = cache_grad_sampling_loc[0]; + *(grad_sampling_loc + 1) = cache_grad_sampling_loc[1]; + *grad_attn_weight = cache_grad_attn_weight[0]; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v1(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + extern __shared__ int _s[]; + scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; + scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + if (tid == 0) + { + scalar_t _grad_w=cache_grad_sampling_loc[0], _grad_h=cache_grad_sampling_loc[1], _grad_a=cache_grad_attn_weight[0]; + int sid=2; + for (unsigned int tid = 1; tid < blockDim.x; ++tid) + { + _grad_w += cache_grad_sampling_loc[sid]; + _grad_h += cache_grad_sampling_loc[sid + 1]; + _grad_a += cache_grad_attn_weight[tid]; + sid += 2; + } + + + *grad_sampling_loc = _grad_w; + *(grad_sampling_loc + 1) = _grad_h; + *grad_attn_weight = _grad_a; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v2(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + extern __shared__ int _s[]; + scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; + scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s=blockDim.x/2, spre=blockDim.x; s>0; s>>=1, spre>>=1) + { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; + if (tid + (s << 1) < spre) + { + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + (s << 1)]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2 + (s << 1)]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1 + (s << 1)]; + } + } + __syncthreads(); + } + + if (tid == 0) + { + *grad_sampling_loc = cache_grad_sampling_loc[0]; + *(grad_sampling_loc + 1) = cache_grad_sampling_loc[1]; + *grad_attn_weight = cache_grad_attn_weight[0]; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v2_multi_blocks(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + extern __shared__ int _s[]; + scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; + scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s=blockDim.x/2, spre=blockDim.x; s>0; s>>=1, spre>>=1) + { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; + if (tid + (s << 1) < spre) + { + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + (s << 1)]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2 + (s << 1)]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1 + (s << 1)]; + } + } + __syncthreads(); + } + + if (tid == 0) + { + atomicAdd(grad_sampling_loc, cache_grad_sampling_loc[0]); + atomicAdd(grad_sampling_loc + 1, cache_grad_sampling_loc[1]); + atomicAdd(grad_attn_weight, cache_grad_attn_weight[0]); + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +__global__ void ms_deformable_col2im_gpu_kernel_gm(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear_gm( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + grad_sampling_loc, grad_attn_weight); + } + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +void ms_deformable_im2col_cuda(cudaStream_t stream, + const scalar_t* data_value, + const int64_t* data_spatial_shapes, + const int64_t* data_level_start_index, + const scalar_t* data_sampling_loc, + const scalar_t* data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t* data_col) +{ + const int num_kernels = batch_size * num_query * num_heads * channels; + const int num_actual_kernels = batch_size * num_query * num_heads * channels; + const int num_threads = CUDA_NUM_THREADS; + ms_deformable_im2col_gpu_kernel + <<>>( + num_kernels, data_value, data_spatial_shapes, data_level_start_index, data_sampling_loc, data_attn_weight, + batch_size, spatial_size, num_heads, channels, num_levels, num_query, num_point, data_col); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in ms_deformable_im2col_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void ms_deformable_col2im_cuda(cudaStream_t stream, + const scalar_t* grad_col, + const scalar_t* data_value, + const int64_t * data_spatial_shapes, + const int64_t * data_level_start_index, + const scalar_t * data_sampling_loc, + const scalar_t * data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t* grad_value, + scalar_t* grad_sampling_loc, + scalar_t* grad_attn_weight) +{ + const int num_threads = (channels > CUDA_NUM_THREADS)?CUDA_NUM_THREADS:channels; + const int num_kernels = batch_size * num_query * num_heads * channels; + const int num_actual_kernels = batch_size * num_query * num_heads * channels; + if (channels > 1024) + { + if ((channels & 1023) == 0) + { + ms_deformable_col2im_gpu_kernel_shm_reduce_v2_multi_blocks + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + else + { + ms_deformable_col2im_gpu_kernel_gm + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + } + else{ + switch(channels) + { + case 1: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 2: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 4: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 8: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 16: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 32: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 64: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 128: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 256: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 512: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 1024: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + default: + if (channels < 64) + { + ms_deformable_col2im_gpu_kernel_shm_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + else + { + ms_deformable_col2im_gpu_kernel_shm_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + } + } + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in ms_deformable_col2im_cuda: %s\n", cudaGetErrorString(err)); + } + +} \ No newline at end of file diff --git a/approach/ovod/APE/ape/layers/csrc/cuda_version.cu b/approach/ovod/APE/ape/layers/csrc/cuda_version.cu new file mode 100644 index 0000000000000000000000000000000000000000..5bc79464b17cc95e0a47bd547758e9c17e0fcf1c --- /dev/null +++ b/approach/ovod/APE/ape/layers/csrc/cuda_version.cu @@ -0,0 +1,7 @@ +#include + +namespace ape { +int get_cudart_version() { + return CUDART_VERSION; +} +} // namespace ape diff --git a/approach/ovod/APE/ape/layers/csrc/vision.cpp b/approach/ovod/APE/ape/layers/csrc/vision.cpp new file mode 100644 index 0000000000000000000000000000000000000000..30f4ad57d610d14f4fc659cd743001aa3e8a101c --- /dev/null +++ b/approach/ovod/APE/ape/layers/csrc/vision.cpp @@ -0,0 +1,80 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved + +#include +#include "MsDeformAttn/ms_deform_attn.h" + +namespace ape { + +#if defined(WITH_CUDA) || defined(WITH_HIP) +extern int get_cudart_version(); +#endif + +std::string get_cuda_version() { +#if defined(WITH_CUDA) || defined(WITH_HIP) + std::ostringstream oss; + +#if defined(WITH_CUDA) + oss << "CUDA "; +#else + oss << "HIP "; +#endif + + // copied from + // https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/cuda/detail/CUDAHooks.cpp#L231 + auto printCudaStyleVersion = [&](int v) { + oss << (v / 1000) << "." << (v / 10 % 100); + if (v % 10 != 0) { + oss << "." << (v % 10); + } + }; + printCudaStyleVersion(get_cudart_version()); + return oss.str(); +#else // neither CUDA nor HIP + return std::string("not available"); +#endif +} + +bool has_cuda() { +#if defined(WITH_CUDA) + return true; +#else + return false; +#endif +} + +// similar to +// https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/Version.cpp +std::string get_compiler_version() { + std::ostringstream ss; +#if defined(__GNUC__) +#ifndef __clang__ + +#if ((__GNUC__ <= 4) && (__GNUC_MINOR__ <= 8)) +#error "GCC >= 4.9 is required!" +#endif + + { ss << "GCC " << __GNUC__ << "." << __GNUC_MINOR__; } +#endif +#endif + +#if defined(__clang_major__) + { + ss << "clang " << __clang_major__ << "." << __clang_minor__ << "." + << __clang_patchlevel__; + } +#endif + +#if defined(_MSC_VER) + { ss << "MSVC " << _MSC_FULL_VER; } +#endif + return ss.str(); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { +} + +TORCH_LIBRARY(ape, m) { + m.def("ms_deform_attn_forward", &ms_deform_attn_forward); + m.def("ms_deform_attn_backward", &ms_deform_attn_backward); +} +} // namespace ape diff --git a/approach/ovod/APE/ape/layers/fuse_helper.py b/approach/ovod/APE/ape/layers/fuse_helper.py new file mode 100644 index 0000000000000000000000000000000000000000..3fa800862c30209292d39b27efe2c7f87446122c --- /dev/null +++ b/approach/ovod/APE/ape/layers/fuse_helper.py @@ -0,0 +1,230 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from timm.models.layers import DropPath + + +class BiMultiHeadAttention(nn.Module): + def __init__( + self, + v_dim, + l_dim, + embed_dim, + num_heads, + dropout=0.1, + stable_softmax_2d=False, + clamp_min_for_underflow=True, + clamp_max_for_overflow=True, + use_attention_mask_v=False, + ): + super(BiMultiHeadAttention, self).__init__() + + self.embed_dim = embed_dim + self.num_heads = num_heads + self.head_dim = embed_dim // num_heads + self.v_dim = v_dim + self.l_dim = l_dim + + assert ( + self.head_dim * self.num_heads == self.embed_dim + ), f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`: {self.num_heads})." + self.scale = self.head_dim ** (-0.5) + self.dropout = dropout + + self.v_proj = nn.Linear(self.v_dim, self.embed_dim) + self.l_proj = nn.Linear(self.l_dim, self.embed_dim) + self.values_v_proj = nn.Linear(self.v_dim, self.embed_dim) + self.values_l_proj = nn.Linear(self.l_dim, self.embed_dim) + + self.out_v_proj = nn.Linear(self.embed_dim, self.v_dim) + self.out_l_proj = nn.Linear(self.embed_dim, self.l_dim) + + self.stable_softmax_2d = stable_softmax_2d + self.clamp_min_for_underflow = clamp_min_for_underflow + self.clamp_max_for_overflow = clamp_max_for_overflow + self.use_attention_mask_v = use_attention_mask_v + + self._reset_parameters() + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def _reset_parameters(self): + nn.init.xavier_uniform_(self.v_proj.weight) + self.v_proj.bias.data.fill_(0) + nn.init.xavier_uniform_(self.l_proj.weight) + self.l_proj.bias.data.fill_(0) + nn.init.xavier_uniform_(self.values_v_proj.weight) + self.values_v_proj.bias.data.fill_(0) + nn.init.xavier_uniform_(self.values_l_proj.weight) + self.values_l_proj.bias.data.fill_(0) + nn.init.xavier_uniform_(self.out_v_proj.weight) + self.out_v_proj.bias.data.fill_(0) + nn.init.xavier_uniform_(self.out_l_proj.weight) + self.out_l_proj.bias.data.fill_(0) + + def forward(self, v, l, attention_mask_v=None, attention_mask_l=None): + bsz, tgt_len, _ = v.size() + + query_states = self.v_proj(v) * self.scale + key_states = self._shape(self.l_proj(l), -1, bsz) + value_v_states = self._shape(self.values_v_proj(v), -1, bsz) + value_l_states = self._shape(self.values_l_proj(l), -1, bsz) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + key_states = key_states.view(*proj_shape) + value_v_states = value_v_states.view(*proj_shape) + value_l_states = value_l_states.view(*proj_shape) + + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is {attn_weights.size()}" + ) + + if self.stable_softmax_2d: + attn_weights = attn_weights - attn_weights.max() + + if self.clamp_min_for_underflow: + attn_weights = torch.clamp( + attn_weights, min=-50000 + ) # Do not increase -50000, data type half has quite limited range + if self.clamp_max_for_overflow: + attn_weights = torch.clamp( + attn_weights, max=50000 + ) # Do not increase 50000, data type half has quite limited range + + attn_weights_T = attn_weights.transpose(1, 2) + attn_weights_l = attn_weights_T - torch.max(attn_weights_T, dim=-1, keepdim=True)[0] + if self.clamp_min_for_underflow: + attn_weights_l = torch.clamp( + attn_weights_l, min=-50000 + ) # Do not increase -50000, data type half has quite limited range + if self.clamp_max_for_overflow: + attn_weights_l = torch.clamp( + attn_weights_l, max=50000 + ) # Do not increase 50000, data type half has quite limited range + + # mask vison for language + if attention_mask_v is not None and self.use_attention_mask_v: + attention_mask_v = ( + attention_mask_v[:, None, None, :].repeat(1, self.num_heads, 1, 1).flatten(0, 1) + ) + attn_weights_l.masked_fill_(attention_mask_v, float("-inf")) + + attn_weights_l = attn_weights_l.softmax(dim=-1) + + # mask language for vision + if attention_mask_l is not None: + # assert attention_mask_l.dim() == 2 # (bs, seq_len) + # attention_mask = attention_mask_l.unsqueeze(1).unsqueeze(1) # (bs, 1, 1, seq_len) + # attention_mask = attention_mask.expand(bsz, 1, tgt_len, src_len) + # attention_mask = attention_mask.masked_fill(attention_mask == 0, -9e15) + + # if attention_mask.size() != (bsz, 1, tgt_len, src_len): + # raise ValueError(f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}") + # attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + # attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + attention_mask_l = ( + attention_mask_l[:, None, None, :].repeat(1, self.num_heads, 1, 1).flatten(0, 1) + ) + attn_weights.masked_fill_(attention_mask_l, float("-inf")) + + attn_weights_v = attn_weights.softmax(dim=-1) + + attn_probs_v = F.dropout(attn_weights_v, p=self.dropout, training=self.training) + attn_probs_l = F.dropout(attn_weights_l, p=self.dropout, training=self.training) + + attn_output_v = torch.bmm(attn_probs_v, value_l_states) + attn_output_l = torch.bmm(attn_probs_l, value_v_states) + + if attn_output_v.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + raise ValueError( + f"`attn_output_v` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is {attn_output_v.size()}" + ) + + if attn_output_l.size() != (bsz * self.num_heads, src_len, self.head_dim): + raise ValueError( + f"`attn_output_l` should be of size {(bsz, self.num_heads, src_len, self.head_dim)}, but is {attn_output_l.size()}" + ) + + attn_output_v = attn_output_v.view(bsz, self.num_heads, tgt_len, self.head_dim) + attn_output_v = attn_output_v.transpose(1, 2) + attn_output_v = attn_output_v.reshape(bsz, tgt_len, self.embed_dim) + + attn_output_l = attn_output_l.view(bsz, self.num_heads, src_len, self.head_dim) + attn_output_l = attn_output_l.transpose(1, 2) + attn_output_l = attn_output_l.reshape(bsz, src_len, self.embed_dim) + + attn_output_v = self.out_v_proj(attn_output_v) + attn_output_l = self.out_l_proj(attn_output_l) + + return attn_output_v, attn_output_l + + def extra_repr(self): + lines = [ + f"stable_softmax_2d={self.stable_softmax_2d}", + f"clamp_min_for_underflow={self.clamp_min_for_underflow}", + f"clamp_max_for_overflow={self.clamp_max_for_overflow}", + f"use_attention_mask_v={self.use_attention_mask_v}", + ] + return "\n".join(lines) + + +class BiAttentionBlock(nn.Module): + def __init__( + self, + v_dim, + l_dim, + embed_dim, + num_heads, + dropout=0.1, + drop_path=0.0, + init_values=1e-4, + stable_softmax_2d=False, + clamp_min_for_underflow=True, + clamp_max_for_overflow=True, + use_attention_mask_v=False, + ): + """ + Inputs: + embed_dim - Dimensionality of input and attention feature vectors + num_heads - Number of heads to use in the Multi-Head Attention block + dropout - Amount of dropout to apply in the feed-forward network + """ + super(BiAttentionBlock, self).__init__() + + # pre layer norm + self.layer_norm_v = nn.LayerNorm(v_dim) + self.layer_norm_l = nn.LayerNorm(l_dim) + self.attn = BiMultiHeadAttention( + v_dim=v_dim, + l_dim=l_dim, + embed_dim=embed_dim, + num_heads=num_heads, + dropout=dropout, + stable_softmax_2d=stable_softmax_2d, + clamp_min_for_underflow=clamp_min_for_underflow, + clamp_max_for_overflow=clamp_max_for_overflow, + use_attention_mask_v=use_attention_mask_v, + ) + + # add layer scale for training stability + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.gamma_v = nn.Parameter(init_values * torch.ones((v_dim)), requires_grad=True) + self.gamma_l = nn.Parameter(init_values * torch.ones((l_dim)), requires_grad=True) + + def forward(self, v, l, attention_mask_v=None, attention_mask_l=None): + v = self.layer_norm_v(v.float()) + l = self.layer_norm_l(l.float()) + delta_v, delta_l = self.attn( + v, l, attention_mask_v=attention_mask_v, attention_mask_l=attention_mask_l + ) + # v, l = v + delta_v, l + delta_l + v = v + self.drop_path(self.gamma_v * delta_v) + l = l + self.drop_path(self.gamma_l * delta_l) + return v, l diff --git a/approach/ovod/APE/ape/layers/multi_scale_deform_attn.py b/approach/ovod/APE/ape/layers/multi_scale_deform_attn.py new file mode 100644 index 0000000000000000000000000000000000000000..4e12e8f76e093c745c3dbd6fa612ba287e55cdf1 --- /dev/null +++ b/approach/ovod/APE/ape/layers/multi_scale_deform_attn.py @@ -0,0 +1,423 @@ +# coding=utf-8 +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from: +# https://github.com/fundamentalvision/Deformable-DETR/blob/main/models/ops/functions/ms_deform_attn_func.py +# https://github.com/fundamentalvision/Deformable-DETR/blob/main/models/ops/modules/ms_deform_attn.py +# https://github.com/open-mmlab/mmcv/blob/master/mmcv/ops/multi_scale_deform_attn.py +# ------------------------------------------------------------------------------------------------ + +import math +import warnings +from typing import Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.init import constant_, xavier_uniform_ + + +# helpers +def _is_power_of_2(n): + if (not isinstance(n, int)) or (n < 0): + raise ValueError("invalid input for _is_power_of_2: {} (type: {})".format(n, type(n))) + return (n & (n - 1) == 0) and n != 0 + + +class MultiScaleDeformableAttnFunction(Function): + @staticmethod + def forward( + ctx, + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + im2col_step, + ): + ctx.im2col_step = im2col_step + output = torch.ops.ape.ms_deform_attn_forward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + ctx.im2col_step, + ) + ctx.save_for_backward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + ) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + ( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + ) = ctx.saved_tensors + grad_value, grad_sampling_loc, grad_attn_weight = torch.ops.ape.ms_deform_attn_backward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + grad_output, + ctx.im2col_step, + ) + + return grad_value, None, None, grad_sampling_loc, grad_attn_weight, None + + +def multi_scale_deformable_attn_pytorch( + value: torch.Tensor, + value_spatial_shapes: torch.Tensor, + sampling_locations: torch.Tensor, + attention_weights: torch.Tensor, +) -> torch.Tensor: + + bs, _, num_heads, embed_dims = value.shape + _, num_queries, num_heads, num_levels, num_points, _ = sampling_locations.shape + value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1) + sampling_grids = 2 * sampling_locations - 1 + sampling_value_list = [] + for level, (H_, W_) in enumerate(value_spatial_shapes): + # bs, H_*W_, num_heads, embed_dims -> + # bs, H_*W_, num_heads*embed_dims -> + # bs, num_heads*embed_dims, H_*W_ -> + # bs*num_heads, embed_dims, H_, W_ + value_l_ = ( + value_list[level].flatten(2).transpose(1, 2).reshape(bs * num_heads, embed_dims, H_, W_) + ) + # bs, num_queries, num_heads, num_points, 2 -> + # bs, num_heads, num_queries, num_points, 2 -> + # bs*num_heads, num_queries, num_points, 2 + sampling_grid_l_ = sampling_grids[:, :, :, level].transpose(1, 2).flatten(0, 1) + # bs*num_heads, embed_dims, num_queries, num_points + sampling_value_l_ = F.grid_sample( + value_l_, sampling_grid_l_, mode="bilinear", padding_mode="zeros", align_corners=False + ) + sampling_value_list.append(sampling_value_l_) + # (bs, num_queries, num_heads, num_levels, num_points) -> + # (bs, num_heads, num_queries, num_levels, num_points) -> + # (bs, num_heads, 1, num_queries, num_levels*num_points) + attention_weights = attention_weights.transpose(1, 2).reshape( + bs * num_heads, 1, num_queries, num_levels * num_points + ) + output = ( + (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights) + .sum(-1) + .view(bs, num_heads * embed_dims, num_queries) + ) + return output.transpose(1, 2).contiguous() + + +class MultiScaleDeformableAttention(nn.Module): + """Multi-Scale Deformable Attention Module used in Deformable-DETR + + `Deformable DETR: Deformable Transformers for End-to-End Object Detection. + `_. + + Args: + embed_dim (int): The embedding dimension of Attention. Default: 256. + num_heads (int): The number of attention heads. Default: 8. + num_levels (int): The number of feature map used in Attention. Default: 4. + num_points (int): The number of sampling points for each query + in each head. Default: 4. + img2col_steps (int): The step used in image_to_column. Defualt: 64. + dropout (float): Dropout layer used in output. Default: 0.1. + batch_first (bool): if ``True``, then the input and output tensor will be + provided as `(bs, n, embed_dim)`. Default: False. `(n, bs, embed_dim)` + """ + + def __init__( + self, + embed_dim: int = 256, + num_heads: int = 8, + num_levels: int = 4, + num_points: int = 4, + img2col_step: int = 64, + dropout: float = 0.1, + batch_first: bool = False, + pytorch_attn: bool = False, + ): + super().__init__() + if embed_dim % num_heads != 0: + raise ValueError( + "embed_dim must be divisible by num_heads, but got {} and {}".format( + embed_dim, num_heads + ) + ) + head_dim = embed_dim // num_heads + + self.dropout = nn.Dropout(dropout) + self.batch_first = batch_first + + if not _is_power_of_2(head_dim): + warnings.warn( + """ + You'd better set d_model in MSDeformAttn to make sure that + each dim of the attention head a power of 2, which is more efficient. + """ + ) + + self.im2col_step = img2col_step + self.embed_dim = embed_dim + self.num_heads = num_heads + self.num_levels = num_levels + self.num_points = num_points + self.sampling_offsets = nn.Linear(embed_dim, num_heads * num_levels * num_points * 2) + self.attention_weights = nn.Linear(embed_dim, num_heads * num_levels * num_points) + self.value_proj = nn.Linear(embed_dim, embed_dim) + self.output_proj = nn.Linear(embed_dim, embed_dim) + + self.init_weights() + + self.pytorch_attn = pytorch_attn + + def init_weights(self): + """ + Default initialization for Parameters of Module. + """ + constant_(self.sampling_offsets.weight.data, 0.0) + thetas = torch.arange(self.num_heads, dtype=torch.float32) * ( + 2.0 * math.pi / self.num_heads + ) + grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) + grid_init = ( + (grid_init / grid_init.abs().max(-1, keepdim=True)[0]) + .view(self.num_heads, 1, 1, 2) + .repeat(1, self.num_levels, self.num_points, 1) + ) + for i in range(self.num_points): + grid_init[:, :, i, :] *= i + 1 + with torch.no_grad(): + self.sampling_offsets.bias = nn.Parameter(grid_init.view(-1)) + constant_(self.attention_weights.weight.data, 0.0) + constant_(self.attention_weights.bias.data, 0.0) + xavier_uniform_(self.value_proj.weight.data) + constant_(self.value_proj.bias.data, 0.0) + xavier_uniform_(self.output_proj.weight.data) + constant_(self.output_proj.bias.data, 0.0) + + def forward( + self, + query: torch.Tensor, + key: Optional[torch.Tensor] = None, + value: Optional[torch.Tensor] = None, + identity: Optional[torch.Tensor] = None, + query_pos: Optional[torch.Tensor] = None, + key_padding_mask: Optional[torch.Tensor] = None, + reference_points: Optional[torch.Tensor] = None, + spatial_shapes: Optional[torch.Tensor] = None, + level_start_index: Optional[torch.Tensor] = None, + **kwargs + ) -> torch.Tensor: + + """Forward Function of MultiScaleDeformableAttention + + Args: + query (torch.Tensor): Query embeddings with shape + `(num_query, bs, embed_dim)` + key (torch.Tensor): Key embeddings with shape + `(num_key, bs, embed_dim)` + value (torch.Tensor): Value embeddings with shape + `(num_key, bs, embed_dim)` + identity (torch.Tensor): The tensor used for addition, with the + same shape as `query`. Default: None. If None, `query` will be + used. + query_pos (torch.Tensor): The position embedding for `query`. Default: None. + key_padding_mask (torch.Tensor): ByteTensor for `query`, with shape `(bs, num_key)`, + indicating which elements within `key` to be ignored in attention. + reference_points (torch.Tensor): The normalized reference points + with shape `(bs, num_query, num_levels, 2)`, + all elements is range in [0, 1], top-left (0, 0), + bottom-right (1, 1), including padding are. + or `(N, Length_{query}, num_levels, 4)`, add additional + two dimensions `(h, w)` to form reference boxes. + spatial_shapes (torch.Tensor): Spatial shape of features in different levels. + With shape `(num_levels, 2)`, last dimension represents `(h, w)`. + level_start_index (torch.Tensor): The start index of each level. A tensor with + shape `(num_levels, )` which can be represented as + `[0, h_0 * w_0, h_0 * w_0 + h_1 * w_1, ...]`. + + Returns: + torch.Tensor: forward results with shape `(num_query, bs, embed_dim)` + """ + + if value is None: + value = query + + if identity is None: + identity = query + if query_pos is not None: + query = query + query_pos + + if not self.batch_first: + # change to (bs, num_query ,embed_dims) + query = query.permute(1, 0, 2) + value = value.permute(1, 0, 2) + + bs, num_query, _ = query.shape + bs, num_value, _ = value.shape + + assert (spatial_shapes[:, 0] * spatial_shapes[:, 1]).sum() == num_value + + value = self.value_proj(value) + if key_padding_mask is not None: + value = value.masked_fill(key_padding_mask[..., None], float(0)) + value = value.view(bs, num_value, self.num_heads, -1) + sampling_offsets = self.sampling_offsets(query).view( + bs, num_query, self.num_heads, self.num_levels, self.num_points, 2 + ) + attention_weights = self.attention_weights(query).view( + bs, num_query, self.num_heads, self.num_levels * self.num_points + ) + attention_weights = attention_weights.softmax(-1) + attention_weights = attention_weights.view( + bs, + num_query, + self.num_heads, + self.num_levels, + self.num_points, + ) + + # bs, num_query, num_heads, num_levels, num_points, 2 + if reference_points.shape[-1] == 2: + offset_normalizer = torch.stack([spatial_shapes[..., 1], spatial_shapes[..., 0]], -1) + sampling_locations = ( + reference_points[:, :, None, :, None, :] + + sampling_offsets / offset_normalizer[None, None, None, :, None, :] + ) + elif reference_points.shape[-1] == 4: + sampling_locations = ( + reference_points[:, :, None, :, None, :2] + + sampling_offsets + / self.num_points + * reference_points[:, :, None, :, None, 2:] + * 0.5 + ) + else: + raise ValueError( + "Last dim of reference_points must be 2 or 4, but get {} instead.".format( + reference_points.shape[-1] + ) + ) + + # the original impl for fp32 training + if torch.cuda.is_available() and value.is_cuda and not self.pytorch_attn: + if torch.jit.is_scripting() or torch.jit.is_tracing(): + output = torch.ops.ape.ms_deform_attn_forward( + # value.to(torch.float32), + value, + spatial_shapes, + level_start_index, + # sampling_locations.to(torch.float32), + sampling_locations.to(value.dtype), + # attention_weights.to(torch.float32), + attention_weights.to(value.dtype), + self.im2col_step, + ) + else: + output = MultiScaleDeformableAttnFunction.apply( + # value.to(torch.float32), + value, + spatial_shapes, + level_start_index, + # sampling_locations.to(torch.float32), + sampling_locations.to(value.dtype), + # attention_weights.to(torch.float32), + attention_weights.to(value.dtype), + self.im2col_step, + ) + else: + output = multi_scale_deformable_attn_pytorch( + value, spatial_shapes, sampling_locations, attention_weights + ) + + if value.dtype == torch.float16: + output = output.to(torch.float16) + + output = self.output_proj(output) + + if not self.batch_first: + output = output.permute(1, 0, 2) + + return self.dropout(output) + identity + + +def create_dummy_class(klass, dependency, message=""): + """ + When a dependency of a class is not available, create a dummy class which throws ImportError + when used. + + Args: + klass (str): name of the class. + dependency (str): name of the dependency. + message: extra message to print + Returns: + class: a class object + """ + err = "Cannot import '{}', therefore '{}' is not available.".format(dependency, klass) + if message: + err = err + " " + message + + class _DummyMetaClass(type): + # throw error on class attribute access + def __getattr__(_, __): # noqa: B902 + raise ImportError(err) + + class _Dummy(object, metaclass=_DummyMetaClass): + # throw error on constructor + def __init__(self, *args, **kwargs): + raise ImportError(err) + + return _Dummy + + +def create_dummy_func(func, dependency, message=""): + """ + When a dependency of a function is not available, create a dummy function which throws + ImportError when used. + + Args: + func (str): name of the function. + dependency (str or list[str]): name(s) of the dependency. + message: extra message to print + Returns: + function: a function object + """ + err = "Cannot import '{}', therefore '{}' is not available.".format(dependency, func) + if message: + err = err + " " + message + + if isinstance(dependency, (list, tuple)): + dependency = ",".join(dependency) + + def _dummy(*args, **kwargs): + raise ImportError(err) + + return _dummy + + +try: + from ape import _C +except ImportError: + # TODO: register ops natively so there is no need to import _C. + _msg = "ape is not compiled successfully, please build following the instructions!" + _args = ("ape._C", _msg) + MultiScaleDeformableAttention = create_dummy_class( # noqa + "MultiScaleDeformableAttention", *_args + ) diff --git a/approach/ovod/APE/ape/layers/vision_language_align.py b/approach/ovod/APE/ape/layers/vision_language_align.py new file mode 100644 index 0000000000000000000000000000000000000000..f0511c3cb6e008e9e5c700e70f84986ade95eff2 --- /dev/null +++ b/approach/ovod/APE/ape/layers/vision_language_align.py @@ -0,0 +1,61 @@ +import math + +import torch +import torch.nn.functional as F +from torch import nn + + +class VisionLanguageAlign(nn.Module): + def __init__( + self, embed_dim, embed_dim_language, prior_prob=0.01, log_scale=0.0, clamp_dot_product=True + ): + super().__init__() + # initialize the bias for focal loss + bias_value = -math.log((1 - prior_prob) / prior_prob) + + # dot product soft token head + self.dot_product_projection_image = nn.Identity() + self.dot_product_projection_text = nn.Linear( + embed_dim_language, embed_dim, bias=True + ) # 768 -> 256 + self.log_scale = nn.Parameter(torch.Tensor([log_scale]), requires_grad=True) + self.bias_lang = nn.Parameter(torch.zeros(embed_dim_language), requires_grad=True) # (768，) + self.bias0 = nn.Parameter(torch.Tensor([bias_value]), requires_grad=True) # size (1,) + + self.clamp_dot_product = clamp_dot_product + + def forward(self, x, embedding): + """ + x: visual features (bs, num_query, 256) + embedding: language features (bs, L, 768) + """ + embedding = embedding.to(x.dtype) + + # norm + embedding = F.normalize(embedding, p=2, dim=-1) # (bs, L, 768) L is maximum sentence length + dot_product_proj_tokens = self.dot_product_projection_text(embedding / 2.0) # 768 -> 256 + dot_product_proj_tokens_bias = ( + torch.matmul(embedding, self.bias_lang) + self.bias0 + ) # (bs, L, 768) x (768, ) + (1, ) -> (bs, L) + + dot_product_proj_queries = self.dot_product_projection_image(x) # (bs, num_query, 256) + A = dot_product_proj_queries.shape[1] # num_query + bias = dot_product_proj_tokens_bias.unsqueeze(1).repeat(1, A, 1) # (bs, num_query, L) + + dot_product_logit = ( + torch.matmul(dot_product_proj_queries, dot_product_proj_tokens.transpose(-1, -2)) + / self.log_scale.exp() + ) + bias # (bs, num_query, 256) x (bs, 256, L) -> (bs, num_query, L) + if self.clamp_dot_product: + dot_product_logit = torch.clamp(dot_product_logit, max=50000) + dot_product_logit = torch.clamp(dot_product_logit, min=-50000) + return dot_product_logit + + +class StillClassifier(nn.Module): + def __init__(self, hidden_dim): + super().__init__() + self.body = nn.Linear(hidden_dim, 1) + + def forward(self, x, lang_feat=None): + return self.body(x) diff --git a/approach/ovod/APE/ape/layers/vision_language_fusion.py b/approach/ovod/APE/ape/layers/vision_language_fusion.py new file mode 100644 index 0000000000000000000000000000000000000000..bbb46aaac2dd75ed2b6265ad8723c592320c7916 --- /dev/null +++ b/approach/ovod/APE/ape/layers/vision_language_fusion.py @@ -0,0 +1,53 @@ +import torch +import torch.utils.checkpoint as checkpoint + +from .fuse_helper import BiAttentionBlock + + +class VisionLanguageFusion(torch.nn.Module): + """ + Early Fusion Module + """ + + def __init__( + self, + v_dim, + l_dim, + embed_dim, + num_heads, + dropout=0.1, + drop_path=0.0, + init_values=1e-4, + stable_softmax_2d=False, + clamp_min_for_underflow=True, + clamp_max_for_overflow=True, + use_checkpoint=False, + use_attention_mask_v=False, + ): + super(VisionLanguageFusion, self).__init__() + self.use_checkpoint = use_checkpoint + + # early fusion module + # bi-direction (text->image, image->text) + self.b_attn = BiAttentionBlock( + v_dim=v_dim, + l_dim=l_dim, + embed_dim=embed_dim, + num_heads=num_heads, + dropout=dropout, + drop_path=drop_path, + init_values=init_values, + stable_softmax_2d=stable_softmax_2d, + clamp_min_for_underflow=clamp_min_for_underflow, + clamp_max_for_overflow=clamp_max_for_overflow, + use_attention_mask_v=use_attention_mask_v, + ) + + def forward(self, v, l, attention_mask_v=None, attention_mask_l=None): + if self.use_checkpoint and self.training: + return checkpoint.checkpoint(self.b_attn, v, l, attention_mask_v, attention_mask_l) + else: + return self.b_attn(v, l, attention_mask_v, attention_mask_l) + + def extra_repr(self): + return f"use_checkpoint={self.use_checkpoint}" diff --git a/approach/ovod/APE/ape/layers/zero_shot_fc.py b/approach/ovod/APE/ape/layers/zero_shot_fc.py new file mode 100644 index 0000000000000000000000000000000000000000..e19622ad9bdd86b6d3b0c5505ae5a704ccda4a8b --- /dev/null +++ b/approach/ovod/APE/ape/layers/zero_shot_fc.py @@ -0,0 +1,162 @@ +import logging +import math + +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F + +# from sota.modeling.text import build_clip_text_encoder, get_clip_embeddings +# from ..modeling.text import build_clip_text_encoder, get_clip_embeddings + +logger = logging.getLogger(__name__) + + +class ZeroShotFC(nn.Module): + def __init__( + self, + input_size, + *, + num_classes: int, + zs_weight_path: str, + zs_weight_dim: int = 512, + use_bias: float = 0.0, + norm_weight: bool = True, + norm_temperature: float = 50.0, + use_project: bool = True, + use_sigmoid_ce: bool, + prior_prob: float = 0.01, + zs_vocabulary: str = "", + text_model: str = "", + ): + super().__init__() + + # assert use_sigmoid_ce + # assert cls_agnostic_bbox_reg + + self.norm_weight = norm_weight + self.norm_temperature = norm_temperature + self.use_project = use_project + self.zs_weight_dim = zs_weight_dim + + self.use_bias = use_bias < 0 + if self.use_bias: + self.cls_bias = nn.Parameter(torch.ones(1) * use_bias, requires_grad=True) + + if self.use_project: + self.linear = nn.Linear(input_size, zs_weight_dim) + + if use_sigmoid_ce: + bias_value = -math.log((1 - prior_prob) / prior_prob) + else: + bias_value = 0 + torch.nn.init.constant_(self.linear.bias, bias_value) + torch.nn.init.normal_(self.linear.weight, std=0.01) + + if len(zs_vocabulary) > 0: + from sota.modeling.text import get_clip_embeddings + + logger.info("Generating weight for " + zs_vocabulary) + zs_vocabulary = zs_vocabulary.split(",") + num_classes = len(zs_vocabulary) + zs_weight = get_clip_embeddings(text_model, zs_vocabulary) + zs_weight = zs_weight.permute(1, 0).contiguous() + elif zs_weight_path == "rand": + zs_weight = torch.randn((zs_weight_dim, num_classes)) + nn.init.normal_(zs_weight, std=0.01) + elif zs_weight_path == "zeros": + zs_weight = torch.zeros((zs_weight_dim, num_classes)) + elif zs_weight_path == "online": + from sota.modeling.text import build_clip_text_encoder + + zs_weight = torch.zeros((zs_weight_dim, num_classes)) + self.text_encoder = build_clip_text_encoder(text_model, pretrain=True) + self.text_encoder.eval() + else: + logger.info("Loading " + zs_weight_path) + zs_weight = ( + torch.tensor(np.load(zs_weight_path), dtype=torch.float32) + .permute(1, 0) + .contiguous() + ) + logger.info(f"Loaded zs_weight {zs_weight.size()}") + + zs_weight = torch.cat([zs_weight, zs_weight.new_zeros((self.zs_weight_dim, 1))], dim=1) + logger.info(f"Cated zs_weight {zs_weight.size()}") + + if self.norm_weight: + zs_weight = F.normalize(zs_weight, p=2, dim=0) + + if zs_weight_path == "rand": + self.zs_weight = nn.Parameter(zs_weight, requires_grad=True) + else: + self.register_buffer("zs_weight", zs_weight) + + assert ( + self.zs_weight.shape[1] == num_classes + 1 + ), f"zs_weight={self.zs_weight.shape} v.s. num_classes={num_classes}" + + def forward(self, x, classifier=None): + """ + Inputs: + x: B x D or B x N x D + classifier: C x D + """ + x_shape = x.shape + if len(x_shape) == 3: + x = x.reshape(x_shape[0] * x_shape[1], x_shape[2]) + assert x.dim() == 2 + + if self.use_project: + x = self.linear(x) + if classifier is not None: + if isinstance(classifier, str): + from sota.modeling.text import get_clip_embeddings + + zs_weight = get_clip_embeddings( + self.text_encoder, classifier, prompt="", device=x.device + ) + else: + zs_weight = classifier + zs_weight = zs_weight.permute(1, 0).contiguous() + zs_weight = torch.cat([zs_weight, zs_weight.new_zeros((self.zs_weight_dim, 1))], dim=1) + if self.norm_weight: + zs_weight = F.normalize(zs_weight, p=2, dim=0) + else: + zs_weight = self.zs_weight + if self.norm_weight: + x = self.norm_temperature * F.normalize(x, p=2, dim=1) + x = torch.mm(x, zs_weight) + if self.use_bias: + x = x + self.cls_bias + + if len(x_shape) == 3: + x = x.reshape(x_shape[:2] + zs_weight.shape[1:]) + return x + + def set_predictor(self, param_or_path): + if type(param_or_path) == str: + logger.info("Loading " + param_or_path) + zs_weight = ( + torch.tensor(np.load(param_or_path), dtype=torch.float32).permute(1, 0).contiguous() + ) + else: + zs_weight = param_or_path.permute(1, 0).contiguous() + logger.info(f"Loaded zs_weight {zs_weight.size()}") + + zs_weight = torch.cat([zs_weight, zs_weight.new_zeros((self.zs_weight_dim, 1))], dim=1) + logger.info(f"Cated zs_weight {zs_weight.size()}") + + if self.norm_weight: + zs_weight = F.normalize(zs_weight, p=2, dim=0) + + zs_weight = zs_weight.to(self.zs_weight.device) + self.zs_weight = zs_weight + + def extra_repr(self): + extra_repr = "" + valtype = (int, float, bool, str, dict, list) + for attribute, value in self.__dict__.items(): + if type(value) in valtype: + extra_repr += "{}={}, ".format(attribute, value) + return extra_repr[:-2] diff --git a/approach/ovod/APE/ape/model_zoo/__init__.py b/approach/ovod/APE/ape/model_zoo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1144331fee61ad472dd46fb785a275e51f9c99f7 --- /dev/null +++ b/approach/ovod/APE/ape/model_zoo/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +""" +Model Zoo API for Detectron2: a collection of functions to create common model architectures +listed in `MODEL_ZOO.md `_, +and optionally load their pre-trained weights. +""" + +from .model_zoo import get, get_checkpoint_url, get_config, get_config_file + +__all__ = ["get_checkpoint_url", "get", "get_config_file", "get_config"] diff --git a/approach/ovod/APE/ape/model_zoo/model_zoo.py b/approach/ovod/APE/ape/model_zoo/model_zoo.py new file mode 100644 index 0000000000000000000000000000000000000000..ca88803253d329c5ab62d03e4b92d62325ff1690 --- /dev/null +++ b/approach/ovod/APE/ape/model_zoo/model_zoo.py @@ -0,0 +1,214 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import os +from typing import Optional + +import pkg_resources +import torch + +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import CfgNode, LazyConfig, get_cfg, instantiate +from detectron2.modeling import build_model + + +class _ModelZooUrls(object): + """ + Mapping from names to officially released Detectron2 pre-trained models. + """ + + S3_PREFIX = "https://dl.fbaipublicfiles.com/detectron2/" + + # format: {config_path.yaml} -> model_id/model_final_{commit}.pkl + CONFIG_PATH_TO_URL_SUFFIX = { + # COCO Detection with Faster R-CNN + "COCO-Detection/faster_rcnn_R_50_C4_1x": "137257644/model_final_721ade.pkl", + "COCO-Detection/faster_rcnn_R_50_DC5_1x": "137847829/model_final_51d356.pkl", + "COCO-Detection/faster_rcnn_R_50_FPN_1x": "137257794/model_final_b275ba.pkl", + "COCO-Detection/faster_rcnn_R_50_C4_3x": "137849393/model_final_f97cb7.pkl", + "COCO-Detection/faster_rcnn_R_50_DC5_3x": "137849425/model_final_68d202.pkl", + "COCO-Detection/faster_rcnn_R_50_FPN_3x": "137849458/model_final_280758.pkl", + "COCO-Detection/faster_rcnn_R_101_C4_3x": "138204752/model_final_298dad.pkl", + "COCO-Detection/faster_rcnn_R_101_DC5_3x": "138204841/model_final_3e0943.pkl", + "COCO-Detection/faster_rcnn_R_101_FPN_3x": "137851257/model_final_f6e8b1.pkl", + "COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x": "139173657/model_final_68b088.pkl", + # COCO Detection with RetinaNet + "COCO-Detection/retinanet_R_50_FPN_1x": "190397773/model_final_bfca0b.pkl", + "COCO-Detection/retinanet_R_50_FPN_3x": "190397829/model_final_5bd44e.pkl", + "COCO-Detection/retinanet_R_101_FPN_3x": "190397697/model_final_971ab9.pkl", + # COCO Detection with RPN and Fast R-CNN + "COCO-Detection/rpn_R_50_C4_1x": "137258005/model_final_450694.pkl", + "COCO-Detection/rpn_R_50_FPN_1x": "137258492/model_final_02ce48.pkl", + "COCO-Detection/fast_rcnn_R_50_FPN_1x": "137635226/model_final_e5f7ce.pkl", + # COCO Instance Segmentation Baselines with Mask R-CNN + "COCO-InstanceSegmentation/mask_rcnn_R_50_C4_1x": "137259246/model_final_9243eb.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_1x": "137260150/model_final_4f86c3.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x": "137260431/model_final_a54504.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x": "137849525/model_final_4ce675.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_3x": "137849551/model_final_84107b.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x": "137849600/model_final_f10217.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_101_C4_3x": "138363239/model_final_a2914c.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_101_DC5_3x": "138363294/model_final_0464b7.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x": "138205316/model_final_a3ec72.pkl", + "COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x": "139653917/model_final_2d9806.pkl", # noqa + # New baselines using Large-Scale Jitter and Longer Training Schedule + "new_baselines/mask_rcnn_R_50_FPN_100ep_LSJ": "42047764/model_final_bb69de.pkl", + "new_baselines/mask_rcnn_R_50_FPN_200ep_LSJ": "42047638/model_final_89a8d3.pkl", + "new_baselines/mask_rcnn_R_50_FPN_400ep_LSJ": "42019571/model_final_14d201.pkl", + "new_baselines/mask_rcnn_R_101_FPN_100ep_LSJ": "42025812/model_final_4f7b58.pkl", + "new_baselines/mask_rcnn_R_101_FPN_200ep_LSJ": "42131867/model_final_0bb7ae.pkl", + "new_baselines/mask_rcnn_R_101_FPN_400ep_LSJ": "42073830/model_final_f96b26.pkl", + "new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_100ep_LSJ": "42047771/model_final_b7fbab.pkl", # noqa + "new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_200ep_LSJ": "42132721/model_final_5d87c1.pkl", # noqa + "new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_400ep_LSJ": "42025447/model_final_f1362d.pkl", # noqa + "new_baselines/mask_rcnn_regnety_4gf_dds_FPN_100ep_LSJ": "42047784/model_final_6ba57e.pkl", # noqa + "new_baselines/mask_rcnn_regnety_4gf_dds_FPN_200ep_LSJ": "42047642/model_final_27b9c1.pkl", # noqa + "new_baselines/mask_rcnn_regnety_4gf_dds_FPN_400ep_LSJ": "42045954/model_final_ef3a80.pkl", # noqa + # COCO Person Keypoint Detection Baselines with Keypoint R-CNN + "COCO-Keypoints/keypoint_rcnn_R_50_FPN_1x": "137261548/model_final_04e291.pkl", + "COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x": "137849621/model_final_a6e10b.pkl", + "COCO-Keypoints/keypoint_rcnn_R_101_FPN_3x": "138363331/model_final_997cc7.pkl", + "COCO-Keypoints/keypoint_rcnn_X_101_32x8d_FPN_3x": "139686956/model_final_5ad38f.pkl", + # COCO Panoptic Segmentation Baselines with Panoptic FPN + "COCO-PanopticSegmentation/panoptic_fpn_R_50_1x": "139514544/model_final_dbfeb4.pkl", + "COCO-PanopticSegmentation/panoptic_fpn_R_50_3x": "139514569/model_final_c10459.pkl", + "COCO-PanopticSegmentation/panoptic_fpn_R_101_3x": "139514519/model_final_cafdb1.pkl", + # LVIS Instance Segmentation Baselines with Mask R-CNN + "LVISv0.5-InstanceSegmentation/mask_rcnn_R_50_FPN_1x": "144219072/model_final_571f7c.pkl", # noqa + "LVISv0.5-InstanceSegmentation/mask_rcnn_R_101_FPN_1x": "144219035/model_final_824ab5.pkl", # noqa + "LVISv0.5-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_1x": "144219108/model_final_5e3439.pkl", # noqa + # Cityscapes & Pascal VOC Baselines + "Cityscapes/mask_rcnn_R_50_FPN": "142423278/model_final_af9cf5.pkl", + "PascalVOC-Detection/faster_rcnn_R_50_C4": "142202221/model_final_b1acc2.pkl", + # Other Settings + "Misc/mask_rcnn_R_50_FPN_1x_dconv_c3-c5": "138602867/model_final_65c703.pkl", + "Misc/mask_rcnn_R_50_FPN_3x_dconv_c3-c5": "144998336/model_final_821d0b.pkl", + "Misc/cascade_mask_rcnn_R_50_FPN_1x": "138602847/model_final_e9d89b.pkl", + "Misc/cascade_mask_rcnn_R_50_FPN_3x": "144998488/model_final_480dd8.pkl", + "Misc/mask_rcnn_R_50_FPN_3x_syncbn": "169527823/model_final_3b3c51.pkl", + "Misc/mask_rcnn_R_50_FPN_3x_gn": "138602888/model_final_dc5d9e.pkl", + "Misc/scratch_mask_rcnn_R_50_FPN_3x_gn": "138602908/model_final_01ca85.pkl", + "Misc/scratch_mask_rcnn_R_50_FPN_9x_gn": "183808979/model_final_da7b4c.pkl", + "Misc/scratch_mask_rcnn_R_50_FPN_9x_syncbn": "184226666/model_final_5ce33e.pkl", + "Misc/panoptic_fpn_R_101_dconv_cascade_gn_3x": "139797668/model_final_be35db.pkl", + "Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv": "18131413/model_0039999_e76410.pkl", # noqa + # D1 Comparisons + "Detectron1-Comparisons/faster_rcnn_R_50_FPN_noaug_1x": "137781054/model_final_7ab50c.pkl", # noqa + "Detectron1-Comparisons/mask_rcnn_R_50_FPN_noaug_1x": "137781281/model_final_62ca52.pkl", # noqa + "Detectron1-Comparisons/keypoint_rcnn_R_50_FPN_1x": "137781195/model_final_cce136.pkl", + } + + @staticmethod + def query(config_path: str) -> Optional[str]: + """ + Args: + config_path: relative config filename + """ + name = config_path.replace(".yaml", "").replace(".py", "") + if name in _ModelZooUrls.CONFIG_PATH_TO_URL_SUFFIX: + suffix = _ModelZooUrls.CONFIG_PATH_TO_URL_SUFFIX[name] + return _ModelZooUrls.S3_PREFIX + name + "/" + suffix + return None + + +def get_checkpoint_url(config_path): + """ + Returns the URL to the model trained using the given config + + Args: + config_path (str): config file name relative to detectron2's "configs/" + directory, e.g., "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml" + + Returns: + str: a URL to the model + """ + url = _ModelZooUrls.query(config_path) + if url is None: + raise RuntimeError("Pretrained model for {} is not available!".format(config_path)) + return url + + +def get_config_file(config_path): + """ + Returns path to a builtin config file. + + Args: + config_path (str): config file name relative to detectron2's "configs/" + directory, e.g., "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml" + + Returns: + str: the real path to the config file. + """ + cfg_file = pkg_resources.resource_filename( + "ape.model_zoo", os.path.join("configs", config_path) + ) + if not os.path.exists(cfg_file): + raise RuntimeError("{} not available in Model Zoo!".format(config_path)) + return cfg_file + + +def get_config(config_path, trained: bool = False): + """ + Returns a config object for a model in model zoo. + + Args: + config_path (str): config file name relative to detectron2's "configs/" + directory, e.g., "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml" + trained (bool): If True, will set ``MODEL.WEIGHTS`` to trained model zoo weights. + If False, the checkpoint specified in the config file's ``MODEL.WEIGHTS`` is used + instead; this will typically (though not always) initialize a subset of weights using + an ImageNet pre-trained model, while randomly initializing the other weights. + + Returns: + CfgNode or omegaconf.DictConfig: a config object + """ + cfg_file = get_config_file(config_path) + if cfg_file.endswith(".yaml"): + cfg = get_cfg() + cfg.merge_from_file(cfg_file) + if trained: + cfg.MODEL.WEIGHTS = get_checkpoint_url(config_path) + return cfg + elif cfg_file.endswith(".py"): + cfg = LazyConfig.load(cfg_file) + if trained: + url = get_checkpoint_url(config_path) + if "train" in cfg and "init_checkpoint" in cfg.train: + cfg.train.init_checkpoint = url + else: + raise NotImplementedError + return cfg + + +def get(config_path, trained: bool = False, device: Optional[str] = None): + """ + Get a model specified by relative path under Detectron2's official ``configs/`` directory. + + Args: + config_path (str): config file name relative to detectron2's "configs/" + directory, e.g., "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml" + trained (bool): see :func:`get_config`. + device (str or None): overwrite the device in config, if given. + + Returns: + nn.Module: a detectron2 model. Will be in training mode. + + Example: + :: + from detectron2 import model_zoo + model = model_zoo.get("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml", trained=True) + """ + cfg = get_config(config_path, trained) + if device is None and not torch.cuda.is_available(): + device = "cpu" + if device is not None and isinstance(cfg, CfgNode): + cfg.MODEL.DEVICE = device + + if isinstance(cfg, CfgNode): + model = build_model(cfg) + DetectionCheckpointer(model).load(cfg.MODEL.WEIGHTS) + else: + model = instantiate(cfg.model) + if device is not None: + model = model.to(device) + if "train" in cfg and "init_checkpoint" in cfg.train: + DetectionCheckpointer(model).load(cfg.train.init_checkpoint) + return model diff --git a/approach/ovod/APE/ape/modeling/__init__.py b/approach/ovod/APE/ape/modeling/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/approach/ovod/APE/ape/modeling/ape_deta/__init__.py b/approach/ovod/APE/ape/modeling/ape_deta/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..451fc319b9307e49548857e22bad130a873df0c9 --- /dev/null +++ b/approach/ovod/APE/ape/modeling/ape_deta/__init__.py @@ -0,0 +1,16 @@ +from .ape_deta import SomeThing +from .assigner import Stage1Assigner, Stage2Assigner +from .deformable_criterion import DeformableCriterion +from .deformable_detr import DeformableDETR +from .deformable_detr_segm import DeformableDETRSegm +from .deformable_detr_segm_vl import DeformableDETRSegmVL +from .deformable_transformer import ( + DeformableDetrTransformer, + DeformableDetrTransformerDecoder, + DeformableDetrTransformerEncoder, +) +from .deformable_transformer_vl import ( + DeformableDetrTransformerDecoderVL, + DeformableDetrTransformerEncoderVL, + DeformableDetrTransformerVL, +) diff --git a/approach/ovod/APE/ape/modeling/ape_deta/ape_deta.py b/approach/ovod/APE/ape/modeling/ape_deta/ape_deta.py new file mode 100644 index 0000000000000000000000000000000000000000..f3e30acac5998b41968c01aa7742fc9d916b5bbf --- /dev/null +++ b/approach/ovod/APE/ape/modeling/ape_deta/ape_deta.py @@ -0,0 +1,40 @@ +import copy +import math +from typing import Dict, List, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import fvcore.nn.weight_init as weight_init +from detectron2.layers import Conv2d, ShapeSpec, get_norm, move_device_like +from detectron2.modeling import GeneralizedRCNN +from detectron2.modeling.postprocessing import detector_postprocess, sem_seg_postprocess +from detectron2.modeling.roi_heads.fast_rcnn import fast_rcnn_inference +from detectron2.structures import BitMasks, Boxes, ImageList, Instances +from detrex.layers import MLP, box_cxcywh_to_xyxy, box_xyxy_to_cxcywh +from detrex.utils import inverse_sigmoid +from torchvision.ops.boxes import batched_nms + + +class SomeThing(nn.Module): + def __init__( + self, + model_vision, + model_language, + **kwargs, + ): + super().__init__(**kwargs) + + self.model_vision = model_vision + self.model_language = model_language + + self.model_vision.set_model_language(self.model_language) + del self.model_language + + def forward(self, batched_inputs, do_postprocess=True): + losses = self.model_vision(batched_inputs, do_postprocess=do_postprocess) + return losses + + def set_eval_dataset(self, dataset_name): + self.model_vision.set_eval_dataset(dataset_name) diff --git a/approach/ovod/APE/ape/modeling/ape_deta/assigner.py b/approach/ovod/APE/ape/modeling/ape_deta/assigner.py new file mode 100644 index 0000000000000000000000000000000000000000..926006dd0f10cb2cb9af488b1fcd5f9d59d6a413 --- /dev/null +++ b/approach/ovod/APE/ape/modeling/ape_deta/assigner.py @@ -0,0 +1,364 @@ +from typing import List + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from detrex.layers import box_cxcywh_to_xyxy, box_iou, box_xyxy_to_cxcywh, generalized_box_iou + + +def nonzero_tuple(x): + """ + A 'as_tuple=True' version of torch.nonzero to support torchscript. + because of https://github.com/pytorch/pytorch/issues/38718 + """ + if torch.jit.is_scripting(): + if x.dim() == 0: + return x.unsqueeze(0).nonzero().unbind(1) + return x.nonzero().unbind(1) + else: + return x.nonzero(as_tuple=True) + + +class Matcher(object): + """ + This class assigns to each predicted "element" (e.g., a box) a ground-truth + element. Each predicted element will have exactly zero or one matches; each + ground-truth element may be matched to zero or more predicted elements. + + The matching is determined by the MxN match_quality_matrix, that characterizes + how well each (ground-truth, prediction)-pair match each other. For example, + if the elements are boxes, this matrix may contain box intersection-over-union + overlap values. + + The matcher returns (a) a vector of length N containing the index of the + ground-truth element m in [0, M) that matches to prediction n in [0, N). + (b) a vector of length N containing the labels for each prediction. + """ + + def __init__( + self, thresholds: List[float], labels: List[int], allow_low_quality_matches: bool = False + ): + """ + Args: + thresholds (list): a list of thresholds used to stratify predictions + into levels. + labels (list): a list of values to label predictions belonging at + each level. A label can be one of {-1, 0, 1} signifying + {ignore, negative class, positive class}, respectively. + allow_low_quality_matches (bool): if True, produce additional matches + for predictions with maximum match quality lower than high_threshold. + See set_low_quality_matches_ for more details. + + For example, + thresholds = [0.3, 0.5] + labels = [0, -1, 1] + All predictions with iou < 0.3 will be marked with 0 and + thus will be considered as false positives while training. + All predictions with 0.3 <= iou < 0.5 will be marked with -1 and + thus will be ignored. + All predictions with 0.5 <= iou will be marked with 1 and + thus will be considered as true positives. + """ + thresholds = thresholds[:] + assert thresholds[0] > 0 + thresholds.insert(0, -float("inf")) + thresholds.append(float("inf")) + assert all( + [low <= high for (low, high) in zip(thresholds[:-1], thresholds[1:])] + ), thresholds + assert all([l in [-1, 0, 1] for l in labels]) + assert len(labels) == len(thresholds) - 1 + self.thresholds = thresholds + self.labels = labels + self.allow_low_quality_matches = allow_low_quality_matches + + def __call__(self, match_quality_matrix): + """ + Args: + match_quality_matrix (Tensor[float]): an MxN tensor, containing the + pairwise quality between M ground-truth elements and N predicted + elements. All elements must be >= 0 (due to the us of `torch.nonzero` + for selecting indices in :meth:`set_low_quality_matches_`). + + Returns: + matches (Tensor[int64]): a vector of length N, where matches[i] is a matched + ground-truth index in [0, M) + match_labels (Tensor[int8]): a vector of length N, where pred_labels[i] indicates + whether a prediction is a true or false positive or ignored + """ + assert match_quality_matrix.dim() == 2 + if match_quality_matrix.numel() == 0: + default_matches = match_quality_matrix.new_full( + (match_quality_matrix.size(1),), 0, dtype=torch.int64 + ) + default_match_labels = match_quality_matrix.new_full( + (match_quality_matrix.size(1),), self.labels[0], dtype=torch.int8 + ) + return default_matches, default_match_labels + + assert torch.all(match_quality_matrix >= 0) + + matched_vals, matches = match_quality_matrix.max(dim=0) + + match_labels = matches.new_full(matches.size(), 1, dtype=torch.int8) + + for (l, low, high) in zip(self.labels, self.thresholds[:-1], self.thresholds[1:]): + low_high = (matched_vals >= low) & (matched_vals < high) + match_labels[low_high] = l + + if self.allow_low_quality_matches: + self.set_low_quality_matches_(match_labels, match_quality_matrix) + + return matches, match_labels + + def set_low_quality_matches_(self, match_labels, match_quality_matrix): + """ + Produce additional matches for predictions that have only low-quality matches. + Specifically, for each ground-truth G find the set of predictions that have + maximum overlap with it (including ties); for each prediction in that set, if + it is unmatched, then match it to the ground-truth G. + + This function implements the RPN assignment case (i) in Sec. 3.1.2 of + :paper:`Faster R-CNN`. + """ + highest_quality_foreach_gt, _ = match_quality_matrix.max(dim=1) + _, pred_inds_with_highest_quality = nonzero_tuple( + match_quality_matrix == highest_quality_foreach_gt[:, None] + ) + match_labels[pred_inds_with_highest_quality] = 1 + + +def subsample_labels( + labels: torch.Tensor, num_samples: int, positive_fraction: float, bg_label: int +): + """ + Return `num_samples` (or fewer, if not enough found) + random samples from `labels` which is a mixture of positives & negatives. + It will try to return as many positives as possible without + exceeding `positive_fraction * num_samples`, and then try to + fill the remaining slots with negatives. + + Args: + labels (Tensor): (N, ) label vector with values: + * -1: ignore + * bg_label: background ("negative") class + * otherwise: one or more foreground ("positive") classes + num_samples (int): The total number of labels with value >= 0 to return. + Values that are not sampled will be filled with -1 (ignore). + positive_fraction (float): The number of subsampled labels with values > 0 + is `min(num_positives, int(positive_fraction * num_samples))`. The number + of negatives sampled is `min(num_negatives, num_samples - num_positives_sampled)`. + In order words, if there are not enough positives, the sample is filled with + negatives. If there are also not enough negatives, then as many elements are + sampled as is possible. + bg_label (int): label index of background ("negative") class. + + Returns: + pos_idx, neg_idx (Tensor): + 1D vector of indices. The total length of both is `num_samples` or fewer. + """ + positive = nonzero_tuple((labels != -1) & (labels != bg_label))[0] + negative = nonzero_tuple(labels == bg_label)[0] + + num_pos = int(num_samples * positive_fraction) + num_pos = min(positive.numel(), num_pos) + num_neg = num_samples - num_pos + num_neg = min(negative.numel(), num_neg) + + perm1 = torch.randperm(positive.numel(), device=positive.device)[:num_pos] + perm2 = torch.randperm(negative.numel(), device=negative.device)[:num_neg] + + pos_idx = positive[perm1] + neg_idx = negative[perm2] + return pos_idx, neg_idx + + +def sample_topk_per_gt(pr_inds, gt_inds, iou, k): + if len(gt_inds) == 0: + return pr_inds, gt_inds + gt_inds2, counts = gt_inds.unique(return_counts=True) + scores, pr_inds2 = iou[gt_inds2].topk(k, dim=1) + gt_inds2 = gt_inds2[:, None].repeat(1, k) + + pr_inds3 = torch.cat([pr[:c] for c, pr in zip(counts, pr_inds2)]) + gt_inds3 = torch.cat([gt[:c] for c, gt in zip(counts, gt_inds2)]) + return pr_inds3, gt_inds3 + + +class Stage2Assigner(nn.Module): + def __init__(self, num_queries, num_classes, max_k=4): + super().__init__() + self.positive_fraction = 0.25 + self.num_classes = num_classes + self.batch_size_per_image = num_queries + self.proposal_matcher = Matcher( + thresholds=[0.6], labels=[0, 1], allow_low_quality_matches=True + ) + self.k = max_k + + def _sample_proposals( + self, matched_idxs: torch.Tensor, matched_labels: torch.Tensor, gt_classes: torch.Tensor + ): + """ + Based on the matching between N proposals and M groundtruth, + sample the proposals and set their classification labels. + + Args: + matched_idxs (Tensor): a vector of length N, each is the best-matched + gt index in [0, M) for each proposal. + matched_labels (Tensor): a vector of length N, the matcher's label + (one of cfg.MODEL.ROI_HEADS.IOU_LABELS) for each proposal. + gt_classes (Tensor): a vector of length M. + + Returns: + Tensor: a vector of indices of sampled proposals. Each is in [0, N). + Tensor: a vector of the same length, the classification label for + each sampled proposal. Each sample is labeled as either a category in + [0, num_classes) or the background (num_classes). + """ + has_gt = gt_classes.numel() > 0 + if has_gt: + gt_classes = gt_classes[matched_idxs] + gt_classes[matched_labels == 0] = self.num_classes + gt_classes[matched_labels == -1] = -1 + else: + gt_classes = torch.zeros_like(matched_idxs) + self.num_classes + + sampled_fg_idxs, sampled_bg_idxs = subsample_labels( + gt_classes, self.batch_size_per_image, self.positive_fraction, self.num_classes + ) + + sampled_idxs = torch.cat([sampled_fg_idxs, sampled_bg_idxs], dim=0) + return sampled_idxs, gt_classes[sampled_idxs] + + def forward(self, outputs, targets, return_cost_matrix=False): + + bs = len(targets) + indices = [] + ious = [] + for b in range(bs): + iou, _ = box_iou( + box_cxcywh_to_xyxy(targets[b]["boxes"]), + box_cxcywh_to_xyxy(outputs["init_reference"][b].detach()), + ) + if not torch.all(iou >= 0): + print("iou", iou, iou.max(), iou.min()) + print("targets[b][boxes]", targets[b]["boxes"]) + print( + "outputs[init_reference][b]", + outputs["init_reference"][b], + outputs["init_reference"][b].max(), + outputs["init_reference"][b].min(), + ) + print("outputs", outputs) + matched_idxs, matched_labels = self.proposal_matcher( + iou + ) # proposal_id -> highest_iou_gt_id, proposal_id -> [1 if iou > 0.6, 0 ow] + ( + sampled_idxs, + sampled_gt_classes, + ) = self._sample_proposals( # list of sampled proposal_ids, sampled_id -> [0, num_classes)+[bg_label] + matched_idxs, matched_labels, targets[b]["labels"] + ) + pos_pr_inds = sampled_idxs[sampled_gt_classes != self.num_classes] + pos_gt_inds = matched_idxs[pos_pr_inds] + pos_pr_inds, pos_gt_inds = self.postprocess_indices(pos_pr_inds, pos_gt_inds, iou) + indices.append((pos_pr_inds, pos_gt_inds)) + ious.append(iou) + if return_cost_matrix: + return indices, ious + return indices + + def postprocess_indices(self, pr_inds, gt_inds, iou): + return sample_topk_per_gt(pr_inds, gt_inds, iou, self.k) + + def __repr__(self, _repr_indent=8): + head = "Matcher " + self.__class__.__name__ + body = [] + for attribute, value in self.__dict__.items(): + if attribute.startswith("_"): + continue + body.append("{}: {}".format(attribute, value)) + lines = [head] + [" " * _repr_indent + line for line in body] + return "\n".join(lines) + + +class Stage1Assigner(nn.Module): + def __init__(self, t_low=0.3, t_high=0.7, max_k=4): + super().__init__() + self.positive_fraction = 0.5 + self.batch_size_per_image = 256 + self.k = max_k + self.t_low = t_low + self.t_high = t_high + self.anchor_matcher = Matcher( + thresholds=[t_low, t_high], labels=[0, -1, 1], allow_low_quality_matches=True + ) + + def _subsample_labels(self, label): + """ + Randomly sample a subset of positive and negative examples, and overwrite + the label vector to the ignore value (-1) for all elements that are not + included in the sample. + + Args: + labels (Tensor): a vector of -1, 0, 1. Will be modified in-place and returned. + """ + pos_idx, neg_idx = subsample_labels( + label, self.batch_size_per_image, self.positive_fraction, 0 + ) + label.fill_(-1) + label.scatter_(0, pos_idx, 1) + label.scatter_(0, neg_idx, 0) + return label + + def forward(self, outputs, targets, return_cost_matrix=False): + bs = len(targets) + indices = [] + ious = [] + for b in range(bs): + anchors = outputs["anchors"][b] + if len(targets[b]["boxes"]) == 0: + indices.append( + ( + torch.tensor([], dtype=torch.long, device=anchors.device), + torch.tensor([], dtype=torch.long, device=anchors.device), + ) + ) + continue + iou, _ = box_iou( + box_cxcywh_to_xyxy(targets[b]["boxes"]), + box_cxcywh_to_xyxy(anchors), + ) + matched_idxs, matched_labels = self.anchor_matcher( + iou + ) # proposal_id -> highest_iou_gt_id, proposal_id -> [1 if iou > 0.7, 0 if iou < 0.3, -1 ow] + matched_labels = self._subsample_labels(matched_labels) + + all_pr_inds = torch.arange(len(anchors)).to(matched_labels.device) + pos_pr_inds = all_pr_inds[matched_labels == 1] + pos_gt_inds = matched_idxs[pos_pr_inds] + pos_ious = iou[pos_gt_inds, pos_pr_inds] + pos_pr_inds, pos_gt_inds = self.postprocess_indices(pos_pr_inds, pos_gt_inds, iou) + pos_pr_inds, pos_gt_inds = pos_pr_inds.to(anchors.device), pos_gt_inds.to( + anchors.device + ) + indices.append((pos_pr_inds, pos_gt_inds)) + ious.append(iou) + if return_cost_matrix: + return indices, ious + return indices + + def postprocess_indices(self, pr_inds, gt_inds, iou): + return sample_topk_per_gt(pr_inds, gt_inds, iou, self.k) + + def __repr__(self, _repr_indent=8): + head = "Matcher " + self.__class__.__name__ + body = [] + for attribute, value in self.__dict__.items(): + if attribute.startswith("_"): + continue + body.append("{}: {}".format(attribute, value)) + lines = [head] + [" " * _repr_indent + line for line in body] + return "\n".join(lines) diff --git a/approach/ovod/APE/ape/modeling/ape_deta/deformable_criterion.py b/approach/ovod/APE/ape/modeling/ape_deta/deformable_criterion.py new file mode 100644 index 0000000000000000000000000000000000000000..6ac62ec782ed21cb684e6e935d516f96f8d71c2b --- /dev/null +++ b/approach/ovod/APE/ape/modeling/ape_deta/deformable_criterion.py @@ -0,0 +1,609 @@ +import copy +import logging +from typing import Callable, List, Optional + +import torch +import torch.nn.functional as F + +from detectron2.projects.point_rend.point_features import ( + get_uncertain_point_coords_with_randomness, + point_sample, +) +from detrex.layers import box_cxcywh_to_xyxy, box_iou, generalized_box_iou +from detrex.modeling import SetCriterion +from detrex.modeling.criterion.criterion import sigmoid_focal_loss +from detrex.modeling.losses import dice_loss +from detrex.utils import get_world_size, is_dist_avail_and_initialized + +from .misc import nested_tensor_from_tensor_list + +logger = logging.getLogger(__name__) + + +def sigmoid_ce_loss( + inputs: torch.Tensor, + targets: torch.Tensor, + num_masks: float, +): + """ + Args: + inputs: A float tensor of arbitrary shape. + The predictions for each example. + targets: A float tensor with the same shape as inputs. Stores the binary + classification label for each element in inputs + (0 for the negative class and 1 for the positive class). + Returns: + Loss tensor + """ + loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction="none") + + return loss.mean(1).sum() / num_masks + + +def calculate_uncertainty(logits): + """ + We estimate uncerainty as L1 distance between 0.0 and the logit prediction in 'logits' for the + foreground class in `classes`. + Args: + logits (Tensor): A tensor of shape (R, 1, ...) for class-specific or + class-agnostic, where R is the total number of predicted masks in all images and C is + the number of foreground classes. The values are logits. + Returns: + scores (Tensor): A tensor of shape (R, 1, ...) that contains uncertainty scores with + the most uncertain locations having the highest uncertainty score. + """ + assert logits.shape[1] == 1 + gt_class_logits = logits.clone() + return -(torch.abs(gt_class_logits)) + + +class DeformableCriterion(SetCriterion): + """This class computes the loss for Deformable-DETR + and two-stage Deformable-DETR + """ + + def __init__( + self, + num_classes, + matcher, + matcher_stage1, + matcher_stage2, + weight_dict, + losses: List[str] = ["class", "boxes"], + eos_coef: float = 0.1, + loss_class_type: str = "focal_loss", + alpha: float = 0.25, + gamma: float = 2.0, + use_fed_loss: bool = False, + get_fed_loss_cls_weights: Optional[Callable] = None, + fed_loss_num_classes: int = 50, + fed_loss_pad_type: str = None, + num_points: int = 12544, + oversample_ratio: float = 3.0, + importance_sample_ratio: float = 0.75, + train_positive_proposal_only: bool = False, + ): + super(DeformableCriterion, self).__init__( + num_classes=num_classes, + matcher=matcher, + weight_dict=weight_dict, + losses=losses, + eos_coef=eos_coef, + loss_class_type=loss_class_type, + alpha=alpha, + gamma=gamma, + ) + + self.matcher_stage1 = matcher_stage1 + self.matcher_stage2 = matcher_stage2 + + self.use_fed_loss = use_fed_loss + if self.use_fed_loss: + fed_loss_cls_weights = get_fed_loss_cls_weights() + logger.info( + f"fed_loss_cls_weights: {fed_loss_cls_weights.size()} num_classes: {num_classes}" + ) + + if len(fed_loss_cls_weights) < num_classes: + if fed_loss_pad_type == "max": + fed_loss_pad_value = fed_loss_cls_weights.max().item() + elif fed_loss_pad_type == "max1000": + fed_loss_pad_value = fed_loss_cls_weights.max().item() * 1000 + elif fed_loss_pad_type == "mean": + fed_loss_pad_value = fed_loss_cls_weights.mean().item() + elif fed_loss_pad_type == "median": + fed_loss_pad_value = fed_loss_cls_weights.median().item() + elif fed_loss_pad_type == "cat": + fed_loss_pad_classes = torch.arange(len(fed_loss_cls_weights), num_classes) + self.register_buffer("fed_loss_pad_classes", fed_loss_pad_classes) + fed_loss_pad_value = 0 + else: + fed_loss_pad_value = torch.kthvalue( + fed_loss_cls_weights, int(num_classes * 7.0 / 10) + )[0].item() + + logger.info( + f"pad fed_loss_cls_weights with type {fed_loss_pad_type} and value {fed_loss_pad_value}" + ) + if getattr(self, "fed_loss_pad_classes", None) is not None: + logger.info(f"pad fed_loss_classes with {self.fed_loss_pad_classes}") + fed_loss_cls_weights = torch.cat( + ( + fed_loss_cls_weights, + fed_loss_cls_weights.new_full( + (num_classes - len(fed_loss_cls_weights),), + fed_loss_pad_value, + ), + ), + dim=0, + ) + + logger.info(f"fed_loss_cls_weights: {fed_loss_cls_weights[-100:]}") + logger.info( + f"fed_loss_cls_weights: {fed_loss_cls_weights.size()} num_classes: {num_classes}" + ) + + assert ( + len(fed_loss_cls_weights) == self.num_classes + ), "Please check the provided fed_loss_cls_weights. Their size should match num_classes" + self.register_buffer("fed_loss_cls_weights", fed_loss_cls_weights) + self.fed_loss_num_classes = fed_loss_num_classes + + self.num_points = num_points + self.oversample_ratio = oversample_ratio + self.importance_sample_ratio = importance_sample_ratio + + self.train_positive_proposal_only = train_positive_proposal_only + self.alpha_old = self.alpha + + def get_fed_loss_classes(self, gt_classes, num_fed_loss_classes, num_classes, weight): + """ + Args: + gt_classes: a long tensor of shape R that contains the gt class label of each proposal. + num_fed_loss_classes: minimum number of classes to keep when calculating federated loss. + Will sample negative classes if number of unique gt_classes is smaller than this value. + num_classes: number of foreground classes + weight: probabilities used to sample negative classes + + Returns: + Tensor: + classes to keep when calculating the federated loss, including both unique gt + classes and sampled negative classes. + """ + unique_gt_classes = torch.unique(gt_classes) + prob = unique_gt_classes.new_ones(num_classes + 1).float() + prob[-1] = 0 + if len(unique_gt_classes) < num_fed_loss_classes: + prob[:num_classes] = weight.float().clone() + prob[unique_gt_classes] = 0 + sampled_negative_classes = torch.multinomial( + prob, num_fed_loss_classes - len(unique_gt_classes), replacement=False + ) + fed_loss_classes = torch.cat([unique_gt_classes, sampled_negative_classes]) + else: + fed_loss_classes = unique_gt_classes + return fed_loss_classes + + def loss_labels(self, outputs, targets, indices, num_boxes): + """Classification loss (Binary focal loss) + targets dicts must contain the key "labels" containing a tensor of dim [nb_target_boxes] + """ + assert "pred_logits" in outputs + src_logits = outputs["pred_logits"] + + if self.loss_class_type == "ce_loss": + num_classes = src_logits.shape[2] - 1 + elif self.loss_class_type == "focal_loss": + num_classes = src_logits.shape[2] + + idx = self._get_src_permutation_idx(indices) + target_classes_o = torch.cat([t["labels"][J] for t, (_, J) in zip(targets, indices)]) + target_classes = torch.full( + src_logits.shape[:2], + num_classes, + dtype=torch.int64, + device=src_logits.device, + ) + target_classes[idx] = target_classes_o + + if self.loss_class_type == "ce_loss": + loss_class = F.cross_entropy( + src_logits.transpose(1, 2), target_classes, self.empty_weight + ) + elif ( + self.loss_class_type == "focal_loss" + and self.use_fed_loss + and num_classes == len(self.fed_loss_cls_weights) + ): + target_classes_onehot = torch.zeros( + [src_logits.shape[0], src_logits.shape[1], src_logits.shape[2] + 1], + dtype=src_logits.dtype, + layout=src_logits.layout, + device=src_logits.device, + ) + target_classes_onehot.scatter_(2, target_classes.unsqueeze(-1), 1) + target_classes_onehot = target_classes_onehot[:, :, :-1] + fed_loss_classes = self.get_fed_loss_classes( + target_classes_o, + num_fed_loss_classes=self.fed_loss_num_classes, + num_classes=target_classes_onehot.shape[2], + weight=self.fed_loss_cls_weights, + ) + + if getattr(self, "fed_loss_pad_classes", None) is not None: + fed_loss_classes = torch.cat([fed_loss_classes, self.fed_loss_pad_classes]) + fed_loss_classes = torch.unique(fed_loss_classes) + + loss_class = ( + sigmoid_focal_loss( + src_logits[:, :, fed_loss_classes], + target_classes_onehot[:, :, fed_loss_classes], + num_boxes=num_boxes, + alpha=self.alpha, + gamma=self.gamma, + ) + * src_logits.shape[1] + ) + elif self.loss_class_type == "focal_loss": + target_classes_onehot = torch.zeros( + [src_logits.shape[0], src_logits.shape[1], src_logits.shape[2] + 1], + dtype=src_logits.dtype, + layout=src_logits.layout, + device=src_logits.device, + ) + target_classes_onehot.scatter_(2, target_classes.unsqueeze(-1), 1) + target_classes_onehot = target_classes_onehot[:, :, :-1] + loss_class = ( + sigmoid_focal_loss( + src_logits, + target_classes_onehot, + num_boxes=num_boxes, + alpha=self.alpha, + gamma=self.gamma, + ) + * src_logits.shape[1] + ) + + if not torch.isfinite(loss_class): + print("loss_class", loss_class) + print("outputs", outputs) + print("targets", targets) + print("indices", indices) + print("num_boxes", num_boxes) + + losses = {"loss_class": loss_class} + + return losses + + def loss_anchor_ious(self, outputs, targets, indices, num_boxes): + assert "pred_logits" in outputs + src_logits = outputs["pred_logits"] + + ious = torch.cat([t["ious"][J, I] for t, (I, J) in zip(targets, indices)]) + predictions = torch.cat([p[I] for p, (I, _) in zip(src_logits, indices)]) + + predictions = predictions.squeeze(1) + + loss_iou = F.mse_loss(predictions, ious, size_average=None, reduce=None, reduction="mean") + + losses = {"loss_iou": loss_iou} + + return losses + + def loss_pred_ious(self, outputs, targets, indices, num_boxes): + assert "pred_boxes" in outputs + idx = self._get_src_permutation_idx(indices) + src_boxes = outputs["pred_boxes"][idx] + target_boxes = torch.cat([t["boxes"][i] for t, (_, i) in zip(targets, indices)], dim=0) + + iou, _ = box_iou( + box_cxcywh_to_xyxy(target_boxes), + box_cxcywh_to_xyxy(src_boxes), + ) + ious = iou[range(len(iou)), range(len(iou))] + + assert "pred_logits" in outputs + src_logits = outputs["pred_logits"][idx] + src_logits = src_logits.squeeze(1) + + loss_iou = F.mse_loss(src_logits, ious, size_average=None, reduce=None, reduction="mean") + + losses = {"loss_iou": loss_iou} + + return losses + + def loss_boxes(self, outputs, targets, indices, num_boxes): + """Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss + targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4] + The target boxes are expected in format (center_x, center_y, w, h), normalized by the image size. + """ + assert "pred_boxes" in outputs + idx = self._get_src_permutation_idx(indices) + src_boxes = outputs["pred_boxes"][idx] + target_boxes = torch.cat([t["boxes"][i] for t, (_, i) in zip(targets, indices)], dim=0) + + loss_bbox = F.l1_loss(src_boxes, target_boxes, reduction="none") + + losses = {} + losses["loss_bbox"] = loss_bbox.sum() / num_boxes + + loss_giou = 1 - torch.diag( + generalized_box_iou( + box_cxcywh_to_xyxy(src_boxes), + box_cxcywh_to_xyxy(target_boxes), + ) + ) + losses["loss_giou"] = loss_giou.sum() / num_boxes + + return losses + + def loss_boxes_panoptic(self, outputs, targets, indices, num_boxes): + """Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss + targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4] + The target boxes are expected in format (center_x, center_y, w, h), normalized by the image size. + """ + assert "pred_boxes" in outputs + idx = self._get_src_permutation_idx(indices) + src_boxes = outputs["pred_boxes"][idx] + target_boxes = torch.cat([t["boxes"][i] for t, (_, i) in zip(targets, indices)], dim=0) + + if "is_thing" in targets[0]: + is_thing = torch.cat([t["is_thing"][i] for t, (_, i) in zip(targets, indices)], dim=0) + if is_thing.sum() == 0: # no gt + losses = {} + losses["loss_bbox"] = src_boxes.sum() * 0.0 + losses["loss_giou"] = src_boxes.sum() * 0.0 + return losses + target_boxes = target_boxes[is_thing] + src_boxes = src_boxes[is_thing] + + loss_bbox = F.l1_loss(src_boxes, target_boxes, reduction="none") + + losses = {} + losses["loss_bbox"] = loss_bbox.sum() / num_boxes + + loss_giou = 1 - torch.diag( + generalized_box_iou( + box_cxcywh_to_xyxy(src_boxes), + box_cxcywh_to_xyxy(target_boxes), + ) + ) + losses["loss_giou"] = loss_giou.sum() / num_boxes + + return losses + + def loss_masks(self, outputs, targets, indices, num_boxes): + """Compute the losses related to the masks: the focal loss and the dice loss. + targets dicts must contain the key "masks" containing a tensor of dim [nb_target_boxes, h, w] + """ + assert "pred_masks" in outputs + if outputs["pred_masks"] is None: + return {} + src_idx = self._get_src_permutation_idx(indices) + tgt_idx = self._get_tgt_permutation_idx(indices) + + max_mask_num = 128 * len(indices) + if src_idx[0].size(0) > max_mask_num: + perm = torch.sort(torch.randperm(src_idx[0].size(0))[:max_mask_num])[0] + + src_idx = (src_idx[0][perm], src_idx[1][perm]) + tgt_idx = (tgt_idx[0][perm], tgt_idx[1][perm]) + + src_masks = outputs["pred_masks"] + src_masks = src_masks[src_idx] + masks = [t["masks"] for t in targets] + target_masks, valid = nested_tensor_from_tensor_list(masks).decompose() + + if target_masks.size(1) == 0: # no gt + losses = {} + losses["loss_mask"] = src_masks.sum() * 0.0 + losses["loss_dice"] = src_masks.sum() * 0.0 + return losses + + target_masks = target_masks.to(src_masks) + target_masks = target_masks[tgt_idx] + + src_masks = F.interpolate( + src_masks[:, None], size=target_masks.shape[-2:], mode="bilinear", align_corners=False + ) + src_masks = src_masks[:, 0].flatten(1) + + target_masks = target_masks.flatten(1) + target_masks = target_masks.view(src_masks.shape) + + losses = { + "loss_mask": sigmoid_focal_loss(src_masks, target_masks, num_boxes), + "loss_dice": dice_loss( + src_masks.sigmoid(), target_masks, reduction="mean", avg_factor=num_boxes + ), + } + del src_masks + del target_masks + return losses + + def loss_masks_maskdino(self, outputs, targets, indices, num_boxes): + """Compute the losses related to the masks: the focal loss and the dice loss. + targets dicts must contain the key "masks" containing a tensor of dim [nb_target_boxes, h, w] + """ + assert "pred_masks" in outputs + if outputs["pred_masks"] is None: + return {} + src_idx = self._get_src_permutation_idx(indices) + tgt_idx = self._get_tgt_permutation_idx(indices) + src_masks = outputs["pred_masks"] + if not isinstance(src_masks, torch.Tensor): + mask_embeds = src_masks["mask_embeds"] + mask_features = src_masks["mask_features"] + src_masks = torch.cat( + [ + torch.einsum("qc,chw->qhw", mask_embeds[i][src], mask_features[i]) + for i, (src, _) in enumerate(indices) + ], + dim=0, + ) + else: + src_masks = src_masks[src_idx] + masks = [t["masks"] for t in targets] + target_masks, valid = nested_tensor_from_tensor_list(masks).decompose() + + if target_masks.size(1) == 0: # no gt + losses = {} + losses["loss_mask_maskdino"] = src_masks.sum() * 0.0 + losses["loss_dice_maskdino"] = src_masks.sum() * 0.0 + return losses + + target_masks = target_masks.to(src_masks) + target_masks = target_masks[tgt_idx] + + src_masks = src_masks[:, None] + target_masks = target_masks[:, None] + + with torch.no_grad(): + point_coords = get_uncertain_point_coords_with_randomness( + src_masks, + lambda logits: calculate_uncertainty(logits), + self.num_points, + self.oversample_ratio, + self.importance_sample_ratio, + ) + point_labels = point_sample( + target_masks, + point_coords, + align_corners=False, + ).squeeze(1) + + point_logits = point_sample( + src_masks, + point_coords, + align_corners=False, + ).squeeze(1) + + losses = { + "loss_mask_maskdino": sigmoid_ce_loss(point_logits, point_labels, num_boxes), + "loss_dice_maskdino": dice_loss( + point_logits.sigmoid(), point_labels, reduction="mean", avg_factor=num_boxes + ), + } + + del src_masks + del target_masks + return losses + + def get_loss(self, loss, outputs, targets, indices, num_boxes, **kwargs): + loss_map = { + "class": self.loss_labels, + "boxes": self.loss_boxes, + "boxes_panoptic": self.loss_boxes_panoptic, + "masks": self.loss_masks, + "masks_maskdino": self.loss_masks_maskdino, + "anchor_iou": self.loss_anchor_ious, + "pred_iou": self.loss_pred_ious, + } + assert loss in loss_map, f"do you really want to compute {loss} loss?" + return loss_map[loss](outputs, targets, indices, num_boxes, **kwargs) + + def forward(self, outputs, targets): + outputs_without_aux = { + k: v for k, v in outputs.items() if k != "aux_outputs" and k != "enc_outputs" + } + + if self.matcher_stage2 is not None: + indices = self.matcher_stage2(outputs_without_aux, targets) + else: + indices = self.matcher(outputs_without_aux, targets) + + num_boxes = sum(len(t["labels"]) for t in targets) + num_boxes = torch.as_tensor( + [num_boxes], dtype=torch.float, device=next(iter(outputs.values())).device + ) + if is_dist_avail_and_initialized(): + torch.distributed.all_reduce(num_boxes) + num_boxes = torch.clamp(num_boxes / get_world_size(), min=1).item() + + if "is_thing" in targets[0] and False: + unique_classes = torch.cat([t["labels"] for t in targets], dim=0) + is_thing = torch.cat([t["is_thing"][i] for t, (_, i) in zip(targets, indices)], dim=0) + all_classes = torch.cat([t["labels"][i] for t, (_, i) in zip(targets, indices)], dim=0) + thing_classes = all_classes[is_thing] + stuff_classes = all_classes[~is_thing] + + print( + "thing_classes", + 1.0 * len(thing_classes) / max(len(torch.unique(thing_classes)), 1), + "stuff_classes", + 1.0 * len(stuff_classes) / max(len(torch.unique(stuff_classes)), 1), + ) + + losses = {} + for loss in self.losses: + if loss == "pred_iou" or loss == "anchor_iou": + continue + kwargs = {} + losses.update(self.get_loss(loss, outputs, targets, indices, num_boxes, **kwargs)) + + if "aux_outputs" in outputs: + for i, aux_outputs in enumerate(outputs["aux_outputs"]): + if self.matcher_stage2 is not None: + pass + else: + indices = self.matcher(aux_outputs, targets) + for loss in self.losses: + if loss == "masks": + continue + if loss == "pred_iou" or loss == "anchor_iou": + continue + l_dict = self.get_loss(loss, aux_outputs, targets, indices, num_boxes, **kwargs) + l_dict = {k + f"_{i}": v for k, v in l_dict.items()} + losses.update(l_dict) + + if "enc_outputs" in outputs: + if self.train_positive_proposal_only: + self.alpha = 1.0 + enc_outputs = outputs["enc_outputs"] + bin_targets = copy.deepcopy(targets) + for bt in bin_targets: + bt["labels"] = torch.zeros_like(bt["labels"]) + if "is_thing" in bt: + del bt["is_thing"] + if self.matcher_stage1 is not None: + indices, ious = self.matcher_stage1( + enc_outputs, bin_targets, return_cost_matrix=True + ) + for bt, iou in zip(bin_targets, ious): + bt["ious"] = iou + else: + indices = self.matcher(enc_outputs, bin_targets) + for loss in self.losses: + if loss == "masks": + continue + if loss == "masks_maskdino": + continue + if loss == "class" and ("pred_iou" in losses or "anchor_iou" in losses): + continue + l_dict = self.get_loss(loss, enc_outputs, bin_targets, indices, num_boxes, **kwargs) + l_dict = {k + "_enc": v for k, v in l_dict.items()} + losses.update(l_dict) + if self.train_positive_proposal_only: + self.alpha = self.alpha_old + + return losses + + def __repr__(self): + head = "Criterion " + self.__class__.__name__ + body = [ + "matcher: {}".format(self.matcher.__repr__(_repr_indent=8)), + "matcher_stage1: {}".format(self.matcher_stage1), + "matcher_stage2: {}".format(self.matcher_stage2), + "losses: {}".format(self.losses), + "loss_class_type: {}".format(self.loss_class_type), + "weight_dict: {}".format(self.weight_dict), + "num_classes: {}".format(self.num_classes), + "eos_coef: {}".format(self.eos_coef), + "focal loss alpha: {}".format(self.alpha), + "focal loss gamma: {}".format(self.gamma), + "use_fed_loss: {}".format(self.use_fed_loss), + "fed_loss_num_classes: {}".format(self.fed_loss_num_classes), + ] + _repr_indent = 4 + lines = [head] + [" " * _repr_indent + line for line in body] + return "\n".join(lines) diff --git a/approach/ovod/APE/ape/modeling/ape_deta/deformable_detr.py b/approach/ovod/APE/ape/modeling/ape_deta/deformable_detr.py new file mode 100644 index 0000000000000000000000000000000000000000..ced5680efd9ef1d3f913f4972a2e4c31cb675ea0 --- /dev/null +++ b/approach/ovod/APE/ape/modeling/ape_deta/deformable_detr.py @@ -0,0 +1,604 @@ +import copy +import logging +import math +from typing import Dict, List, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ape.layers import VisionLanguageAlign, ZeroShotFC +from detectron2.layers import move_device_like +from detectron2.modeling import GeneralizedRCNN, detector_postprocess +from detectron2.modeling.roi_heads.fast_rcnn import fast_rcnn_inference +from detectron2.structures import Boxes, ImageList, Instances +from detrex.layers import MLP, box_cxcywh_to_xyxy, box_xyxy_to_cxcywh +from detrex.utils import inverse_sigmoid +from torchvision.ops.boxes import batched_nms + +logger = logging.getLogger(__name__) + + +class DeformableDETR(nn.Module): + """Implements the Deformable DETR model. + + Code is modified from the `official github repo + `_. + + More details can be found in the `paper + `_ . + + Args: + backbone (nn.Module): the backbone module. + position_embedding (nn.Module): the position embedding module. + neck (nn.Module): the neck module. + transformer (nn.Module): the transformer module. + embed_dim (int): the dimension of the embedding. + num_classes (int): Number of total categories. + num_queries (int): Number of proposal dynamic anchor boxes in Transformer + criterion (nn.Module): Criterion for calculating the total losses. + pixel_mean (List[float]): Pixel mean value for image normalization. + Default: [123.675, 116.280, 103.530]. + pixel_std (List[float]): Pixel std value for image normalization. + Default: [58.395, 57.120, 57.375]. + aux_loss (bool): whether to use auxiliary loss. Default: True. + with_box_refine (bool): whether to use box refinement. Default: False. + as_two_stage (bool): whether to use two-stage. Default: False. + select_box_nums_for_evaluation (int): the number of topk candidates + slected at postprocess for evaluation. Default: 100. + + """ + + def __init__( + self, + backbone, + position_embedding, + neck, + transformer, + embed_dim, + num_classes, + num_queries, + criterion, + pixel_mean: Tuple[float], + pixel_std: Tuple[float], + aux_loss=True, + with_box_refine=False, + as_two_stage=False, + select_box_nums_for_evaluation=100, + select_box_nums_for_evaluation_list: list = None, + input_format: Optional[str] = None, + vis_period: int = 0, + output_dir: Optional[str] = None, + dataset_names: List[str] = [], + dataset_metas: List[str] = [], + dataset_prompts: List[str] = None, + embed_dim_language: int = 512, + text_feature_batch_repeat: bool = True, + text_feature_bank: bool = False, + text_feature_bank_reset: bool = False, + text_feature_reduce_type: str = "last", + text_feature_reduce_before_fusion: bool = True, + expression_cumulative_gt_class: bool = True, + test_nms_thresh: float = 0.7, + test_score_thresh: float = 0.0, + last_class_embed_use_mlp: bool = False, + openset_classifier: str = "VisionLanguageAlign", + ): + super().__init__() + self.backbone = backbone + self.position_embedding = position_embedding + + self.neck = neck + + self.num_queries = num_queries + if not as_two_stage: + self.query_embedding = nn.Embedding(num_queries, embed_dim * 2) + + self.transformer = transformer + + self.num_classes = num_classes + if criterion[0].loss_class_type == "ce_loss": + self.class_embed = nn.Linear(embed_dim, num_classes + 1) + else: + self.class_embed = nn.Linear(embed_dim, num_classes) + self.bbox_embed = MLP(embed_dim, embed_dim, 4, 3) + + self.aux_loss = aux_loss + self.criterion = nn.ModuleList(criterion) + + self.with_box_refine = with_box_refine + self.as_two_stage = as_two_stage + + prior_prob = 0.01 + bias_value = -math.log((1 - prior_prob) / prior_prob) + if criterion[0].loss_class_type == "ce_loss": + self.class_embed.bias.data = torch.ones(num_classes + 1) * bias_value + else: + self.class_embed.bias.data = torch.ones(num_classes) * bias_value + nn.init.constant_(self.bbox_embed.layers[-1].weight.data, 0) + nn.init.constant_(self.bbox_embed.layers[-1].bias.data, 0) + if self.neck is not None: + for _, neck_layer in self.neck.named_modules(): + if isinstance(neck_layer, nn.Conv2d): + nn.init.xavier_uniform_(neck_layer.weight, gain=1) + nn.init.constant_(neck_layer.bias, 0) + + self.text_feature_batch_repeat = text_feature_batch_repeat + if openset_classifier == "ZeroShotFC": + del self.class_embed + self.class_embed = ZeroShotFC( + input_size=embed_dim, + num_classes=num_classes, + zs_weight_path="zeros", + zs_weight_dim=embed_dim_language, + use_bias=0.0, + norm_weight=True, + norm_temperature=50.0, + use_project=True, + use_sigmoid_ce=True, + prior_prob=0.01, + zs_vocabulary="", + text_model="", + ) + + if openset_classifier == "VisionLanguageAlign": + del self.class_embed + self.class_embed = VisionLanguageAlign(embed_dim, embed_dim_language) + + num_pred = ( + (transformer.decoder.num_layers + 1) if as_two_stage else transformer.decoder.num_layers + ) + if with_box_refine: + self.class_embed = nn.ModuleList( + [copy.deepcopy(self.class_embed) for i in range(num_pred)] + ) + self.bbox_embed = nn.ModuleList( + [copy.deepcopy(self.bbox_embed) for i in range(num_pred)] + ) + nn.init.constant_(self.bbox_embed[0].layers[-1].bias.data[2:], -2.0) + self.transformer.decoder.bbox_embed = self.bbox_embed + else: + nn.init.constant_(self.bbox_embed.layers[-1].bias.data[2:], -2.0) + self.class_embed = nn.ModuleList([self.class_embed for _ in range(num_pred)]) + self.bbox_embed = nn.ModuleList([self.bbox_embed for _ in range(num_pred)]) + self.transformer.decoder.bbox_embed = None + + if as_two_stage: + self.transformer.decoder.class_embed = self.class_embed + if True: + prior_prob = 0.01 + bias_value = -math.log((1 - prior_prob) / prior_prob) + if criterion[0].loss_class_type == "ce_loss": + self.transformer.decoder.class_embed[-1] = nn.Linear(embed_dim, num_classes + 1) + self.transformer.decoder.class_embed[-1].bias.data = ( + torch.ones(num_classes + 1) * bias_value + ) + else: + self.transformer.decoder.class_embed[-1] = nn.Linear(embed_dim, 1) + self.transformer.decoder.class_embed[-1].bias.data = torch.ones(1) * bias_value + if last_class_embed_use_mlp: + self.transformer.decoder.class_embed[-1] = MLP(embed_dim, embed_dim, 1, 3) + self.transformer.decoder.class_embed[-1].layers[-1].bias.data = ( + torch.ones(1) * bias_value + ) + for box_embed in self.bbox_embed: + nn.init.constant_(box_embed.layers[-1].bias.data[2:], 0.0) + + if self.transformer.proposal_ambiguous: + self.transformer.decoder.bbox_embed_ambiguous = nn.ModuleList( + [ + copy.deepcopy(self.transformer.decoder.bbox_embed[-1]) + for _ in range(self.transformer.proposal_ambiguous) + ] + ) + self.transformer.decoder.class_embed_ambiguous = nn.ModuleList( + [ + copy.deepcopy(self.transformer.decoder.class_embed[-1]) + for _ in range(self.transformer.proposal_ambiguous) + ] + ) + + if False: + self.transformer.decoder.bbox_embed_2 = copy.deepcopy( + self.transformer.decoder.bbox_embed[-1] + ) + self.transformer.decoder.class_embed_2 = copy.deepcopy( + self.transformer.decoder.class_embed[-1] + ) + + self.transformer.decoder.bbox_embed_3 = copy.deepcopy( + self.transformer.decoder.bbox_embed[-1] + ) + self.transformer.decoder.class_embed_3 = copy.deepcopy( + self.transformer.decoder.class_embed[-1] + ) + + self.select_box_nums_for_evaluation = select_box_nums_for_evaluation + self.select_box_nums_for_evaluation_list = select_box_nums_for_evaluation_list + + self.test_topk_per_image = self.select_box_nums_for_evaluation + self.test_nms_thresh = test_nms_thresh + self.test_score_thresh = test_score_thresh + + self.input_format = input_format + self.vis_period = vis_period + if vis_period > 0: + assert input_format is not None, "input_format is required for visualization!" + + self.register_buffer("pixel_mean", torch.tensor(pixel_mean).view(-1, 1, 1), False) + self.register_buffer("pixel_std", torch.tensor(pixel_std).view(-1, 1, 1), False) + assert ( + self.pixel_mean.shape == self.pixel_std.shape + ), f"{self.pixel_mean} and {self.pixel_std} have different shapes!" + + self.output_dir = output_dir + + self.dataset_names = dataset_names + from detectron2.data.catalog import MetadataCatalog + + if isinstance(dataset_metas, str): + dataset_metas = [ + dataset_metas, + ] + self.metadata_list = [copy.deepcopy(MetadataCatalog.get(d)) for d in dataset_metas] + + self.dataset_prompts = dataset_prompts + self.dataset_entities = [] + for i, metadata in enumerate(self.metadata_list): + if "stuffonly" in metadata.name: + del metadata.thing_classes + + if ( + metadata.get("thing_classes", None) is not None + and metadata.get("stuff_classes", None) is not None + ): + self.dataset_entities.append("thing+stuff") + elif metadata.get("thing_classes", None) is not None: + self.dataset_entities.append("thing") + elif metadata.get("stuff_classes", None) is not None: + self.dataset_entities.append("stuff") + else: + self.dataset_entities.append("thing") + + logger.info("dataset_id: " + str(i)) + logger.info("dataset_name: " + metadata.name) + logger.info("thing_classes: " + str(metadata.get("thing_classes", None))) + logger.info("stuff_classes: " + str(metadata.get("stuff_classes", None))) + logger.info("dataset_entity: " + self.dataset_entities[i]) + + self.dataset_name_to_idx = {k: i for i, k in enumerate(self.dataset_names)} + self.dataset_name_to_entity = { + k: i for i, k in zip(self.dataset_entities, self.dataset_names) + } + + self.eval_dataset_id = -1 + self.eval_dataset_entity = "" + + self.text_feature_bank = text_feature_bank + self.text_feature_bank_reset = text_feature_bank_reset + if self.text_feature_bank: + features_phrase_bank = torch.zeros( + ( + len(self.criterion), + max([ctr.num_classes for ctr in self.criterion]), + embed_dim_language, + ), + dtype=torch.float, + device=self.device, + ) + self.register_buffer("features_phrase_bank", features_phrase_bank, False) + + self.text_feature_reduce_type = text_feature_reduce_type + self.text_feature_reduce_before_fusion = text_feature_reduce_before_fusion + self.expression_cumulative_gt_class = expression_cumulative_gt_class + self.embed_dim_language = embed_dim_language + + @property + def device(self): + return self.pixel_mean.device + + def _move_to_current_device(self, x): + return move_device_like(x, self.pixel_mean) + + def forward(self, batched_inputs, do_postprocess=True): + images = self.preprocess_image(batched_inputs) + + batch_size, _, H, W = images.tensor.shape + img_masks = images.tensor.new_ones(batch_size, H, W) + for image_id, image_size in enumerate(images.image_sizes): + img_masks[image_id, : image_size[0], : image_size[1]] = 0 + + features = self.backbone(images.tensor) # output feature dict + + if self.neck is not None: + multi_level_feats = self.neck({f: features[f] for f in self.neck.in_features}) + else: + multi_level_feats = [feat for feat_name, feat in features.items()] + multi_level_masks = [] + multi_level_position_embeddings = [] + for feat in multi_level_feats: + multi_level_masks.append( + F.interpolate(img_masks[None], size=feat.shape[-2:]).to(torch.bool).squeeze(0) + ) + multi_level_position_embeddings.append( + self.position_embedding(multi_level_masks[-1]).to(images.tensor.dtype) + ) + + query_embeds = None + if not self.as_two_stage: + query_embeds = self.query_embedding.weight + + ( + inter_states, + init_reference, + inter_references, + enc_outputs_class, + enc_outputs_coord_unact, + anchors, + memory, + ) = self.transformer( + multi_level_feats, multi_level_masks, multi_level_position_embeddings, query_embeds + ) + + outputs_classes = [] + outputs_coords = [] + for lvl in range(inter_states.shape[0]): + if lvl == 0: + reference = init_reference + else: + reference = inter_references[lvl - 1] + reference = inverse_sigmoid(reference) + outputs_class = self.class_embed[lvl](inter_states[lvl]) + tmp = self.bbox_embed[lvl](inter_states[lvl]) + if reference.shape[-1] == 4: + tmp += reference + else: + assert reference.shape[-1] == 2 + tmp[..., :2] += reference + outputs_coord = tmp.sigmoid() + outputs_classes.append(outputs_class) + outputs_coords.append(outputs_coord) + outputs_class = torch.stack(outputs_classes) + outputs_coord = torch.stack(outputs_coords) + + output = { + "pred_logits": outputs_class[-1], + "pred_boxes": outputs_coord[-1], + "init_reference": init_reference, + } + if self.aux_loss: + output["aux_outputs"] = self._set_aux_loss(outputs_class, outputs_coord) + + if self.as_two_stage: + enc_outputs_coord = enc_outputs_coord_unact.sigmoid() + output["enc_outputs"] = { + "pred_logits": enc_outputs_class, + "pred_boxes": enc_outputs_coord, + "anchors": anchors, + } + + if self.training: + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + targets = self.prepare_targets(gt_instances) + loss_dict = self.criterion(output, targets) + weight_dict = self.criterion.weight_dict + for k in loss_dict.keys(): + if k in weight_dict: + loss_dict[k] *= weight_dict[k] + return loss_dict + else: + del features + del multi_level_feats + + box_cls = output["pred_logits"] + box_pred = output["pred_boxes"] + results, filter_inds = self.inference(box_cls, box_pred, images.image_sizes) + + if do_postprocess: + assert not torch.jit.is_scripting(), "Scripting is not supported for postprocess." + return GeneralizedRCNN._postprocess(results, batched_inputs, images.image_sizes) + return results + + @torch.jit.unused + def _set_aux_loss(self, outputs_class, outputs_coord): + return [ + {"pred_logits": a, "pred_boxes": b} + for a, b in zip(outputs_class[:-1], outputs_coord[:-1]) + ] + + def inference(self, box_cls, box_pred, image_sizes): + """ + Arguments: + box_cls (Tensor): tensor of shape (batch_size, num_queries, K). + The tensor predicts the classification probability for each query. + box_pred (Tensor): tensors of shape (batch_size, num_queries, 4). + The tensor predicts 4-vector (x,y,w,h) box + regression values for every queryx + image_sizes (List[torch.Size]): the input image sizes + + Returns: + results (List[Instances]): a list of #images elements. + """ + + if True: + return NMSPostProcess()( + {"pred_logits": box_cls, "pred_boxes": box_pred}, + torch.tensor([list(x) for x in image_sizes], device=self.device), + self.select_box_nums_for_evaluation, + ) + + scores = torch.cat( + ( + box_cls.sigmoid(), + torch.zeros((box_cls.size(0), box_cls.size(1), 1), device=self.device), + ), + dim=2, + ) + + boxes = box_cxcywh_to_xyxy(box_pred) + + img_h, img_w = torch.tensor(image_sizes, device=self.device).unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) + boxes = boxes * scale_fct[:, None, :] + + boxes = boxes.unbind(0) + scores = scores.unbind(0) + image_shapes = image_sizes + + self.test_topk_per_image = self.select_box_nums_for_evaluation + self.test_nms_thresh = 0.7 + self.test_score_thresh = 0.05 + + return fast_rcnn_inference( + boxes, + scores, + image_shapes, + self.test_score_thresh, + self.test_nms_thresh, + self.test_topk_per_image, + ) + + assert len(box_cls) == len(image_sizes) + results = [] + + prob = box_cls.sigmoid() + topk_values, topk_indexes = torch.topk( + prob.view(box_cls.shape[0], -1), self.select_box_nums_for_evaluation, dim=1 + ) + scores = topk_values + topk_boxes = torch.div(topk_indexes, box_cls.shape[2], rounding_mode="floor") + labels = topk_indexes % box_cls.shape[2] + + boxes = torch.gather(box_pred, 1, topk_boxes.unsqueeze(-1).repeat(1, 1, 4)) + + for i, (scores_per_image, labels_per_image, box_pred_per_image, image_size) in enumerate( + zip(scores, labels, boxes, image_sizes) + ): + result = Instances(image_size) + result.pred_boxes = Boxes(box_cxcywh_to_xyxy(box_pred_per_image)) + result.pred_boxes.scale(scale_x=image_size[1], scale_y=image_size[0]) + result.scores = scores_per_image + result.pred_classes = labels_per_image + results.append(result) + return results, topk_indexes + + def prepare_targets(self, targets): + new_targets = [] + for targets_per_image in targets: + h, w = targets_per_image.image_size + image_size_xyxy = torch.as_tensor([w, h, w, h], dtype=torch.float, device=self.device) + gt_classes = targets_per_image.gt_classes + gt_boxes = targets_per_image.gt_boxes.tensor / image_size_xyxy + gt_boxes = box_xyxy_to_cxcywh(gt_boxes) + new_targets.append({"labels": gt_classes, "boxes": gt_boxes}) + return new_targets + + def preprocess_image(self, batched_inputs): + images = [self._move_to_current_device(x["image"]) for x in batched_inputs] + images = [x.to(self.pixel_mean.dtype) for x in images] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors( + images, + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ) + return images + + @staticmethod + def _postprocess(instances, batched_inputs: List[Dict[str, torch.Tensor]], image_sizes): + """ + Rescale the output instances to the target size. + """ + processed_results = [] + for results_per_image, input_per_image, image_size in zip( + instances, batched_inputs, image_sizes + ): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + r = detector_postprocess(results_per_image, height, width) + processed_results.append({"instances": r}) + return processed_results + + def set_eval_dataset(self, dataset_name): + for d in self.dataset_names: + if sum([dd in dataset_name for dd in d.split("+")]): + self.eval_dataset_id = self.dataset_name_to_idx[d] + self.eval_dataset_entity = self.dataset_name_to_entity[d] + break + else: + self.eval_dataset_id = -1 + self.eval_dataset_entity = "" + + logger.info( + "Setting eval dataset to: " + + str(d) + + " " + + str(dataset_name) + + " " + + str(self.eval_dataset_id) + ) + logger.info( + "Setting eval entity to: " + + str(d) + + " " + + str(dataset_name) + + " " + + str(self.eval_dataset_entity) + ) + + +class NMSPostProcess(nn.Module): + """This module converts the model's output into the format expected by the coco api""" + + @torch.no_grad() + def forward(self, outputs, target_sizes, select_box_nums_for_evaluation): + """Perform the computation + Parameters: + outputs: raw outputs of the model + target_sizes: tensor of dimension [batch_size x 2] containing the size of each images of the batch + For evaluation, this must be the original image size (before any data augmentation) + For visualization, this should be the image size after data augment, but before padding + """ + out_logits, out_bbox = outputs["pred_logits"], outputs["pred_boxes"] + bs, n_queries, n_cls = out_logits.shape + + assert len(out_logits) == len(target_sizes) + assert target_sizes.shape[1] == 2 + + prob = out_logits.sigmoid() + + all_scores = prob.view(bs, n_queries * n_cls).to(out_logits.device) + all_indexes = torch.arange(n_queries * n_cls)[None].repeat(bs, 1).to(out_logits.device) + all_boxes = torch.div(all_indexes, out_logits.shape[2], rounding_mode="trunc") + all_labels = all_indexes % out_logits.shape[2] + + boxes = box_cxcywh_to_xyxy(out_bbox) + boxes = torch.gather(boxes, 1, all_boxes.unsqueeze(-1).repeat(1, 1, 4)) + + img_h, img_w = target_sizes.unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) + boxes = boxes * scale_fct[:, None, :] + + results = [] + keep_inds_all = [] + for b in range(bs): + box = boxes[b] + score = all_scores[b] + lbls = all_labels[b] + + pre_topk = score.topk(10000).indices + box = box[pre_topk] + score = score[pre_topk] + lbls = lbls[pre_topk] + + keep_inds = batched_nms(box, score, lbls, 0.7)[:select_box_nums_for_evaluation] + + result = Instances(target_sizes[b]) + result.pred_boxes = Boxes(box[keep_inds]) + result.scores = score[keep_inds] + result.pred_classes = lbls[keep_inds] + results.append(result) + + keep_inds_all.append(keep_inds) + + return results, keep_inds_all diff --git a/approach/ovod/APE/ape/modeling/ape_deta/deformable_detr_segm.py b/approach/ovod/APE/ape/modeling/ape_deta/deformable_detr_segm.py new file mode 100644 index 0000000000000000000000000000000000000000..2b5a33569d0c77c0ad3edb1a5232788b668b9eda --- /dev/null +++ b/approach/ovod/APE/ape/modeling/ape_deta/deformable_detr_segm.py @@ -0,0 +1,1624 @@ +import copy +import math +import os +import time +from typing import Dict, List, Optional, Tuple + +import cv2 +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +import fvcore.nn.weight_init as weight_init +from ape.modeling.text import utils as text_utils +from detectron2.data.detection_utils import convert_image_to_rgb +from detectron2.layers import Conv2d, ShapeSpec, get_norm, move_device_like +from detectron2.modeling import GeneralizedRCNN +from detectron2.modeling.meta_arch.panoptic_fpn import combine_semantic_and_instance_outputs +from detectron2.modeling.postprocessing import detector_postprocess, sem_seg_postprocess +from detectron2.structures import BitMasks, Boxes, ImageList, Instances +from detectron2.utils.events import get_event_storage +from detectron2.utils.memory import retry_if_cuda_oom +from detrex.layers import MLP, box_cxcywh_to_xyxy, box_xyxy_to_cxcywh +from detrex.utils import inverse_sigmoid +from torchvision.ops.boxes import batched_nms + +from .deformable_detr import DeformableDETR +from .fast_rcnn import fast_rcnn_inference +from .segmentation import MaskHeadSmallConv, MHAttentionMap + + +class DeformableDETRSegm(DeformableDETR): + """Implements the Deformable DETR model. + + Code is modified from the `official github repo + `_. + + More details can be found in the `paper + `_ . + + Args: + backbone (nn.Module): the backbone module. + position_embedding (nn.Module): the position embedding module. + neck (nn.Module): the neck module. + transformer (nn.Module): the transformer module. + embed_dim (int): the dimension of the embedding. + num_classes (int): Number of total categories. + num_queries (int): Number of proposal dynamic anchor boxes in Transformer + criterion (nn.Module): Criterion for calculating the total losses. + pixel_mean (List[float]): Pixel mean value for image normalization. + Default: [123.675, 116.280, 103.530]. + pixel_std (List[float]): Pixel std value for image normalization. + Default: [58.395, 57.120, 57.375]. + aux_loss (bool): whether to use auxiliary loss. Default: True. + with_box_refine (bool): whether to use box refinement. Default: False. + as_two_stage (bool): whether to use two-stage. Default: False. + select_box_nums_for_evaluation (int): the number of topk candidates + slected at postprocess for evaluation. Default: 100. + + """ + + def __init__( + self, + instance_on: bool = True, + semantic_on: bool = False, + panoptic_on: bool = False, + freeze_detr=False, + input_shapes=[], + mask_in_features=[], + mask_encode_level=0, + stuff_dataset_learn_thing: bool = True, + stuff_prob_thing: float = -1.0, + name_prompt_fusion_type: str = "none", + name_prompt_fusion_text: bool = None, + test_mask_on: bool = True, + semantic_post_nms: bool = True, + panoptic_post_nms: bool = True, + aux_mask: bool = False, + panoptic_configs: dict = { + "prob": 0.1, + "pano_temp": 0.06, + "transform_eval": True, + "object_mask_threshold": 0.01, + "overlap_threshold": 0.4, + }, + **kwargs, + ): + super().__init__(**kwargs) + + self.instance_on = instance_on + self.semantic_on = semantic_on + self.panoptic_on = panoptic_on + + if freeze_detr: + for p in self.parameters(): + p.requires_grad_(False) + + self.input_shapes = input_shapes + self.mask_in_features = mask_in_features + self.mask_encode_level = mask_encode_level + + hidden_dim = self.transformer.embed_dim + norm = "GN" + use_bias = False + + assert len(self.mask_in_features) == 1 + in_channels = [self.input_shapes[feat_name].channels for feat_name in self.mask_in_features] + in_channel = in_channels[0] + + self.lateral_conv = Conv2d( + in_channel, + hidden_dim, + kernel_size=1, + stride=1, + bias=use_bias, + padding=0, + norm=get_norm(norm, hidden_dim), + ) + self.output_conv = Conv2d( + hidden_dim, + hidden_dim, + kernel_size=3, + stride=1, + bias=use_bias, + padding=1, + norm=get_norm(norm, hidden_dim), + activation=F.relu, + ) + self.mask_conv = Conv2d( + hidden_dim, hidden_dim, kernel_size=1, stride=1, bias=use_bias, padding=0 + ) + + self.mask_embed = MLP(hidden_dim, hidden_dim, hidden_dim, 3) + self.aux_mask = aux_mask + if self.aux_mask: + self.mask_embed = nn.ModuleList( + [copy.deepcopy(self.mask_embed) for i in range(len(self.class_embed) - 1)] + ) + + weight_init.c2_xavier_fill(self.lateral_conv) + weight_init.c2_xavier_fill(self.output_conv) + weight_init.c2_xavier_fill(self.mask_conv) + + self.stuff_dataset_learn_thing = stuff_dataset_learn_thing + self.stuff_prob_thing = stuff_prob_thing + self.test_mask_on = test_mask_on + self.semantic_post_nms = semantic_post_nms + self.panoptic_post_nms = panoptic_post_nms + self.panoptic_configs = panoptic_configs + + self.name_prompt_fusion_type = name_prompt_fusion_type + self.name_prompt_fusion_text = name_prompt_fusion_text + if name_prompt_fusion_type == "learnable": + self.name_prompt_fusion_feature = nn.Parameter( + torch.Tensor(1, 1, self.embed_dim_language) + ) + nn.init.normal_(self.name_prompt_fusion_feature) + elif name_prompt_fusion_type == "zero": + self.name_prompt_fusion_feature = nn.Parameter( + torch.zeros(1, 1, self.embed_dim_language), requires_grad=False + ) + else: + self.name_prompt_fusion_feature = None + + def forward(self, batched_inputs, do_postprocess=True): + if self.training: + if "dataset_id" in batched_inputs[0]: + dataset_ids = [x["dataset_id"] for x in batched_inputs] + assert len(set(dataset_ids)) == 1, dataset_ids + dataset_id = dataset_ids[0] + else: + dataset_id = 0 + else: + dataset_id = self.eval_dataset_id + + if dataset_id >= 0: + prompt = self.dataset_prompts[dataset_id] + elif "prompt" in batched_inputs[0]: + prompt = batched_inputs[0]["prompt"] + else: + prompt = "name" + + if prompt == "expression": + for x in batched_inputs: + if isinstance(x["expressions"], List): + pass + else: + x["expressions"] = [x["expressions"]] + assert all([len(xx) > 0 for xx in x["expressions"]]) + assert all([isinstance(xx, str) for xx in x["expressions"]]) + self.test_topk_per_image = 1 + else: + self.test_topk_per_image = self.select_box_nums_for_evaluation + if self.select_box_nums_for_evaluation_list is not None: + self.test_topk_per_image = self.select_box_nums_for_evaluation_list[dataset_id] + + if self.training and prompt == "phrase": + gt_num = torch.tensor([len(input["instances"]) for input in batched_inputs]).to( + self.device + ) + gt_classes = torch.arange(gt_num.sum()).to(self.device) + gt_cumsum = torch.cumsum(gt_num, dim=0).to(self.device) + for i, input in enumerate(batched_inputs): + if i == 0: + input["instances"].gt_classes = gt_classes[: gt_cumsum[i]] + else: + input["instances"].gt_classes = gt_classes[gt_cumsum[i - 1] : gt_cumsum[i]] + if self.training and prompt == "expression": + gt_num = torch.tensor([len(input["instances"]) for input in batched_inputs]).to( + self.device + ) + gt_classes = torch.arange(gt_num.sum()).to(self.device) + gt_cumsum = torch.cumsum(gt_num, dim=0).to(self.device) + for i, input in enumerate(batched_inputs): + if i == 0: + input["instances"].gt_classes = gt_classes[: gt_cumsum[i]] + else: + input["instances"].gt_classes = gt_classes[gt_cumsum[i - 1] : gt_cumsum[i]] + + if not self.expression_cumulative_gt_class: + input["instances"].gt_classes *= 0 + + if prompt == "text": + texts = [x["text_prompt"] for x in batched_inputs] + text_promp_text_list = [x.strip() for x in ",".join(texts).split(",")] + text_promp_text_list = [x for x in text_promp_text_list if len(x) > 0] + + if any([True if x.count(" ") >= 1 else False for x in text_promp_text_list]): + prompt = "phrase" + else: + prompt = "name" + else: + text_promp_text_list = None + + if prompt == "name": + if text_promp_text_list: + text_list = text_promp_text_list + cache = False + elif dataset_id >= 0: + text_list = get_text_list( + self.metadata_list[dataset_id], self.dataset_entities[dataset_id] + ) + cache = True + else: + text_list = [] + for metadata, dataset_entity in zip(self.metadata_list, self.dataset_entities): + text_list += get_text_list(metadata, dataset_entity) + text_list = text_list[:1203+365+601] + text_list = text_list[:1203] + cache = True + + # from detectron2.data.catalog import MetadataCatalog + # metadata = MetadataCatalog.get("coco_2017_train_panoptic_separated") + # text_list = get_text_list(metadata, "thing+stuff") + + outputs_l = self.model_language.forward_text(text_list, cache=cache) + if "last_hidden_state_eot" in outputs_l: + features_l = outputs_l["last_hidden_state_eot"] + else: + features_l = text_utils.reduce_language_feature( + outputs_l["last_hidden_state"], + outputs_l["attention_mask"], + reduce_type=self.text_feature_reduce_type, + ) + attention_mask_l = None + + if ( + dataset_id >= 0 + and self.dataset_entities[dataset_id] == "stuff" + and self.metadata_list[dataset_id].get("stuff_classes")[0] == "things" + and not self.stuff_dataset_learn_thing + ): + features_l[0, :] *= 0 + if self.training: + for i, input in enumerate(batched_inputs): + input["instances"] = input["instances"][input["instances"].gt_classes > 0] + + if self.text_feature_batch_repeat or True: + features_l = features_l.unsqueeze(0).repeat(len(batched_inputs), 1, 1) + else: + features_l = features_l.unsqueeze(1) + + elif prompt == "phrase" or prompt == "expression": + if text_promp_text_list: + text_list = text_promp_text_list + elif prompt == "phrase": + text_list = [phrase for x in batched_inputs for phrase in x["instances"].phrases] + elif prompt == "expression": + text_list = [xx for x in batched_inputs for xx in x["expressions"]] + + outputs_l = self.model_language.forward_text(text_list) + + if self.text_feature_reduce_before_fusion: + if "last_hidden_state_eot" in outputs_l: + features_l = outputs_l["last_hidden_state_eot"] + else: + features_l = text_utils.reduce_language_feature( + outputs_l["last_hidden_state"], + outputs_l["attention_mask"], + reduce_type=self.text_feature_reduce_type, + ) + attention_mask_l = None + + if ( + self.text_feature_bank + and not self.text_feature_bank_reset + and dataset_id >= 0 + and dataset_id < len(self.metadata_list) + ): + features_l = torch.cat( + [features_l, self.features_phrase_bank[dataset_id]], dim=0 + ) + features_l = features_l[ + : max(len(text_list), self.criterion[dataset_id].num_classes) + ] + self.features_phrase_bank[ + dataset_id, : self.criterion[dataset_id].num_classes + ] = features_l[: self.criterion[dataset_id].num_classes] + elif self.text_feature_bank and self.text_feature_bank_reset: + features_l = torch.cat( + [features_l, self.features_phrase_bank[dataset_id] * 0], dim=0 + ) + features_l = features_l[ + : max(len(text_list), self.criterion[dataset_id].num_classes) + ] + + if self.text_feature_batch_repeat: + features_l = features_l.unsqueeze(0).repeat(len(batched_inputs), 1, 1) + else: + features_l = features_l.unsqueeze(1) + else: + features_l = outputs_l["last_hidden_state"] + attention_mask_l = outputs_l["attention_mask"] + + start_time = time.perf_counter() + images = self.preprocess_image(batched_inputs) + + batch_size, _, H, W = images.tensor.shape + img_masks = images.tensor.new_ones(batch_size, H, W) + for image_id, image_size in enumerate(images.image_sizes): + img_masks[image_id, : image_size[0], : image_size[1]] = 0 + self.preprocess_time = time.perf_counter() - start_time + + start_time = time.perf_counter() + features = self.backbone(images.tensor) # output feature dict + self.backbone_time = time.perf_counter() - start_time + + if self.neck is not None: + multi_level_feats = self.neck({f: features[f] for f in self.neck.in_features}) + else: + multi_level_feats = [feat for feat_name, feat in features.items()] + multi_level_masks = [] + multi_level_position_embeddings = [] + spatial_shapes = [] + for feat in multi_level_feats: + multi_level_masks.append( + F.interpolate(img_masks[None], size=feat.shape[-2:]).to(torch.bool).squeeze(0) + ) + multi_level_position_embeddings.append( + self.position_embedding(multi_level_masks[-1]).to(images.tensor.dtype) + ) + + bs, c, h, w = feat.shape + spatial_shape = (h, w) + spatial_shapes.append(spatial_shape) + + query_embeds = None + if not self.as_two_stage: + query_embeds = self.query_embedding.weight + + start_time = time.perf_counter() + ( + inter_states, + init_reference, + inter_references, + enc_outputs_class, + enc_outputs_coord_unact, + anchors, + memory, + ) = self.transformer( + multi_level_feats, multi_level_masks, multi_level_position_embeddings, query_embeds + ) + self.transformer_time = time.perf_counter() - start_time + + mask_features = self.maskdino_mask_features(memory, features, multi_level_masks) + + outputs_classes = [] + outputs_coords = [] + outputs_masks = [] + for lvl in range(inter_states.shape[0]): + if lvl == 0: + reference = init_reference + else: + reference = inter_references[lvl - 1] + reference = inverse_sigmoid(reference) + if prompt == "name": + outputs_class = self.class_embed[lvl](inter_states[lvl], features_l) + elif prompt == "phrase" or prompt == "expression": + outputs_class = self.class_embed[lvl](inter_states[lvl], features_l) + else: + outputs_class = self.class_embed[lvl](inter_states[lvl]) + tmp = self.bbox_embed[lvl](inter_states[lvl]) + if reference.shape[-1] == 4: + tmp += reference + else: + assert reference.shape[-1] == 2 + tmp[..., :2] += reference + outputs_coord = tmp.sigmoid() + outputs_classes.append(outputs_class) + outputs_coords.append(outputs_coord) + + if self.aux_mask: + mask_embeds = self.mask_embed[lvl](inter_states[lvl]) + else: + mask_embeds = self.mask_embed(inter_states[lvl]) + outputs_mask = torch.einsum("bqc,bchw->bqhw", mask_embeds, mask_features) + outputs_masks.append(outputs_mask) + outputs_class = torch.stack(outputs_classes) + outputs_coord = torch.stack(outputs_coords) + outputs_mask = outputs_masks + outputs_mask[-1] += 0.0 * sum(outputs_mask) + + output = { + "pred_logits": outputs_class[-1], + "pred_boxes": outputs_coord[-1], + "pred_masks": outputs_mask[-1], + "init_reference": init_reference, + } + if self.aux_loss: + output["aux_outputs"] = self._set_aux_loss( + outputs_class, + outputs_coord, + outputs_mask, + ) + + if self.as_two_stage: + enc_outputs_coord = enc_outputs_coord_unact.sigmoid() + output["enc_outputs"] = { + "pred_logits": enc_outputs_class, + "pred_boxes": enc_outputs_coord, + "anchors": anchors, + "spatial_shapes": spatial_shapes, + "image_tensor_size": images.tensor.size()[2:], + } + + if ( + self.vis_period > 0 + and self.training + and get_event_storage().iter % self.vis_period == self.vis_period - 1 + ): + self.visualize_training(batched_inputs, output, images, dataset_id) + self.visualize_training_enc_output(batched_inputs, output, images, dataset_id) + self.visualize_training_enc_output_nonms(batched_inputs, output, images, dataset_id) + self.visualize_training_init_reference(batched_inputs, output, images, dataset_id) + + if self.training: + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + targets = self.prepare_targets(gt_instances) + + if ( + self.vis_period > 0 + and self.training + and get_event_storage().iter % self.vis_period == self.vis_period - 1 + ): + enc_outputs = output["enc_outputs"] + bin_targets = copy.deepcopy(targets) + for bt in bin_targets: + bt["labels"] = torch.zeros_like(bt["labels"]) + if self.criterion[dataset_id].matcher_stage1 is not None: + tmp1 = self.criterion[dataset_id].matcher_stage1.positive_fraction + tmp2 = self.criterion[dataset_id].matcher_stage1.batch_size_per_image + self.criterion[dataset_id].matcher_stage1.positive_fraction = 1.0 + self.criterion[dataset_id].matcher_stage1.batch_size_per_image = 5120000 + indices, ious = self.criterion[dataset_id].matcher_stage1( + enc_outputs, bin_targets, return_cost_matrix=True + ) + self.criterion[dataset_id].matcher_stage1.positive_fraction = tmp1 + self.criterion[dataset_id].matcher_stage1.batch_size_per_image = tmp2 + + self.visualize_training_enc_output_pos( + batched_inputs, output, images, dataset_id, indices + ) + + if self.criterion[dataset_id].matcher_stage2 is not None: + indices = self.criterion[dataset_id].matcher_stage2(output, targets) + + self.visualize_training_init_reference_pos( + batched_inputs, output, images, dataset_id, indices + ) + + loss_dict = self.criterion[dataset_id](output, targets) + + weight_dict = self.criterion[dataset_id].weight_dict + for k in loss_dict.keys(): + if k in weight_dict: + loss_dict[k] *= weight_dict[k] + return loss_dict + else: + + box_cls = output["pred_logits"] + box_pred = output["pred_boxes"] + mask_pred = output["pred_masks"] + + start_time = time.perf_counter() + + iter_func = retry_if_cuda_oom(F.interpolate) + mask_pred = iter_func( + mask_pred, size=images.tensor.size()[2:], mode="bilinear", align_corners=False + ) + + merged_results = [{} for _ in range(box_cls.size(0))] + if self.instance_on and not ( + self.eval_dataset_entity and "thing" not in self.eval_dataset_entity + ): + if dataset_id >= 0 and dataset_id < len(self.metadata_list): + if is_thing_stuff_overlap(self.metadata_list[dataset_id]): + thing_id = self.metadata_list[ + dataset_id + ].thing_dataset_id_to_contiguous_id.values() + thing_id = torch.Tensor(list(thing_id)).to(torch.long).to(self.device) + + detector_box_cls = torch.zeros_like(box_cls) + detector_box_cls += float("-inf") + detector_box_cls[..., thing_id] = box_cls[..., thing_id] + else: + num_thing_classes = len(self.metadata_list[dataset_id].thing_classes) + detector_box_cls = box_cls[..., :num_thing_classes] + else: + detector_box_cls = box_cls + + use_sigmoid = True + detector_results, filter_inds = self.inference( + detector_box_cls, box_pred, images.image_sizes, use_sigmoid=use_sigmoid + ) + + if self.test_mask_on: + detector_mask_preds = [ + x[filter_ind] for x, filter_ind in zip(mask_pred, filter_inds) + ] + + for result, box_mask in zip(detector_results, detector_mask_preds): + box_mask = box_mask.sigmoid() > 0.5 + box_mask = BitMasks(box_mask).crop_and_resize( + result.pred_boxes.tensor.to(box_mask.device), 128 + ) + result.pred_masks = ( + box_mask.to(result.pred_boxes.tensor.device) + .unsqueeze(1) + .to(dtype=torch.float32) + ) + + if do_postprocess: + assert ( + not torch.jit.is_scripting() + ), "Scripting is not supported for postprocess." + detector_results = DeformableDETRSegm._postprocess_instance( + detector_results, batched_inputs, images.image_sizes + ) + for merged_result, detector_result in zip(merged_results, detector_results): + merged_result.update(detector_result) + + else: + detector_results = None + + if self.semantic_on and not ( + self.eval_dataset_entity and "stuff" not in self.eval_dataset_entity + ): + + semantic_mask_pred = mask_pred.clone() + semantic_box_cls = get_stuff_score( + box_cls, self.metadata_list[dataset_id], self.dataset_entities[dataset_id] + ) + + if self.semantic_post_nms: + _, filter_inds = self.inference(semantic_box_cls, box_pred, images.image_sizes) + semantic_box_cls = torch.stack( + [x[filter_ind] for x, filter_ind in zip(semantic_box_cls, filter_inds)], + dim=0, + ) + semantic_mask_pred = torch.stack( + [x[filter_ind] for x, filter_ind in zip(semantic_mask_pred, filter_inds)], + dim=0, + ) + + if do_postprocess: + assert ( + not torch.jit.is_scripting() + ), "Scripting is not supported for postprocess." + semantic_results = DeformableDETRSegm._postprocess_semantic( + semantic_box_cls, semantic_mask_pred, batched_inputs, images + ) + if ( + dataset_id >= 0 + and self.dataset_entities[dataset_id] == "stuff" + and self.metadata_list[dataset_id].get("stuff_classes")[0] == "things" + and self.stuff_prob_thing > 0 + ): + for semantic_result in semantic_results: + semantic_result["sem_seg"][0, ...] = math.log( + self.stuff_prob_thing / (1 - self.stuff_prob_thing) + ) + for merged_result, semantic_result in zip(merged_results, semantic_results): + merged_result.update(semantic_result) + + else: + semantic_results = None + + if self.panoptic_on and not ( + self.eval_dataset_entity and "thing+stuff" not in self.eval_dataset_entity + ): + assert dataset_id >= 0 and dataset_id < len(self.metadata_list) + if do_postprocess: + assert ( + not torch.jit.is_scripting() + ), "Scripting is not supported for postprocess." + if True: + if self.panoptic_post_nms: + _, filter_inds = self.inference(box_cls, box_pred, images.image_sizes) + panoptic_mask_pred = [ + x[filter_ind] for x, filter_ind in zip(mask_pred, filter_inds) + ] + panoptic_box_cls = [ + x[filter_ind] for x, filter_ind in zip(box_cls, filter_inds) + ] + + panoptic_results = DeformableDETRSegm._postprocess_panoptic( + panoptic_box_cls, + panoptic_mask_pred, + batched_inputs, + images, + self.metadata_list[dataset_id], + self.panoptic_configs, + ) + else: + panoptic_results = [] + self.combine_overlap_thresh = 0.5 + self.combine_stuff_area_thresh = 4096 + self.combine_instances_score_thresh = 0.5 + for detector_result, semantic_result in zip( + detector_results, semantic_results + ): + detector_r = detector_result["instances"] + sem_seg_r = semantic_result["sem_seg"] + panoptic_r = combine_semantic_and_instance_outputs( + detector_r, + sem_seg_r.argmax(dim=0), + self.combine_overlap_thresh, + self.combine_stuff_area_thresh, + self.combine_instances_score_thresh, + ) + panoptic_results.append({"panoptic_seg": panoptic_r}) + for merged_result, panoptic_result in zip(merged_results, panoptic_results): + merged_result.update(panoptic_result) + + else: + panoptic_results = None + + self.postprocess_time = time.perf_counter() - start_time + + if do_postprocess: + return merged_results + + return detector_results, semantic_results, panoptic_results + + def maskdino_mask_features(self, encode_feats, multi_level_feats, multi_level_masks): + start_idx = sum( + [mask.shape[1] * mask.shape[2] for mask in multi_level_masks[: self.mask_encode_level]] + ) + end_idx = sum( + [ + mask.shape[1] * mask.shape[2] + for mask in multi_level_masks[: self.mask_encode_level + 1] + ] + ) + b, h, w = multi_level_masks[self.mask_encode_level].size() + + encode_feats = encode_feats[:, start_idx:end_idx, :] + encode_feats = encode_feats.permute(0, 2, 1).reshape(b, -1, h, w) + + x = [multi_level_feats[f] for f in self.mask_in_features] + x = x[0] + x = self.lateral_conv(x) + x = x + F.interpolate(encode_feats, size=x.shape[-2:], mode="bilinear", align_corners=False) + x = self.output_conv(x) + mask_features = self.mask_conv(x) + + return mask_features + + @torch.jit.unused + def _set_aux_loss(self, outputs_class, outputs_coord, outputs_mask): + return [ + {"pred_logits": a, "pred_boxes": b, "pred_masks": c} + for a, b, c in zip(outputs_class[:-1], outputs_coord[:-1], outputs_mask[:-1]) + ] + + def inference(self, box_cls, box_pred, image_sizes, use_sigmoid=True): + """ + Arguments: + box_cls (Tensor): tensor of shape (batch_size, num_queries, K). + The tensor predicts the classification probability for each query. + box_pred (Tensor): tensors of shape (batch_size, num_queries, 4). + The tensor predicts 4-vector (x,y,w,h) box + regression values for every queryx + image_sizes (List[torch.Size]): the input image sizes + + Returns: + results (List[Instances]): a list of #images elements. + """ + + if True: + + if use_sigmoid: + scores = torch.cat( + ( + box_cls.sigmoid(), + torch.zeros((box_cls.size(0), box_cls.size(1), 1), device=self.device), + ), + dim=2, + ) + else: + scores = torch.cat( + ( + box_cls, + torch.zeros((box_cls.size(0), box_cls.size(1), 1), device=self.device), + ), + dim=2, + ) + + boxes = box_cxcywh_to_xyxy(box_pred) + + img_h = torch.tensor([image_size[0] for image_size in image_sizes], device=self.device) + img_w = torch.tensor([image_size[1] for image_size in image_sizes], device=self.device) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) + boxes = boxes * scale_fct[:, None, :] + + boxes = boxes.unbind(0) + scores = scores.unbind(0) + image_shapes = image_sizes + + results, filter_inds = fast_rcnn_inference( + boxes, + scores, + image_shapes, + self.test_score_thresh, + self.test_nms_thresh, + self.test_topk_per_image, + ) + + return results, filter_inds + + assert len(box_cls) == len(image_sizes) + results = [] + + prob = box_cls.sigmoid() + topk_values, topk_indexes = torch.topk( + prob.view(box_cls.shape[0], -1), self.select_box_nums_for_evaluation, dim=1 + ) + scores = topk_values + topk_boxes = torch.div(topk_indexes, box_cls.shape[2], rounding_mode="floor") + labels = topk_indexes % box_cls.shape[2] + + boxes = torch.gather(box_pred, 1, topk_boxes.unsqueeze(-1).repeat(1, 1, 4)) + + for i, (scores_per_image, labels_per_image, box_pred_per_image, image_size) in enumerate( + zip(scores, labels, boxes, image_sizes) + ): + result = Instances(image_size) + result.pred_boxes = Boxes(box_cxcywh_to_xyxy(box_pred_per_image)) + result.pred_boxes.scale(scale_x=image_size[1], scale_y=image_size[0]) + result.scores = scores_per_image + result.pred_classes = labels_per_image + results.append(result) + return results, topk_indexes + + def prepare_targets(self, targets): + new_targets = [] + for targets_per_image in targets: + h, w = targets_per_image.image_size + image_size_xyxy = torch.as_tensor([w, h, w, h], dtype=torch.float, device=self.device) + gt_classes = targets_per_image.gt_classes + gt_boxes = targets_per_image.gt_boxes.tensor / image_size_xyxy + gt_boxes = box_xyxy_to_cxcywh(gt_boxes) + + if not targets_per_image.has("gt_masks"): + gt_masks = torch.zeros((0, h, w), dtype=torch.bool) + else: + gt_masks = targets_per_image.gt_masks + + if not isinstance(gt_masks, torch.Tensor): + if isinstance(gt_masks, BitMasks): + gt_masks = gt_masks.tensor + else: + gt_masks = BitMasks.from_polygon_masks(gt_masks, h, w).tensor + + gt_masks = self._move_to_current_device(gt_masks) + gt_masks = ImageList.from_tensors( + [gt_masks], + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ).tensor.squeeze(0) + + new_targets.append({"labels": gt_classes, "boxes": gt_boxes, "masks": gt_masks}) + + if targets_per_image.has("is_thing"): + new_targets[-1]["is_thing"] = targets_per_image.is_thing + + return new_targets + + def preprocess_image(self, batched_inputs): + images = [self._move_to_current_device(x["image"]) for x in batched_inputs] + images = [x.to(self.pixel_mean.dtype) for x in images] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors( + images, + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ) + return images + + @staticmethod + def _postprocess_instance( + instances, batched_inputs: List[Dict[str, torch.Tensor]], image_sizes + ): + """ + Rescale the output instances to the target size. + """ + processed_results = [] + for results_per_image, input_per_image, image_size in zip( + instances, batched_inputs, image_sizes + ): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + r = detector_postprocess(results_per_image, height, width) + processed_results.append({"instances": r.to("cpu")}) + return processed_results + + @staticmethod + def _postprocess_semantic( + mask_clses, + mask_preds, + batched_inputs: List[Dict[str, torch.Tensor]], + images, + pano_temp=0.06, + transform_eval=True, + ): + processed_results = [] + for mask_cls, mask_pred, input_per_image, image_size in zip( + mask_clses, mask_preds, batched_inputs, images.image_sizes + ): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + + T = pano_temp + mask_cls = mask_cls.sigmoid() + + if transform_eval: + mask_cls = F.softmax(mask_cls / T, dim=-1) # already sigmoid + mask_pred = mask_pred.sigmoid() + if mask_cls.size(1) > 1000: + mask_cls = mask_cls.cpu() + mask_pred = mask_pred.cpu() + result = torch.einsum("qc,qhw->chw", mask_cls, mask_pred) + + r = sem_seg_postprocess(result, image_size, height, width) + processed_results.append({"sem_seg": r}) + return processed_results + + @staticmethod + def _postprocess_panoptic( + mask_clses, + mask_preds, + batched_inputs: List[Dict[str, torch.Tensor]], + images, + metadata, + panoptic_configs, + ): + prob = panoptic_configs["prob"] + pano_temp = panoptic_configs["pano_temp"] + transform_eval = panoptic_configs["transform_eval"] + object_mask_threshold = panoptic_configs["object_mask_threshold"] + overlap_threshold = panoptic_configs["overlap_threshold"] + + processed_results = [] + for mask_cls, mask_pred, input_per_image, image_size in zip( + mask_clses, mask_preds, batched_inputs, images.image_sizes + ): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + + mask_pred = sem_seg_postprocess(mask_pred, image_size, height, width) + + T = pano_temp + scores, labels = mask_cls.sigmoid().max(-1) + mask_pred = mask_pred.sigmoid() + keep = scores > object_mask_threshold + if transform_eval: + scores, labels = F.softmax(mask_cls.sigmoid() / T, dim=-1).max(-1) + cur_scores = scores[keep] + cur_classes = labels[keep] + cur_masks = mask_pred[keep] + cur_prob_masks = cur_scores.view(-1, 1, 1) * cur_masks + + panoptic_seg = torch.zeros((height, width), dtype=torch.int32, device=cur_masks.device) + segments_info = [] + + current_segment_id = 0 + + if cur_masks.size(0) > 0: + + cur_mask_ids = cur_prob_masks.argmax(0) + + stuff_memory_list = {} + for k in range(cur_classes.shape[0]): + pred_class = cur_classes[k].item() + isthing = pred_class in metadata.thing_dataset_id_to_contiguous_id.values() + mask_area = (cur_mask_ids == k).sum().item() + original_area = (cur_masks[k] >= prob).sum().item() + mask = (cur_mask_ids == k) & (cur_masks[k] >= prob) + + if mask_area > 0 and original_area > 0 and mask.sum().item() > 0: + if mask_area / original_area < overlap_threshold: + continue + + if not isthing: + if int(pred_class) in stuff_memory_list.keys(): + panoptic_seg[mask] = stuff_memory_list[int(pred_class)] + continue + else: + stuff_memory_list[int(pred_class)] = current_segment_id + 1 + + current_segment_id += 1 + panoptic_seg[mask] = current_segment_id + + if not isthing and metadata.get("stuff_classes")[0] == "things": + pred_class = int(pred_class) - len(metadata.thing_classes) + 1 + + segments_info.append( + { + "id": current_segment_id, + "isthing": bool(isthing), + "category_id": int(pred_class), + } + ) + + processed_results.append({"panoptic_seg": (panoptic_seg, segments_info)}) + return processed_results + + @torch.no_grad() + def visualize_training( + self, batched_inputs, output, images, dataset_id, suffix="", do_nms=True + ): + if self.output_dir is None: + return + if self.training: + storage = get_event_storage() + os.makedirs(self.output_dir + "/training", exist_ok=True) + else: + os.makedirs(self.output_dir + "/inference", exist_ok=True) + + pred_logits = output["pred_logits"] + pred_boxes = output["pred_boxes"] + pred_masks = output["pred_masks"] + + thing_classes = self.metadata_list[dataset_id].get("thing_classes", []) + stuff_classes = self.metadata_list[dataset_id].get("stuff_classes", []) + if len(thing_classes) > 0 and len(stuff_classes) > 0 and stuff_classes[0] == "things": + stuff_classes = stuff_classes[1:] + if is_thing_stuff_overlap(self.metadata_list[dataset_id]): + class_names = ( + thing_classes if len(thing_classes) > len(stuff_classes) else stuff_classes + ) + else: + class_names = thing_classes + stuff_classes + + if "instances" in batched_inputs[0] and batched_inputs[0]["instances"].has("phrases"): + class_names = [phrase for x in batched_inputs for phrase in x["instances"].phrases] + [ + "unknown" + ] * 1000 + if "expressions" in batched_inputs[0] and self.expression_cumulative_gt_class: + class_names = [x["expressions"] for x in batched_inputs] + ["unknown"] * 1000 + + num_thing_classes = len(class_names) + pred_logits = pred_logits[..., :num_thing_classes] + + if pred_masks is not None: + pred_masks = [ + F.interpolate( + pred_mask.float().cpu().unsqueeze(0), + size=images.tensor.size()[2:], + mode="bilinear", + align_corners=False, + ).squeeze(0) + if pred_mask.size(0) > 0 + else pred_mask + for pred_mask in pred_masks + ] + else: + pred_masks = [ + torch.zeros(pred_box.size(0), image_size[0], image_size[1]) + for pred_box, image_size in zip(pred_boxes, images.image_sizes) + ] + + if do_nms: + results, filter_inds = self.inference(pred_logits, pred_boxes, images.image_sizes) + pred_masks = [ + pred_mask[filter_ind.cpu()] + for pred_mask, filter_ind in zip(pred_masks, filter_inds) + ] + for result, pred_mask in zip(results, pred_masks): + result.pred_masks = pred_mask.sigmoid() > 0.5 + else: + results = [] + for pred_logit, pred_box, pred_mask, image_size in zip( + pred_logits, pred_boxes, pred_masks, images.image_sizes + ): + result = Instances(image_size) + result.pred_boxes = Boxes(pred_box) + result.scores = pred_logit[:, 0] + result.pred_classes = torch.zeros( + len(pred_box), dtype=torch.int64, device=pred_logit.device + ) + result.pred_masks = pred_mask.sigmoid() > 0.5 + + results.append(result) + + from detectron2.utils.visualizer import Visualizer + + for input, result in zip(batched_inputs, results): + + if "expressions" in batched_inputs[0] and not self.expression_cumulative_gt_class: + class_names = [input["expressions"]] + ["unknown"] * 1000 + + img = input["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format) + v_gt = Visualizer(img, None) + + if "instances" in input: + labels = [ + "{}".format(class_names[gt_class]) for gt_class in input["instances"].gt_classes + ] + v_gt = v_gt.overlay_instances( + boxes=input["instances"].gt_boxes, + masks=input["instances"].gt_masks + if input["instances"].has("gt_masks") + else None, + labels=labels, + ) + else: + v_gt = v_gt.output + anno_img = v_gt.get_image() + + labels = [ + "{}_{:.0f}%".format(class_names[pred_class], score * 100) + for pred_class, score in zip(result.pred_classes.cpu(), result.scores.cpu()) + ] + v_pred = Visualizer(img, None) + v_pred = v_pred.overlay_instances( + boxes=result.pred_boxes.tensor.clone().detach().cpu().numpy(), + labels=labels, + masks=result.pred_masks[:, : img.shape[0], : img.shape[1]] + .clone() + .detach() + .cpu() + .numpy() + if result.has("pred_masks") + else None, + ) + pred_img = v_pred.get_image() + + vis_img = np.concatenate((anno_img, pred_img), axis=1) + + if result.has("pred_texts"): + labels = [ + "{}".format(text) for text, score in zip(result.pred_texts, result.scores.cpu()) + ] + v_pred = Visualizer(img, None) + v_pred = v_pred.overlay_instances( + boxes=result.pred_boxes.tensor.clone().detach().cpu().numpy(), + labels=labels, + masks=result.pred_masks.clone().detach().cpu().numpy(), + ) + pred_img = v_pred.get_image() + vis_img = np.concatenate((vis_img, pred_img), axis=1) + + basename = os.path.basename(input["file_name"]) + if self.training: + cv2.imwrite( + os.path.join( + self.output_dir, "training", str(storage.iter) + suffix + "_" + basename + ), + vis_img[:, :, ::-1], + ) + else: + cv2.imwrite( + os.path.join(self.output_dir, "inference", suffix + basename), + vis_img[:, :, ::-1], + ) + + @torch.no_grad() + def visualize_inference_panoptic(self, batched_inputs, results, dataset_id): + if self.output_dir is None: + return + if self.training: + storage = get_event_storage() + os.makedirs(self.output_dir + "/training", exist_ok=True) + else: + os.makedirs(self.output_dir + "/inference", exist_ok=True) + + from detectron2.utils.visualizer import Visualizer + + for input, result in zip(batched_inputs, results): + + img = input["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format) + + height = input["height"] + width = input["width"] + img = cv2.resize(img, (width, height)) + + v_gt = Visualizer(img, self.metadata_list[dataset_id]) + + if "instances" in input: + labels = [ + "{}".format(class_names[gt_class]) for gt_class in input["instances"].gt_classes + ] + v_gt = v_gt.overlay_instances( + boxes=input["instances"].gt_boxes, + masks=input["instances"].gt_masks + if input["instances"].has("gt_masks") + else None, + labels=labels, + ) + else: + v_gt = v_gt.output + anno_img = v_gt.get_image() + + v_pred = Visualizer(img, self.metadata_list[dataset_id]) + + panoptic_seg, segments_info = result["panoptic_seg"] + v_pred = v_pred.draw_panoptic_seg_predictions(panoptic_seg.cpu(), segments_info) + pred_img = v_pred.get_image() + + vis_img = np.concatenate((anno_img, pred_img), axis=1) + + basename = os.path.basename(input["file_name"]) + if self.training: + cv2.imwrite( + os.path.join( + self.output_dir, "training", str(storage.iter) + "_pan_" + basename + ), + vis_img[:, :, ::-1], + ) + else: + cv2.imwrite( + os.path.join(self.output_dir, "inference", "pan_" + basename), + vis_img[:, :, ::-1], + ) + + @torch.no_grad() + def visualize_training_enc_output(self, batched_inputs, output, images, dataset_id, suffix=""): + if self.output_dir is None: + return + if self.training: + storage = get_event_storage() + os.makedirs(self.output_dir + "/training", exist_ok=True) + else: + os.makedirs(self.output_dir + "/inference", exist_ok=True) + + pred_logits = output["enc_outputs"]["pred_logits"] + pred_boxes = output["enc_outputs"]["pred_boxes"] + + results, filter_inds = self.inference(pred_logits, pred_boxes, images.image_sizes) + + from detectron2.utils.visualizer import Visualizer + + for input, result in zip(batched_inputs, results): + img = input["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format) + v_gt = Visualizer(img, None) + if "instances" in input: + v_gt = v_gt.overlay_instances( + boxes=input["instances"].gt_boxes, + ) + else: + v_gt = v_gt.output + anno_img = v_gt.get_image() + + labels = [ + "{}_{:.0f}%".format(pred_class, score * 100) + for pred_class, score in zip(result.pred_classes.cpu(), result.scores.cpu()) + ] + v_pred = Visualizer(img, None) + v_pred = v_pred.overlay_instances( + boxes=result.pred_boxes.tensor.clone().detach().cpu().numpy(), + labels=labels, + ) + pred_img = v_pred.get_image() + + vis_img = np.concatenate((anno_img, pred_img), axis=1) + + basename = os.path.basename(input["file_name"]) + if self.training: + cv2.imwrite( + os.path.join( + self.output_dir, + "training", + str(storage.iter) + suffix + "_enc_output_" + basename, + ), + vis_img[:, :, ::-1], + ) + else: + cv2.imwrite( + os.path.join(self.output_dir, "inference", suffix + "enc_output_" + basename), + vis_img[:, :, ::-1], + ) + + def visualize_training_enc_output_nonms( + self, batched_inputs, output, images, dataset_id, suffix="" + ): + if self.output_dir is None: + return + if self.training: + storage = get_event_storage() + os.makedirs(self.output_dir + "/training", exist_ok=True) + else: + os.makedirs(self.output_dir + "/inference", exist_ok=True) + + pred_logits = output["enc_outputs"]["pred_logits"] + pred_boxes = output["enc_outputs"]["pred_boxes"] + + image_sizes = images.image_sizes + pred_boxes = box_cxcywh_to_xyxy(pred_boxes) + + img_h = torch.tensor([image_size[0] for image_size in image_sizes], device=self.device) + img_w = torch.tensor([image_size[1] for image_size in image_sizes], device=self.device) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) + pred_boxes = pred_boxes * scale_fct[:, None, :] + + pred_boxes = pred_boxes.unbind(0) + pred_logits = pred_logits.unbind(0) + + from detectron2.utils.visualizer import Visualizer + + for input, pred_box, pred_logit in zip(batched_inputs, pred_boxes, pred_logits): + img = input["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format) + v_gt = Visualizer(img, None) + if "instances" in input: + v_gt = v_gt.overlay_instances( + boxes=input["instances"].gt_boxes, + ) + else: + v_gt = v_gt.output + anno_img = v_gt.get_image() + + keep = pred_logit.sigmoid() > 0.1 + if keep.sum() == 0: + continue + pred_box = pred_box[keep.squeeze()] + pred_logit = pred_logit[keep.squeeze()] + + labels = [ + "{:.0f}%".format(score * 100) for score in pred_logit.squeeze().cpu().tolist() + ] + v_pred = Visualizer(img, None) + v_pred = v_pred.overlay_instances( + boxes=pred_box.clone().detach().cpu().numpy(), + labels=labels, + ) + pred_img = v_pred.get_image() + + vis_img = np.concatenate((anno_img, pred_img), axis=1) + + basename = os.path.basename(input["file_name"]) + if self.training: + cv2.imwrite( + os.path.join( + self.output_dir, + "training", + str(storage.iter) + suffix + "_enc_output_nonms_" + basename, + ), + vis_img[:, :, ::-1], + ) + else: + cv2.imwrite( + os.path.join( + self.output_dir, "inference", suffix + "enc_output_nonms_" + basename + ), + vis_img[:, :, ::-1], + ) + + @torch.no_grad() + def visualize_training_init_reference( + self, batched_inputs, output, images, dataset_id, suffix="" + ): + if self.output_dir is None: + return + if self.training: + storage = get_event_storage() + os.makedirs(self.output_dir + "/training", exist_ok=True) + else: + os.makedirs(self.output_dir + "/inference", exist_ok=True) + + pred_boxes = output["init_reference"] + + image_sizes = images.image_sizes + pred_boxes = box_cxcywh_to_xyxy(pred_boxes) + + img_h = torch.tensor([image_size[0] for image_size in image_sizes], device=self.device) + img_w = torch.tensor([image_size[1] for image_size in image_sizes], device=self.device) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) + pred_boxes = pred_boxes * scale_fct[:, None, :] + + pred_boxes = pred_boxes.unbind(0) + + from detectron2.utils.visualizer import Visualizer + + for input, pred_box in zip(batched_inputs, pred_boxes): + img = input["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format) + v_gt = Visualizer(img, None) + if "instances" in input: + v_gt = v_gt.overlay_instances( + boxes=input["instances"].gt_boxes, + ) + else: + v_gt = v_gt.output + anno_img = v_gt.get_image() + + v_pred = Visualizer(img, None) + v_pred = v_pred.overlay_instances( + boxes=pred_box.clone().detach().cpu().numpy(), + ) + pred_img = v_pred.get_image() + + vis_img = np.concatenate((anno_img, pred_img), axis=1) + + basename = os.path.basename(input["file_name"]) + if self.training: + cv2.imwrite( + os.path.join( + self.output_dir, + "training", + str(storage.iter) + suffix + "_init_reference_" + basename, + ), + vis_img[:, :, ::-1], + ) + else: + cv2.imwrite( + os.path.join( + self.output_dir, "inference", suffix + "init_reference_" + basename + ), + vis_img[:, :, ::-1], + ) + + @torch.no_grad() + def visualize_training_enc_output_pos( + self, batched_inputs, output, images, dataset_id, indices, suffix="" + ): + if self.output_dir is None: + return + if self.training: + storage = get_event_storage() + os.makedirs(self.output_dir + "/training", exist_ok=True) + else: + os.makedirs(self.output_dir + "/inference", exist_ok=True) + + anchors = output["enc_outputs"]["anchors"] + + image_sizes = images.image_sizes + anchors = box_cxcywh_to_xyxy(anchors) + + img_h, img_w = torch.tensor(image_sizes, device=self.device).unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) + anchors = anchors * scale_fct[:, None, :] + + anchors = anchors.unbind(0) + + from detectron2.utils.visualizer import Visualizer + + for input, anchor, indice in zip(batched_inputs, anchors, indices): + img = input["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format) + v_gt = Visualizer(img, None) + if "instances" in input: + labels = ["{}".format(idx) for idx in range(len(input["instances"]))] + v_gt = v_gt.overlay_instances( + boxes=input["instances"].gt_boxes, + masks=input["instances"].gt_masks + if input["instances"].has("gt_masks") + else None, + labels=labels, + ) + else: + v_gt = v_gt.output + anno_img = v_gt.get_image() + + v_pred = Visualizer(img, None) + v_pred = v_pred.overlay_instances( + boxes=anchor.clone().detach().cpu().numpy(), + ) + pred_img = v_pred.get_image() + + anchor = anchor[indice[0], :] + labels = ["{}".format(idx) for idx in indice[1]] + v_pred = Visualizer(img, None) + v_pred = v_pred.overlay_instances( + boxes=anchor.clone().detach().cpu().numpy(), + labels=labels, + ) + pred_img2 = v_pred.get_image() + + vis_img = np.concatenate((anno_img, pred_img, pred_img2), axis=1) + + basename = os.path.basename(input["file_name"]) + if self.training: + cv2.imwrite( + os.path.join( + self.output_dir, + "training", + str(storage.iter) + suffix + "_enc_output_pos_" + basename, + ), + vis_img[:, :, ::-1], + ) + else: + cv2.imwrite( + os.path.join( + self.output_dir, "inference", suffix + "enc_output_pos_" + basename + ), + vis_img[:, :, ::-1], + ) + + @torch.no_grad() + def visualize_training_init_reference_pos( + self, batched_inputs, output, images, dataset_id, indices, suffix="" + ): + if self.output_dir is None: + return + if self.training: + storage = get_event_storage() + os.makedirs(self.output_dir + "/training", exist_ok=True) + else: + os.makedirs(self.output_dir + "/inference", exist_ok=True) + + pred_boxes = output["init_reference"] + + image_sizes = images.image_sizes + pred_boxes = box_cxcywh_to_xyxy(pred_boxes) + + img_h = torch.tensor([image_size[0] for image_size in image_sizes], device=self.device) + img_w = torch.tensor([image_size[1] for image_size in image_sizes], device=self.device) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) + pred_boxes = pred_boxes * scale_fct[:, None, :] + + pred_boxes = pred_boxes.unbind(0) + + from detectron2.utils.visualizer import Visualizer + + for input, pred_box, indice in zip(batched_inputs, pred_boxes, indices): + img = input["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format) + v_gt = Visualizer(img, None) + if "instances" in input: + labels = ["{}".format(idx) for idx in range(len(input["instances"]))] + v_gt = v_gt.overlay_instances( + boxes=input["instances"].gt_boxes, + masks=input["instances"].gt_masks + if input["instances"].has("gt_masks") + else None, + labels=labels, + ) + else: + v_gt = v_gt.output + anno_img = v_gt.get_image() + + pred_box = pred_box[indice[0]] + labels = ["{}".format(idx) for idx in indice[1]] + v_pred = Visualizer(img, None) + v_pred = v_pred.overlay_instances( + boxes=pred_box.clone().detach().cpu().numpy(), + labels=labels, + ) + pred_img = v_pred.get_image() + + vis_img = np.concatenate((anno_img, pred_img), axis=1) + + basename = os.path.basename(input["file_name"]) + if self.training: + cv2.imwrite( + os.path.join( + self.output_dir, + "training", + str(storage.iter) + suffix + "_init_reference_pos_" + basename, + ), + vis_img[:, :, ::-1], + ) + else: + cv2.imwrite( + os.path.join( + self.output_dir, "inference", suffix + "init_reference_pos_" + basename + ), + vis_img[:, :, ::-1], + ) + + def set_model_language(self, model_language): + self.model_language = model_language + + +class NMSPostProcess(nn.Module): + """This module converts the model's output into the format expected by the coco api""" + + @torch.no_grad() + def forward(self, outputs, target_sizes, select_box_nums_for_evaluation): + """Perform the computation + Parameters: + outputs: raw outputs of the model + target_sizes: tensor of dimension [batch_size x 2] containing the size of each images of the batch + For evaluation, this must be the original image size (before any data augmentation) + For visualization, this should be the image size after data augment, but before padding + """ + out_logits, out_bbox = outputs["pred_logits"], outputs["pred_boxes"] + out_mask = outputs["pred_masks"] + bs, n_queries, n_cls = out_logits.shape + + assert len(out_logits) == len(target_sizes) + assert target_sizes.shape[1] == 2 + + prob = out_logits.sigmoid() + + all_scores = prob.view(bs, n_queries * n_cls).to(out_logits.device) + all_indexes = torch.arange(n_queries * n_cls)[None].repeat(bs, 1).to(out_logits.device) + all_boxes = torch.div(all_indexes, out_logits.shape[2], rounding_mode="trunc") + all_labels = all_indexes % out_logits.shape[2] + + boxes = box_cxcywh_to_xyxy(out_bbox) + boxes = torch.gather(boxes, 1, all_boxes.unsqueeze(-1).repeat(1, 1, 4)) + + img_h, img_w = target_sizes.unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) + boxes = boxes * scale_fct[:, None, :] + + results = [] + keep_inds_all = [] + for b in range(bs): + box = boxes[b] + score = all_scores[b] + lbls = all_labels[b] + mask = out_mask[b] + + pre_topk = score.topk(10000).indices + box = box[pre_topk] + score = score[pre_topk] + lbls = lbls[pre_topk] + + keep_inds = batched_nms(box, score, lbls, 0.7)[:select_box_nums_for_evaluation] + + result = Instances(target_sizes[b]) + result.pred_boxes = Boxes(box[keep_inds]) + result.scores = score[keep_inds] + result.pred_classes = lbls[keep_inds] + results.append(result) + + keep_inds_all.append(keep_inds) + + return results, keep_inds_all + + +def is_thing_stuff_overlap(metadata): + thing_classes = metadata.get("thing_classes", []) + stuff_classes = metadata.get("stuff_classes", []) + if len(thing_classes) == 0 or len(stuff_classes) == 0: + return False + + if set(thing_classes).issubset(set(stuff_classes)) or set(stuff_classes).issubset( + set(thing_classes) + ): + return True + else: + return False + + +def get_text_list(metadata, dataset_entity): + thing_classes = metadata.get("thing_classes", []) + stuff_classes = metadata.get("stuff_classes", []) + + if dataset_entity == "thing+stuff" and stuff_classes[0] == "things": + text_list = list(thing_classes) + list(stuff_classes[1:]) + + elif dataset_entity == "thing+stuff" and is_thing_stuff_overlap(metadata): + text_list = thing_classes if len(thing_classes) > len(stuff_classes) else stuff_classes + + elif dataset_entity == "thing+stuff": + text_list = list(thing_classes) + list(stuff_classes) + + elif dataset_entity == "stuff": + text_list = list(stuff_classes) + + elif dataset_entity == "thing": + text_list = list(thing_classes) + + return text_list + + +def get_stuff_score(box_cls, metadata, dataset_entity): + thing_classes = metadata.get("thing_classes", []) + stuff_classes = metadata.get("stuff_classes", []) + + semantic_box_cls = box_cls.clone() + + if dataset_entity == "thing+stuff" and stuff_classes[0] == "things": + num_thing_classes = len(thing_classes) + + semantic_box_cls_0 = box_cls[..., :num_thing_classes] + semantic_box_cls_1 = box_cls[..., num_thing_classes:] + semantic_box_cls_0, _ = semantic_box_cls_0.min(dim=2, keepdim=True) + semantic_box_cls = torch.cat([semantic_box_cls_0, semantic_box_cls_1], dim=2) + + if dataset_entity == "thing+stuff" and is_thing_stuff_overlap(metadata): + semantic_box_cls = box_cls.clone() + + if dataset_entity == "stuff": + semantic_box_cls = box_cls.clone() + + return semantic_box_cls diff --git a/approach/ovod/APE/ape/modeling/ape_deta/deformable_detr_segm_vl.py b/approach/ovod/APE/ape/modeling/ape_deta/deformable_detr_segm_vl.py new file mode 100644 index 0000000000000000000000000000000000000000..a8572e06954b502829fdb2c8db832230cd74901c --- /dev/null +++ b/approach/ovod/APE/ape/modeling/ape_deta/deformable_detr_segm_vl.py @@ -0,0 +1,1264 @@ +import copy +import math +import os +import time +from typing import Dict, List, Optional, Tuple + +import cv2 +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +import fvcore.nn.weight_init as weight_init +from ape.modeling.text import utils as text_utils +from detectron2.data.detection_utils import convert_image_to_rgb +from detectron2.layers import Conv2d, ShapeSpec, get_norm, move_device_like +from detectron2.modeling import GeneralizedRCNN +from detectron2.modeling.meta_arch.panoptic_fpn import combine_semantic_and_instance_outputs +from detectron2.modeling.postprocessing import detector_postprocess, sem_seg_postprocess +from detectron2.structures import BitMasks, Boxes, ImageList, Instances +from detectron2.utils.events import get_event_storage +from detectron2.utils.memory import retry_if_cuda_oom +from detrex.layers import MLP, box_cxcywh_to_xyxy, box_xyxy_to_cxcywh +from detrex.utils import inverse_sigmoid +from torchvision.ops.boxes import batched_nms + +from .deformable_detr import DeformableDETR +from .fast_rcnn import fast_rcnn_inference +from .segmentation import MaskHeadSmallConv, MHAttentionMap + + +class DeformableDETRSegmVL(DeformableDETR): + """Implements the Deformable DETR model. + + Code is modified from the `official github repo + `_. + + More details can be found in the `paper + `_ . + + Args: + backbone (nn.Module): the backbone module. + position_embedding (nn.Module): the position embedding module. + neck (nn.Module): the neck module. + transformer (nn.Module): the transformer module. + embed_dim (int): the dimension of the embedding. + num_classes (int): Number of total categories. + num_queries (int): Number of proposal dynamic anchor boxes in Transformer + criterion (nn.Module): Criterion for calculating the total losses. + pixel_mean (List[float]): Pixel mean value for image normalization. + Default: [123.675, 116.280, 103.530]. + pixel_std (List[float]): Pixel std value for image normalization. + Default: [58.395, 57.120, 57.375]. + aux_loss (bool): whether to use auxiliary loss. Default: True. + with_box_refine (bool): whether to use box refinement. Default: False. + as_two_stage (bool): whether to use two-stage. Default: False. + select_box_nums_for_evaluation (int): the number of topk candidates + slected at postprocess for evaluation. Default: 100. + + """ + + def __init__( + self, + instance_on: bool = True, + semantic_on: bool = False, + panoptic_on: bool = False, + freeze_detr=False, + input_shapes=[], + mask_in_features=[], + mask_encode_level=0, + stuff_dataset_learn_thing: bool = True, + stuff_prob_thing: float = -1.0, + name_prompt_fusion_type: str = "none", + name_prompt_fusion_text: bool = None, + test_mask_on: bool = True, + semantic_post_nms: bool = True, + panoptic_post_nms: bool = True, + aux_mask: bool = False, + panoptic_configs: dict = { + "prob": 0.1, + "pano_temp": 0.06, + "transform_eval": True, + "object_mask_threshold": 0.01, + "overlap_threshold": 0.4, + }, + **kwargs, + ): + super().__init__(**kwargs) + + self.instance_on = instance_on + self.semantic_on = semantic_on + self.panoptic_on = panoptic_on + + if freeze_detr: + for p in self.parameters(): + p.requires_grad_(False) + + self.input_shapes = input_shapes + self.mask_in_features = mask_in_features + self.mask_encode_level = mask_encode_level + + hidden_dim = self.transformer.embed_dim + norm = "GN" + use_bias = False + + assert len(self.mask_in_features) == 1 + in_channels = [self.input_shapes[feat_name].channels for feat_name in self.mask_in_features] + in_channel = in_channels[0] + + self.lateral_conv = Conv2d( + in_channel, + hidden_dim, + kernel_size=1, + stride=1, + bias=use_bias, + padding=0, + norm=get_norm(norm, hidden_dim), + ) + self.output_conv = Conv2d( + hidden_dim, + hidden_dim, + kernel_size=3, + stride=1, + bias=use_bias, + padding=1, + norm=get_norm(norm, hidden_dim), + activation=F.relu, + ) + self.mask_conv = Conv2d( + hidden_dim, hidden_dim, kernel_size=1, stride=1, bias=use_bias, padding=0 + ) + + self.mask_embed = MLP(hidden_dim, hidden_dim, hidden_dim, 3) + self.aux_mask = aux_mask + if self.aux_mask: + self.mask_embed = nn.ModuleList( + [copy.deepcopy(self.mask_embed) for i in range(len(self.class_embed) - 1)] + ) + + weight_init.c2_xavier_fill(self.lateral_conv) + weight_init.c2_xavier_fill(self.output_conv) + weight_init.c2_xavier_fill(self.mask_conv) + + self.stuff_dataset_learn_thing = stuff_dataset_learn_thing + self.stuff_prob_thing = stuff_prob_thing + self.test_mask_on = test_mask_on + self.semantic_post_nms = semantic_post_nms + self.panoptic_post_nms = panoptic_post_nms + self.panoptic_configs = panoptic_configs + + self.name_prompt_fusion_type = name_prompt_fusion_type + self.name_prompt_fusion_text = name_prompt_fusion_text + if name_prompt_fusion_type == "learnable": + self.name_prompt_fusion_feature = nn.Parameter( + torch.Tensor(1, 1, self.embed_dim_language) + ) + nn.init.normal_(self.name_prompt_fusion_feature) + elif name_prompt_fusion_type == "zero": + self.name_prompt_fusion_feature = nn.Parameter( + torch.zeros(1, 1, self.embed_dim_language), requires_grad=False + ) + else: + self.name_prompt_fusion_feature = None + + def forward(self, batched_inputs, do_postprocess=True): + if self.training: + if "dataset_id" in batched_inputs[0]: + dataset_ids = [x["dataset_id"] for x in batched_inputs] + assert len(set(dataset_ids)) == 1, dataset_ids + dataset_id = dataset_ids[0] + else: + dataset_id = 0 + else: + dataset_id = self.eval_dataset_id + + if dataset_id >= 0: + prompt = self.dataset_prompts[dataset_id] + elif "prompt" in batched_inputs[0]: + prompt = batched_inputs[0]["prompt"] + else: + prompt = "name" + + if prompt == "expression": + for x in batched_inputs: + if isinstance(x["expressions"], List): + pass + else: + x["expressions"] = [x["expressions"]] + assert all([len(xx) > 0 for xx in x["expressions"]]) + assert all([isinstance(xx, str) for xx in x["expressions"]]) + self.test_topk_per_image = 1 + else: + self.test_topk_per_image = self.select_box_nums_for_evaluation + if self.select_box_nums_for_evaluation_list is not None: + self.test_topk_per_image = self.select_box_nums_for_evaluation_list[dataset_id] + + if self.training and prompt == "phrase": + gt_num = torch.tensor([len(input["instances"]) for input in batched_inputs]).to( + self.device + ) + gt_classes = torch.arange(gt_num.sum()).to(self.device) + gt_cumsum = torch.cumsum(gt_num, dim=0).to(self.device) + for i, input in enumerate(batched_inputs): + if i == 0: + input["instances"].gt_classes = gt_classes[: gt_cumsum[i]] + else: + input["instances"].gt_classes = gt_classes[gt_cumsum[i - 1] : gt_cumsum[i]] + if self.training and prompt == "expression": + gt_num = torch.tensor([len(input["instances"]) for input in batched_inputs]).to( + self.device + ) + gt_classes = torch.arange(gt_num.sum()).to(self.device) + gt_cumsum = torch.cumsum(gt_num, dim=0).to(self.device) + for i, input in enumerate(batched_inputs): + if i == 0: + input["instances"].gt_classes = gt_classes[: gt_cumsum[i]] + else: + input["instances"].gt_classes = gt_classes[gt_cumsum[i - 1] : gt_cumsum[i]] + + if not self.expression_cumulative_gt_class: + input["instances"].gt_classes *= 0 + + if prompt == "text": + texts = [x["text_prompt"] for x in batched_inputs] + text_promp_text_list = [x.strip() for x in ",".join(texts).split(",")] + text_promp_text_list = [x for x in text_promp_text_list if len(x) > 0] + + if any([True if x.count(" ") >= 1 else False for x in text_promp_text_list]): + prompt = "phrase" + else: + prompt = "name" + else: + text_promp_text_list = None + + if prompt == "name": + if text_promp_text_list: + text_list = text_promp_text_list + cache = False + elif dataset_id >= 0: + text_list = get_text_list( + self.metadata_list[dataset_id], self.dataset_entities[dataset_id] + ) + cache = True + else: + text_list = [] + for metadata, dataset_entity in zip(self.metadata_list, self.dataset_entities): + text_list += get_text_list(metadata, dataset_entity) + text_list = text_list[:1203+365+601] + text_list = text_list[:1203] + cache = True + + # from detectron2.data.catalog import MetadataCatalog + # metadata = MetadataCatalog.get("coco_2017_train_panoptic_separated") + # text_list = get_text_list(metadata, "thing+stuff") + + outputs_l = self.model_language.forward_text(text_list, cache=cache) + if "last_hidden_state_eot" in outputs_l: + features_l = outputs_l["last_hidden_state_eot"] + else: + features_l = text_utils.reduce_language_feature( + outputs_l["last_hidden_state"], + outputs_l["attention_mask"], + reduce_type=self.text_feature_reduce_type, + ) + attention_mask_l = None + + if ( + dataset_id >= 0 + and self.dataset_entities[dataset_id] == "stuff" + and self.metadata_list[dataset_id].get("stuff_classes")[0] == "things" + and not self.stuff_dataset_learn_thing + ): + features_l[0, :] *= 0 + if self.training: + for i, input in enumerate(batched_inputs): + input["instances"] = input["instances"][input["instances"].gt_classes > 0] + + if self.text_feature_batch_repeat or True: + features_l = features_l.unsqueeze(0).repeat(len(batched_inputs), 1, 1) + else: + features_l = features_l.unsqueeze(1) + + elif prompt == "phrase" or prompt == "expression": + if text_promp_text_list: + text_list = text_promp_text_list + elif prompt == "phrase": + text_list = [phrase for x in batched_inputs for phrase in x["instances"].phrases] + elif prompt == "expression": + text_list = [xx for x in batched_inputs for xx in x["expressions"]] + + outputs_l = self.model_language.forward_text(text_list) + + if self.text_feature_reduce_before_fusion: + if "last_hidden_state_eot" in outputs_l: + features_l = outputs_l["last_hidden_state_eot"] + else: + features_l = text_utils.reduce_language_feature( + outputs_l["last_hidden_state"], + outputs_l["attention_mask"], + reduce_type=self.text_feature_reduce_type, + ) + attention_mask_l = None + + if ( + self.text_feature_bank + and not self.text_feature_bank_reset + and dataset_id >= 0 + and dataset_id < len(self.metadata_list) + ): + features_l = torch.cat( + [features_l, self.features_phrase_bank[dataset_id]], dim=0 + ) + features_l = features_l[ + : max(len(text_list), self.criterion[dataset_id].num_classes) + ] + self.features_phrase_bank[ + dataset_id, : self.criterion[dataset_id].num_classes + ] = features_l[: self.criterion[dataset_id].num_classes] + elif self.text_feature_bank and self.text_feature_bank_reset: + features_l = torch.cat( + [features_l, self.features_phrase_bank[dataset_id] * 0], dim=0 + ) + features_l = features_l[ + : max(len(text_list), self.criterion[dataset_id].num_classes) + ] + + if self.text_feature_batch_repeat: + features_l = features_l.unsqueeze(0).repeat(len(batched_inputs), 1, 1) + else: + features_l = features_l.unsqueeze(1) + else: + features_l = outputs_l["last_hidden_state"] + attention_mask_l = outputs_l["attention_mask"] + + if prompt == "name": + if ( + self.name_prompt_fusion_text is not None + and self.name_prompt_fusion_text[dataset_id] + ): + features_l_fusion = features_l + else: + if self.name_prompt_fusion_feature is not None: + features_l_fusion = self.name_prompt_fusion_feature.repeat( + len(batched_inputs), 1, 1 + ) + else: + features_l_fusion = None + attention_mask_l_fusion = None + elif prompt == "phrase" or prompt == "expression": + features_l_fusion = features_l + attention_mask_l_fusion = attention_mask_l + if self.name_prompt_fusion_feature is not None: + features_l_fusion += 0.0 * self.name_prompt_fusion_feature + + start_time = time.perf_counter() + images = self.preprocess_image(batched_inputs) + + batch_size, _, H, W = images.tensor.shape + img_masks = images.tensor.new_ones(batch_size, H, W) + for image_id, image_size in enumerate(images.image_sizes): + img_masks[image_id, : image_size[0], : image_size[1]] = 0 + self.preprocess_time = time.perf_counter() - start_time + + start_time = time.perf_counter() + features = self.backbone(images.tensor) # output feature dict + self.backbone_time = time.perf_counter() - start_time + + if self.neck is not None: + multi_level_feats = self.neck({f: features[f] for f in self.neck.in_features}) + else: + multi_level_feats = [feat for feat_name, feat in features.items()] + multi_level_masks = [] + multi_level_position_embeddings = [] + spatial_shapes = [] + for feat in multi_level_feats: + multi_level_masks.append( + F.interpolate(img_masks[None], size=feat.shape[-2:]).to(torch.bool).squeeze(0) + ) + multi_level_position_embeddings.append( + self.position_embedding(multi_level_masks[-1]).to(images.tensor.dtype) + ) + + bs, c, h, w = feat.shape + spatial_shape = (h, w) + spatial_shapes.append(spatial_shape) + + if not self.training and "mask_prompt" in batched_inputs[0]: + masks_prompt = [self._move_to_current_device(x["mask_prompt"]) for x in batched_inputs] + masks_prompt = [x.to(self.pixel_mean.dtype) for x in masks_prompt] + masks_prompt = ImageList.from_tensors( + masks_prompt, + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ) + masks_prompt = masks_prompt.tensor + if masks_prompt.sum() == 0: + masks_prompt[...] = 255 + + multi_level_masks_prompt = [] + for feat in multi_level_feats: + multi_level_masks_prompt.append( + F.interpolate(masks_prompt[None], size=feat.shape[-2:], mode="bilinear") + .to(torch.bool) + .squeeze(0) + ) + for mask_prompt in multi_level_masks_prompt: + print("mask_prompt", mask_prompt.sum(), mask_prompt.size()) + else: + multi_level_masks_prompt = None + + query_embeds = None + if not self.as_two_stage: + query_embeds = self.query_embedding.weight + + start_time = time.perf_counter() + ( + inter_states, + init_reference, + inter_references, + enc_outputs_class, + enc_outputs_coord_unact, + anchors, + memory, + features_l_fusion, + ) = self.transformer( + multi_level_feats, + multi_level_masks, + multi_level_position_embeddings, + query_embeds, + features_l_fusion, + attention_mask_l_fusion, + multi_level_masks_prompt, + ) + self.transformer_time = time.perf_counter() - start_time + + mask_features = self.maskdino_mask_features(memory, features, multi_level_masks) + + if prompt == "name": + features_l = 1.0 * features_l + 0.0 * features_l_fusion + elif prompt == "phrase" or prompt == "expression": + features_l = 0.0 * features_l + 1.0 * features_l_fusion + + if not self.text_feature_reduce_before_fusion: + features_l = text_utils.reduce_language_feature( + features_l, attention_mask_l, reduce_type=self.text_feature_reduce_type + ) + attention_mask_l = None + + if self.text_feature_bank: + features_l = torch.cat( + [features_l, self.features_phrase_bank[dataset_id]], dim=0 + ) + features_l = features_l[: self.criterion[dataset_id].num_classes] + self.features_phrase_bank[ + dataset_id, : self.criterion[dataset_id].num_classes + ] = features_l + elif self.text_feature_bank and not self.training: + features_l = torch.cat( + ( + features_l, + torch.zeros( + (self.criterion[dataset_id].num_classes - 1, features_l.size(1)), + dtype=features_l.dtype, + device=self.device, + ), + ), + dim=0, + ) + + if self.text_feature_batch_repeat: + features_l = features_l.unsqueeze(0).repeat(len(batched_inputs), 1, 1) + else: + features_l = features_l.unsqueeze(1) + + outputs_classes = [] + outputs_coords = [] + outputs_masks = [] + for lvl in range(inter_states.shape[0]): + if lvl == 0: + reference = init_reference + else: + reference = inter_references[lvl - 1] + reference = inverse_sigmoid(reference) + if prompt == "name": + outputs_class = self.class_embed[lvl](inter_states[lvl], features_l) + elif prompt == "phrase" or prompt == "expression": + outputs_class = self.class_embed[lvl](inter_states[lvl], features_l) + else: + outputs_class = self.class_embed[lvl](inter_states[lvl]) + tmp = self.bbox_embed[lvl](inter_states[lvl]) + if reference.shape[-1] == 4: + tmp += reference + else: + assert reference.shape[-1] == 2 + tmp[..., :2] += reference + outputs_coord = tmp.sigmoid() + outputs_classes.append(outputs_class) + outputs_coords.append(outputs_coord) + + if self.aux_mask: + mask_embeds = self.mask_embed[lvl](inter_states[lvl]) + else: + mask_embeds = self.mask_embed(inter_states[lvl]) + outputs_mask = torch.einsum("bqc,bchw->bqhw", mask_embeds, mask_features) + outputs_masks.append(outputs_mask) + outputs_class = torch.stack(outputs_classes) + outputs_coord = torch.stack(outputs_coords) + + outputs_mask = outputs_masks + outputs_mask[-1] += 0.0 * sum(outputs_mask) + + output = { + "pred_logits": outputs_class[-1], + "pred_boxes": outputs_coord[-1], + "pred_masks": outputs_mask[-1], + "init_reference": init_reference, + } + if self.aux_loss: + output["aux_outputs"] = self._set_aux_loss( + outputs_class, + outputs_coord, + outputs_mask, + ) + + if self.as_two_stage: + enc_outputs_coord = enc_outputs_coord_unact.sigmoid() + output["enc_outputs"] = { + "pred_logits": enc_outputs_class, + "pred_boxes": enc_outputs_coord, + "anchors": anchors, + "spatial_shapes": spatial_shapes, + "image_tensor_size": images.tensor.size()[2:], + } + + if ( + self.vis_period > 0 + and self.training + and get_event_storage().iter % self.vis_period == self.vis_period - 1 + ): + self.visualize_training(batched_inputs, output, images, dataset_id) + self.visualize_training_enc_output(batched_inputs, output, images, dataset_id) + + if self.training: + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + targets = self.prepare_targets(gt_instances) + + loss_dict = self.criterion[dataset_id](output, targets) + + weight_dict = self.criterion[dataset_id].weight_dict + for k in loss_dict.keys(): + if k in weight_dict: + loss_dict[k] *= weight_dict[k] + return loss_dict + else: + + box_cls = output["pred_logits"] + box_pred = output["pred_boxes"] + mask_pred = output["pred_masks"] + + start_time = time.perf_counter() + + iter_func = retry_if_cuda_oom(F.interpolate) + mask_pred = iter_func( + mask_pred, size=images.tensor.size()[2:], mode="bilinear", align_corners=False + ) + + merged_results = [{} for _ in range(box_cls.size(0))] + if self.instance_on and not ( + self.eval_dataset_entity and "thing" not in self.eval_dataset_entity + ): + if dataset_id >= 0 and dataset_id < len(self.metadata_list): + if is_thing_stuff_overlap(self.metadata_list[dataset_id]): + thing_id = self.metadata_list[ + dataset_id + ].thing_dataset_id_to_contiguous_id.values() + thing_id = torch.Tensor(list(thing_id)).to(torch.long).to(self.device) + + detector_box_cls = torch.zeros_like(box_cls) + detector_box_cls += float("-inf") + detector_box_cls[..., thing_id] = box_cls[..., thing_id] + else: + num_thing_classes = len(self.metadata_list[dataset_id].thing_classes) + detector_box_cls = box_cls[..., :num_thing_classes] + else: + detector_box_cls = box_cls + + use_sigmoid = True + detector_results, filter_inds = self.inference( + detector_box_cls, box_pred, images.image_sizes, use_sigmoid=use_sigmoid + ) + + if self.test_mask_on: + detector_mask_preds = [ + x[filter_ind] for x, filter_ind in zip(mask_pred, filter_inds) + ] + + for result, box_mask in zip(detector_results, detector_mask_preds): + box_mask = box_mask.sigmoid() > 0.5 + box_mask = BitMasks(box_mask).crop_and_resize( + result.pred_boxes.tensor.to(box_mask.device), 128 + ) + result.pred_masks = ( + box_mask.to(result.pred_boxes.tensor.device) + .unsqueeze(1) + .to(dtype=torch.float32) + ) + + if do_postprocess: + assert ( + not torch.jit.is_scripting() + ), "Scripting is not supported for postprocess." + detector_results = DeformableDETRSegmVL._postprocess_instance( + detector_results, batched_inputs, images.image_sizes + ) + for merged_result, detector_result in zip(merged_results, detector_results): + merged_result.update(detector_result) + + else: + detector_results = None + + if self.semantic_on and not ( + self.eval_dataset_entity and "stuff" not in self.eval_dataset_entity + ): + + semantic_mask_pred = mask_pred.clone() + semantic_box_cls = get_stuff_score( + box_cls, self.metadata_list[dataset_id], self.dataset_entities[dataset_id] + ) + + if self.semantic_post_nms: + _, filter_inds = self.inference(semantic_box_cls, box_pred, images.image_sizes) + semantic_box_cls = torch.stack( + [x[filter_ind] for x, filter_ind in zip(semantic_box_cls, filter_inds)], + dim=0, + ) + semantic_mask_pred = torch.stack( + [x[filter_ind] for x, filter_ind in zip(semantic_mask_pred, filter_inds)], + dim=0, + ) + + if do_postprocess: + assert ( + not torch.jit.is_scripting() + ), "Scripting is not supported for postprocess." + semantic_results = DeformableDETRSegmVL._postprocess_semantic( + semantic_box_cls, semantic_mask_pred, batched_inputs, images + ) + if ( + dataset_id >= 0 + and self.dataset_entities[dataset_id] == "stuff" + and self.metadata_list[dataset_id].get("stuff_classes")[0] == "things" + and self.stuff_prob_thing > 0 + ): + for semantic_result in semantic_results: + semantic_result["sem_seg"][0, ...] = math.log( + self.stuff_prob_thing / (1 - self.stuff_prob_thing) + ) + for merged_result, semantic_result in zip(merged_results, semantic_results): + merged_result.update(semantic_result) + + else: + semantic_results = None + + if self.panoptic_on and not ( + self.eval_dataset_entity and "thing+stuff" not in self.eval_dataset_entity + ): + assert dataset_id >= 0 and dataset_id < len(self.metadata_list) + if do_postprocess: + assert ( + not torch.jit.is_scripting() + ), "Scripting is not supported for postprocess." + if True: + if self.panoptic_post_nms: + _, filter_inds = self.inference(box_cls, box_pred, images.image_sizes) + panoptic_mask_pred = [ + x[filter_ind] for x, filter_ind in zip(mask_pred, filter_inds) + ] + panoptic_box_cls = [ + x[filter_ind] for x, filter_ind in zip(box_cls, filter_inds) + ] + + panoptic_results = DeformableDETRSegmVL._postprocess_panoptic( + panoptic_box_cls, + panoptic_mask_pred, + batched_inputs, + images, + self.metadata_list[dataset_id], + self.panoptic_configs, + ) + else: + panoptic_results = [] + self.combine_overlap_thresh = 0.5 + self.combine_stuff_area_thresh = 4096 + self.combine_instances_score_thresh = 0.5 + for detector_result, semantic_result in zip( + detector_results, semantic_results + ): + detector_r = detector_result["instances"] + sem_seg_r = semantic_result["sem_seg"] + panoptic_r = combine_semantic_and_instance_outputs( + detector_r, + sem_seg_r.argmax(dim=0), + self.combine_overlap_thresh, + self.combine_stuff_area_thresh, + self.combine_instances_score_thresh, + ) + panoptic_results.append({"panoptic_seg": panoptic_r}) + for merged_result, panoptic_result in zip(merged_results, panoptic_results): + merged_result.update(panoptic_result) + + else: + panoptic_results = None + + self.postprocess_time = time.perf_counter() - start_time + + if do_postprocess: + return merged_results + + return detector_results, semantic_results, panoptic_results + + def maskdino_mask_features(self, encode_feats, multi_level_feats, multi_level_masks): + start_idx = sum( + [mask.shape[1] * mask.shape[2] for mask in multi_level_masks[: self.mask_encode_level]] + ) + end_idx = sum( + [ + mask.shape[1] * mask.shape[2] + for mask in multi_level_masks[: self.mask_encode_level + 1] + ] + ) + b, h, w = multi_level_masks[self.mask_encode_level].size() + + encode_feats = encode_feats[:, start_idx:end_idx, :] + encode_feats = encode_feats.permute(0, 2, 1).reshape(b, -1, h, w) + + x = [multi_level_feats[f] for f in self.mask_in_features] + x = x[0] + x = self.lateral_conv(x) + x = x + F.interpolate(encode_feats, size=x.shape[-2:], mode="bilinear", align_corners=False) + x = self.output_conv(x) + mask_features = self.mask_conv(x) + + return mask_features + + @torch.jit.unused + def _set_aux_loss(self, outputs_class, outputs_coord, outputs_mask): + return [ + {"pred_logits": a, "pred_boxes": b, "pred_masks": c} + for a, b, c in zip(outputs_class[:-1], outputs_coord[:-1], outputs_mask[:-1]) + ] + + def inference(self, box_cls, box_pred, image_sizes, use_sigmoid=True): + """ + Arguments: + box_cls (Tensor): tensor of shape (batch_size, num_queries, K). + The tensor predicts the classification probability for each query. + box_pred (Tensor): tensors of shape (batch_size, num_queries, 4). + The tensor predicts 4-vector (x,y,w,h) box + regression values for every queryx + image_sizes (List[torch.Size]): the input image sizes + + Returns: + results (List[Instances]): a list of #images elements. + """ + + if use_sigmoid: + scores = torch.cat( + ( + box_cls.sigmoid(), + torch.zeros((box_cls.size(0), box_cls.size(1), 1), device=self.device), + ), + dim=2, + ) + else: + scores = torch.cat( + ( + box_cls, + torch.zeros((box_cls.size(0), box_cls.size(1), 1), device=self.device), + ), + dim=2, + ) + + boxes = box_cxcywh_to_xyxy(box_pred) + + img_h = torch.tensor([image_size[0] for image_size in image_sizes], device=self.device) + img_w = torch.tensor([image_size[1] for image_size in image_sizes], device=self.device) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) + boxes = boxes * scale_fct[:, None, :] + + boxes = boxes.unbind(0) + scores = scores.unbind(0) + image_shapes = image_sizes + + results, filter_inds = fast_rcnn_inference( + boxes, + scores, + image_shapes, + self.test_score_thresh, + self.test_nms_thresh, + self.test_topk_per_image, + ) + + return results, filter_inds + + def prepare_targets(self, targets): + new_targets = [] + for targets_per_image in targets: + h, w = targets_per_image.image_size + image_size_xyxy = torch.as_tensor([w, h, w, h], dtype=torch.float, device=self.device) + gt_classes = targets_per_image.gt_classes + gt_boxes = targets_per_image.gt_boxes.tensor / image_size_xyxy + gt_boxes = box_xyxy_to_cxcywh(gt_boxes) + + if not targets_per_image.has("gt_masks"): + gt_masks = torch.zeros((0, h, w), dtype=torch.bool) + else: + gt_masks = targets_per_image.gt_masks + + if not isinstance(gt_masks, torch.Tensor): + if isinstance(gt_masks, BitMasks): + gt_masks = gt_masks.tensor + else: + gt_masks = BitMasks.from_polygon_masks(gt_masks, h, w).tensor + + gt_masks = self._move_to_current_device(gt_masks) + gt_masks = ImageList.from_tensors( + [gt_masks], + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ).tensor.squeeze(0) + + new_targets.append({"labels": gt_classes, "boxes": gt_boxes, "masks": gt_masks}) + + if targets_per_image.has("is_thing"): + new_targets[-1]["is_thing"] = targets_per_image.is_thing + + return new_targets + + def preprocess_image(self, batched_inputs): + images = [self._move_to_current_device(x["image"]) for x in batched_inputs] + images = [x.to(self.pixel_mean.dtype) for x in images] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors( + images, + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ) + return images + + @staticmethod + def _postprocess_instance( + instances, batched_inputs: List[Dict[str, torch.Tensor]], image_sizes + ): + """ + Rescale the output instances to the target size. + """ + processed_results = [] + for results_per_image, input_per_image, image_size in zip( + instances, batched_inputs, image_sizes + ): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + r = detector_postprocess(results_per_image, height, width) + processed_results.append({"instances": r.to("cpu")}) + return processed_results + + @staticmethod + def _postprocess_semantic( + mask_clses, + mask_preds, + batched_inputs: List[Dict[str, torch.Tensor]], + images, + pano_temp=0.06, + transform_eval=True, + ): + processed_results = [] + for mask_cls, mask_pred, input_per_image, image_size in zip( + mask_clses, mask_preds, batched_inputs, images.image_sizes + ): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + + T = pano_temp + mask_cls = mask_cls.sigmoid() + + if transform_eval: + mask_cls = F.softmax(mask_cls / T, dim=-1) # already sigmoid + mask_pred = mask_pred.sigmoid() + if mask_cls.size(1) > 1000: + mask_cls = mask_cls.cpu() + mask_pred = mask_pred.cpu() + result = torch.einsum("qc,qhw->chw", mask_cls, mask_pred) + + if True and False: + num_thing_classes = len( + metadata.get( + "thing_classes", + [ + "things", + ], + ) + ) + + result_0 = result[:num_thing_classes, ...] + result_1 = result[num_thing_classes:, ...] + result_0 = result_0.mean(dim=0, keepdim=True) + result = torch.cat([result_0, result_1], dim=0) + + r = sem_seg_postprocess(result, image_size, height, width) + processed_results.append({"sem_seg": r}) + return processed_results + + @staticmethod + def _postprocess_panoptic( + mask_clses, + mask_preds, + batched_inputs: List[Dict[str, torch.Tensor]], + images, + metadata, + panoptic_configs, + ): + prob = panoptic_configs["prob"] + pano_temp = panoptic_configs["pano_temp"] + transform_eval = panoptic_configs["transform_eval"] + object_mask_threshold = panoptic_configs["object_mask_threshold"] + overlap_threshold = panoptic_configs["overlap_threshold"] + + processed_results = [] + for mask_cls, mask_pred, input_per_image, image_size in zip( + mask_clses, mask_preds, batched_inputs, images.image_sizes + ): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + + mask_pred = sem_seg_postprocess(mask_pred, image_size, height, width) + + T = pano_temp + scores, labels = mask_cls.sigmoid().max(-1) + mask_pred = mask_pred.sigmoid() + keep = scores > object_mask_threshold + if transform_eval: + scores, labels = F.softmax(mask_cls.sigmoid() / T, dim=-1).max(-1) + cur_scores = scores[keep] + cur_classes = labels[keep] + cur_masks = mask_pred[keep] + cur_prob_masks = cur_scores.view(-1, 1, 1) * cur_masks + + panoptic_seg = torch.zeros((height, width), dtype=torch.int32, device=cur_masks.device) + segments_info = [] + + current_segment_id = 0 + + if cur_masks.size(0) > 0: + + cur_mask_ids = cur_prob_masks.argmax(0) + + stuff_memory_list = {} + for k in range(cur_classes.shape[0]): + pred_class = cur_classes[k].item() + isthing = pred_class in metadata.thing_dataset_id_to_contiguous_id.values() + mask_area = (cur_mask_ids == k).sum().item() + original_area = (cur_masks[k] >= prob).sum().item() + mask = (cur_mask_ids == k) & (cur_masks[k] >= prob) + + if mask_area > 0 and original_area > 0 and mask.sum().item() > 0: + if mask_area / original_area < overlap_threshold: + continue + + if not isthing: + if int(pred_class) in stuff_memory_list.keys(): + panoptic_seg[mask] = stuff_memory_list[int(pred_class)] + continue + else: + stuff_memory_list[int(pred_class)] = current_segment_id + 1 + + current_segment_id += 1 + panoptic_seg[mask] = current_segment_id + + if not isthing and metadata.get("stuff_classes")[0] == "things": + pred_class = int(pred_class) - len(metadata.thing_classes) + 1 + + segments_info.append( + { + "id": current_segment_id, + "isthing": bool(isthing), + "category_id": int(pred_class), + } + ) + + processed_results.append({"panoptic_seg": (panoptic_seg, segments_info)}) + return processed_results + + @torch.no_grad() + def visualize_training( + self, batched_inputs, output, images, dataset_id, suffix="", do_nms=True + ): + if self.output_dir is None: + return + if self.training: + storage = get_event_storage() + os.makedirs(self.output_dir + "/training", exist_ok=True) + else: + os.makedirs(self.output_dir + "/inference", exist_ok=True) + + pred_logits = output["pred_logits"] + pred_boxes = output["pred_boxes"] + pred_masks = output["pred_masks"] + + thing_classes = self.metadata_list[dataset_id].get("thing_classes", []) + stuff_classes = self.metadata_list[dataset_id].get("stuff_classes", []) + if len(thing_classes) > 0 and len(stuff_classes) > 0 and stuff_classes[0] == "things": + stuff_classes = stuff_classes[1:] + if is_thing_stuff_overlap(self.metadata_list[dataset_id]): + class_names = ( + thing_classes if len(thing_classes) > len(stuff_classes) else stuff_classes + ) + else: + class_names = thing_classes + stuff_classes + + if "instances" in batched_inputs[0] and batched_inputs[0]["instances"].has("phrases"): + class_names = [phrase for x in batched_inputs for phrase in x["instances"].phrases] + [ + "unknown" + ] * 1000 + if "expressions" in batched_inputs[0] and self.expression_cumulative_gt_class: + class_names = [xx for x in batched_inputs for xx in x["expressions"]] + [ + "unknown" + ] * 1000 + + num_thing_classes = len(class_names) + pred_logits = pred_logits[..., :num_thing_classes] + + if pred_masks is not None: + pred_masks = [ + F.interpolate( + pred_mask.float().cpu().unsqueeze(0), + size=images.tensor.size()[2:], + mode="bilinear", + align_corners=False, + ).squeeze(0) + if pred_mask.size(0) > 0 + else pred_mask + for pred_mask in pred_masks + ] + else: + pred_masks = [ + torch.zeros(pred_box.size(0), image_size[0], image_size[1]) + for pred_box, image_size in zip(pred_boxes, images.image_sizes) + ] + + if do_nms: + results, filter_inds = self.inference(pred_logits, pred_boxes, images.image_sizes) + pred_masks = [ + pred_mask[filter_ind.cpu()] + for pred_mask, filter_ind in zip(pred_masks, filter_inds) + ] + for result, pred_mask in zip(results, pred_masks): + result.pred_masks = pred_mask.sigmoid() > 0.5 + else: + results = [] + for pred_logit, pred_box, pred_mask, image_size in zip( + pred_logits, pred_boxes, pred_masks, images.image_sizes + ): + result = Instances(image_size) + result.pred_boxes = Boxes(pred_box) + result.scores = pred_logit[:, 0] + result.pred_classes = torch.zeros( + len(pred_box), dtype=torch.int64, device=pred_logit.device + ) + result.pred_masks = pred_mask.sigmoid() > 0.5 + + results.append(result) + + from detectron2.utils.visualizer import Visualizer + + for input, result in zip(batched_inputs, results): + + if "expressions" in batched_inputs[0] and not self.expression_cumulative_gt_class: + class_names = [xx for xx in input["expressions"]] + ["unknown"] * 1000 + + img = input["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format) + v_gt = Visualizer(img, None) + + if "instances" in input: + labels = [ + "{}".format(class_names[gt_class]) for gt_class in input["instances"].gt_classes + ] + v_gt = v_gt.overlay_instances( + boxes=input["instances"].gt_boxes, + masks=input["instances"].gt_masks + if input["instances"].has("gt_masks") + else None, + labels=labels, + ) + else: + v_gt = v_gt.output + anno_img = v_gt.get_image() + + labels = [ + "{}_{:.0f}%".format(class_names[pred_class], score * 100) + for pred_class, score in zip(result.pred_classes.cpu(), result.scores.cpu()) + ] + v_pred = Visualizer(img, None) + v_pred = v_pred.overlay_instances( + boxes=result.pred_boxes.tensor.clone().detach().cpu().numpy(), + labels=labels, + masks=result.pred_masks[:, : img.shape[0], : img.shape[1]] + .clone() + .detach() + .cpu() + .numpy() + if result.has("pred_masks") + else None, + ) + pred_img = v_pred.get_image() + + vis_img = np.concatenate((anno_img, pred_img), axis=1) + + if result.has("pred_texts"): + labels = [ + "{}".format(text) for text, score in zip(result.pred_texts, result.scores.cpu()) + ] + v_pred = Visualizer(img, None) + v_pred = v_pred.overlay_instances( + boxes=result.pred_boxes.tensor.clone().detach().cpu().numpy(), + labels=labels, + masks=result.pred_masks.clone().detach().cpu().numpy(), + ) + pred_img = v_pred.get_image() + vis_img = np.concatenate((vis_img, pred_img), axis=1) + + basename = os.path.basename(input["file_name"]) + if self.training: + cv2.imwrite( + os.path.join( + self.output_dir, "training", str(storage.iter) + suffix + "_" + basename + ), + vis_img[:, :, ::-1], + ) + else: + cv2.imwrite( + os.path.join(self.output_dir, "inference", suffix + basename), + vis_img[:, :, ::-1], + ) + + @torch.no_grad() + def visualize_training_enc_output(self, batched_inputs, output, images, dataset_id, suffix=""): + if self.output_dir is None: + return + if self.training: + storage = get_event_storage() + os.makedirs(self.output_dir + "/training", exist_ok=True) + else: + os.makedirs(self.output_dir + "/inference", exist_ok=True) + + pred_logits = output["enc_outputs"]["pred_logits"] + pred_boxes = output["enc_outputs"]["pred_boxes"] + + results, filter_inds = self.inference(pred_logits, pred_boxes, images.image_sizes) + + from detectron2.utils.visualizer import Visualizer + + for input, result in zip(batched_inputs, results): + img = input["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format) + v_gt = Visualizer(img, None) + if "instances" in input: + v_gt = v_gt.overlay_instances( + boxes=input["instances"].gt_boxes, + ) + else: + v_gt = v_gt.output + anno_img = v_gt.get_image() + + labels = [ + "{}_{:.0f}%".format(pred_class, score * 100) + for pred_class, score in zip(result.pred_classes.cpu(), result.scores.cpu()) + ] + v_pred = Visualizer(img, None) + v_pred = v_pred.overlay_instances( + boxes=result.pred_boxes.tensor.clone().detach().cpu().numpy(), + labels=labels, + ) + pred_img = v_pred.get_image() + + vis_img = np.concatenate((anno_img, pred_img), axis=1) + + basename = os.path.basename(input["file_name"]) + if self.training: + cv2.imwrite( + os.path.join( + self.output_dir, + "training", + str(storage.iter) + suffix + "_enc_output_" + basename, + ), + vis_img[:, :, ::-1], + ) + else: + cv2.imwrite( + os.path.join(self.output_dir, "inference", suffix + "enc_output_" + basename), + vis_img[:, :, ::-1], + ) + + def set_model_language(self, model_language): + self.model_language = model_language + + +def is_thing_stuff_overlap(metadata): + thing_classes = metadata.get("thing_classes", []) + stuff_classes = metadata.get("stuff_classes", []) + if len(thing_classes) == 0 or len(stuff_classes) == 0: + return False + + if set(thing_classes).issubset(set(stuff_classes)) or set(stuff_classes).issubset( + set(thing_classes) + ): + return True + else: + return False + + +def get_text_list(metadata, dataset_entity): + thing_classes = metadata.get("thing_classes", []) + stuff_classes = metadata.get("stuff_classes", []) + + if dataset_entity == "thing+stuff" and stuff_classes[0] == "things": + text_list = list(thing_classes) + list(stuff_classes[1:]) + + elif dataset_entity == "thing+stuff" and is_thing_stuff_overlap(metadata): + text_list = thing_classes if len(thing_classes) > len(stuff_classes) else stuff_classes + + elif dataset_entity == "thing+stuff": + text_list = list(thing_classes) + list(stuff_classes) + + elif dataset_entity == "stuff": + text_list = list(stuff_classes) + + elif dataset_entity == "thing": + text_list = list(thing_classes) + + return text_list + + +def get_stuff_score(box_cls, metadata, dataset_entity): + thing_classes = metadata.get("thing_classes", []) + stuff_classes = metadata.get("stuff_classes", []) + + semantic_box_cls = box_cls.clone() + + if dataset_entity == "thing+stuff" and stuff_classes[0] == "things": + num_thing_classes = len(thing_classes) + + semantic_box_cls_0 = box_cls[..., :num_thing_classes] + semantic_box_cls_1 = box_cls[..., num_thing_classes:] + semantic_box_cls_0, _ = semantic_box_cls_0.min(dim=2, keepdim=True) + semantic_box_cls = torch.cat([semantic_box_cls_0, semantic_box_cls_1], dim=2) + + if dataset_entity == "thing+stuff" and is_thing_stuff_overlap(metadata): + semantic_box_cls = box_cls.clone() + + if dataset_entity == "stuff": + semantic_box_cls = box_cls.clone() + + return semantic_box_cls diff --git a/approach/ovod/APE/ape/modeling/ape_deta/deformable_transformer.py b/approach/ovod/APE/ape/modeling/ape_deta/deformable_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..60d88941221222ca304ae61103d799e0e100f3b3 --- /dev/null +++ b/approach/ovod/APE/ape/modeling/ape_deta/deformable_transformer.py @@ -0,0 +1,623 @@ +import math + +import torch +import torch.nn as nn + +from ape.layers import MultiScaleDeformableAttention +from detrex.layers import ( + FFN, + BaseTransformerLayer, + MultiheadAttention, + TransformerLayerSequence, + box_cxcywh_to_xyxy, +) +from detrex.utils import inverse_sigmoid +from torchvision.ops.boxes import batched_nms + + +class DeformableDetrTransformerEncoder(TransformerLayerSequence): + def __init__( + self, + embed_dim: int = 256, + num_heads: int = 8, + feedforward_dim: int = 1024, + attn_dropout: float = 0.1, + ffn_dropout: float = 0.1, + num_layers: int = 6, + post_norm: bool = False, + num_feature_levels: int = 4, + use_act_checkpoint: bool = False, + pytorch_attn=False, + ): + super(DeformableDetrTransformerEncoder, self).__init__( + transformer_layers=BaseTransformerLayer( + attn=MultiScaleDeformableAttention( + embed_dim=embed_dim, + num_heads=num_heads, + dropout=attn_dropout, + batch_first=True, + num_levels=num_feature_levels, + pytorch_attn=pytorch_attn, + ), + ffn=FFN( + embed_dim=embed_dim, + feedforward_dim=feedforward_dim, + output_dim=embed_dim, + num_fcs=2, + ffn_drop=ffn_dropout, + ), + norm=nn.LayerNorm(embed_dim), + operation_order=("self_attn", "norm", "ffn", "norm"), + ), + num_layers=num_layers, + ) + self.embed_dim = self.layers[0].embed_dim + self.pre_norm = self.layers[0].pre_norm + + if post_norm: + self.post_norm_layer = nn.LayerNorm(self.embed_dim) + else: + self.post_norm_layer = None + + if use_act_checkpoint: + from fairscale.nn.checkpoint import checkpoint_wrapper + + for i, layer in enumerate(self.layers): + layer = checkpoint_wrapper(layer) + self.layers[i] = layer + + def forward( + self, + query, + key, + value, + query_pos=None, + key_pos=None, + attn_masks=None, + query_key_padding_mask=None, + key_padding_mask=None, + **kwargs, + ): + + for layer in self.layers: + query = layer( + query, + key, + value, + query_pos=query_pos, + attn_masks=attn_masks, + query_key_padding_mask=query_key_padding_mask, + key_padding_mask=key_padding_mask, + **kwargs, + ) + + if self.post_norm_layer is not None: + query = self.post_norm_layer(query) + return query + + +class DeformableDetrTransformerDecoder(TransformerLayerSequence): + def __init__( + self, + embed_dim: int = 256, + num_heads: int = 8, + feedforward_dim: int = 1024, + attn_dropout: float = 0.1, + ffn_dropout: float = 0.1, + num_layers: int = 6, + return_intermediate: bool = True, + num_feature_levels: int = 4, + use_act_checkpoint: bool = False, + pytorch_attn=False, + ): + super(DeformableDetrTransformerDecoder, self).__init__( + transformer_layers=BaseTransformerLayer( + attn=[ + MultiheadAttention( + embed_dim=embed_dim, + num_heads=num_heads, + attn_drop=attn_dropout, + batch_first=True, + ), + MultiScaleDeformableAttention( + embed_dim=embed_dim, + num_heads=num_heads, + dropout=attn_dropout, + batch_first=True, + num_levels=num_feature_levels, + pytorch_attn=pytorch_attn, + ), + ], + ffn=FFN( + embed_dim=embed_dim, + feedforward_dim=feedforward_dim, + output_dim=embed_dim, + ffn_drop=ffn_dropout, + ), + norm=nn.LayerNorm(embed_dim), + operation_order=("self_attn", "norm", "cross_attn", "norm", "ffn", "norm"), + ), + num_layers=num_layers, + ) + self.return_intermediate = return_intermediate + + self.bbox_embed = None + self.class_embed = None + + if use_act_checkpoint: + from fairscale.nn.checkpoint import checkpoint_wrapper + + for i, layer in enumerate(self.layers): + layer = checkpoint_wrapper(layer) + self.layers[i] = layer + + def forward( + self, + query, + key, + value, + query_pos=None, + key_pos=None, + attn_masks=None, + query_key_padding_mask=None, + key_padding_mask=None, + reference_points=None, + valid_ratios=None, + **kwargs, + ): + output = query + + intermediate = [] + intermediate_reference_points = [] + for layer_idx, layer in enumerate(self.layers): + if reference_points.shape[-1] == 4: + reference_points_input = ( + reference_points[:, :, None] + * torch.cat([valid_ratios, valid_ratios], -1)[:, None] + ) + else: + assert reference_points.shape[-1] == 2 + reference_points_input = reference_points[:, :, None] * valid_ratios[:, None] + + output = layer( + output, + key, + value, + query_pos=query_pos, + key_pos=key_pos, + attn_masks=attn_masks, + query_key_padding_mask=query_key_padding_mask, + key_padding_mask=key_padding_mask, + reference_points=reference_points_input, + **kwargs, + ) + + if self.bbox_embed is not None: + tmp = self.bbox_embed[layer_idx](output) + if reference_points.shape[-1] == 4: + new_reference_points = tmp + inverse_sigmoid(reference_points) + new_reference_points = new_reference_points.sigmoid() + else: + assert reference_points.shape[-1] == 2 + new_reference_points = tmp + new_reference_points[..., :2] = tmp[..., :2] + inverse_sigmoid(reference_points) + new_reference_points = new_reference_points.sigmoid() + reference_points = new_reference_points.detach() + + if self.return_intermediate: + intermediate.append(output) + intermediate_reference_points.append(reference_points) + + if self.return_intermediate: + return torch.stack(intermediate), torch.stack(intermediate_reference_points) + + return output, reference_points + + +class DeformableDetrTransformer(nn.Module): + """Transformer module for Deformable DETR + + Args: + encoder (nn.Module): encoder module. + decoder (nn.Module): decoder module. + as_two_stage (bool): whether to use two-stage transformer. Default False. + num_feature_levels (int): number of feature levels. Default 4. + two_stage_num_proposals (int): number of proposals in two-stage transformer. Default 300. + Only used when as_two_stage is True. + """ + + def __init__( + self, + encoder=None, + decoder=None, + num_feature_levels=4, + as_two_stage=False, + two_stage_num_proposals=300, + assign_first_stage=False, + pre_nms_topk=1000, + nms_thresh_enc=0.9, + proposal_ambiguous=0, + ): + super(DeformableDetrTransformer, self).__init__() + self.encoder = encoder + self.decoder = decoder + self.num_feature_levels = num_feature_levels + self.as_two_stage = as_two_stage + self.two_stage_num_proposals = two_stage_num_proposals + self.assign_first_stage = assign_first_stage + self.pre_nms_topk = pre_nms_topk + self.nms_thresh_enc = nms_thresh_enc + self.proposal_ambiguous = proposal_ambiguous + + self.embed_dim = self.encoder.embed_dim + + self.level_embeds = nn.Parameter(torch.Tensor(self.num_feature_levels, self.embed_dim)) + + if self.as_two_stage: + self.enc_output = nn.Linear(self.embed_dim, self.embed_dim) + self.enc_output_norm = nn.LayerNorm(self.embed_dim) + self.pos_trans = nn.Linear(self.embed_dim * 2, self.embed_dim * 2) + self.pos_trans_norm = nn.LayerNorm(self.embed_dim * 2) + self.pix_trans = nn.Linear(self.embed_dim, self.embed_dim) + self.pix_trans_norm = nn.LayerNorm(self.embed_dim) + else: + self.reference_points = nn.Linear(self.embed_dim, 2) + + self.init_weights() + + def init_weights(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + for m in self.modules(): + if isinstance(m, MultiScaleDeformableAttention): + m.init_weights() + if not self.as_two_stage: + nn.init.xavier_normal_(self.reference_points.weight.data, gain=1.0) + nn.init.constant_(self.reference_points.bias.data, 0.0) + nn.init.normal_(self.level_embeds) + + def gen_encoder_output_proposals(self, memory, memory_padding_mask, spatial_shapes): + N, S, C = memory.shape + proposals = [] + _cur = 0 + level_ids = [] + for lvl, (H, W) in enumerate(spatial_shapes): + mask_flatten_ = memory_padding_mask[:, _cur : (_cur + H * W)].view(N, H, W, 1) + valid_H = torch.sum(~mask_flatten_[:, :, 0, 0], 1) + valid_W = torch.sum(~mask_flatten_[:, 0, :, 0], 1) + + grid_y, grid_x = torch.meshgrid( + torch.linspace(0, H - 1, H, dtype=torch.float32, device=memory.device), + torch.linspace(0, W - 1, W, dtype=torch.float32, device=memory.device), + ) + grid = torch.cat([grid_x.unsqueeze(-1), grid_y.unsqueeze(-1)], -1) + + scale = torch.cat([valid_W.unsqueeze(-1), valid_H.unsqueeze(-1)], 1).view(N, 1, 1, 2) + grid = (grid.unsqueeze(0).expand(N, -1, -1, -1) + 0.5) / scale + wh = torch.ones_like(grid) * 0.05 * (2.0**lvl) + proposal = torch.cat((grid, wh), -1).view(N, -1, 4) + proposals.append(proposal) + _cur += H * W + level_ids.append(grid.new_ones(H * W, dtype=torch.long) * lvl) + + output_proposals = torch.cat(proposals, 1) + output_proposals_valid = ((output_proposals > 0.01) & (output_proposals < 0.99)).all( + -1, keepdim=True + ) + output_proposals = torch.log(output_proposals / (1 - output_proposals)) + output_proposals = output_proposals.masked_fill( + memory_padding_mask.unsqueeze(-1), float("inf") + ) + output_proposals = output_proposals.masked_fill(~output_proposals_valid, float("inf")) + + output_memory = memory + output_memory = output_memory.masked_fill(memory_padding_mask.unsqueeze(-1), float(0)) + output_memory = output_memory.masked_fill(~output_proposals_valid, float(0)) + output_memory = self.enc_output_norm(self.enc_output(output_memory)) + level_ids = torch.cat(level_ids) + output_proposals = output_proposals.to(output_memory.dtype) + return output_memory, output_proposals, level_ids + + @staticmethod + def get_reference_points(spatial_shapes, valid_ratios, device): + """Get the reference points used in decoder. + + Args: + spatial_shapes (Tensor): The shape of all + feature maps, has shape (num_level, 2). + valid_ratios (Tensor): The ratios of valid + points on the feature map, has shape + (bs, num_levels, 2) + device (obj:`device`): The device where + reference_points should be. + + Returns: + Tensor: reference points used in decoder, has \ + shape (bs, num_keys, num_levels, 2). + """ + reference_points_list = [] + for lvl, (H, W) in enumerate(spatial_shapes): + ref_y, ref_x = torch.meshgrid( + torch.linspace(0.5, H - 0.5, H, dtype=torch.float32, device=device), + torch.linspace(0.5, W - 0.5, W, dtype=torch.float32, device=device), + ) + ref_y = ref_y.reshape(-1)[None] / (valid_ratios[:, None, lvl, 1] * H) + ref_x = ref_x.reshape(-1)[None] / (valid_ratios[:, None, lvl, 0] * W) + ref = torch.stack((ref_x, ref_y), -1) + reference_points_list.append(ref) + reference_points = torch.cat(reference_points_list, 1) + reference_points = reference_points[:, :, None] * valid_ratios[:, None] + return reference_points + + def get_valid_ratio(self, mask): + """Get the valid ratios of feature maps of all levels.""" + _, H, W = mask.shape + valid_H = torch.sum(~mask[:, :, 0], 1) + valid_W = torch.sum(~mask[:, 0, :], 1) + valid_ratio_h = valid_H.float() / H + valid_ratio_w = valid_W.float() / W + valid_ratio = torch.stack([valid_ratio_w, valid_ratio_h], -1) + return valid_ratio + + def get_proposal_pos_embed(self, proposals, num_pos_feats=128, temperature=10000): + """Get the position embedding of proposal.""" + scale = 2 * math.pi + dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=proposals.device) + dim_t = temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / num_pos_feats) + proposals = proposals.sigmoid() * scale + pos = proposals[:, :, :, None] / dim_t + pos = torch.stack((pos[:, :, :, 0::2].sin(), pos[:, :, :, 1::2].cos()), dim=4).flatten(2) + return pos + + def forward( + self, + multi_level_feats, + multi_level_masks, + multi_level_pos_embeds, + query_embed, + **kwargs, + ): + assert self.as_two_stage or query_embed is not None + + feat_flatten = [] + mask_flatten = [] + lvl_pos_embed_flatten = [] + spatial_shapes = [] + for lvl, (feat, mask, pos_embed) in enumerate( + zip(multi_level_feats, multi_level_masks, multi_level_pos_embeds) + ): + bs, c, h, w = feat.shape + spatial_shape = (h, w) + spatial_shapes.append(spatial_shape) + + feat = feat.flatten(2).transpose(1, 2) # bs, hw, c + mask = mask.flatten(1) + pos_embed = pos_embed.flatten(2).transpose(1, 2) # bs, hw, c + lvl_pos_embed = pos_embed + self.level_embeds[lvl].view(1, 1, -1) + lvl_pos_embed_flatten.append(lvl_pos_embed) + feat_flatten.append(feat) + mask_flatten.append(mask) + feat_flatten = torch.cat(feat_flatten, 1) + mask_flatten = torch.cat(mask_flatten, 1) + lvl_pos_embed_flatten = torch.cat(lvl_pos_embed_flatten, 1) + spatial_shapes = torch.as_tensor( + spatial_shapes, dtype=torch.long, device=feat_flatten.device + ) + level_start_index = torch.cat( + (spatial_shapes.new_zeros((1,)), spatial_shapes.prod(1).cumsum(0)[:-1]) + ) + valid_ratios = torch.stack([self.get_valid_ratio(m) for m in multi_level_masks], 1) + valid_ratios = valid_ratios.to(feat_flatten.dtype) + + reference_points = self.get_reference_points( + spatial_shapes, valid_ratios, device=feat.device + ) + reference_points = reference_points.to(feat_flatten.dtype) + + memory = self.encoder( + query=feat_flatten, + key=None, + value=None, + query_pos=lvl_pos_embed_flatten, + query_key_padding_mask=mask_flatten, + spatial_shapes=spatial_shapes, + reference_points=reference_points, + level_start_index=level_start_index, + valid_ratios=valid_ratios, + **kwargs, + ) + + bs, _, c = memory.shape + if self.as_two_stage: + output_memory, output_proposals, level_ids = self.gen_encoder_output_proposals( + memory, mask_flatten, spatial_shapes + ) + + enc_outputs_class = self.decoder.class_embed[self.decoder.num_layers](output_memory) + enc_outputs_coord_unact = ( + self.decoder.bbox_embed[self.decoder.num_layers](output_memory) + output_proposals + ) + + if self.proposal_ambiguous: + enc_outputs_class_ambiguous = torch.stack( + [ + enc_outputs_class, + ] + + [x(output_memory) for x in self.decoder.class_embed_ambiguous], + dim=1, + ) + enc_outputs_coord_unact_ambiguous = torch.stack( + [ + enc_outputs_coord_unact, + ] + + [ + x(output_memory) + output_proposals + for x in self.decoder.bbox_embed_ambiguous + ], + dim=1, + ) + + indices = torch.argmax(enc_outputs_class_ambiguous, dim=1, keepdim=True) + enc_outputs_class = torch.gather( + enc_outputs_class_ambiguous, dim=1, index=indices + ).squeeze(dim=1) + enc_outputs_coord_unact = torch.gather( + enc_outputs_coord_unact_ambiguous, dim=1, index=indices.repeat(1, 1, 1, 4) + ).squeeze(dim=1) + + if False: + + enc_outputs_class_3 = self.decoder.class_embed_3(output_memory) + enc_outputs_coord_unact_3 = ( + self.decoder.bbox_embed_3(output_memory) + output_proposals + ) + + enc_outputs_class_2 = self.decoder.class_embed_2(output_memory) + enc_outputs_coord_unact_2 = ( + self.decoder.bbox_embed_2(output_memory) + output_proposals + ) + + enc_outputs_class_1 = enc_outputs_class + enc_outputs_coord_unact_1 = enc_outputs_coord_unact + + enc_outputs_class = torch.stack( + [enc_outputs_class_1, enc_outputs_class_2, enc_outputs_class_3], dim=1 + ) + enc_outputs_coord_unact = torch.stack( + [ + enc_outputs_coord_unact_1, + enc_outputs_coord_unact_2, + enc_outputs_coord_unact_3, + ], + dim=1, + ) + indices = torch.argmax(enc_outputs_class, dim=1, keepdim=True) + enc_outputs_class = torch.gather(enc_outputs_class, dim=1, index=indices).squeeze( + dim=1 + ) + enc_outputs_coord_unact = torch.gather( + enc_outputs_coord_unact, dim=1, index=indices.repeat(1, 1, 1, 4) + ).squeeze(dim=1) + + topk = self.two_stage_num_proposals + + proposal_logit = enc_outputs_class[..., 0] + + if self.assign_first_stage: + proposal_boxes = box_cxcywh_to_xyxy(enc_outputs_coord_unact.sigmoid()).clamp(0, 1) + topk_proposals = [] + for b in range(bs): + prop_boxes_b = proposal_boxes[b] + prop_logits_b = proposal_logit[b] + + pre_nms_topk = self.pre_nms_topk + pre_nms_inds = [] + for lvl in range(len(spatial_shapes)): + lvl_mask = level_ids == lvl + pre_nms_inds.append( + torch.topk( + prop_logits_b.sigmoid() * lvl_mask, + min(pre_nms_topk, prop_logits_b.size(0)), + )[1] + ) + pre_nms_inds = torch.cat(pre_nms_inds) + + post_nms_inds = batched_nms( + prop_boxes_b[pre_nms_inds], + prop_logits_b[pre_nms_inds], + level_ids[pre_nms_inds], + self.nms_thresh_enc, + ) + keep_inds = pre_nms_inds[post_nms_inds] + + if len(keep_inds) < self.two_stage_num_proposals: + print( + f"[WARNING] nms proposals ({len(keep_inds)}) < {self.two_stage_num_proposals}, running naive topk" + ) + keep_inds = torch.topk( + proposal_logit[b], min(topk, proposal_logit[b].size(0)) + )[1] + + q_per_l = topk // len(spatial_shapes) + is_level_ordered = ( + level_ids[keep_inds][None] + == torch.arange(len(spatial_shapes), device=level_ids.device)[:, None] + ) # LS + keep_inds_mask = is_level_ordered & ( + is_level_ordered.cumsum(1) <= q_per_l + ) # LS + keep_inds_mask = keep_inds_mask.any(0) # S + + if keep_inds_mask.sum() < topk: + num_to_add = topk - keep_inds_mask.sum() + pad_inds = (~keep_inds_mask).nonzero()[:num_to_add] + keep_inds_mask[pad_inds] = True + + keep_inds_topk = keep_inds[keep_inds_mask] + topk_proposals.append(keep_inds_topk) + topk_proposals = torch.stack(topk_proposals) + else: + topk_proposals = torch.topk(proposal_logit, topk, dim=1)[1] + + topk_coords_unact = torch.gather( + enc_outputs_coord_unact, 1, topk_proposals.unsqueeze(-1).repeat(1, 1, 4) + ) + topk_coords_unact = topk_coords_unact.detach() + reference_points = topk_coords_unact.sigmoid() + init_reference_out = reference_points + pos_trans_out = self.pos_trans_norm( + self.pos_trans( + self.get_proposal_pos_embed(topk_coords_unact).to(topk_coords_unact.dtype) + ) + ) + query_pos, query = torch.split(pos_trans_out, c, dim=2) + + topk_feats = torch.stack( + [output_memory[b][topk_proposals[b]] for b in range(bs)] + ).detach() + query = query + self.pix_trans_norm(self.pix_trans(topk_feats)) + else: + query_pos, query = torch.split(query_embed, c, dim=1) + query_pos = query_pos.unsqueeze(0).expand(bs, -1, -1) + query = query.unsqueeze(0).expand(bs, -1, -1) + reference_points = self.reference_points(query_pos).sigmoid() + init_reference_out = reference_points + + if self.proposal_ambiguous and False: + enc_outputs_class = torch.stack([enc_outputs_class_1, enc_outputs_class_2], dim=1) + enc_outputs_coord_unact = torch.stack( + [enc_outputs_coord_unact_1, enc_outputs_coord_unact_2], dim=1 + ) + + indices = torch.argmax(enc_outputs_class, dim=1, keepdim=True) + enc_outputs_class = torch.gather(enc_outputs_class, dim=1, index=indices).squeeze(dim=1) + enc_outputs_coord_unact = torch.gather( + enc_outputs_coord_unact, dim=1, index=indices.repeat(1, 1, 1, 4) + ).squeeze(dim=1) + + inter_states, inter_references = self.decoder( + query=query, # bs, num_queries, embed_dims + key=None, # bs, num_tokens, embed_dims + value=memory, # bs, num_tokens, embed_dims + query_pos=query_pos, + key_padding_mask=mask_flatten, # bs, num_tokens + reference_points=reference_points, # num_queries, 4 + spatial_shapes=spatial_shapes, # nlvl, 2 + level_start_index=level_start_index, # nlvl + valid_ratios=valid_ratios, # bs, nlvl, 2 + **kwargs, + ) + + inter_references_out = inter_references + if self.as_two_stage: + return ( + inter_states, + init_reference_out, + inter_references_out, + enc_outputs_class, + enc_outputs_coord_unact, + output_proposals.sigmoid(), + memory, + ) + return inter_states, init_reference_out, inter_references_out, None, None, None, memory diff --git a/approach/ovod/APE/ape/modeling/ape_deta/deformable_transformer_vl.py b/approach/ovod/APE/ape/modeling/ape_deta/deformable_transformer_vl.py new file mode 100644 index 0000000000000000000000000000000000000000..17681daa75f6575ec6a198a734c98d24967ec1d3 --- /dev/null +++ b/approach/ovod/APE/ape/modeling/ape_deta/deformable_transformer_vl.py @@ -0,0 +1,678 @@ +import copy +import math + +import torch +import torch.nn as nn + +from ape.layers import MultiScaleDeformableAttention +from detrex.layers import ( + FFN, + BaseTransformerLayer, + MultiheadAttention, + TransformerLayerSequence, + box_cxcywh_to_xyxy, +) +from detrex.utils import inverse_sigmoid +from torchvision.ops.boxes import batched_nms + + +class DeformableDetrTransformerEncoderVL(TransformerLayerSequence): + def __init__( + self, + embed_dim: int = 256, + num_heads: int = 8, + feedforward_dim: int = 1024, + attn_dropout: float = 0.1, + ffn_dropout: float = 0.1, + num_layers: int = 6, + post_norm: bool = False, + num_feature_levels: int = 4, + vl_layer=None, + use_act_checkpoint=False, + pytorch_attn=False, + ): + super(DeformableDetrTransformerEncoderVL, self).__init__( + transformer_layers=BaseTransformerLayer( + attn=MultiScaleDeformableAttention( + embed_dim=embed_dim, + num_heads=num_heads, + dropout=attn_dropout, + batch_first=True, + num_levels=num_feature_levels, + pytorch_attn=pytorch_attn, + ), + ffn=FFN( + embed_dim=embed_dim, + feedforward_dim=feedforward_dim, + output_dim=embed_dim, + num_fcs=2, + ffn_drop=ffn_dropout, + ), + norm=nn.LayerNorm(embed_dim), + operation_order=("self_attn", "norm", "ffn", "norm"), + ), + num_layers=num_layers, + ) + self.embed_dim = self.layers[0].embed_dim + self.pre_norm = self.layers[0].pre_norm + + if post_norm: + self.post_norm_layer = nn.LayerNorm(self.embed_dim) + else: + self.post_norm_layer = None + + self.vl_layers = nn.ModuleList([copy.deepcopy(vl_layer) for _ in range(num_layers)]) + + if use_act_checkpoint: + from fairscale.nn.checkpoint import checkpoint_wrapper + + for i, layer in enumerate(self.layers): + layer = checkpoint_wrapper(layer) + self.layers[i] = layer + + def forward( + self, + query, + key, + value, + query_l, + attention_mask_l, + query_pos=None, + key_pos=None, + attn_masks=None, + query_key_padding_mask=None, + key_padding_mask=None, + **kwargs, + ): + + for vl_layer, layer in zip(self.vl_layers, self.layers): + if vl_layer is not None and query_l is not None: + query, query_l = vl_layer( + query, + query_l, + attention_mask_v=query_key_padding_mask, + attention_mask_l=attention_mask_l, + ) + query = layer( + query, + key, + value, + query_pos=query_pos, + attn_masks=attn_masks, + query_key_padding_mask=query_key_padding_mask, + key_padding_mask=key_padding_mask, + **kwargs, + ) + + if self.post_norm_layer is not None: + query = self.post_norm_layer(query) + if query_l is None: + query_l = sum([_.sum() for _ in self.vl_layers.parameters()]) * 0.0 + return query, query_l + + +class DeformableDetrTransformerDecoderVL(TransformerLayerSequence): + def __init__( + self, + embed_dim: int = 256, + num_heads: int = 8, + feedforward_dim: int = 1024, + attn_dropout: float = 0.1, + ffn_dropout: float = 0.1, + num_layers: int = 6, + return_intermediate: bool = True, + num_feature_levels: int = 4, + use_act_checkpoint: bool = False, + look_forward_twice: bool = False, + pytorch_attn=False, + ): + super(DeformableDetrTransformerDecoderVL, self).__init__( + transformer_layers=BaseTransformerLayer( + attn=[ + MultiheadAttention( + embed_dim=embed_dim, + num_heads=num_heads, + attn_drop=attn_dropout, + batch_first=True, + ), + MultiScaleDeformableAttention( + embed_dim=embed_dim, + num_heads=num_heads, + dropout=attn_dropout, + batch_first=True, + num_levels=num_feature_levels, + pytorch_attn=pytorch_attn, + ), + ], + ffn=FFN( + embed_dim=embed_dim, + feedforward_dim=feedforward_dim, + output_dim=embed_dim, + ffn_drop=ffn_dropout, + ), + norm=nn.LayerNorm(embed_dim), + operation_order=("self_attn", "norm", "cross_attn", "norm", "ffn", "norm"), + ), + num_layers=num_layers, + ) + self.return_intermediate = return_intermediate + + self.bbox_embed = None + self.class_embed = None + + if use_act_checkpoint: + from fairscale.nn.checkpoint import checkpoint_wrapper + + for i, layer in enumerate(self.layers): + layer = checkpoint_wrapper(layer) + self.layers[i] = layer + + self.look_forward_twice = look_forward_twice + + def forward( + self, + query, + key, + value, + query_pos=None, + key_pos=None, + attn_masks=None, + query_key_padding_mask=None, + key_padding_mask=None, + reference_points=None, + valid_ratios=None, + **kwargs, + ): + output = query + + intermediate = [] + intermediate_reference_points = [] + for layer_idx, layer in enumerate(self.layers): + if reference_points.shape[-1] == 4: + reference_points_input = ( + reference_points[:, :, None] + * torch.cat([valid_ratios, valid_ratios], -1)[:, None] + ) + else: + assert reference_points.shape[-1] == 2 + reference_points_input = reference_points[:, :, None] * valid_ratios[:, None] + + output = layer( + output, + key, + value, + query_pos=query_pos, + key_pos=key_pos, + attn_masks=attn_masks, + query_key_padding_mask=query_key_padding_mask, + key_padding_mask=key_padding_mask, + reference_points=reference_points_input, + **kwargs, + ) + + if self.bbox_embed is not None: + tmp = self.bbox_embed[layer_idx](output) + if reference_points.shape[-1] == 4: + new_reference_points = tmp + inverse_sigmoid(reference_points) + new_reference_points = new_reference_points.sigmoid() + else: + assert reference_points.shape[-1] == 2 + new_reference_points = tmp + new_reference_points[..., :2] = tmp[..., :2] + inverse_sigmoid(reference_points) + new_reference_points = new_reference_points.sigmoid() + reference_points = new_reference_points.detach() + + if self.return_intermediate: + intermediate.append(output) + intermediate_reference_points.append( + new_reference_points if self.look_forward_twice else reference_points + ) + + if self.return_intermediate: + return torch.stack(intermediate), torch.stack(intermediate_reference_points) + + return output, reference_points + + +class DeformableDetrTransformerVL(nn.Module): + """Transformer module for Deformable DETR + + Args: + encoder (nn.Module): encoder module. + decoder (nn.Module): decoder module. + as_two_stage (bool): whether to use two-stage transformer. Default False. + num_feature_levels (int): number of feature levels. Default 4. + two_stage_num_proposals (int): number of proposals in two-stage transformer. Default 300. + Only used when as_two_stage is True. + """ + + def __init__( + self, + encoder=None, + decoder=None, + num_feature_levels=4, + as_two_stage=False, + two_stage_num_proposals=300, + assign_first_stage=False, + pre_nms_topk=1000, + nms_thresh_enc=0.9, + proposal_ambiguous=0, + ): + super(DeformableDetrTransformerVL, self).__init__() + self.encoder = encoder + self.decoder = decoder + self.num_feature_levels = num_feature_levels + self.as_two_stage = as_two_stage + self.two_stage_num_proposals = two_stage_num_proposals + self.assign_first_stage = assign_first_stage + self.pre_nms_topk = pre_nms_topk + self.nms_thresh_enc = nms_thresh_enc + self.proposal_ambiguous = proposal_ambiguous + + self.embed_dim = self.encoder.embed_dim + + self.level_embeds = nn.Parameter(torch.Tensor(self.num_feature_levels, self.embed_dim)) + + if self.as_two_stage: + self.enc_output = nn.Linear(self.embed_dim, self.embed_dim) + self.enc_output_norm = nn.LayerNorm(self.embed_dim) + self.pos_trans = nn.Linear(self.embed_dim * 2, self.embed_dim * 2) + self.pos_trans_norm = nn.LayerNorm(self.embed_dim * 2) + self.pix_trans = nn.Linear(self.embed_dim, self.embed_dim) + self.pix_trans_norm = nn.LayerNorm(self.embed_dim) + else: + self.reference_points = nn.Linear(self.embed_dim, 2) + + self.init_weights() + + def init_weights(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + for m in self.modules(): + if isinstance(m, MultiScaleDeformableAttention): + m.init_weights() + if not self.as_two_stage: + nn.init.xavier_normal_(self.reference_points.weight.data, gain=1.0) + nn.init.constant_(self.reference_points.bias.data, 0.0) + nn.init.normal_(self.level_embeds) + + def gen_encoder_output_proposals( + self, memory, memory_padding_mask, spatial_shapes, mask_prompt_flatten + ): + N, S, C = memory.shape + proposals = [] + _cur = 0 + level_ids = [] + for lvl, (H, W) in enumerate(spatial_shapes): + mask_flatten_ = memory_padding_mask[:, _cur : (_cur + H * W)].view(N, H, W, 1) + valid_H = torch.sum(~mask_flatten_[:, :, 0, 0], 1) + valid_W = torch.sum(~mask_flatten_[:, 0, :, 0], 1) + + grid_y, grid_x = torch.meshgrid( + torch.linspace(0, H - 1, H, dtype=torch.float32, device=memory.device), + torch.linspace(0, W - 1, W, dtype=torch.float32, device=memory.device), + ) + grid = torch.cat([grid_x.unsqueeze(-1), grid_y.unsqueeze(-1)], -1) + + scale = torch.cat([valid_W.unsqueeze(-1), valid_H.unsqueeze(-1)], 1).view(N, 1, 1, 2) + grid = (grid.unsqueeze(0).expand(N, -1, -1, -1) + 0.5) / scale + wh = torch.ones_like(grid) * 0.05 * (2.0**lvl) + proposal = torch.cat((grid, wh), -1).view(N, -1, 4) + proposals.append(proposal) + _cur += H * W + level_ids.append(grid.new_ones(H * W, dtype=torch.long) * lvl) + + output_proposals = torch.cat(proposals, 1) + output_proposals_valid = ((output_proposals > 0.01) & (output_proposals < 0.99)).all( + -1, keepdim=True + ) + output_proposals = torch.log(output_proposals / (1 - output_proposals)) + output_proposals = output_proposals.masked_fill( + memory_padding_mask.unsqueeze(-1), float("inf") + ) + output_proposals = output_proposals.masked_fill(~output_proposals_valid, float("inf")) + if mask_prompt_flatten is not None: + output_proposals = output_proposals.masked_fill( + ~mask_prompt_flatten.unsqueeze(-1), float("inf") + ) + + output_memory = memory + output_memory = output_memory.masked_fill(memory_padding_mask.unsqueeze(-1), float(0)) + output_memory = output_memory.masked_fill(~output_proposals_valid, float(0)) + if mask_prompt_flatten is not None: + output_memory = output_memory.masked_fill(~mask_prompt_flatten.unsqueeze(-1), float(0)) + output_memory = self.enc_output_norm(self.enc_output(output_memory)) + level_ids = torch.cat(level_ids) + output_proposals = output_proposals.to(output_memory.dtype) + return output_memory, output_proposals, level_ids + + @staticmethod + def get_reference_points(spatial_shapes, valid_ratios, device): + """Get the reference points used in decoder. + + Args: + spatial_shapes (Tensor): The shape of all + feature maps, has shape (num_level, 2). + valid_ratios (Tensor): The ratios of valid + points on the feature map, has shape + (bs, num_levels, 2) + device (obj:`device`): The device where + reference_points should be. + + Returns: + Tensor: reference points used in decoder, has \ + shape (bs, num_keys, num_levels, 2). + """ + reference_points_list = [] + for lvl, (H, W) in enumerate(spatial_shapes): + ref_y, ref_x = torch.meshgrid( + torch.linspace(0.5, H - 0.5, H, dtype=torch.float32, device=device), + torch.linspace(0.5, W - 0.5, W, dtype=torch.float32, device=device), + ) + ref_y = ref_y.reshape(-1)[None] / (valid_ratios[:, None, lvl, 1] * H) + ref_x = ref_x.reshape(-1)[None] / (valid_ratios[:, None, lvl, 0] * W) + ref = torch.stack((ref_x, ref_y), -1) + reference_points_list.append(ref) + reference_points = torch.cat(reference_points_list, 1) + reference_points = reference_points[:, :, None] * valid_ratios[:, None] + return reference_points + + def get_valid_ratio(self, mask): + """Get the valid ratios of feature maps of all levels.""" + _, H, W = mask.shape + valid_H = torch.sum(~mask[:, :, 0], 1) + valid_W = torch.sum(~mask[:, 0, :], 1) + valid_ratio_h = valid_H.float() / H + valid_ratio_w = valid_W.float() / W + valid_ratio = torch.stack([valid_ratio_w, valid_ratio_h], -1) + return valid_ratio + + def get_proposal_pos_embed(self, proposals, num_pos_feats=128, temperature=10000): + """Get the position embedding of proposal.""" + scale = 2 * math.pi + dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=proposals.device) + dim_t = temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / num_pos_feats) + proposals = proposals.sigmoid() * scale + pos = proposals[:, :, :, None] / dim_t + pos = torch.stack((pos[:, :, :, 0::2].sin(), pos[:, :, :, 1::2].cos()), dim=4).flatten(2) + return pos + + def forward( + self, + multi_level_feats, + multi_level_masks, + multi_level_pos_embeds, + query_embed, + query_l, + attention_mask_l, + multi_level_masks_prompt, + **kwargs, + ): + assert self.as_two_stage or query_embed is not None + + feat_flatten = [] + mask_flatten = [] + lvl_pos_embed_flatten = [] + spatial_shapes = [] + for lvl, (feat, mask, pos_embed) in enumerate( + zip(multi_level_feats, multi_level_masks, multi_level_pos_embeds) + ): + bs, c, h, w = feat.shape + spatial_shape = (h, w) + spatial_shapes.append(spatial_shape) + + feat = feat.flatten(2).transpose(1, 2) # bs, hw, c + mask = mask.flatten(1) + pos_embed = pos_embed.flatten(2).transpose(1, 2) # bs, hw, c + lvl_pos_embed = pos_embed + self.level_embeds[lvl].view(1, 1, -1) + lvl_pos_embed_flatten.append(lvl_pos_embed) + feat_flatten.append(feat) + mask_flatten.append(mask) + feat_flatten = torch.cat(feat_flatten, 1) + mask_flatten = torch.cat(mask_flatten, 1) + lvl_pos_embed_flatten = torch.cat(lvl_pos_embed_flatten, 1) + spatial_shapes = torch.as_tensor( + spatial_shapes, dtype=torch.long, device=feat_flatten.device + ) + level_start_index = torch.cat( + (spatial_shapes.new_zeros((1,)), spatial_shapes.prod(1).cumsum(0)[:-1]) + ) + valid_ratios = torch.stack([self.get_valid_ratio(m) for m in multi_level_masks], 1) + valid_ratios = valid_ratios.to(feat_flatten.dtype) + + if multi_level_masks_prompt is not None: + mask_prompt_flatten = [] + for mask_prompt in multi_level_masks_prompt: + mask_prompt = mask_prompt.flatten(1) + mask_prompt_flatten.append(mask_prompt) + mask_prompt_flatten = torch.cat(mask_prompt_flatten, 1) + else: + mask_prompt_flatten = None + + reference_points = self.get_reference_points( + spatial_shapes, valid_ratios, device=feat.device + ) + reference_points = reference_points.to(feat_flatten.dtype) + + memory, query_l = self.encoder( + query=feat_flatten, + key=None, + value=None, + query_l=query_l, + attention_mask_l=attention_mask_l, + query_pos=lvl_pos_embed_flatten, + query_key_padding_mask=mask_flatten, + spatial_shapes=spatial_shapes, + reference_points=reference_points, + level_start_index=level_start_index, + valid_ratios=valid_ratios, + **kwargs, + ) + + bs, _, c = memory.shape + if self.as_two_stage: + output_memory, output_proposals, level_ids = self.gen_encoder_output_proposals( + memory, + mask_flatten, + spatial_shapes, + mask_prompt_flatten, + ) + + enc_outputs_class = self.decoder.class_embed[self.decoder.num_layers](output_memory) + enc_outputs_coord_unact = ( + self.decoder.bbox_embed[self.decoder.num_layers](output_memory) + output_proposals + ) + + if self.proposal_ambiguous: + enc_outputs_class_ambiguous = torch.stack( + [ + enc_outputs_class, + ] + + [x(output_memory) for x in self.decoder.class_embed_ambiguous], + dim=1, + ) + enc_outputs_coord_unact_ambiguous = torch.stack( + [ + enc_outputs_coord_unact, + ] + + [ + x(output_memory) + output_proposals + for x in self.decoder.bbox_embed_ambiguous + ], + dim=1, + ) + + indices = torch.argmax(enc_outputs_class_ambiguous, dim=1, keepdim=True) + enc_outputs_class = torch.gather( + enc_outputs_class_ambiguous, dim=1, index=indices + ).squeeze(dim=1) + enc_outputs_coord_unact = torch.gather( + enc_outputs_coord_unact_ambiguous, dim=1, index=indices.repeat(1, 1, 1, 4) + ).squeeze(dim=1) + + if False: + + enc_outputs_class_3 = self.decoder.class_embed_3(output_memory) + enc_outputs_coord_unact_3 = ( + self.decoder.bbox_embed_3(output_memory) + output_proposals + ) + + enc_outputs_class_2 = self.decoder.class_embed_2(output_memory) + enc_outputs_coord_unact_2 = ( + self.decoder.bbox_embed_2(output_memory) + output_proposals + ) + + enc_outputs_class_1 = enc_outputs_class + enc_outputs_coord_unact_1 = enc_outputs_coord_unact + + enc_outputs_class = torch.stack( + [enc_outputs_class_1, enc_outputs_class_2, enc_outputs_class_3], dim=1 + ) + enc_outputs_coord_unact = torch.stack( + [ + enc_outputs_coord_unact_1, + enc_outputs_coord_unact_2, + enc_outputs_coord_unact_3, + ], + dim=1, + ) + indices = torch.argmax(enc_outputs_class, dim=1, keepdim=True) + enc_outputs_class = torch.gather(enc_outputs_class, dim=1, index=indices).squeeze( + dim=1 + ) + enc_outputs_coord_unact = torch.gather( + enc_outputs_coord_unact, dim=1, index=indices.repeat(1, 1, 1, 4) + ).squeeze(dim=1) + + topk = self.two_stage_num_proposals + + proposal_logit = enc_outputs_class[..., 0] + + if self.assign_first_stage: + proposal_boxes = box_cxcywh_to_xyxy(enc_outputs_coord_unact.sigmoid()).clamp(0, 1) + topk_proposals = [] + for b in range(bs): + prop_boxes_b = proposal_boxes[b] + prop_logits_b = proposal_logit[b] + + pre_nms_topk = self.pre_nms_topk + pre_nms_inds = [] + for lvl in range(len(spatial_shapes)): + lvl_mask = level_ids == lvl + pre_nms_inds.append( + torch.topk( + prop_logits_b.sigmoid() * lvl_mask, + min(pre_nms_topk, prop_logits_b.size(0)), + )[1] + ) + pre_nms_inds = torch.cat(pre_nms_inds) + + post_nms_inds = batched_nms( + prop_boxes_b[pre_nms_inds], + prop_logits_b[pre_nms_inds], + level_ids[pre_nms_inds], + self.nms_thresh_enc, + ) + keep_inds = pre_nms_inds[post_nms_inds] + + if len(keep_inds) < self.two_stage_num_proposals: + print( + f"[WARNING] nms proposals ({len(keep_inds)}) < {self.two_stage_num_proposals}, running naive topk" + ) + keep_inds = torch.topk( + proposal_logit[b], min(topk, proposal_logit[b].size(0)) + )[1] + + q_per_l = topk // len(spatial_shapes) + is_level_ordered = ( + level_ids[keep_inds][None] + == torch.arange(len(spatial_shapes), device=level_ids.device)[:, None] + ) # LS + keep_inds_mask = is_level_ordered & ( + is_level_ordered.cumsum(1) <= q_per_l + ) # LS + keep_inds_mask = keep_inds_mask.any(0) # S + + if keep_inds_mask.sum() < topk: + num_to_add = topk - keep_inds_mask.sum() + pad_inds = (~keep_inds_mask).nonzero()[:num_to_add] + keep_inds_mask[pad_inds] = True + + keep_inds_topk = keep_inds[keep_inds_mask] + topk_proposals.append(keep_inds_topk) + topk_proposals = torch.stack(topk_proposals) + else: + topk_proposals = torch.topk(proposal_logit, topk, dim=1)[1] + + topk_coords_unact = torch.gather( + enc_outputs_coord_unact, 1, topk_proposals.unsqueeze(-1).repeat(1, 1, 4) + ) + topk_coords_unact = topk_coords_unact.detach() + reference_points = topk_coords_unact.sigmoid() + init_reference_out = reference_points + pos_trans_out = self.pos_trans_norm( + self.pos_trans( + self.get_proposal_pos_embed(topk_coords_unact).to(topk_coords_unact.dtype) + ) + ) + query_pos, query = torch.split(pos_trans_out, c, dim=2) + + topk_feats = torch.stack( + [output_memory[b][topk_proposals[b]] for b in range(bs)] + ).detach() + query = query + self.pix_trans_norm(self.pix_trans(topk_feats)) + else: + query_pos, query = torch.split(query_embed, c, dim=1) + query_pos = query_pos.unsqueeze(0).expand(bs, -1, -1) + query = query.unsqueeze(0).expand(bs, -1, -1) + reference_points = self.reference_points(query_pos).sigmoid() + init_reference_out = reference_points + + if self.proposal_ambiguous and False: + enc_outputs_class = torch.stack([enc_outputs_class_1, enc_outputs_class_2], dim=1) + enc_outputs_coord_unact = torch.stack( + [enc_outputs_coord_unact_1, enc_outputs_coord_unact_2], dim=1 + ) + + indices = torch.argmax(enc_outputs_class, dim=1, keepdim=True) + enc_outputs_class = torch.gather(enc_outputs_class, dim=1, index=indices).squeeze(dim=1) + enc_outputs_coord_unact = torch.gather( + enc_outputs_coord_unact, dim=1, index=indices.repeat(1, 1, 1, 4) + ).squeeze(dim=1) + + inter_states, inter_references = self.decoder( + query=query, # bs, num_queries, embed_dims + key=None, # bs, num_tokens, embed_dims + value=memory, # bs, num_tokens, embed_dims + query_pos=query_pos, + key_padding_mask=mask_flatten, # bs, num_tokens + reference_points=reference_points, # num_queries, 4 + spatial_shapes=spatial_shapes, # nlvl, 2 + level_start_index=level_start_index, # nlvl + valid_ratios=valid_ratios, # bs, nlvl, 2 + **kwargs, + ) + + inter_references_out = inter_references + if self.as_two_stage: + return ( + inter_states, + init_reference_out, + inter_references_out, + enc_outputs_class, + enc_outputs_coord_unact, + output_proposals.sigmoid(), + memory, + query_l, + ) + return ( + inter_states, + init_reference_out, + inter_references_out, + None, + None, + None, + memory, + query_l, + ) diff --git a/approach/ovod/APE/ape/modeling/ape_deta/fast_rcnn.py b/approach/ovod/APE/ape/modeling/ape_deta/fast_rcnn.py new file mode 100644 index 0000000000000000000000000000000000000000..239e396957aeebae970dea5e10a3485634f0586d --- /dev/null +++ b/approach/ovod/APE/ape/modeling/ape_deta/fast_rcnn.py @@ -0,0 +1,201 @@ +import warnings +from typing import List, Tuple + +import torch + +from detectron2.layers import batched_nms +from detectron2.structures import Boxes, Instances + +__all__ = [ + "fast_rcnn_inference", +] + + +""" +Shape shorthand in this module: + + N: number of images in the minibatch + R: number of ROIs, combined over all images, in the minibatch + Ri: number of ROIs in image i + K: number of foreground classes. E.g.,there are 80 foreground classes in COCO. + +Naming convention: + + deltas: refers to the 4-d (dx, dy, dw, dh) deltas that parameterize the box2box + transform (see :class:`box_regression.Box2BoxTransform`). + + pred_class_logits: predicted class scores in [-inf, +inf]; use + softmax(pred_class_logits) to estimate P(class). + + gt_classes: ground-truth classification labels in [0, K], where [0, K) represent + foreground object classes and K represents the background class. + + pred_proposal_deltas: predicted box2box transform deltas for transforming proposals + to detection box predictions. + + gt_proposal_deltas: ground-truth box2box transform deltas +""" + + +def fast_rcnn_inference( + boxes: List[torch.Tensor], + scores: List[torch.Tensor], + image_shapes: List[Tuple[int, int]], + score_thresh: float, + nms_thresh: float, + topk_per_image: int, + use_soft_nms: bool = False, + soft_nms_method: str = "linear", + soft_nms_iou_threshold: float = 0.3, + soft_nms_sigma: float = 0.5, + soft_nms_class_wise: bool = False, +): + """ + Call `fast_rcnn_inference_single_image` for all images. + + Args: + boxes (list[Tensor]): A list of Tensors of predicted class-specific or class-agnostic + boxes for each image. Element i has shape (Ri, K * 4) if doing + class-specific regression, or (Ri, 4) if doing class-agnostic + regression, where Ri is the number of predicted objects for image i. + This is compatible with the output of :meth:`FastRCNNOutputLayers.predict_boxes`. + scores (list[Tensor]): A list of Tensors of predicted class scores for each image. + Element i has shape (Ri, K + 1), where Ri is the number of predicted objects + for image i. Compatible with the output of :meth:`FastRCNNOutputLayers.predict_probs`. + image_shapes (list[tuple]): A list of (width, height) tuples for each image in the batch. + score_thresh (float): Only return detections with a confidence score exceeding this + threshold. + nms_thresh (float): The threshold to use for box non-maximum suppression. Value in [0, 1]. + topk_per_image (int): The number of top scoring detections to return. Set < 0 to return + all detections. + + Returns: + instances: (list[Instances]): A list of N instances, one for each image in the batch, + that stores the topk most confidence detections. + kept_indices: (list[Tensor]): A list of 1D tensor of length of N, each element indicates + the corresponding boxes/scores index in [0, Ri) from the input, for image i. + """ + result_per_image = [ + fast_rcnn_inference_single_image( + boxes_per_image, + scores_per_image, + image_shape, + score_thresh, + nms_thresh, + topk_per_image, + use_soft_nms=use_soft_nms, + soft_nms_method=soft_nms_method, + soft_nms_iou_threshold=soft_nms_iou_threshold, + soft_nms_sigma=soft_nms_sigma, + soft_nms_class_wise=soft_nms_class_wise, + ) + for scores_per_image, boxes_per_image, image_shape in zip(scores, boxes, image_shapes) + ] + return [x[0] for x in result_per_image], [x[1] for x in result_per_image] + + +def fast_rcnn_inference_single_image( + boxes, + scores, + image_shape: Tuple[int, int], + score_thresh: float, + nms_thresh: float, + topk_per_image: int, + use_soft_nms: bool = False, + soft_nms_method: str = "linear", + soft_nms_iou_threshold: float = 0.3, + soft_nms_sigma: float = 0.5, + soft_nms_class_wise: bool = False, +): + """ + Single-image inference. Return bounding-box detection results by thresholding + on scores and applying non-maximum suppression (NMS). + + Args: + Same as `fast_rcnn_inference`, but with boxes, scores, and image shapes + per image. + + Returns: + Same as `fast_rcnn_inference`, but for only one image. + """ + valid_mask = torch.isfinite(boxes).all(dim=1) & torch.isfinite(scores).all(dim=1) + if not valid_mask.all(): + boxes = boxes[valid_mask] + scores = scores[valid_mask] + + scores = scores[:, :-1] + num_bbox_reg_classes = boxes.shape[1] // 4 + boxes = Boxes(boxes.reshape(-1, 4)) + boxes.clip(image_shape) + boxes = boxes.tensor.view(-1, num_bbox_reg_classes, 4) # R x C x 4 + + filter_mask = scores > score_thresh # R x K + filter_inds = filter_mask.nonzero() + if num_bbox_reg_classes == 1: + boxes = boxes[filter_inds[:, 0], 0] + else: + boxes = boxes[filter_mask] + scores = scores[filter_mask] + + if use_soft_nms: + from mmcv.ops import soft_nms + + if not soft_nms_class_wise: + dets, keep = soft_nms( + boxes=boxes, + scores=scores, + iou_threshold=soft_nms_iou_threshold, + sigma=soft_nms_sigma, + min_score=1e-3, + method=soft_nms_method, + ) + boxes, scores = dets[:, :4], dets[:, -1] + else: + try: + max_coordinate = boxes.max() + except: + print(boxes.shape) # empty + warnings.warn("setting max_coordinate to 0") + max_coordinate = 0 + idxs = filter_inds[:, 1] + offsets = idxs.to(boxes) * (max_coordinate + torch.tensor(1).to(boxes)) + boxes_for_nms = boxes + offsets[:, None] + dets, keep = soft_nms( + boxes=boxes_for_nms, + scores=scores, + iou_threshold=soft_nms_iou_threshold, + sigma=soft_nms_sigma, + min_score=1e-3, + method=soft_nms_method, + ) + + if topk_per_image >= 0: + keep = keep[:topk_per_image] + boxes, scores, filter_inds = boxes[keep], scores[keep], filter_inds[keep] + + result = Instances(image_shape) + result.pred_boxes = Boxes(boxes) + result.scores = scores + result.pred_classes = filter_inds[:, 1] + return result, filter_inds[:, 0] + + boxes = boxes[keep] + scores = dets[:, -1] # scores are updated in soft-nms + + result = Instances(image_shape) + result.pred_boxes = Boxes(boxes) + result.scores = scores + filter_inds = filter_inds[keep] + result.pred_classes = filter_inds[:, 1] + return result, filter_inds[:, 0] + + keep = batched_nms(boxes, scores, filter_inds[:, 1], nms_thresh) + if topk_per_image >= 0: + keep = keep[:topk_per_image] + boxes, scores, filter_inds = boxes[keep], scores[keep], filter_inds[keep] + + result = Instances(image_shape) + result.pred_boxes = Boxes(boxes) + result.scores = scores + result.pred_classes = filter_inds[:, 1] + return result, filter_inds[:, 0] diff --git a/approach/ovod/APE/ape/modeling/ape_deta/misc.py b/approach/ovod/APE/ape/modeling/ape_deta/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..d697fe8e922426116ce1c1be1782d8d3b8674149 --- /dev/null +++ b/approach/ovod/APE/ape/modeling/ape_deta/misc.py @@ -0,0 +1,469 @@ +""" +Misc functions, including distributed helpers. + +Mostly copy-paste from torchvision references. +""" +import datetime +import os +import pickle +import subprocess +import time +from collections import defaultdict, deque +from typing import List, Optional + +import torch +import torch.distributed as dist +from packaging import version +from torch import Tensor + +import torchvision + +if version.parse(torchvision.__version__) < version.parse("0.7"): + from torchvision.ops import _new_empty_tensor + from torchvision.ops.misc import _output_size + + +class SmoothedValue(object): + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], dtype=torch.float64, device="cuda") + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + return self.total / self.count + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, + avg=self.avg, + global_avg=self.global_avg, + max=self.max, + value=self.value, + ) + + +def all_gather(data): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors) + Args: + data: any picklable object + Returns: + list[data]: list of data gathered from each rank + """ + world_size = get_world_size() + if world_size == 1: + return [data] + + buffer = pickle.dumps(data) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to("cuda") + + local_size = torch.tensor([tensor.numel()], device="cuda") + size_list = [torch.tensor([0], device="cuda") for _ in range(world_size)] + dist.all_gather(size_list, local_size) + size_list = [int(size.item()) for size in size_list] + max_size = max(size_list) + + tensor_list = [] + for _ in size_list: + tensor_list.append(torch.empty((max_size,), dtype=torch.uint8, device="cuda")) + if local_size != max_size: + padding = torch.empty(size=(max_size - local_size,), dtype=torch.uint8, device="cuda") + tensor = torch.cat((tensor, padding), dim=0) + dist.all_gather(tensor_list, tensor) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + + return data_list + + +def reduce_dict(input_dict, average=True): + """ + Args: + input_dict (dict): all the values will be reduced + average (bool): whether to do average or sum + Reduce the values in the dictionary from all processes so that all processes + have the averaged results. Returns a dict with the same fields as + input_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return input_dict + with torch.no_grad(): + names = [] + values = [] + for k in sorted(input_dict.keys()): + names.append(k) + values.append(input_dict[k]) + values = torch.stack(values, dim=0) + dist.all_reduce(values) + if average: + values /= world_size + reduced_dict = {k: v for k, v in zip(names, values)} + return reduced_dict + + +class MetricLogger(object): + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError("'{}' object has no attribute '{}'".format(type(self).__name__, attr)) + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + loss_str.append("{}: {}".format(name, str(meter))) + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, print_freq, header=None): + i = 0 + if not header: + header = "" + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt="{avg:.4f}") + data_time = SmoothedValue(fmt="{avg:.4f}") + space_fmt = ":" + str(len(str(len(iterable)))) + "d" + if torch.cuda.is_available(): + log_msg = self.delimiter.join( + [ + header, + "[{0" + space_fmt + "}/{1}]", + "eta: {eta}", + "{meters}", + "time: {time}", + "data: {data}", + "max mem: {memory:.0f}", + ] + ) + else: + log_msg = self.delimiter.join( + [ + header, + "[{0" + space_fmt + "}/{1}]", + "eta: {eta}", + "{meters}", + "time: {time}", + "data: {data}", + ] + ) + MB = 1024.0 * 1024.0 + for obj in iterable: + data_time.update(time.time() - end) + yield obj + iter_time.update(time.time() - end) + if i % print_freq == 0 or i == len(iterable) - 1: + eta_seconds = iter_time.global_avg * (len(iterable) - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + if torch.cuda.is_available(): + print( + log_msg.format( + i, + len(iterable), + eta=eta_string, + meters=str(self), + time=str(iter_time), + data=str(data_time), + memory=torch.cuda.max_memory_allocated() / MB, + ) + ) + else: + print( + log_msg.format( + i, + len(iterable), + eta=eta_string, + meters=str(self), + time=str(iter_time), + data=str(data_time), + ) + ) + i += 1 + end = time.time() + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print( + "{} Total time: {} ({:.4f} s / it)".format( + header, total_time_str, total_time / len(iterable) + ) + ) + + +def get_sha(): + cwd = os.path.dirname(os.path.abspath(__file__)) + + def _run(command): + return subprocess.check_output(command, cwd=cwd).decode("ascii").strip() + + sha = "N/A" + diff = "clean" + branch = "N/A" + try: + sha = _run(["git", "rev-parse", "HEAD"]) + subprocess.check_output(["git", "diff"], cwd=cwd) + diff = _run(["git", "diff-index", "HEAD"]) + diff = "has uncommited changes" if diff else "clean" + branch = _run(["git", "rev-parse", "--abbrev-ref", "HEAD"]) + except Exception: + pass + message = f"sha: {sha}, status: {diff}, branch: {branch}" + return message + + +def collate_fn(batch): + batch = list(zip(*batch)) + batch[0] = nested_tensor_from_tensor_list(batch[0]) + return tuple(batch) + + +def _max_by_axis(the_list): + maxes = the_list[0] + for sublist in the_list[1:]: + for index, item in enumerate(sublist): + maxes[index] = max(maxes[index], item) + return maxes + + +class NestedTensor(object): + def __init__(self, tensors, mask: Optional[Tensor]): + self.tensors = tensors + self.mask = mask + + def to(self, device): + cast_tensor = self.tensors.to(device) + mask = self.mask + if mask is not None: + assert mask is not None + cast_mask = mask.to(device) + else: + cast_mask = None + return NestedTensor(cast_tensor, cast_mask) + + def decompose(self): + return self.tensors, self.mask + + def __repr__(self): + return str(self.tensors) + + +def nested_tensor_from_tensor_list(tensor_list: List[Tensor]): + if tensor_list[0].ndim == 3: + if torchvision._is_tracing(): + return _onnx_nested_tensor_from_tensor_list(tensor_list) + + max_size = _max_by_axis([list(img.shape) for img in tensor_list]) + batch_shape = [len(tensor_list)] + max_size + b, c, h, w = batch_shape + dtype = tensor_list[0].dtype + device = tensor_list[0].device + tensor = torch.zeros(batch_shape, dtype=dtype, device=device) + mask = torch.ones((b, h, w), dtype=torch.bool, device=device) + for img, pad_img, m in zip(tensor_list, tensor, mask): + pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + m[: img.shape[1], : img.shape[2]] = False + else: + raise ValueError("not supported") + return NestedTensor(tensor, mask) + + +@torch.jit.unused +def _onnx_nested_tensor_from_tensor_list(tensor_list: List[Tensor]) -> NestedTensor: + max_size = [] + for i in range(tensor_list[0].dim()): + max_size_i = torch.max( + torch.stack([img.shape[i] for img in tensor_list]).to(torch.float32) + ).to(torch.int64) + max_size.append(max_size_i) + max_size = tuple(max_size) + + padded_imgs = [] + padded_masks = [] + for img in tensor_list: + padding = [(s1 - s2) for s1, s2 in zip(max_size, tuple(img.shape))] + padded_img = torch.nn.functional.pad(img, (0, padding[2], 0, padding[1], 0, padding[0])) + padded_imgs.append(padded_img) + + m = torch.zeros_like(img[0], dtype=torch.int, device=img.device) + padded_mask = torch.nn.functional.pad(m, (0, padding[2], 0, padding[1]), "constant", 1) + padded_masks.append(padded_mask.to(torch.bool)) + + tensor = torch.stack(padded_imgs) + mask = torch.stack(padded_masks) + + return NestedTensor(tensor, mask=mask) + + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop("force", False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + if "RANK" in os.environ and "WORLD_SIZE" in os.environ: + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ["WORLD_SIZE"]) + args.gpu = int(os.environ["LOCAL_RANK"]) + elif "SLURM_PROCID" in os.environ: + args.rank = int(os.environ["SLURM_PROCID"]) + args.gpu = args.rank % torch.cuda.device_count() + else: + print("Not using distributed mode") + args.distributed = False + return + + args.distributed = True + + torch.cuda.set_device(args.gpu) + args.dist_backend = "nccl" + print("| distributed init (rank {}): {}".format(args.rank, args.dist_url), flush=True) + torch.distributed.init_process_group( + backend=args.dist_backend, + init_method=args.dist_url, + world_size=args.world_size, + rank=args.rank, + ) + torch.distributed.barrier() + setup_for_distributed(args.rank == 0) + + +@torch.no_grad() +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + if target.numel() == 0: + return [torch.zeros([], device=output.device)] + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def interpolate(input, size=None, scale_factor=None, mode="nearest", align_corners=None): + """ + Equivalent to nn.functional.interpolate, but with support for empty batch sizes. + This will eventually be supported natively by PyTorch, and this + class can go away. + """ + if version.parse(torchvision.__version__) < version.parse("0.7"): + if input.numel() > 0: + return torch.nn.functional.interpolate(input, size, scale_factor, mode, align_corners) + + output_shape = _output_size(2, input, size, scale_factor) + output_shape = list(input.shape[:-2]) + list(output_shape) + return _new_empty_tensor(input, output_shape) + else: + return torchvision.ops.misc.interpolate(input, size, scale_factor, mode, align_corners) diff --git a/approach/ovod/APE/ape/modeling/ape_deta/segmentation.py b/approach/ovod/APE/ape/modeling/ape_deta/segmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..95decb8b7c95d9bd990ee0ecef011a2c3899b72b --- /dev/null +++ b/approach/ovod/APE/ape/modeling/ape_deta/segmentation.py @@ -0,0 +1,378 @@ +""" +This file provides the definition of the convolutional heads used to predict masks, as well as the losses +""" +import io +from collections import defaultdict + +import torch +import torch.nn as nn +import torch.nn.functional as F +from PIL import Image + +from detrex.layers import box_cxcywh_to_xyxy + +try: + from panopticapi.utils import id2rgb, rgb2id +except ImportError: + pass + + +class DETRsegm(nn.Module): + def __init__(self, detr, freeze_detr=False): + super().__init__() + self.detr = detr + + if freeze_detr: + for p in self.parameters(): + p.requires_grad_(False) + + hidden_dim, nheads = detr.transformer.d_model, detr.transformer.nhead + self.bbox_attention = MHAttentionMap(hidden_dim, hidden_dim, nheads, dropout=0) + self.mask_head = MaskHeadSmallConv(hidden_dim + nheads, [1024, 512, 256], hidden_dim) + + def forward(self, samples): + if not isinstance(samples, NestedTensor): + samples = nested_tensor_from_tensor_list(samples) + features, pos = self.detr.backbone(samples) + + bs = features[-1].tensors.shape[0] + + src, mask = features[-1].decompose() + src_proj = self.detr.input_proj(src) + hs, memory = self.detr.transformer(src_proj, mask, self.detr.query_embed.weight, pos[-1]) + + outputs_class = self.detr.class_embed(hs) + outputs_coord = self.detr.bbox_embed(hs).sigmoid() + out = {"pred_logits": outputs_class[-1], "pred_boxes": outputs_coord[-1]} + if self.detr.aux_loss: + out["aux_outputs"] = [ + {"pred_logits": a, "pred_boxes": b} + for a, b in zip(outputs_class[:-1], outputs_coord[:-1]) + ] + + bbox_mask = self.bbox_attention(hs[-1], memory, mask=mask) + + seg_masks = self.mask_head( + src_proj, bbox_mask, [features[2].tensors, features[1].tensors, features[0].tensors] + ) + outputs_seg_masks = seg_masks.view( + bs, self.detr.num_queries, seg_masks.shape[-2], seg_masks.shape[-1] + ) + + out["pred_masks"] = outputs_seg_masks + return out + + +class MaskHeadSmallConv(nn.Module): + """ + Simple convolutional head, using group norm. + Upsampling is done using a FPN approach + """ + + def __init__(self, dim, fpn_dims, context_dim): + super().__init__() + + inter_dims = [ + dim, + context_dim // 2, + context_dim // 4, + context_dim // 8, + context_dim // 16, + context_dim // 64, + ] + self.lay1 = torch.nn.Conv2d(dim, dim, 3, padding=1) + self.gn1 = torch.nn.GroupNorm(8, dim) + self.lay2 = torch.nn.Conv2d(dim, inter_dims[1], 3, padding=1) + self.gn2 = torch.nn.GroupNorm(8, inter_dims[1]) + self.lay3 = torch.nn.Conv2d(inter_dims[1], inter_dims[2], 3, padding=1) + self.gn3 = torch.nn.GroupNorm(8, inter_dims[2]) + self.lay4 = torch.nn.Conv2d(inter_dims[2], inter_dims[3], 3, padding=1) + self.gn4 = torch.nn.GroupNorm(8, inter_dims[3]) + self.lay5 = torch.nn.Conv2d(inter_dims[3], inter_dims[4], 3, padding=1) + self.gn5 = torch.nn.GroupNorm(8, inter_dims[4]) + self.out_lay = torch.nn.Conv2d(inter_dims[4], 1, 3, padding=1) + + self.dim = dim + + self.adapter1 = torch.nn.Conv2d(fpn_dims[0], inter_dims[1], 1) + self.adapter2 = torch.nn.Conv2d(fpn_dims[1], inter_dims[2], 1) + self.adapter3 = torch.nn.Conv2d(fpn_dims[2], inter_dims[3], 1) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_uniform_(m.weight, a=1) + nn.init.constant_(m.bias, 0) + + def forward(self, x, bbox_mask, fpns): + def expand(tensor, length): + return tensor.unsqueeze(1).repeat(1, int(length), 1, 1, 1).flatten(0, 1) + + x = torch.cat([expand(x, bbox_mask.shape[1]), bbox_mask.flatten(0, 1)], 1) + + x = self.lay1(x) + x = self.gn1(x) + x = F.relu(x) + x = self.lay2(x) + x = self.gn2(x) + x = F.relu(x) + + cur_fpn = self.adapter1(fpns[0]) + if cur_fpn.size(0) != x.size(0): + cur_fpn = expand(cur_fpn, x.size(0) / cur_fpn.size(0)) + x = cur_fpn + F.interpolate(x, size=cur_fpn.shape[-2:], mode="nearest") + x = self.lay3(x) + x = self.gn3(x) + x = F.relu(x) + + cur_fpn = self.adapter2(fpns[1]) + if cur_fpn.size(0) != x.size(0): + cur_fpn = expand(cur_fpn, x.size(0) / cur_fpn.size(0)) + x = cur_fpn + F.interpolate(x, size=cur_fpn.shape[-2:], mode="nearest") + x = self.lay4(x) + x = self.gn4(x) + x = F.relu(x) + + cur_fpn = self.adapter3(fpns[2]) + if cur_fpn.size(0) != x.size(0): + cur_fpn = expand(cur_fpn, x.size(0) / cur_fpn.size(0)) + x = cur_fpn + F.interpolate(x, size=cur_fpn.shape[-2:], mode="nearest") + x = self.lay5(x) + x = self.gn5(x) + x = F.relu(x) + + x = self.out_lay(x) + return x + + +class MHAttentionMap(nn.Module): + """This is a 2D attention module, which only returns the attention softmax (no multiplication by value)""" + + def __init__(self, query_dim, hidden_dim, num_heads, dropout=0, bias=True): + super().__init__() + self.num_heads = num_heads + self.hidden_dim = hidden_dim + self.dropout = nn.Dropout(dropout) + + self.q_linear = nn.Linear(query_dim, hidden_dim, bias=bias) + self.k_linear = nn.Linear(query_dim, hidden_dim, bias=bias) + + nn.init.zeros_(self.k_linear.bias) + nn.init.zeros_(self.q_linear.bias) + nn.init.xavier_uniform_(self.k_linear.weight) + nn.init.xavier_uniform_(self.q_linear.weight) + self.normalize_fact = float(hidden_dim / self.num_heads) ** -0.5 + + def forward(self, q, k, mask=None): + q = self.q_linear(q) + k = F.conv2d(k, self.k_linear.weight.unsqueeze(-1).unsqueeze(-1), self.k_linear.bias) + qh = q.view(q.shape[0], q.shape[1], self.num_heads, self.hidden_dim // self.num_heads) + kh = k.view( + k.shape[0], self.num_heads, self.hidden_dim // self.num_heads, k.shape[-2], k.shape[-1] + ) + weights = torch.einsum("bqnc,bnchw->bqnhw", qh * self.normalize_fact, kh) + + if mask is not None: + weights.masked_fill_(mask.unsqueeze(1).unsqueeze(1), float("-inf")) + weights = F.softmax(weights.flatten(2), dim=-1).view_as(weights) + weights = self.dropout(weights) + return weights + + +def dice_loss(inputs, targets, num_boxes): + """ + Compute the DICE loss, similar to generalized IOU for masks + Args: + inputs: A float tensor of arbitrary shape. + The predictions for each example. + targets: A float tensor with the same shape as inputs. Stores the binary + classification label for each element in inputs + (0 for the negative class and 1 for the positive class). + """ + inputs = inputs.sigmoid() + inputs = inputs.flatten(1) + numerator = 2 * (inputs * targets).sum(1) + denominator = inputs.sum(-1) + targets.sum(-1) + loss = 1 - (numerator + 1) / (denominator + 1) + return loss.sum() / num_boxes + + +def sigmoid_focal_loss(inputs, targets, num_boxes, alpha: float = 0.25, gamma: float = 2): + """ + Loss used in RetinaNet for dense detection: https://arxiv.org/abs/1708.02002. + Args: + inputs: A float tensor of arbitrary shape. + The predictions for each example. + targets: A float tensor with the same shape as inputs. Stores the binary + classification label for each element in inputs + (0 for the negative class and 1 for the positive class). + alpha: (optional) Weighting factor in range (0,1) to balance + positive vs negative examples. Default = -1 (no weighting). + gamma: Exponent of the modulating factor (1 - p_t) to + balance easy vs hard examples. + Returns: + Loss tensor + """ + prob = inputs.sigmoid() + ce_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction="none") + p_t = prob * targets + (1 - prob) * (1 - targets) + loss = ce_loss * ((1 - p_t) ** gamma) + + if alpha >= 0: + alpha_t = alpha * targets + (1 - alpha) * (1 - targets) + loss = alpha_t * loss + + return loss.mean(1).sum() / num_boxes + + +class PostProcessSegm(nn.Module): + def __init__(self, threshold=0.5): + super().__init__() + self.threshold = threshold + + @torch.no_grad() + def forward(self, results, outputs, orig_target_sizes, max_target_sizes): + assert len(orig_target_sizes) == len(max_target_sizes) + max_h, max_w = max_target_sizes.max(0)[0].tolist() + outputs_masks = outputs["pred_masks"].squeeze(2) + outputs_masks = F.interpolate( + outputs_masks, size=(max_h, max_w), mode="bilinear", align_corners=False + ) + outputs_masks = (outputs_masks.sigmoid() > self.threshold).cpu() + + for i, (cur_mask, t, tt) in enumerate( + zip(outputs_masks, max_target_sizes, orig_target_sizes) + ): + img_h, img_w = t[0], t[1] + results[i]["masks"] = cur_mask[:, :img_h, :img_w].unsqueeze(1) + results[i]["masks"] = F.interpolate( + results[i]["masks"].float(), size=tuple(tt.tolist()), mode="nearest" + ).byte() + + return results + + +class PostProcessPanoptic(nn.Module): + """This class converts the output of the model to the final panoptic result, in the format expected by the + coco panoptic API""" + + def __init__(self, is_thing_map, threshold=0.85): + """ + Parameters: + is_thing_map: This is a whose keys are the class ids, and the values a boolean indicating whether + the class is a thing (True) or a stuff (False) class + threshold: confidence threshold: segments with confidence lower than this will be deleted + """ + super().__init__() + self.threshold = threshold + self.is_thing_map = is_thing_map + + def forward(self, outputs, processed_sizes, target_sizes=None): + """This function computes the panoptic prediction from the model's predictions. + Parameters: + outputs: This is a dict coming directly from the model. See the model doc for the content. + processed_sizes: This is a list of tuples (or torch tensors) of sizes of the images that were passed to the + model, ie the size after data augmentation but before batching. + target_sizes: This is a list of tuples (or torch tensors) corresponding to the requested final size + of each prediction. If left to None, it will default to the processed_sizes + """ + if target_sizes is None: + target_sizes = processed_sizes + assert len(processed_sizes) == len(target_sizes) + out_logits, raw_masks, raw_boxes = ( + outputs["pred_logits"], + outputs["pred_masks"], + outputs["pred_boxes"], + ) + assert len(out_logits) == len(raw_masks) == len(target_sizes) + preds = [] + + def to_tuple(tup): + if isinstance(tup, tuple): + return tup + return tuple(tup.cpu().tolist()) + + for cur_logits, cur_masks, cur_boxes, size, target_size in zip( + out_logits, raw_masks, raw_boxes, processed_sizes, target_sizes + ): + scores, labels = cur_logits.softmax(-1).max(-1) + keep = labels.ne(outputs["pred_logits"].shape[-1] - 1) & (scores > self.threshold) + cur_scores, cur_classes = cur_logits.softmax(-1).max(-1) + cur_scores = cur_scores[keep] + cur_classes = cur_classes[keep] + cur_masks = cur_masks[keep] + cur_masks = F.interpolate(cur_masks[None], to_tuple(size), mode="bilinear").squeeze(0) + cur_boxes = box_cxcywh_to_xyxy(cur_boxes[keep]) + + h, w = cur_masks.shape[-2:] + assert len(cur_boxes) == len(cur_classes) + + cur_masks = cur_masks.flatten(1) + stuff_equiv_classes = defaultdict(lambda: []) + for k, label in enumerate(cur_classes): + if not self.is_thing_map[label.item()]: + stuff_equiv_classes[label.item()].append(k) + + def get_ids_area(masks, scores, dedup=False): + + m_id = masks.transpose(0, 1).softmax(-1) + + if m_id.shape[-1] == 0: + m_id = torch.zeros((h, w), dtype=torch.long, device=m_id.device) + else: + m_id = m_id.argmax(-1).view(h, w) + + if dedup: + for equiv in stuff_equiv_classes.values(): + if len(equiv) > 1: + for eq_id in equiv: + m_id.masked_fill_(m_id.eq(eq_id), equiv[0]) + + final_h, final_w = to_tuple(target_size) + + seg_img = Image.fromarray(id2rgb(m_id.view(h, w).cpu().numpy())) + seg_img = seg_img.resize(size=(final_w, final_h), resample=Image.NEAREST) + + np_seg_img = ( + torch.ByteTensor(torch.ByteStorage.from_buffer(seg_img.tobytes())) + .view(final_h, final_w, 3) + .numpy() + ) + m_id = torch.from_numpy(rgb2id(np_seg_img)) + + area = [] + for i in range(len(scores)): + area.append(m_id.eq(i).sum().item()) + return area, seg_img + + area, seg_img = get_ids_area(cur_masks, cur_scores, dedup=True) + if cur_classes.numel() > 0: + while True: + filtered_small = torch.as_tensor( + [area[i] <= 4 for i, c in enumerate(cur_classes)], + dtype=torch.bool, + device=keep.device, + ) + if filtered_small.any().item(): + cur_scores = cur_scores[~filtered_small] + cur_classes = cur_classes[~filtered_small] + cur_masks = cur_masks[~filtered_small] + area, seg_img = get_ids_area(cur_masks, cur_scores) + else: + break + + else: + cur_classes = torch.ones(1, dtype=torch.long, device=cur_classes.device) + + segments_info = [] + for i, a in enumerate(area): + cat = cur_classes[i].item() + segments_info.append( + {"id": i, "isthing": self.is_thing_map[cat], "category_id": cat, "area": a} + ) + del cur_classes + + with io.BytesIO() as out: + seg_img.save(out, format="PNG") + predictions = {"png_string": out.getvalue(), "segments_info": segments_info} + preds.append(predictions) + return preds diff --git a/approach/ovod/APE/ape/modeling/deta/__init__.py b/approach/ovod/APE/ape/modeling/deta/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..184b9e7515a0d69b075a051c0c2f5e63a6dca41d --- /dev/null +++ b/approach/ovod/APE/ape/modeling/deta/__init__.py @@ -0,0 +1,9 @@ +from .assigner import Stage1Assigner, Stage2Assigner +from .deformable_criterion import DeformableCriterion +from .deformable_detr import DeformableDETR +from .deformable_detr_segm import DeformableDETRSegm +from .deformable_transformer import ( + DeformableDetrTransformer, + DeformableDetrTransformerDecoder, + DeformableDetrTransformerEncoder, +) diff --git a/approach/ovod/APE/ape/modeling/deta/assigner.py b/approach/ovod/APE/ape/modeling/deta/assigner.py new file mode 100644 index 0000000000000000000000000000000000000000..a6ee56c1142672a274f8c6c6b2874dc31f960f06 --- /dev/null +++ b/approach/ovod/APE/ape/modeling/deta/assigner.py @@ -0,0 +1,358 @@ +from typing import List + +import torch +import torch.nn as nn + +from ape.utils.box_ops import box_cxcywh_to_xyxy, box_iou, box_xyxy_to_cxcywh, generalized_box_iou + + +def nonzero_tuple(x): + """ + A 'as_tuple=True' version of torch.nonzero to support torchscript. + because of https://github.com/pytorch/pytorch/issues/38718 + """ + if torch.jit.is_scripting(): + if x.dim() == 0: + return x.unsqueeze(0).nonzero().unbind(1) + return x.nonzero().unbind(1) + else: + return x.nonzero(as_tuple=True) + + +class Matcher(object): + """ + This class assigns to each predicted "element" (e.g., a box) a ground-truth + element. Each predicted element will have exactly zero or one matches; each + ground-truth element may be matched to zero or more predicted elements. + + The matching is determined by the MxN match_quality_matrix, that characterizes + how well each (ground-truth, prediction)-pair match each other. For example, + if the elements are boxes, this matrix may contain box intersection-over-union + overlap values. + + The matcher returns (a) a vector of length N containing the index of the + ground-truth element m in [0, M) that matches to prediction n in [0, N). + (b) a vector of length N containing the labels for each prediction. + """ + + def __init__( + self, thresholds: List[float], labels: List[int], allow_low_quality_matches: bool = False + ): + """ + Args: + thresholds (list): a list of thresholds used to stratify predictions + into levels. + labels (list): a list of values to label predictions belonging at + each level. A label can be one of {-1, 0, 1} signifying + {ignore, negative class, positive class}, respectively. + allow_low_quality_matches (bool): if True, produce additional matches + for predictions with maximum match quality lower than high_threshold. + See set_low_quality_matches_ for more details. + + For example, + thresholds = [0.3, 0.5] + labels = [0, -1, 1] + All predictions with iou < 0.3 will be marked with 0 and + thus will be considered as false positives while training. + All predictions with 0.3 <= iou < 0.5 will be marked with -1 and + thus will be ignored. + All predictions with 0.5 <= iou will be marked with 1 and + thus will be considered as true positives. + """ + thresholds = thresholds[:] + assert thresholds[0] > 0 + thresholds.insert(0, -float("inf")) + thresholds.append(float("inf")) + assert all( + [low <= high for (low, high) in zip(thresholds[:-1], thresholds[1:])] + ), thresholds + assert all([l in [-1, 0, 1] for l in labels]) + assert len(labels) == len(thresholds) - 1 + self.thresholds = thresholds + self.labels = labels + self.allow_low_quality_matches = allow_low_quality_matches + + def __call__(self, match_quality_matrix): + """ + Args: + match_quality_matrix (Tensor[float]): an MxN tensor, containing the + pairwise quality between M ground-truth elements and N predicted + elements. All elements must be >= 0 (due to the us of `torch.nonzero` + for selecting indices in :meth:`set_low_quality_matches_`). + + Returns: + matches (Tensor[int64]): a vector of length N, where matches[i] is a matched + ground-truth index in [0, M) + match_labels (Tensor[int8]): a vector of length N, where pred_labels[i] indicates + whether a prediction is a true or false positive or ignored + """ + assert match_quality_matrix.dim() == 2 + if match_quality_matrix.numel() == 0: + default_matches = match_quality_matrix.new_full( + (match_quality_matrix.size(1),), 0, dtype=torch.int64 + ) + default_match_labels = match_quality_matrix.new_full( + (match_quality_matrix.size(1),), self.labels[0], dtype=torch.int8 + ) + return default_matches, default_match_labels + + assert torch.all(match_quality_matrix >= 0) + + matched_vals, matches = match_quality_matrix.max(dim=0) + + match_labels = matches.new_full(matches.size(), 1, dtype=torch.int8) + + for (l, low, high) in zip(self.labels, self.thresholds[:-1], self.thresholds[1:]): + low_high = (matched_vals >= low) & (matched_vals < high) + match_labels[low_high] = l + + if self.allow_low_quality_matches: + self.set_low_quality_matches_(match_labels, match_quality_matrix) + + return matches, match_labels + + def set_low_quality_matches_(self, match_labels, match_quality_matrix): + """ + Produce additional matches for predictions that have only low-quality matches. + Specifically, for each ground-truth G find the set of predictions that have + maximum overlap with it (including ties); for each prediction in that set, if + it is unmatched, then match it to the ground-truth G. + + This function implements the RPN assignment case (i) in Sec. 3.1.2 of + :paper:`Faster R-CNN`. + """ + highest_quality_foreach_gt, _ = match_quality_matrix.max(dim=1) + _, pred_inds_with_highest_quality = nonzero_tuple( + match_quality_matrix == highest_quality_foreach_gt[:, None] + ) + match_labels[pred_inds_with_highest_quality] = 1 + + +def subsample_labels( + labels: torch.Tensor, num_samples: int, positive_fraction: float, bg_label: int +): + """ + Return `num_samples` (or fewer, if not enough found) + random samples from `labels` which is a mixture of positives & negatives. + It will try to return as many positives as possible without + exceeding `positive_fraction * num_samples`, and then try to + fill the remaining slots with negatives. + + Args: + labels (Tensor): (N, ) label vector with values: + * -1: ignore + * bg_label: background ("negative") class + * otherwise: one or more foreground ("positive") classes + num_samples (int): The total number of labels with value >= 0 to return. + Values that are not sampled will be filled with -1 (ignore). + positive_fraction (float): The number of subsampled labels with values > 0 + is `min(num_positives, int(positive_fraction * num_samples))`. The number + of negatives sampled is `min(num_negatives, num_samples - num_positives_sampled)`. + In order words, if there are not enough positives, the sample is filled with + negatives. If there are also not enough negatives, then as many elements are + sampled as is possible. + bg_label (int): label index of background ("negative") class. + + Returns: + pos_idx, neg_idx (Tensor): + 1D vector of indices. The total length of both is `num_samples` or fewer. + """ + positive = nonzero_tuple((labels != -1) & (labels != bg_label))[0] + negative = nonzero_tuple(labels == bg_label)[0] + + num_pos = int(num_samples * positive_fraction) + num_pos = min(positive.numel(), num_pos) + num_neg = num_samples - num_pos + num_neg = min(negative.numel(), num_neg) + + perm1 = torch.randperm(positive.numel(), device=positive.device)[:num_pos] + perm2 = torch.randperm(negative.numel(), device=negative.device)[:num_neg] + + pos_idx = positive[perm1] + neg_idx = negative[perm2] + return pos_idx, neg_idx + + +def sample_topk_per_gt(pr_inds, gt_inds, iou, k): + if len(gt_inds) == 0: + return pr_inds, gt_inds + gt_inds2, counts = gt_inds.unique(return_counts=True) + scores, pr_inds2 = iou[gt_inds2].topk(k, dim=1) + gt_inds2 = gt_inds2[:, None].repeat(1, k) + + pr_inds3 = torch.cat([pr[:c] for c, pr in zip(counts, pr_inds2)]) + gt_inds3 = torch.cat([gt[:c] for c, gt in zip(counts, gt_inds2)]) + return pr_inds3, gt_inds3 + + +class Stage2Assigner(nn.Module): + def __init__(self, num_queries, num_classes, max_k=4): + super().__init__() + self.positive_fraction = 0.25 + self.num_classes = num_classes + self.batch_size_per_image = num_queries + self.proposal_matcher = Matcher( + thresholds=[0.6], labels=[0, 1], allow_low_quality_matches=True + ) + self.k = max_k + + def _sample_proposals( + self, matched_idxs: torch.Tensor, matched_labels: torch.Tensor, gt_classes: torch.Tensor + ): + """ + Based on the matching between N proposals and M groundtruth, + sample the proposals and set their classification labels. + + Args: + matched_idxs (Tensor): a vector of length N, each is the best-matched + gt index in [0, M) for each proposal. + matched_labels (Tensor): a vector of length N, the matcher's label + (one of cfg.MODEL.ROI_HEADS.IOU_LABELS) for each proposal. + gt_classes (Tensor): a vector of length M. + + Returns: + Tensor: a vector of indices of sampled proposals. Each is in [0, N). + Tensor: a vector of the same length, the classification label for + each sampled proposal. Each sample is labeled as either a category in + [0, num_classes) or the background (num_classes). + """ + has_gt = gt_classes.numel() > 0 + if has_gt: + gt_classes = gt_classes[matched_idxs] + gt_classes[matched_labels == 0] = self.num_classes + gt_classes[matched_labels == -1] = -1 + else: + gt_classes = torch.zeros_like(matched_idxs) + self.num_classes + + sampled_fg_idxs, sampled_bg_idxs = subsample_labels( + gt_classes, self.batch_size_per_image, self.positive_fraction, self.num_classes + ) + + sampled_idxs = torch.cat([sampled_fg_idxs, sampled_bg_idxs], dim=0) + return sampled_idxs, gt_classes[sampled_idxs] + + def forward(self, outputs, targets, return_cost_matrix=False): + + bs = len(targets) + indices = [] + ious = [] + for b in range(bs): + iou, _ = box_iou( + box_cxcywh_to_xyxy(targets[b]["boxes"]), + box_cxcywh_to_xyxy(outputs["init_reference"][b].detach()), + ) + if not torch.all(iou >= 0): + print("iou", iou, iou.max(), iou.min()) + print("targets[b][boxes]", targets[b]["boxes"]) + print( + "outputs[init_reference][b]", + outputs["init_reference"][b], + outputs["init_reference"][b].max(), + outputs["init_reference"][b].min(), + ) + matched_idxs, matched_labels = self.proposal_matcher( + iou + ) # proposal_id -> highest_iou_gt_id, proposal_id -> [1 if iou > 0.6, 0 ow] + ( + sampled_idxs, + sampled_gt_classes, + ) = self._sample_proposals( # list of sampled proposal_ids, sampled_id -> [0, num_classes)+[bg_label] + matched_idxs, matched_labels, targets[b]["labels"] + ) + pos_pr_inds = sampled_idxs[sampled_gt_classes != self.num_classes] + pos_gt_inds = matched_idxs[pos_pr_inds] + pos_pr_inds, pos_gt_inds = self.postprocess_indices(pos_pr_inds, pos_gt_inds, iou) + indices.append((pos_pr_inds, pos_gt_inds)) + ious.append(iou) + if return_cost_matrix: + return indices, ious + return indices + + def postprocess_indices(self, pr_inds, gt_inds, iou): + return sample_topk_per_gt(pr_inds, gt_inds, iou, self.k) + + def __repr__(self, _repr_indent=8): + head = "Matcher " + self.__class__.__name__ + body = [] + for attribute, value in self.__dict__.items(): + if attribute.startswith("_"): + continue + body.append("{}: {}".format(attribute, value)) + lines = [head] + [" " * _repr_indent + line for line in body] + return "\n".join(lines) + + +class Stage1Assigner(nn.Module): + def __init__(self, t_low=0.3, t_high=0.7, max_k=4): + super().__init__() + self.positive_fraction = 0.5 + self.batch_size_per_image = 256 + self.k = max_k + self.t_low = t_low + self.t_high = t_high + self.anchor_matcher = Matcher( + thresholds=[t_low, t_high], labels=[0, -1, 1], allow_low_quality_matches=True + ) + + def _subsample_labels(self, label): + """ + Randomly sample a subset of positive and negative examples, and overwrite + the label vector to the ignore value (-1) for all elements that are not + included in the sample. + + Args: + labels (Tensor): a vector of -1, 0, 1. Will be modified in-place and returned. + """ + pos_idx, neg_idx = subsample_labels( + label, self.batch_size_per_image, self.positive_fraction, 0 + ) + label.fill_(-1) + label.scatter_(0, pos_idx, 1) + label.scatter_(0, neg_idx, 0) + return label + + def forward(self, outputs, targets): + bs = len(targets) + indices = [] + for b in range(bs): + anchors = outputs["anchors"][b] + if len(targets[b]["boxes"]) == 0: + indices.append( + ( + torch.tensor([], dtype=torch.long, device=anchors.device), + torch.tensor([], dtype=torch.long, device=anchors.device), + ) + ) + continue + iou, _ = box_iou( + box_cxcywh_to_xyxy(targets[b]["boxes"]), + box_cxcywh_to_xyxy(anchors), + ) + matched_idxs, matched_labels = self.anchor_matcher( + iou + ) # proposal_id -> highest_iou_gt_id, proposal_id -> [1 if iou > 0.7, 0 if iou < 0.3, -1 ow] + matched_labels = self._subsample_labels(matched_labels) + + all_pr_inds = torch.arange(len(anchors)) + pos_pr_inds = all_pr_inds[matched_labels == 1] + pos_gt_inds = matched_idxs[pos_pr_inds] + pos_ious = iou[pos_gt_inds, pos_pr_inds] + pos_pr_inds, pos_gt_inds = self.postprocess_indices(pos_pr_inds, pos_gt_inds, iou) + pos_pr_inds, pos_gt_inds = pos_pr_inds.to(anchors.device), pos_gt_inds.to( + anchors.device + ) + indices.append((pos_pr_inds, pos_gt_inds)) + return indices + + def postprocess_indices(self, pr_inds, gt_inds, iou): + return sample_topk_per_gt(pr_inds, gt_inds, iou, self.k) + + def __repr__(self, _repr_indent=8): + head = "Matcher " + self.__class__.__name__ + body = [] + for attribute, value in self.__dict__.items(): + if attribute.startswith("_"): + continue + body.append("{}: {}".format(attribute, value)) + lines = [head] + [" " * _repr_indent + line for line in body] + return "\n".join(lines) diff --git a/approach/ovod/APE/ape/modeling/deta/deformable_criterion.py b/approach/ovod/APE/ape/modeling/deta/deformable_criterion.py new file mode 100644 index 0000000000000000000000000000000000000000..d0c5a80ca19a6dd388cbed8637ca52f15ded0edc --- /dev/null +++ b/approach/ovod/APE/ape/modeling/deta/deformable_criterion.py @@ -0,0 +1,533 @@ +import copy +import logging +from typing import Callable, List, Optional + +import torch +import torch.nn.functional as F + +from detectron2.projects.point_rend.point_features import ( + get_uncertain_point_coords_with_randomness, + point_sample, +) +from detrex.layers import box_cxcywh_to_xyxy, generalized_box_iou +from detrex.modeling import SetCriterion +from detrex.modeling.criterion.criterion import sigmoid_focal_loss +from detrex.modeling.losses import dice_loss +from detrex.utils import get_world_size, is_dist_avail_and_initialized + +from .misc import nested_tensor_from_tensor_list + +logger = logging.getLogger(__name__) + + +def sigmoid_ce_loss( + inputs: torch.Tensor, + targets: torch.Tensor, + num_masks: float, +): + """ + Args: + inputs: A float tensor of arbitrary shape. + The predictions for each example. + targets: A float tensor with the same shape as inputs. Stores the binary + classification label for each element in inputs + (0 for the negative class and 1 for the positive class). + Returns: + Loss tensor + """ + loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction="none") + + return loss.mean(1).sum() / num_masks + + +def calculate_uncertainty(logits): + """ + We estimate uncerainty as L1 distance between 0.0 and the logit prediction in 'logits' for the + foreground class in `classes`. + Args: + logits (Tensor): A tensor of shape (R, 1, ...) for class-specific or + class-agnostic, where R is the total number of predicted masks in all images and C is + the number of foreground classes. The values are logits. + Returns: + scores (Tensor): A tensor of shape (R, 1, ...) that contains uncertainty scores with + the most uncertain locations having the highest uncertainty score. + """ + assert logits.shape[1] == 1 + gt_class_logits = logits.clone() + return -(torch.abs(gt_class_logits)) + + +class DeformableCriterion(SetCriterion): + """This class computes the loss for Deformable-DETR + and two-stage Deformable-DETR + """ + + def __init__( + self, + num_classes, + matcher, + matcher_stage1, + matcher_stage2, + weight_dict, + losses: List[str] = ["class", "boxes"], + eos_coef: float = 0.1, + loss_class_type: str = "focal_loss", + alpha: float = 0.25, + gamma: float = 2.0, + use_fed_loss: bool = False, + get_fed_loss_cls_weights: Optional[Callable] = None, + fed_loss_num_classes: int = 50, + fed_loss_pad_type: str = None, + num_points: int = 12544, + oversample_ratio: float = 3.0, + importance_sample_ratio: float = 0.75, + ): + super(DeformableCriterion, self).__init__( + num_classes=num_classes, + matcher=matcher, + weight_dict=weight_dict, + losses=losses, + eos_coef=eos_coef, + loss_class_type=loss_class_type, + alpha=alpha, + gamma=gamma, + ) + + self.matcher_stage1 = matcher_stage1 + self.matcher_stage2 = matcher_stage2 + + self.use_fed_loss = use_fed_loss + if self.use_fed_loss: + fed_loss_cls_weights = get_fed_loss_cls_weights() + logger.info( + f"fed_loss_cls_weights: {fed_loss_cls_weights.size()} num_classes: {num_classes}" + ) + + if len(fed_loss_cls_weights) < num_classes: + if fed_loss_pad_type == "max": + fed_loss_pad_value = fed_loss_cls_weights.max().item() + elif fed_loss_pad_type == "max1000": + fed_loss_pad_value = fed_loss_cls_weights.max().item() * 1000 + elif fed_loss_pad_type == "mean": + fed_loss_pad_value = fed_loss_cls_weights.mean().item() + elif fed_loss_pad_type == "median": + fed_loss_pad_value = fed_loss_cls_weights.median().item() + elif fed_loss_pad_type == "cat": + fed_loss_pad_classes = torch.arange(len(fed_loss_cls_weights), num_classes) + self.register_buffer("fed_loss_pad_classes", fed_loss_pad_classes) + fed_loss_pad_value = 0 + else: + fed_loss_pad_value = torch.kthvalue( + fed_loss_cls_weights, int(num_classes * 7.0 / 10) + )[0].item() + + logger.info( + f"pad fed_loss_cls_weights with type {fed_loss_pad_type} and value {fed_loss_pad_value}" + ) + if getattr(self, "fed_loss_pad_classes", None) is not None: + logger.info(f"pad fed_loss_classes with {self.fed_loss_pad_classes}") + fed_loss_cls_weights = torch.cat( + ( + fed_loss_cls_weights, + fed_loss_cls_weights.new_full( + (num_classes - len(fed_loss_cls_weights),), + fed_loss_pad_value, + ), + ), + dim=0, + ) + + logger.info(f"fed_loss_cls_weights: {fed_loss_cls_weights[-100:]}") + logger.info( + f"fed_loss_cls_weights: {fed_loss_cls_weights.size()} num_classes: {num_classes}" + ) + + assert ( + len(fed_loss_cls_weights) == self.num_classes + ), "Please check the provided fed_loss_cls_weights. Their size should match num_classes" + self.register_buffer("fed_loss_cls_weights", fed_loss_cls_weights) + self.fed_loss_num_classes = fed_loss_num_classes + + self.num_points = num_points + self.oversample_ratio = oversample_ratio + self.importance_sample_ratio = importance_sample_ratio + + def get_fed_loss_classes(self, gt_classes, num_fed_loss_classes, num_classes, weight): + """ + Args: + gt_classes: a long tensor of shape R that contains the gt class label of each proposal. + num_fed_loss_classes: minimum number of classes to keep when calculating federated loss. + Will sample negative classes if number of unique gt_classes is smaller than this value. + num_classes: number of foreground classes + weight: probabilities used to sample negative classes + + Returns: + Tensor: + classes to keep when calculating the federated loss, including both unique gt + classes and sampled negative classes. + """ + unique_gt_classes = torch.unique(gt_classes) + prob = unique_gt_classes.new_ones(num_classes + 1).float() + prob[-1] = 0 + if len(unique_gt_classes) < num_fed_loss_classes: + prob[:num_classes] = weight.float().clone() + prob[unique_gt_classes] = 0 + sampled_negative_classes = torch.multinomial( + prob, num_fed_loss_classes - len(unique_gt_classes), replacement=False + ) + fed_loss_classes = torch.cat([unique_gt_classes, sampled_negative_classes]) + else: + fed_loss_classes = unique_gt_classes + return fed_loss_classes + + def loss_labels(self, outputs, targets, indices, num_boxes): + """Classification loss (Binary focal loss) + targets dicts must contain the key "labels" containing a tensor of dim [nb_target_boxes] + """ + assert "pred_logits" in outputs + src_logits = outputs["pred_logits"] + + if self.loss_class_type == "ce_loss": + num_classes = src_logits.shape[2] - 1 + elif self.loss_class_type == "focal_loss": + num_classes = src_logits.shape[2] + + idx = self._get_src_permutation_idx(indices) + target_classes_o = torch.cat([t["labels"][J] for t, (_, J) in zip(targets, indices)]) + target_classes = torch.full( + src_logits.shape[:2], + num_classes, + dtype=torch.int64, + device=src_logits.device, + ) + target_classes[idx] = target_classes_o + + if self.loss_class_type == "ce_loss": + loss_class = F.cross_entropy( + src_logits.transpose(1, 2), target_classes, self.empty_weight + ) + elif ( + self.loss_class_type == "focal_loss" + and self.use_fed_loss + and num_classes == len(self.fed_loss_cls_weights) + ): + target_classes_onehot = torch.zeros( + [src_logits.shape[0], src_logits.shape[1], src_logits.shape[2] + 1], + dtype=src_logits.dtype, + layout=src_logits.layout, + device=src_logits.device, + ) + target_classes_onehot.scatter_(2, target_classes.unsqueeze(-1), 1) + target_classes_onehot = target_classes_onehot[:, :, :-1] + fed_loss_classes = self.get_fed_loss_classes( + target_classes_o, + num_fed_loss_classes=self.fed_loss_num_classes, + num_classes=target_classes_onehot.shape[2], + weight=self.fed_loss_cls_weights, + ) + + if getattr(self, "fed_loss_pad_classes", None) is not None: + fed_loss_classes = torch.cat([fed_loss_classes, self.fed_loss_pad_classes]) + fed_loss_classes = torch.unique(fed_loss_classes) + + loss_class = ( + sigmoid_focal_loss( + src_logits[:, :, fed_loss_classes], + target_classes_onehot[:, :, fed_loss_classes], + num_boxes=num_boxes, + alpha=self.alpha, + gamma=self.gamma, + ) + * src_logits.shape[1] + ) + elif self.loss_class_type == "focal_loss": + target_classes_onehot = torch.zeros( + [src_logits.shape[0], src_logits.shape[1], src_logits.shape[2] + 1], + dtype=src_logits.dtype, + layout=src_logits.layout, + device=src_logits.device, + ) + target_classes_onehot.scatter_(2, target_classes.unsqueeze(-1), 1) + target_classes_onehot = target_classes_onehot[:, :, :-1] + loss_class = ( + sigmoid_focal_loss( + src_logits, + target_classes_onehot, + num_boxes=num_boxes, + alpha=self.alpha, + gamma=self.gamma, + ) + * src_logits.shape[1] + ) + + losses = {"loss_class": loss_class} + + return losses + + def loss_boxes(self, outputs, targets, indices, num_boxes): + """Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss + targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4] + The target boxes are expected in format (center_x, center_y, w, h), normalized by the image size. + """ + assert "pred_boxes" in outputs + idx = self._get_src_permutation_idx(indices) + src_boxes = outputs["pred_boxes"][idx] + target_boxes = torch.cat([t["boxes"][i] for t, (_, i) in zip(targets, indices)], dim=0) + + loss_bbox = F.l1_loss(src_boxes, target_boxes, reduction="none") + + losses = {} + losses["loss_bbox"] = loss_bbox.sum() / num_boxes + + loss_giou = 1 - torch.diag( + generalized_box_iou( + box_cxcywh_to_xyxy(src_boxes), + box_cxcywh_to_xyxy(target_boxes), + ) + ) + losses["loss_giou"] = loss_giou.sum() / num_boxes + + return losses + + def loss_boxes_panoptic(self, outputs, targets, indices, num_boxes): + """Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss + targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4] + The target boxes are expected in format (center_x, center_y, w, h), normalized by the image size. + """ + assert "pred_boxes" in outputs + idx = self._get_src_permutation_idx(indices) + src_boxes = outputs["pred_boxes"][idx] + target_boxes = torch.cat([t["boxes"][i] for t, (_, i) in zip(targets, indices)], dim=0) + + if "is_thing" in targets[0]: + is_thing = torch.cat([t["is_thing"][i] for t, (_, i) in zip(targets, indices)], dim=0) + if is_thing.sum() == 0: # no gt + losses = {} + losses["loss_bbox"] = src_boxes.sum() * 0.0 + losses["loss_giou"] = src_boxes.sum() * 0.0 + return losses + target_boxes = target_boxes[is_thing] + src_boxes = src_boxes[is_thing] + + loss_bbox = F.l1_loss(src_boxes, target_boxes, reduction="none") + + losses = {} + losses["loss_bbox"] = loss_bbox.sum() / num_boxes + + loss_giou = 1 - torch.diag( + generalized_box_iou( + box_cxcywh_to_xyxy(src_boxes), + box_cxcywh_to_xyxy(target_boxes), + ) + ) + losses["loss_giou"] = loss_giou.sum() / num_boxes + + return losses + + def loss_masks(self, outputs, targets, indices, num_boxes): + """Compute the losses related to the masks: the focal loss and the dice loss. + targets dicts must contain the key "masks" containing a tensor of dim [nb_target_boxes, h, w] + """ + assert "pred_masks" in outputs + src_idx = self._get_src_permutation_idx(indices) + tgt_idx = self._get_tgt_permutation_idx(indices) + src_masks = outputs["pred_masks"] + src_masks = src_masks[src_idx] + masks = [t["masks"] for t in targets] + target_masks, valid = nested_tensor_from_tensor_list(masks).decompose() + + if target_masks.size(1) == 0: # no gt + losses = {} + losses["loss_mask"] = src_masks.sum() * 0.0 + losses["loss_dice"] = src_masks.sum() * 0.0 + return losses + + target_masks = target_masks.to(src_masks) + target_masks = target_masks[tgt_idx] + + src_masks = F.interpolate( + src_masks[:, None], size=target_masks.shape[-2:], mode="bilinear", align_corners=False + ) + src_masks = src_masks[:, 0].flatten(1) + + target_masks = target_masks.flatten(1) + target_masks = target_masks.view(src_masks.shape) + + losses = { + "loss_mask": sigmoid_focal_loss(src_masks, target_masks, num_boxes), + "loss_dice": dice_loss( + src_masks.sigmoid(), target_masks, reduction="mean", avg_factor=num_boxes + ), + } + del src_masks + del target_masks + return losses + + def loss_masks_maskdino(self, outputs, targets, indices, num_boxes): + """Compute the losses related to the masks: the focal loss and the dice loss. + targets dicts must contain the key "masks" containing a tensor of dim [nb_target_boxes, h, w] + """ + assert "pred_masks" in outputs + src_idx = self._get_src_permutation_idx(indices) + tgt_idx = self._get_tgt_permutation_idx(indices) + src_masks = outputs["pred_masks"] + if not isinstance(src_masks, torch.Tensor): + mask_embeds = src_masks["mask_embeds"] + mask_features = src_masks["mask_features"] + src_masks = torch.cat( + [ + torch.einsum("qc,chw->qhw", mask_embeds[i][src], mask_features[i]) + for i, (src, _) in enumerate(indices) + ], + dim=0, + ) + else: + src_masks = src_masks[src_idx] + masks = [t["masks"] for t in targets] + target_masks, valid = nested_tensor_from_tensor_list(masks).decompose() + + if target_masks.size(1) == 0: # no gt + losses = {} + losses["loss_mask_maskdino"] = src_masks.sum() * 0.0 + losses["loss_dice_maskdino"] = src_masks.sum() * 0.0 + return losses + + target_masks = target_masks.to(src_masks) + target_masks = target_masks[tgt_idx] + + src_masks = src_masks[:, None] + target_masks = target_masks[:, None] + + with torch.no_grad(): + point_coords = get_uncertain_point_coords_with_randomness( + src_masks, + lambda logits: calculate_uncertainty(logits), + self.num_points, + self.oversample_ratio, + self.importance_sample_ratio, + ) + point_labels = point_sample( + target_masks, + point_coords, + align_corners=False, + ).squeeze(1) + + point_logits = point_sample( + src_masks, + point_coords, + align_corners=False, + ).squeeze(1) + + losses = { + "loss_mask_maskdino": sigmoid_ce_loss(point_logits, point_labels, num_boxes), + "loss_dice_maskdino": dice_loss( + point_logits.sigmoid(), point_labels, reduction="mean", avg_factor=num_boxes + ), + } + + del src_masks + del target_masks + return losses + + def get_loss(self, loss, outputs, targets, indices, num_boxes, **kwargs): + loss_map = { + "class": self.loss_labels, + "boxes": self.loss_boxes, + "boxes_panoptic": self.loss_boxes_panoptic, + "masks": self.loss_masks, + "masks_maskdino": self.loss_masks_maskdino, + } + assert loss in loss_map, f"do you really want to compute {loss} loss?" + return loss_map[loss](outputs, targets, indices, num_boxes, **kwargs) + + def forward(self, outputs, targets): + outputs_without_aux = { + k: v for k, v in outputs.items() if k != "aux_outputs" and k != "enc_outputs" + } + + if self.matcher_stage2 is not None: + indices = self.matcher_stage2(outputs_without_aux, targets) + else: + indices = self.matcher(outputs_without_aux, targets) + + num_boxes = sum(len(t["labels"]) for t in targets) + num_boxes = torch.as_tensor( + [num_boxes], dtype=torch.float, device=next(iter(outputs.values())).device + ) + if is_dist_avail_and_initialized(): + torch.distributed.all_reduce(num_boxes) + num_boxes = torch.clamp(num_boxes / get_world_size(), min=1).item() + + if "is_thing" in targets[0] and False: + unique_classes = torch.cat([t["labels"] for t in targets], dim=0) + is_thing = torch.cat([t["is_thing"][i] for t, (_, i) in zip(targets, indices)], dim=0) + all_classes = torch.cat([t["labels"][i] for t, (_, i) in zip(targets, indices)], dim=0) + thing_classes = all_classes[is_thing] + stuff_classes = all_classes[~is_thing] + + print( + "thing_classes", + 1.0 * len(thing_classes) / max(len(torch.unique(thing_classes)), 1), + "stuff_classes", + 1.0 * len(stuff_classes) / max(len(torch.unique(stuff_classes)), 1), + ) + + losses = {} + for loss in self.losses: + kwargs = {} + losses.update(self.get_loss(loss, outputs, targets, indices, num_boxes, **kwargs)) + + if "aux_outputs" in outputs: + for i, aux_outputs in enumerate(outputs["aux_outputs"]): + if self.matcher_stage2 is not None: + pass + else: + indices = self.matcher(aux_outputs, targets) + for loss in self.losses: + if loss == "masks": + continue + l_dict = self.get_loss(loss, aux_outputs, targets, indices, num_boxes, **kwargs) + l_dict = {k + f"_{i}": v for k, v in l_dict.items()} + losses.update(l_dict) + + if "enc_outputs" in outputs: + enc_outputs = outputs["enc_outputs"] + bin_targets = copy.deepcopy(targets) + for bt in bin_targets: + bt["labels"] = torch.zeros_like(bt["labels"]) + if "is_thing" in bt: + del bt["is_thing"] + if self.matcher_stage1 is not None: + indices = self.matcher_stage1(enc_outputs, bin_targets) + else: + indices = self.matcher(enc_outputs, bin_targets) + for loss in self.losses: + if loss == "masks": + continue + if loss == "masks_maskdino": + continue + l_dict = self.get_loss(loss, enc_outputs, bin_targets, indices, num_boxes, **kwargs) + l_dict = {k + "_enc": v for k, v in l_dict.items()} + losses.update(l_dict) + + return losses + + def __repr__(self): + head = "Criterion " + self.__class__.__name__ + body = [ + "matcher: {}".format(self.matcher.__repr__(_repr_indent=8)), + "matcher_stage1: {}".format(self.matcher_stage1), + "matcher_stage2: {}".format(self.matcher_stage2), + "losses: {}".format(self.losses), + "loss_class_type: {}".format(self.loss_class_type), + "weight_dict: {}".format(self.weight_dict), + "num_classes: {}".format(self.num_classes), + "eos_coef: {}".format(self.eos_coef), + "focal loss alpha: {}".format(self.alpha), + "focal loss gamma: {}".format(self.gamma), + "use_fed_loss: {}".format(self.use_fed_loss), + "fed_loss_num_classes: {}".format(self.fed_loss_num_classes), + ] + _repr_indent = 4 + lines = [head] + [" " * _repr_indent + line for line in body] + return "\n".join(lines) diff --git a/approach/ovod/APE/ape/modeling/deta/deformable_detr.py b/approach/ovod/APE/ape/modeling/deta/deformable_detr.py new file mode 100644 index 0000000000000000000000000000000000000000..c00deff73e819ccf7aadb2ee9866a35eea40bd00 --- /dev/null +++ b/approach/ovod/APE/ape/modeling/deta/deformable_detr.py @@ -0,0 +1,453 @@ +import copy +import math +from typing import Dict, List, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from detectron2.layers import move_device_like +from detectron2.modeling import GeneralizedRCNN, detector_postprocess +from detectron2.modeling.roi_heads.fast_rcnn import fast_rcnn_inference +from detectron2.structures import Boxes, ImageList, Instances +from detrex.layers import MLP, box_cxcywh_to_xyxy, box_xyxy_to_cxcywh +from detrex.utils import inverse_sigmoid +from torchvision.ops.boxes import batched_nms + + +class DeformableDETR(nn.Module): + """Implements the Deformable DETR model. + + Code is modified from the `official github repo + `_. + + More details can be found in the `paper + `_ . + + Args: + backbone (nn.Module): the backbone module. + position_embedding (nn.Module): the position embedding module. + neck (nn.Module): the neck module. + transformer (nn.Module): the transformer module. + embed_dim (int): the dimension of the embedding. + num_classes (int): Number of total categories. + num_queries (int): Number of proposal dynamic anchor boxes in Transformer + criterion (nn.Module): Criterion for calculating the total losses. + pixel_mean (List[float]): Pixel mean value for image normalization. + Default: [123.675, 116.280, 103.530]. + pixel_std (List[float]): Pixel std value for image normalization. + Default: [58.395, 57.120, 57.375]. + aux_loss (bool): whether to use auxiliary loss. Default: True. + with_box_refine (bool): whether to use box refinement. Default: False. + as_two_stage (bool): whether to use two-stage. Default: False. + select_box_nums_for_evaluation (int): the number of topk candidates + slected at postprocess for evaluation. Default: 100. + + """ + + def __init__( + self, + backbone, + position_embedding, + neck, + transformer, + embed_dim, + num_classes, + num_queries, + criterion, + pixel_mean: Tuple[float], + pixel_std: Tuple[float], + aux_loss=True, + with_box_refine=False, + as_two_stage=False, + select_box_nums_for_evaluation=100, + select_box_nums_for_evaluation_list: list = None, + input_format: Optional[str] = None, + vis_period: int = 0, + output_dir: Optional[str] = None, + dataset_names: List[str] = [], + dataset_metas: List[str] = [], + test_nms_thresh: float = 0.7, + test_score_thresh: float = 0.0, + ): + super().__init__() + self.backbone = backbone + self.position_embedding = position_embedding + + self.neck = neck + + self.num_queries = num_queries + if not as_two_stage: + self.query_embedding = nn.Embedding(num_queries, embed_dim * 2) + + self.transformer = transformer + + self.num_classes = num_classes + if criterion.loss_class_type == "ce_loss": + self.class_embed = nn.Linear(embed_dim, num_classes + 1) + else: + self.class_embed = nn.Linear(embed_dim, num_classes) + self.bbox_embed = MLP(embed_dim, embed_dim, 4, 3) + + self.aux_loss = aux_loss + self.criterion = criterion + + self.with_box_refine = with_box_refine + self.as_two_stage = as_two_stage + + prior_prob = 0.01 + bias_value = -math.log((1 - prior_prob) / prior_prob) + if criterion.loss_class_type == "ce_loss": + self.class_embed.bias.data = torch.ones(num_classes + 1) * bias_value + else: + self.class_embed.bias.data = torch.ones(num_classes) * bias_value + nn.init.constant_(self.bbox_embed.layers[-1].weight.data, 0) + nn.init.constant_(self.bbox_embed.layers[-1].bias.data, 0) + if self.neck is not None: + for _, neck_layer in self.neck.named_modules(): + if isinstance(neck_layer, nn.Conv2d): + nn.init.xavier_uniform_(neck_layer.weight, gain=1) + nn.init.constant_(neck_layer.bias, 0) + + num_pred = ( + (transformer.decoder.num_layers + 1) if as_two_stage else transformer.decoder.num_layers + ) + if with_box_refine: + self.class_embed = nn.ModuleList( + [copy.deepcopy(self.class_embed) for i in range(num_pred)] + ) + self.bbox_embed = nn.ModuleList( + [copy.deepcopy(self.bbox_embed) for i in range(num_pred)] + ) + nn.init.constant_(self.bbox_embed[0].layers[-1].bias.data[2:], -2.0) + self.transformer.decoder.bbox_embed = self.bbox_embed + else: + nn.init.constant_(self.bbox_embed.layers[-1].bias.data[2:], -2.0) + self.class_embed = nn.ModuleList([self.class_embed for _ in range(num_pred)]) + self.bbox_embed = nn.ModuleList([self.bbox_embed for _ in range(num_pred)]) + self.transformer.decoder.bbox_embed = None + + if as_two_stage: + self.transformer.decoder.class_embed = self.class_embed + if True: + prior_prob = 0.01 + bias_value = -math.log((1 - prior_prob) / prior_prob) + if criterion.loss_class_type == "ce_loss": + self.transformer.decoder.class_embed[-1] = nn.Linear(embed_dim, num_classes + 1) + self.transformer.decoder.class_embed[-1].bias.data = ( + torch.ones(num_classes + 1) * bias_value + ) + else: + self.transformer.decoder.class_embed[-1] = nn.Linear(embed_dim, 1) + self.transformer.decoder.class_embed[-1].bias.data = torch.ones(1) * bias_value + for box_embed in self.bbox_embed: + nn.init.constant_(box_embed.layers[-1].bias.data[2:], 0.0) + + self.select_box_nums_for_evaluation = select_box_nums_for_evaluation + self.select_box_nums_for_evaluation_list = select_box_nums_for_evaluation_list + + self.test_topk_per_image = self.select_box_nums_for_evaluation + self.test_nms_thresh = test_nms_thresh + self.test_score_thresh = test_score_thresh + + self.input_format = input_format + self.vis_period = vis_period + if vis_period > 0: + assert input_format is not None, "input_format is required for visualization!" + + self.register_buffer("pixel_mean", torch.tensor(pixel_mean).view(-1, 1, 1), False) + self.register_buffer("pixel_std", torch.tensor(pixel_std).view(-1, 1, 1), False) + assert ( + self.pixel_mean.shape == self.pixel_std.shape + ), f"{self.pixel_mean} and {self.pixel_std} have different shapes!" + + self.output_dir = output_dir + + self.dataset_names = dataset_names + from detectron2.data.catalog import MetadataCatalog + + if isinstance(dataset_metas, str): + dataset_metas = [dataset_metas] + self.metadata_list = [copy.deepcopy(MetadataCatalog.get(d)) for d in dataset_metas] + assert all(x == self.metadata_list[0] for x in self.metadata_list) + self.metadata = self.metadata_list[0] + + @property + def device(self): + return self.pixel_mean.device + + def _move_to_current_device(self, x): + return move_device_like(x, self.pixel_mean) + + def forward(self, batched_inputs, do_postprocess=True): + images = self.preprocess_image(batched_inputs) + + batch_size, _, H, W = images.tensor.shape + img_masks = images.tensor.new_ones(batch_size, H, W) + for image_id, image_size in enumerate(images.image_sizes): + img_masks[image_id, : image_size[0], : image_size[1]] = 0 + + features = self.backbone(images.tensor) # output feature dict + + if self.neck is not None: + multi_level_feats = self.neck(features) + else: + multi_level_feats = [feat for feat_name, feat in features.items()] + multi_level_masks = [] + multi_level_position_embeddings = [] + for feat in multi_level_feats: + multi_level_masks.append( + F.interpolate(img_masks[None], size=feat.shape[-2:]).to(torch.bool).squeeze(0) + ) + multi_level_position_embeddings.append( + self.position_embedding(multi_level_masks[-1]).to(images.tensor.dtype) + ) + + query_embeds = None + if not self.as_two_stage: + query_embeds = self.query_embedding.weight + + ( + inter_states, + init_reference, + inter_references, + enc_outputs_class, + enc_outputs_coord_unact, + anchors, + memory, + ) = self.transformer( + multi_level_feats, multi_level_masks, multi_level_position_embeddings, query_embeds + ) + + outputs_classes = [] + outputs_coords = [] + for lvl in range(inter_states.shape[0]): + if lvl == 0: + reference = init_reference + else: + reference = inter_references[lvl - 1] + reference = inverse_sigmoid(reference) + outputs_class = self.class_embed[lvl](inter_states[lvl]) + tmp = self.bbox_embed[lvl](inter_states[lvl]) + if reference.shape[-1] == 4: + tmp += reference + else: + assert reference.shape[-1] == 2 + tmp[..., :2] += reference + outputs_coord = tmp.sigmoid() + outputs_classes.append(outputs_class) + outputs_coords.append(outputs_coord) + outputs_class = torch.stack(outputs_classes) + outputs_coord = torch.stack(outputs_coords) + + output = { + "pred_logits": outputs_class[-1], + "pred_boxes": outputs_coord[-1], + "init_reference": init_reference, + } + if self.aux_loss: + output["aux_outputs"] = self._set_aux_loss(outputs_class, outputs_coord) + + if self.as_two_stage: + enc_outputs_coord = enc_outputs_coord_unact.sigmoid() + output["enc_outputs"] = { + "pred_logits": enc_outputs_class, + "pred_boxes": enc_outputs_coord, + "anchors": anchors, + } + + if self.training: + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + targets = self.prepare_targets(gt_instances) + loss_dict = self.criterion(output, targets) + weight_dict = self.criterion.weight_dict + for k in loss_dict.keys(): + if k in weight_dict: + loss_dict[k] *= weight_dict[k] + return loss_dict + else: + box_cls = output["pred_logits"] + box_pred = output["pred_boxes"] + results, filter_inds = self.inference(box_cls, box_pred, images.image_sizes) + + if do_postprocess: + assert not torch.jit.is_scripting(), "Scripting is not supported for postprocess." + return GeneralizedRCNN._postprocess(results, batched_inputs, images.image_sizes) + return results + + @torch.jit.unused + def _set_aux_loss(self, outputs_class, outputs_coord): + return [ + {"pred_logits": a, "pred_boxes": b} + for a, b in zip(outputs_class[:-1], outputs_coord[:-1]) + ] + + def inference(self, box_cls, box_pred, image_sizes): + """ + Arguments: + box_cls (Tensor): tensor of shape (batch_size, num_queries, K). + The tensor predicts the classification probability for each query. + box_pred (Tensor): tensors of shape (batch_size, num_queries, 4). + The tensor predicts 4-vector (x,y,w,h) box + regression values for every queryx + image_sizes (List[torch.Size]): the input image sizes + + Returns: + results (List[Instances]): a list of #images elements. + """ + + if True: + return NMSPostProcess()( + {"pred_logits": box_cls, "pred_boxes": box_pred}, + torch.tensor([list(x) for x in image_sizes], device=self.device), + self.select_box_nums_for_evaluation, + ) + + scores = torch.cat( + ( + box_cls.sigmoid(), + torch.zeros((box_cls.size(0), box_cls.size(1), 1), device=self.device), + ), + dim=2, + ) + + boxes = box_cxcywh_to_xyxy(box_pred) + + img_h, img_w = torch.tensor(image_sizes, device=self.device).unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) + boxes = boxes * scale_fct[:, None, :] + + boxes = boxes.unbind(0) + scores = scores.unbind(0) + image_shapes = image_sizes + + self.test_topk_per_image = self.select_box_nums_for_evaluation + self.test_nms_thresh = 0.7 + self.test_score_thresh = 0.05 + + return fast_rcnn_inference( + boxes, + scores, + image_shapes, + self.test_score_thresh, + self.test_nms_thresh, + self.test_topk_per_image, + ) + + assert len(box_cls) == len(image_sizes) + results = [] + + prob = box_cls.sigmoid() + topk_values, topk_indexes = torch.topk( + prob.view(box_cls.shape[0], -1), self.select_box_nums_for_evaluation, dim=1 + ) + scores = topk_values + topk_boxes = torch.div(topk_indexes, box_cls.shape[2], rounding_mode="floor") + labels = topk_indexes % box_cls.shape[2] + + boxes = torch.gather(box_pred, 1, topk_boxes.unsqueeze(-1).repeat(1, 1, 4)) + + for i, (scores_per_image, labels_per_image, box_pred_per_image, image_size) in enumerate( + zip(scores, labels, boxes, image_sizes) + ): + result = Instances(image_size) + result.pred_boxes = Boxes(box_cxcywh_to_xyxy(box_pred_per_image)) + result.pred_boxes.scale(scale_x=image_size[1], scale_y=image_size[0]) + result.scores = scores_per_image + result.pred_classes = labels_per_image + results.append(result) + return results, topk_indexes + + def prepare_targets(self, targets): + new_targets = [] + for targets_per_image in targets: + h, w = targets_per_image.image_size + image_size_xyxy = torch.as_tensor([w, h, w, h], dtype=torch.float, device=self.device) + gt_classes = targets_per_image.gt_classes + gt_boxes = targets_per_image.gt_boxes.tensor / image_size_xyxy + gt_boxes = box_xyxy_to_cxcywh(gt_boxes) + new_targets.append({"labels": gt_classes, "boxes": gt_boxes}) + return new_targets + + def preprocess_image(self, batched_inputs): + images = [self._move_to_current_device(x["image"]) for x in batched_inputs] + images = [x.to(self.pixel_mean.dtype) for x in images] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors( + images, + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ) + return images + + @staticmethod + def _postprocess(instances, batched_inputs: List[Dict[str, torch.Tensor]], image_sizes): + """ + Rescale the output instances to the target size. + """ + processed_results = [] + for results_per_image, input_per_image, image_size in zip( + instances, batched_inputs, image_sizes + ): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + r = detector_postprocess(results_per_image, height, width) + processed_results.append({"instances": r}) + return processed_results + + +class NMSPostProcess(nn.Module): + """This module converts the model's output into the format expected by the coco api""" + + @torch.no_grad() + def forward(self, outputs, target_sizes, select_box_nums_for_evaluation): + """Perform the computation + Parameters: + outputs: raw outputs of the model + target_sizes: tensor of dimension [batch_size x 2] containing the size of each images of the batch + For evaluation, this must be the original image size (before any data augmentation) + For visualization, this should be the image size after data augment, but before padding + """ + out_logits, out_bbox = outputs["pred_logits"], outputs["pred_boxes"] + bs, n_queries, n_cls = out_logits.shape + + assert len(out_logits) == len(target_sizes) + assert target_sizes.shape[1] == 2 + + prob = out_logits.sigmoid() + + all_scores = prob.view(bs, n_queries * n_cls).to(out_logits.device) + all_indexes = torch.arange(n_queries * n_cls)[None].repeat(bs, 1).to(out_logits.device) + all_boxes = torch.div(all_indexes, out_logits.shape[2], rounding_mode="trunc") + all_labels = all_indexes % out_logits.shape[2] + + boxes = box_cxcywh_to_xyxy(out_bbox) + boxes = torch.gather(boxes, 1, all_boxes.unsqueeze(-1).repeat(1, 1, 4)) + + img_h, img_w = target_sizes.unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) + boxes = boxes * scale_fct[:, None, :] + + results = [] + keep_inds_all = [] + for b in range(bs): + box = boxes[b] + score = all_scores[b] + lbls = all_labels[b] + + pre_topk = score.topk(10000).indices + box = box[pre_topk] + score = score[pre_topk] + lbls = lbls[pre_topk] + + keep_inds = batched_nms(box, score, lbls, 0.7)[:select_box_nums_for_evaluation] + + result = Instances(target_sizes[b]) + result.pred_boxes = Boxes(box[keep_inds]) + result.scores = score[keep_inds] + result.pred_classes = lbls[keep_inds] + results.append(result) + + keep_inds_all.append(keep_inds) + + return results, keep_inds_all diff --git a/approach/ovod/APE/ape/modeling/deta/deformable_detr_segm.py b/approach/ovod/APE/ape/modeling/deta/deformable_detr_segm.py new file mode 100644 index 0000000000000000000000000000000000000000..74c8ab30b1cb48c77c1d36907b7e2e8fffdad26a --- /dev/null +++ b/approach/ovod/APE/ape/modeling/deta/deformable_detr_segm.py @@ -0,0 +1,940 @@ +import copy +import math +import os +from typing import Dict, List, Optional, Tuple + +import cv2 +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +import fvcore.nn.weight_init as weight_init +from detectron2.data.detection_utils import convert_image_to_rgb +from detectron2.layers import Conv2d, ShapeSpec, get_norm, move_device_like +from detectron2.modeling import GeneralizedRCNN +from detectron2.modeling.meta_arch.panoptic_fpn import combine_semantic_and_instance_outputs +from detectron2.modeling.postprocessing import detector_postprocess, sem_seg_postprocess +from detectron2.modeling.roi_heads.fast_rcnn import fast_rcnn_inference +from detectron2.structures import BitMasks, Boxes, ImageList, Instances +from detectron2.utils.events import get_event_storage +from detectron2.utils.memory import retry_if_cuda_oom +from detrex.layers import MLP, box_cxcywh_to_xyxy, box_xyxy_to_cxcywh +from detrex.utils import inverse_sigmoid +from torchvision.ops.boxes import batched_nms + +from .deformable_detr import DeformableDETR +from .segmentation import MaskHeadSmallConv, MHAttentionMap + + +class DeformableDETRSegm(DeformableDETR): + """Implements the Deformable DETR model. + + Code is modified from the `official github repo + `_. + + More details can be found in the `paper + `_ . + + Args: + backbone (nn.Module): the backbone module. + position_embedding (nn.Module): the position embedding module. + neck (nn.Module): the neck module. + transformer (nn.Module): the transformer module. + embed_dim (int): the dimension of the embedding. + num_classes (int): Number of total categories. + num_queries (int): Number of proposal dynamic anchor boxes in Transformer + criterion (nn.Module): Criterion for calculating the total losses. + pixel_mean (List[float]): Pixel mean value for image normalization. + Default: [123.675, 116.280, 103.530]. + pixel_std (List[float]): Pixel std value for image normalization. + Default: [58.395, 57.120, 57.375]. + aux_loss (bool): whether to use auxiliary loss. Default: True. + with_box_refine (bool): whether to use box refinement. Default: False. + as_two_stage (bool): whether to use two-stage. Default: False. + select_box_nums_for_evaluation (int): the number of topk candidates + slected at postprocess for evaluation. Default: 100. + + """ + + def __init__( + self, + instance_on: bool = True, + semantic_on: bool = False, + panoptic_on: bool = False, + freeze_detr=False, + input_shapes=[], + mask_in_features=[], + mask_encode_level=0, + stuff_dataset_learn_thing: bool = True, + stuff_prob_thing: float = -1.0, + test_mask_on: bool = True, + semantic_post_nms: bool = True, + panoptic_post_nms: bool = True, + aux_mask: bool = True, + **kwargs, + ): + super().__init__(**kwargs) + + self.instance_on = instance_on + self.semantic_on = semantic_on + self.panoptic_on = panoptic_on + + if freeze_detr: + for p in self.parameters(): + p.requires_grad_(False) + + self.input_shapes = input_shapes + self.mask_in_features = mask_in_features + self.mask_encode_level = mask_encode_level + + hidden_dim = self.transformer.embed_dim + norm = "GN" + use_bias = False + + assert len(self.mask_in_features) == 1 + in_channels = [self.input_shapes[feat_name].channels for feat_name in self.mask_in_features] + in_channel = in_channels[0] + + self.lateral_conv = Conv2d( + in_channel, + hidden_dim, + kernel_size=1, + stride=1, + bias=use_bias, + padding=0, + norm=get_norm(norm, hidden_dim), + ) + self.output_conv = Conv2d( + hidden_dim, + hidden_dim, + kernel_size=3, + stride=1, + bias=use_bias, + padding=1, + norm=get_norm(norm, hidden_dim), + activation=F.relu, + ) + self.mask_conv = Conv2d( + hidden_dim, hidden_dim, kernel_size=1, stride=1, bias=use_bias, padding=0 + ) + + self.mask_embed = MLP(hidden_dim, hidden_dim, hidden_dim, 3) + self.aux_mask = aux_mask + if self.aux_mask: + self.mask_embed = nn.ModuleList( + [copy.deepcopy(self.mask_embed) for i in range(len(self.class_embed) - 1)] + ) + + weight_init.c2_xavier_fill(self.lateral_conv) + weight_init.c2_xavier_fill(self.output_conv) + weight_init.c2_xavier_fill(self.mask_conv) + + self.stuff_dataset_learn_thing = stuff_dataset_learn_thing + self.stuff_prob_thing = stuff_prob_thing + self.test_mask_on = test_mask_on + self.semantic_post_nms = semantic_post_nms + self.panoptic_post_nms = panoptic_post_nms + + def forward(self, batched_inputs, do_postprocess=True): + images = self.preprocess_image(batched_inputs) + + batch_size, _, H, W = images.tensor.shape + img_masks = images.tensor.new_ones(batch_size, H, W) + for image_id, image_size in enumerate(images.image_sizes): + img_masks[image_id, : image_size[0], : image_size[1]] = 0 + + features = self.backbone(images.tensor) # output feature dict + + if self.neck is not None: + multi_level_feats = self.neck({f: features[f] for f in self.neck.in_features}) + else: + multi_level_feats = [feat for feat_name, feat in features.items()] + multi_level_masks = [] + multi_level_position_embeddings = [] + for feat in multi_level_feats: + multi_level_masks.append( + F.interpolate(img_masks[None], size=feat.shape[-2:]).to(torch.bool).squeeze(0) + ) + multi_level_position_embeddings.append( + self.position_embedding(multi_level_masks[-1]).to(images.tensor.dtype) + ) + + query_embeds = None + if not self.as_two_stage: + query_embeds = self.query_embedding.weight + + ( + inter_states, + init_reference, + inter_references, + enc_outputs_class, + enc_outputs_coord_unact, + anchors, + memory, + ) = self.transformer( + multi_level_feats, multi_level_masks, multi_level_position_embeddings, query_embeds + ) + + mask_features = self.maskdino_mask_features(memory, features, multi_level_masks) + + outputs_classes = [] + outputs_coords = [] + outputs_masks = [] + for lvl in range(inter_states.shape[0]): + if lvl == 0: + reference = init_reference + else: + reference = inter_references[lvl - 1] + reference = inverse_sigmoid(reference) + outputs_class = self.class_embed[lvl](inter_states[lvl]) + tmp = self.bbox_embed[lvl](inter_states[lvl]) + if reference.shape[-1] == 4: + tmp += reference + else: + assert reference.shape[-1] == 2 + tmp[..., :2] += reference + outputs_coord = tmp.sigmoid() + outputs_classes.append(outputs_class) + outputs_coords.append(outputs_coord) + + if self.aux_mask: + mask_embeds = self.mask_embed[lvl](inter_states[lvl]) + else: + mask_embeds = self.mask_embed(inter_states[lvl]) + outputs_mask = torch.einsum("bqc,bchw->bqhw", mask_embeds, mask_features) + outputs_masks.append(outputs_mask) + outputs_class = torch.stack(outputs_classes) + outputs_coord = torch.stack(outputs_coords) + outputs_mask = outputs_masks + if self.aux_mask: + outputs_mask[-1] += 0.0 * sum(outputs_mask) + + output = { + "pred_logits": outputs_class[-1], + "pred_boxes": outputs_coord[-1], + "pred_masks": outputs_mask[-1], + "init_reference": init_reference, + } + if self.aux_loss: + output["aux_outputs"] = self._set_aux_loss( + outputs_class, + outputs_coord, + outputs_mask, + ) + + if self.as_two_stage: + enc_outputs_coord = enc_outputs_coord_unact.sigmoid() + output["enc_outputs"] = { + "pred_logits": enc_outputs_class, + "pred_boxes": enc_outputs_coord, + "anchors": anchors, + } + + if ( + self.vis_period > 0 + and self.training + and get_event_storage().iter % self.vis_period == self.vis_period - 1 + ): + self.visualize_training(batched_inputs, output, images) + + if self.training: + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + targets = self.prepare_targets(gt_instances) + + loss_dict = self.criterion(output, targets) + weight_dict = self.criterion.weight_dict + for k in loss_dict.keys(): + if k in weight_dict: + loss_dict[k] *= weight_dict[k] + return loss_dict + else: + + box_cls = output["pred_logits"] + box_pred = output["pred_boxes"] + mask_pred = output["pred_masks"] + + iter_func = retry_if_cuda_oom(F.interpolate) + mask_pred = iter_func( + mask_pred, size=images.tensor.size()[2:], mode="bilinear", align_corners=False + ) + + merged_results = [{} for _ in range(box_cls.size(0))] + if self.instance_on: + if self.metadata is not None: + if is_thing_stuff_overlap(self.metadata): + thing_id = self.metadata.thing_dataset_id_to_contiguous_id.values() + thing_id = torch.Tensor(list(thing_id)).to(torch.long).to(self.device) + + detector_box_cls = torch.zeros_like(box_cls) + detector_box_cls += float("-inf") + detector_box_cls[..., thing_id] = box_cls[..., thing_id] + else: + num_thing_classes = len(self.metadata.thing_classes) + detector_box_cls = box_cls[..., :num_thing_classes] + else: + detector_box_cls = box_cls + + detector_results, filter_inds = self.inference( + detector_box_cls, box_pred, images.image_sizes + ) + + if self.test_mask_on: + detector_mask_preds = [ + x[filter_ind] for x, filter_ind in zip(mask_pred, filter_inds) + ] + + for result, box_mask in zip(detector_results, detector_mask_preds): + box_mask = box_mask.sigmoid() > 0.5 + box_mask = BitMasks(box_mask).crop_and_resize( + result.pred_boxes.tensor.to(box_mask.device), 128 + ) + result.pred_masks = ( + box_mask.to(result.pred_boxes.tensor.device) + .unsqueeze(1) + .to(dtype=torch.float32) + ) + + if do_postprocess: + assert ( + not torch.jit.is_scripting() + ), "Scripting is not supported for postprocess." + detector_results = DeformableDETRSegm._postprocess_instance( + detector_results, batched_inputs, images.image_sizes + ) + for merged_result, detector_result in zip(merged_results, detector_results): + merged_result.update(detector_result) + + else: + detector_results = None + + if self.semantic_on: + + semantic_mask_pred = mask_pred.clone() + + if self.metadata is not None: + if is_thing_stuff_overlap(self.metadata): + semantic_box_cls = box_cls.clone() + + else: + num_thing_classes = len(self.metadata.get("thing_classes", ["things"])) + + semantic_box_cls_0 = box_cls[..., :num_thing_classes] + semantic_box_cls_1 = box_cls[..., num_thing_classes:] + semantic_box_cls_0, _ = semantic_box_cls_0.min(dim=2, keepdim=True) + semantic_box_cls = torch.cat( + [semantic_box_cls_0, semantic_box_cls_1], dim=2 + ) + else: + semantic_box_cls = box_cls.clone() + + if self.semantic_post_nms: + _, filter_inds = self.inference(semantic_box_cls, box_pred, images.image_sizes) + semantic_box_cls = torch.stack( + [x[filter_ind] for x, filter_ind in zip(semantic_box_cls, filter_inds)], + dim=0, + ) + semantic_mask_pred = torch.stack( + [x[filter_ind] for x, filter_ind in zip(semantic_mask_pred, filter_inds)], + dim=0, + ) + + if do_postprocess: + assert ( + not torch.jit.is_scripting() + ), "Scripting is not supported for postprocess." + semantic_results = DeformableDETRSegm._postprocess_semantic( + semantic_box_cls, semantic_mask_pred, batched_inputs, images + ) + for merged_result, semantic_result in zip(merged_results, semantic_results): + if self.stuff_prob_thing > 0: + semantic_result["sem_seg"][0, ...] = math.log( + self.stuff_prob_thing / (1 - self.stuff_prob_thing) + ) + merged_result.update(semantic_result) + + else: + semantic_results = None + + if self.panoptic_on: + assert self.metadata is not None + if do_postprocess: + assert ( + not torch.jit.is_scripting() + ), "Scripting is not supported for postprocess." + if True: + if self.panoptic_post_nms: + _, filter_inds = self.inference(box_cls, box_pred, images.image_sizes) + panoptic_mask_pred = [ + x[filter_ind] for x, filter_ind in zip(mask_pred, filter_inds) + ] + panoptic_box_cls = [ + x[filter_ind] for x, filter_ind in zip(box_cls, filter_inds) + ] + + panoptic_results = DeformableDETRSegm._postprocess_panoptic( + panoptic_box_cls, + panoptic_mask_pred, + batched_inputs, + images, + self.metadata, + ) + else: + panoptic_results = [] + self.combine_overlap_thresh = 0.5 + self.combine_stuff_area_thresh = 4096 + self.combine_instances_score_thresh = 0.5 + for detector_result, semantic_result in zip( + detector_results, semantic_results + ): + detector_r = detector_result["instances"] + sem_seg_r = semantic_result["sem_seg"] + panoptic_r = combine_semantic_and_instance_outputs( + detector_r, + sem_seg_r.argmax(dim=0), + self.combine_overlap_thresh, + self.combine_stuff_area_thresh, + self.combine_instances_score_thresh, + ) + panoptic_results.append({"panoptic_seg": panoptic_r}) + for merged_result, panoptic_result in zip(merged_results, panoptic_results): + merged_result.update(panoptic_result) + + else: + panoptic_results = None + + if do_postprocess: + return merged_results + + return detector_results, semantic_results, panoptic_results + + def maskdino_mask_features(self, encode_feats, multi_level_feats, multi_level_masks): + start_idx = sum( + [mask.shape[1] * mask.shape[2] for mask in multi_level_masks[: self.mask_encode_level]] + ) + end_idx = sum( + [ + mask.shape[1] * mask.shape[2] + for mask in multi_level_masks[: self.mask_encode_level + 1] + ] + ) + b, h, w = multi_level_masks[self.mask_encode_level].size() + + encode_feats = encode_feats[:, start_idx:end_idx, :] + encode_feats = encode_feats.permute(0, 2, 1).reshape(b, -1, h, w) + + x = [multi_level_feats[f] for f in self.mask_in_features] + x = x[0] + x = self.lateral_conv(x) + x = x + F.interpolate(encode_feats, size=x.shape[-2:], mode="bilinear", align_corners=False) + x = self.output_conv(x) + mask_features = self.mask_conv(x) + + return mask_features + + @torch.jit.unused + def _set_aux_loss(self, outputs_class, outputs_coord, outputs_mask): + return [ + {"pred_logits": a, "pred_boxes": b, "pred_masks": c} + for a, b, c in zip(outputs_class[:-1], outputs_coord[:-1], outputs_mask[:-1]) + ] + + def inference(self, box_cls, box_pred, image_sizes): + """ + Arguments: + box_cls (Tensor): tensor of shape (batch_size, num_queries, K). + The tensor predicts the classification probability for each query. + box_pred (Tensor): tensors of shape (batch_size, num_queries, 4). + The tensor predicts 4-vector (x,y,w,h) box + regression values for every queryx + image_sizes (List[torch.Size]): the input image sizes + + Returns: + results (List[Instances]): a list of #images elements. + """ + + if True: + + scores = torch.cat( + ( + box_cls.sigmoid(), + torch.zeros((box_cls.size(0), box_cls.size(1), 1), device=self.device), + ), + dim=2, + ) + + boxes = box_cxcywh_to_xyxy(box_pred) + + img_h, img_w = torch.tensor(image_sizes, device=self.device).unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) + boxes = boxes * scale_fct[:, None, :] + + boxes = boxes.unbind(0) + scores = scores.unbind(0) + image_shapes = image_sizes + + results, filter_inds = fast_rcnn_inference( + boxes, + scores, + image_shapes, + self.test_score_thresh, + self.test_nms_thresh, + self.test_topk_per_image, + ) + + return results, filter_inds + + assert len(box_cls) == len(image_sizes) + results = [] + + prob = box_cls.sigmoid() + topk_values, topk_indexes = torch.topk( + prob.view(box_cls.shape[0], -1), self.select_box_nums_for_evaluation, dim=1 + ) + scores = topk_values + topk_boxes = torch.div(topk_indexes, box_cls.shape[2], rounding_mode="floor") + labels = topk_indexes % box_cls.shape[2] + + boxes = torch.gather(box_pred, 1, topk_boxes.unsqueeze(-1).repeat(1, 1, 4)) + + for i, (scores_per_image, labels_per_image, box_pred_per_image, image_size) in enumerate( + zip(scores, labels, boxes, image_sizes) + ): + result = Instances(image_size) + result.pred_boxes = Boxes(box_cxcywh_to_xyxy(box_pred_per_image)) + result.pred_boxes.scale(scale_x=image_size[1], scale_y=image_size[0]) + result.scores = scores_per_image + result.pred_classes = labels_per_image + results.append(result) + return results, topk_indexes + + def prepare_targets(self, targets): + new_targets = [] + for targets_per_image in targets: + h, w = targets_per_image.image_size + image_size_xyxy = torch.as_tensor([w, h, w, h], dtype=torch.float, device=self.device) + gt_classes = targets_per_image.gt_classes + gt_boxes = targets_per_image.gt_boxes.tensor / image_size_xyxy + gt_boxes = box_xyxy_to_cxcywh(gt_boxes) + + if not targets_per_image.has("gt_masks"): + gt_masks = torch.zeros((0, h, w), dtype=torch.bool) + else: + gt_masks = targets_per_image.gt_masks + + if not isinstance(gt_masks, torch.Tensor): + if isinstance(gt_masks, BitMasks): + gt_masks = gt_masks.tensor + else: + gt_masks = BitMasks.from_polygon_masks(gt_masks, h, w).tensor + + gt_masks = self._move_to_current_device(gt_masks) + gt_masks = ImageList.from_tensors( + [gt_masks], + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ).tensor.squeeze(0) + + new_targets.append({"labels": gt_classes, "boxes": gt_boxes, "masks": gt_masks}) + + if targets_per_image.has("is_thing"): + new_targets[-1]["is_thing"] = targets_per_image.is_thing + + return new_targets + + def preprocess_image(self, batched_inputs): + images = [self._move_to_current_device(x["image"]) for x in batched_inputs] + images = [x.to(self.pixel_mean.dtype) for x in images] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors( + images, + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ) + return images + + @staticmethod + def _postprocess_instance( + instances, batched_inputs: List[Dict[str, torch.Tensor]], image_sizes + ): + """ + Rescale the output instances to the target size. + """ + processed_results = [] + for results_per_image, input_per_image, image_size in zip( + instances, batched_inputs, image_sizes + ): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + r = detector_postprocess(results_per_image, height, width) + processed_results.append({"instances": r.to("cpu")}) + return processed_results + + @staticmethod + def _postprocess_semantic( + mask_clses, + mask_preds, + batched_inputs: List[Dict[str, torch.Tensor]], + images, + pano_temp=0.06, + transform_eval=True, + ): + processed_results = [] + for mask_cls, mask_pred, input_per_image, image_size in zip( + mask_clses, mask_preds, batched_inputs, images.image_sizes + ): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + + T = pano_temp + mask_cls = mask_cls.sigmoid() + + if transform_eval: + mask_cls = F.softmax(mask_cls / T, dim=-1) # already sigmoid + mask_pred = mask_pred.sigmoid() + result = torch.einsum("qc,qhw->chw", mask_cls, mask_pred) + + r = sem_seg_postprocess(result, image_size, height, width) + processed_results.append({"sem_seg": r}) + return processed_results + + @staticmethod + def _postprocess_panoptic( + mask_clses, + mask_preds, + batched_inputs: List[Dict[str, torch.Tensor]], + images, + metadata, + prob=0.5, + pano_temp=0.06, + transform_eval=True, + object_mask_threshold=0.25, + overlap_threshold=0.8, + ): + num_classes = len(metadata.thing_classes) + len(metadata.stuff_classes) - 1 + + object_mask_threshold = 0.01 + overlap_threshold = 0.4 + prob = 0.1 + + processed_results = [] + for mask_cls, mask_pred, input_per_image, image_size in zip( + mask_clses, mask_preds, batched_inputs, images.image_sizes + ): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + + mask_pred = sem_seg_postprocess(mask_pred, image_size, height, width) + + T = pano_temp + scores, labels = mask_cls.sigmoid().max(-1) + mask_pred = mask_pred.sigmoid() + keep = labels.ne(num_classes) & (scores > object_mask_threshold) + if transform_eval: + scores, labels = F.softmax(mask_cls.sigmoid() / T, dim=-1).max(-1) + cur_scores = scores[keep] + cur_classes = labels[keep] + cur_masks = mask_pred[keep] + cur_prob_masks = cur_scores.view(-1, 1, 1) * cur_masks + + panoptic_seg = torch.zeros((height, width), dtype=torch.int32, device=cur_masks.device) + segments_info = [] + + current_segment_id = 0 + + if cur_masks.size(0) > 0: + + cur_mask_ids = cur_prob_masks.argmax(0) + + stuff_memory_list = {} + for k in range(cur_classes.shape[0]): + pred_class = cur_classes[k].item() + isthing = pred_class in metadata.thing_dataset_id_to_contiguous_id.values() + mask_area = (cur_mask_ids == k).sum().item() + original_area = (cur_masks[k] >= prob).sum().item() + mask = (cur_mask_ids == k) & (cur_masks[k] >= prob) + + if mask_area > 0 and original_area > 0 and mask.sum().item() > 0: + if mask_area / original_area < overlap_threshold: + continue + + if not isthing: + if int(pred_class) in stuff_memory_list.keys(): + panoptic_seg[mask] = stuff_memory_list[int(pred_class)] + continue + else: + stuff_memory_list[int(pred_class)] = current_segment_id + 1 + + current_segment_id += 1 + panoptic_seg[mask] = current_segment_id + + if not isthing and not is_thing_stuff_overlap(metadata): + pred_class = int(pred_class) - len(metadata.thing_classes) + 1 + + segments_info.append( + { + "id": current_segment_id, + "isthing": bool(isthing), + "category_id": int(pred_class), + } + ) + + processed_results.append({"panoptic_seg": (panoptic_seg, segments_info)}) + return processed_results + + @torch.no_grad() + def visualize_training(self, batched_inputs, output, images): + if self.output_dir is None: + return + if self.training: + storage = get_event_storage() + os.makedirs(self.output_dir + "/training", exist_ok=True) + else: + os.makedirs(self.output_dir + "/inference", exist_ok=True) + + pred_logits = output["pred_logits"] + pred_boxes = output["pred_boxes"] + pred_masks = output["pred_masks"] + + thing_classes = self.metadata.get("thing_classes", []) + stuff_classes = self.metadata.get("stuff_classes", []) + if len(thing_classes) > 0 and len(stuff_classes) > 0 and stuff_classes[0] == "things": + stuff_classes = stuff_classes[1:] + if is_thing_stuff_overlap(self.metadata): + class_names = ( + thing_classes if len(thing_classes) > len(stuff_classes) else stuff_classes + ) + else: + class_names = thing_classes + stuff_classes + + num_thing_classes = len(class_names) + pred_logits = pred_logits[..., :num_thing_classes] + + if pred_masks is not None: + pred_masks = [ + F.interpolate( + pred_mask.float().cpu().unsqueeze(0), + size=images.tensor.size()[2:], + mode="bilinear", + align_corners=False, + ).squeeze(0) + if pred_mask.size(0) > 0 + else pred_mask + for pred_mask in pred_masks + ] + else: + pred_masks = [ + torch.zeros(pred_box.size(0), image_size[0], image_size[1]) + for pred_box, image_size in zip(pred_boxes, images.image_sizes) + ] + + if True: + results, filter_inds = self.inference(pred_logits, pred_boxes, images.image_sizes) + pred_masks = [ + pred_mask[filter_ind.cpu()] + for pred_mask, filter_ind in zip(pred_masks, filter_inds) + ] + for result, pred_mask in zip(results, pred_masks): + result.pred_masks = pred_mask.sigmoid() > 0.5 + else: + results = [] + for pred_logit, pred_box, pred_mask, image_size in zip( + pred_logits, pred_boxes, pred_masks, images.image_sizes + ): + result = Instances(image_size) + result.pred_boxes = Boxes(pred_box) + result.scores = pred_logit[:, 0] + result.pred_classes = torch.zeros( + len(pred_box), dtype=torch.int64, device=pred_logit.device + ) + result.pred_masks = pred_mask.sigmoid() > 0.5 + + results.append(result) + + from detectron2.utils.visualizer import Visualizer + + for input, result in zip(batched_inputs, results): + + img = input["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format) + v_gt = Visualizer(img, None) + + if "instances" in input: + labels = [ + "{}".format(class_names[gt_class]) for gt_class in input["instances"].gt_classes + ] + v_gt = v_gt.overlay_instances( + boxes=input["instances"].gt_boxes, + masks=input["instances"].gt_masks + if input["instances"].has("gt_masks") + else None, + labels=labels, + ) + else: + v_gt = v_gt.output + anno_img = v_gt.get_image() + + labels = [ + "{}_{:.0f}%".format(class_names[pred_class], score * 100) + for pred_class, score in zip(result.pred_classes.cpu(), result.scores.cpu()) + ] + v_pred = Visualizer(img, None) + v_pred = v_pred.overlay_instances( + boxes=result.pred_boxes.tensor.clone().detach().cpu().numpy(), + labels=labels, + masks=result.pred_masks[:, : img.shape[0], : img.shape[1]] + .clone() + .detach() + .cpu() + .numpy() + if result.has("pred_masks") + else None, + ) + pred_img = v_pred.get_image() + + vis_img = np.concatenate((anno_img, pred_img), axis=1) + + basename = os.path.basename(input["file_name"]) + if self.training: + cv2.imwrite( + os.path.join(self.output_dir, "training", str(storage.iter) + "_" + basename), + vis_img[:, :, ::-1], + ) + else: + cv2.imwrite( + os.path.join(self.output_dir, "inference", basename), vis_img[:, :, ::-1] + ) + + @torch.no_grad() + def visualize_inference_panoptic(self, batched_inputs, results): + if self.output_dir is None: + return + if self.training: + storage = get_event_storage() + os.makedirs(self.output_dir + "/training", exist_ok=True) + else: + os.makedirs(self.output_dir + "/inference", exist_ok=True) + + from detectron2.utils.visualizer import Visualizer + + for input, result in zip(batched_inputs, results): + + img = input["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format) + + height = input["height"] + width = input["width"] + img = cv2.resize(img, (width, height)) + + v_gt = Visualizer(img, self.metadata) + + if "instances" in input: + labels = [ + "{}".format(class_names[gt_class]) for gt_class in input["instances"].gt_classes + ] + v_gt = v_gt.overlay_instances( + boxes=input["instances"].gt_boxes, + masks=input["instances"].gt_masks + if input["instances"].has("gt_masks") + else None, + labels=labels, + ) + else: + v_gt = v_gt.output + anno_img = v_gt.get_image() + + v_pred = Visualizer(img, self.metadata) + + panoptic_seg, segments_info = result["panoptic_seg"] + v_pred = v_pred.draw_panoptic_seg_predictions(panoptic_seg.cpu(), segments_info) + pred_img = v_pred.get_image() + + vis_img = np.concatenate((anno_img, pred_img), axis=1) + + basename = os.path.basename(input["file_name"]) + if self.training: + cv2.imwrite( + os.path.join( + self.output_dir, "training", str(storage.iter) + "_pan_" + basename + ), + vis_img[:, :, ::-1], + ) + else: + cv2.imwrite( + os.path.join(self.output_dir, "inference", "pan_" + basename), + vis_img[:, :, ::-1], + ) + + +class NMSPostProcess(nn.Module): + """This module converts the model's output into the format expected by the coco api""" + + @torch.no_grad() + def forward(self, outputs, target_sizes, select_box_nums_for_evaluation): + """Perform the computation + Parameters: + outputs: raw outputs of the model + target_sizes: tensor of dimension [batch_size x 2] containing the size of each images of the batch + For evaluation, this must be the original image size (before any data augmentation) + For visualization, this should be the image size after data augment, but before padding + """ + out_logits, out_bbox = outputs["pred_logits"], outputs["pred_boxes"] + out_mask = outputs["pred_masks"] + bs, n_queries, n_cls = out_logits.shape + print("PostProcessSegm", out_logits.size(), out_bbox.size(), out_mask.size()) + + assert len(out_logits) == len(target_sizes) + assert target_sizes.shape[1] == 2 + + prob = out_logits.sigmoid() + + all_scores = prob.view(bs, n_queries * n_cls).to(out_logits.device) + all_indexes = torch.arange(n_queries * n_cls)[None].repeat(bs, 1).to(out_logits.device) + all_boxes = torch.div(all_indexes, out_logits.shape[2], rounding_mode="trunc") + all_labels = all_indexes % out_logits.shape[2] + + boxes = box_cxcywh_to_xyxy(out_bbox) + boxes = torch.gather(boxes, 1, all_boxes.unsqueeze(-1).repeat(1, 1, 4)) + + img_h, img_w = target_sizes.unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) + boxes = boxes * scale_fct[:, None, :] + + results = [] + keep_inds_all = [] + for b in range(bs): + box = boxes[b] + score = all_scores[b] + lbls = all_labels[b] + mask = out_mask[b] + + pre_topk = score.topk(10000).indices + box = box[pre_topk] + score = score[pre_topk] + lbls = lbls[pre_topk] + + keep_inds = batched_nms(box, score, lbls, 0.7)[:select_box_nums_for_evaluation] + + result = Instances(target_sizes[b]) + result.pred_boxes = Boxes(box[keep_inds]) + result.scores = score[keep_inds] + result.pred_classes = lbls[keep_inds] + results.append(result) + + keep_inds_all.append(keep_inds) + + return results, keep_inds_all + + +def is_thing_stuff_overlap(metadata): + thing_classes = metadata.get("thing_classes", []) + stuff_classes = metadata.get("stuff_classes", []) + if len(thing_classes) == 0 or len(stuff_classes) == 0: + return False + + if set(thing_classes).issubset(set(stuff_classes)) or set(stuff_classes).issubset( + set(thing_classes) + ): + return True + else: + return False diff --git a/approach/ovod/APE/ape/modeling/deta/deformable_transformer.py b/approach/ovod/APE/ape/modeling/deta/deformable_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..d1911429e129c0de94105ac6c9fededa4f7614ee --- /dev/null +++ b/approach/ovod/APE/ape/modeling/deta/deformable_transformer.py @@ -0,0 +1,524 @@ +import math + +import torch +import torch.nn as nn + +from ape.layers import MultiScaleDeformableAttention +from detrex.layers import FFN # MultiScaleDeformableAttention, +from detrex.layers import ( + BaseTransformerLayer, + MultiheadAttention, + TransformerLayerSequence, + box_cxcywh_to_xyxy, +) +from detrex.utils import inverse_sigmoid +from torchvision.ops.boxes import batched_nms + + +class DeformableDetrTransformerEncoder(TransformerLayerSequence): + def __init__( + self, + embed_dim: int = 256, + num_heads: int = 8, + feedforward_dim: int = 1024, + attn_dropout: float = 0.1, + ffn_dropout: float = 0.1, + num_layers: int = 6, + post_norm: bool = False, + num_feature_levels: int = 4, + ): + super(DeformableDetrTransformerEncoder, self).__init__( + transformer_layers=BaseTransformerLayer( + attn=MultiScaleDeformableAttention( + embed_dim=embed_dim, + num_heads=num_heads, + dropout=attn_dropout, + batch_first=True, + num_levels=num_feature_levels, + ), + ffn=FFN( + embed_dim=embed_dim, + feedforward_dim=feedforward_dim, + output_dim=embed_dim, + num_fcs=2, + ffn_drop=ffn_dropout, + ), + norm=nn.LayerNorm(embed_dim), + operation_order=("self_attn", "norm", "ffn", "norm"), + ), + num_layers=num_layers, + ) + self.embed_dim = self.layers[0].embed_dim + self.pre_norm = self.layers[0].pre_norm + + if post_norm: + self.post_norm_layer = nn.LayerNorm(self.embed_dim) + else: + self.post_norm_layer = None + + def forward( + self, + query, + key, + value, + query_pos=None, + key_pos=None, + attn_masks=None, + query_key_padding_mask=None, + key_padding_mask=None, + **kwargs, + ): + + for layer in self.layers: + query = layer( + query, + key, + value, + query_pos=query_pos, + attn_masks=attn_masks, + query_key_padding_mask=query_key_padding_mask, + key_padding_mask=key_padding_mask, + **kwargs, + ) + + if self.post_norm_layer is not None: + query = self.post_norm_layer(query) + return query + + +class DeformableDetrTransformerDecoder(TransformerLayerSequence): + def __init__( + self, + embed_dim: int = 256, + num_heads: int = 8, + feedforward_dim: int = 1024, + attn_dropout: float = 0.1, + ffn_dropout: float = 0.1, + num_layers: int = 6, + return_intermediate: bool = True, + num_feature_levels: int = 4, + ): + super(DeformableDetrTransformerDecoder, self).__init__( + transformer_layers=BaseTransformerLayer( + attn=[ + MultiheadAttention( + embed_dim=embed_dim, + num_heads=num_heads, + attn_drop=attn_dropout, + batch_first=True, + ), + MultiScaleDeformableAttention( + embed_dim=embed_dim, + num_heads=num_heads, + dropout=attn_dropout, + batch_first=True, + num_levels=num_feature_levels, + ), + ], + ffn=FFN( + embed_dim=embed_dim, + feedforward_dim=feedforward_dim, + output_dim=embed_dim, + ffn_drop=ffn_dropout, + ), + norm=nn.LayerNorm(embed_dim), + operation_order=("self_attn", "norm", "cross_attn", "norm", "ffn", "norm"), + ), + num_layers=num_layers, + ) + self.return_intermediate = return_intermediate + + self.bbox_embed = None + self.class_embed = None + + def forward( + self, + query, + key, + value, + query_pos=None, + key_pos=None, + attn_masks=None, + query_key_padding_mask=None, + key_padding_mask=None, + reference_points=None, + valid_ratios=None, + **kwargs, + ): + output = query + + intermediate = [] + intermediate_reference_points = [] + for layer_idx, layer in enumerate(self.layers): + if reference_points.shape[-1] == 4: + reference_points_input = ( + reference_points[:, :, None] + * torch.cat([valid_ratios, valid_ratios], -1)[:, None] + ) + else: + assert reference_points.shape[-1] == 2 + reference_points_input = reference_points[:, :, None] * valid_ratios[:, None] + + output = layer( + output, + key, + value, + query_pos=query_pos, + key_pos=key_pos, + attn_masks=attn_masks, + query_key_padding_mask=query_key_padding_mask, + key_padding_mask=key_padding_mask, + reference_points=reference_points_input, + **kwargs, + ) + + if self.bbox_embed is not None: + tmp = self.bbox_embed[layer_idx](output) + if reference_points.shape[-1] == 4: + new_reference_points = tmp + inverse_sigmoid(reference_points) + new_reference_points = new_reference_points.sigmoid() + else: + assert reference_points.shape[-1] == 2 + new_reference_points = tmp + new_reference_points[..., :2] = tmp[..., :2] + inverse_sigmoid(reference_points) + new_reference_points = new_reference_points.sigmoid() + reference_points = new_reference_points.detach() + + if self.return_intermediate: + intermediate.append(output) + intermediate_reference_points.append(reference_points) + + if self.return_intermediate: + return torch.stack(intermediate), torch.stack(intermediate_reference_points) + + return output, reference_points + + +class DeformableDetrTransformer(nn.Module): + """Transformer module for Deformable DETR + + Args: + encoder (nn.Module): encoder module. + decoder (nn.Module): decoder module. + as_two_stage (bool): whether to use two-stage transformer. Default False. + num_feature_levels (int): number of feature levels. Default 4. + two_stage_num_proposals (int): number of proposals in two-stage transformer. Default 300. + Only used when as_two_stage is True. + """ + + def __init__( + self, + encoder=None, + decoder=None, + num_feature_levels=4, + as_two_stage=False, + two_stage_num_proposals=300, + assign_first_stage=False, + ): + super(DeformableDetrTransformer, self).__init__() + self.encoder = encoder + self.decoder = decoder + self.num_feature_levels = num_feature_levels + self.as_two_stage = as_two_stage + self.two_stage_num_proposals = two_stage_num_proposals + self.assign_first_stage = assign_first_stage + + self.embed_dim = self.encoder.embed_dim + + self.level_embeds = nn.Parameter(torch.Tensor(self.num_feature_levels, self.embed_dim)) + + if self.as_two_stage: + self.enc_output = nn.Linear(self.embed_dim, self.embed_dim) + self.enc_output_norm = nn.LayerNorm(self.embed_dim) + self.pos_trans = nn.Linear(self.embed_dim * 2, self.embed_dim * 2) + self.pos_trans_norm = nn.LayerNorm(self.embed_dim * 2) + self.pix_trans = nn.Linear(self.embed_dim, self.embed_dim) + self.pix_trans_norm = nn.LayerNorm(self.embed_dim) + else: + self.reference_points = nn.Linear(self.embed_dim, 2) + + self.init_weights() + + def init_weights(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + for m in self.modules(): + if isinstance(m, MultiScaleDeformableAttention): + m.init_weights() + if not self.as_two_stage: + nn.init.xavier_normal_(self.reference_points.weight.data, gain=1.0) + nn.init.constant_(self.reference_points.bias.data, 0.0) + nn.init.normal_(self.level_embeds) + + def gen_encoder_output_proposals(self, memory, memory_padding_mask, spatial_shapes): + N, S, C = memory.shape + proposals = [] + _cur = 0 + level_ids = [] + for lvl, (H, W) in enumerate(spatial_shapes): + mask_flatten_ = memory_padding_mask[:, _cur : (_cur + H * W)].view(N, H, W, 1) + valid_H = torch.sum(~mask_flatten_[:, :, 0, 0], 1) + valid_W = torch.sum(~mask_flatten_[:, 0, :, 0], 1) + + grid_y, grid_x = torch.meshgrid( + torch.linspace(0, H - 1, H, dtype=torch.float32, device=memory.device), + torch.linspace(0, W - 1, W, dtype=torch.float32, device=memory.device), + ) + grid = torch.cat([grid_x.unsqueeze(-1), grid_y.unsqueeze(-1)], -1) + + scale = torch.cat([valid_W.unsqueeze(-1), valid_H.unsqueeze(-1)], 1).view(N, 1, 1, 2) + grid = (grid.unsqueeze(0).expand(N, -1, -1, -1) + 0.5) / scale + wh = torch.ones_like(grid) * 0.05 * (2.0**lvl) + proposal = torch.cat((grid, wh), -1).view(N, -1, 4) + proposals.append(proposal) + _cur += H * W + level_ids.append(grid.new_ones(H * W, dtype=torch.long) * lvl) + + output_proposals = torch.cat(proposals, 1) + output_proposals_valid = ((output_proposals > 0.01) & (output_proposals < 0.99)).all( + -1, keepdim=True + ) + output_proposals = torch.log(output_proposals / (1 - output_proposals)) + output_proposals = output_proposals.masked_fill( + memory_padding_mask.unsqueeze(-1), float("inf") + ) + output_proposals = output_proposals.masked_fill(~output_proposals_valid, float("inf")) + + output_memory = memory + output_memory = output_memory.masked_fill(memory_padding_mask.unsqueeze(-1), float(0)) + output_memory = output_memory.masked_fill(~output_proposals_valid, float(0)) + output_memory = self.enc_output_norm(self.enc_output(output_memory)) + level_ids = torch.cat(level_ids) + output_proposals = output_proposals.to(output_memory.dtype) + return output_memory, output_proposals, level_ids + + @staticmethod + def get_reference_points(spatial_shapes, valid_ratios, device): + """Get the reference points used in decoder. + + Args: + spatial_shapes (Tensor): The shape of all + feature maps, has shape (num_level, 2). + valid_ratios (Tensor): The ratios of valid + points on the feature map, has shape + (bs, num_levels, 2) + device (obj:`device`): The device where + reference_points should be. + + Returns: + Tensor: reference points used in decoder, has \ + shape (bs, num_keys, num_levels, 2). + """ + reference_points_list = [] + for lvl, (H, W) in enumerate(spatial_shapes): + ref_y, ref_x = torch.meshgrid( + torch.linspace(0.5, H - 0.5, H, dtype=torch.float32, device=device), + torch.linspace(0.5, W - 0.5, W, dtype=torch.float32, device=device), + ) + ref_y = ref_y.reshape(-1)[None] / (valid_ratios[:, None, lvl, 1] * H) + ref_x = ref_x.reshape(-1)[None] / (valid_ratios[:, None, lvl, 0] * W) + ref = torch.stack((ref_x, ref_y), -1) + reference_points_list.append(ref) + reference_points = torch.cat(reference_points_list, 1) + reference_points = reference_points[:, :, None] * valid_ratios[:, None] + return reference_points + + def get_valid_ratio(self, mask): + """Get the valid ratios of feature maps of all levels.""" + _, H, W = mask.shape + valid_H = torch.sum(~mask[:, :, 0], 1) + valid_W = torch.sum(~mask[:, 0, :], 1) + valid_ratio_h = valid_H.float() / H + valid_ratio_w = valid_W.float() / W + valid_ratio = torch.stack([valid_ratio_w, valid_ratio_h], -1) + return valid_ratio + + def get_proposal_pos_embed(self, proposals, num_pos_feats=128, temperature=10000): + """Get the position embedding of proposal.""" + scale = 2 * math.pi + dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=proposals.device) + dim_t = temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / num_pos_feats) + proposals = proposals.sigmoid() * scale + pos = proposals[:, :, :, None] / dim_t + pos = torch.stack((pos[:, :, :, 0::2].sin(), pos[:, :, :, 1::2].cos()), dim=4).flatten(2) + return pos + + def forward( + self, + multi_level_feats, + multi_level_masks, + multi_level_pos_embeds, + query_embed, + **kwargs, + ): + assert self.as_two_stage or query_embed is not None + + feat_flatten = [] + mask_flatten = [] + lvl_pos_embed_flatten = [] + spatial_shapes = [] + for lvl, (feat, mask, pos_embed) in enumerate( + zip(multi_level_feats, multi_level_masks, multi_level_pos_embeds) + ): + bs, c, h, w = feat.shape + spatial_shape = (h, w) + spatial_shapes.append(spatial_shape) + + feat = feat.flatten(2).transpose(1, 2) # bs, hw, c + mask = mask.flatten(1) + pos_embed = pos_embed.flatten(2).transpose(1, 2) # bs, hw, c + lvl_pos_embed = pos_embed + self.level_embeds[lvl].view(1, 1, -1) + lvl_pos_embed_flatten.append(lvl_pos_embed) + feat_flatten.append(feat) + mask_flatten.append(mask) + feat_flatten = torch.cat(feat_flatten, 1) + mask_flatten = torch.cat(mask_flatten, 1) + lvl_pos_embed_flatten = torch.cat(lvl_pos_embed_flatten, 1) + spatial_shapes = torch.as_tensor( + spatial_shapes, dtype=torch.long, device=feat_flatten.device + ) + level_start_index = torch.cat( + (spatial_shapes.new_zeros((1,)), spatial_shapes.prod(1).cumsum(0)[:-1]) + ) + valid_ratios = torch.stack([self.get_valid_ratio(m) for m in multi_level_masks], 1) + valid_ratios = valid_ratios.to(feat_flatten.dtype) + + reference_points = self.get_reference_points( + spatial_shapes, valid_ratios, device=feat.device + ) + reference_points = reference_points.to(feat_flatten.dtype) + + memory = self.encoder( + query=feat_flatten, + key=None, + value=None, + query_pos=lvl_pos_embed_flatten, + query_key_padding_mask=mask_flatten, + spatial_shapes=spatial_shapes, + reference_points=reference_points, + level_start_index=level_start_index, + valid_ratios=valid_ratios, + **kwargs, + ) + + bs, _, c = memory.shape + if self.as_two_stage: + output_memory, output_proposals, level_ids = self.gen_encoder_output_proposals( + memory, mask_flatten, spatial_shapes + ) + + enc_outputs_class = self.decoder.class_embed[self.decoder.num_layers](output_memory) + enc_outputs_coord_unact = ( + self.decoder.bbox_embed[self.decoder.num_layers](output_memory) + output_proposals + ) + + topk = self.two_stage_num_proposals + + proposal_logit = enc_outputs_class[..., 0] + + if self.assign_first_stage: + proposal_boxes = box_cxcywh_to_xyxy(enc_outputs_coord_unact.sigmoid()).clamp(0, 1) + topk_proposals = [] + for b in range(bs): + prop_boxes_b = proposal_boxes[b] + prop_logits_b = proposal_logit[b] + + pre_nms_topk = 1000 + pre_nms_inds = [] + for lvl in range(len(spatial_shapes)): + lvl_mask = level_ids == lvl + pre_nms_inds.append( + torch.topk( + prop_logits_b.sigmoid() * lvl_mask, + min(pre_nms_topk, prop_logits_b.size(0)), + )[1] + ) + pre_nms_inds = torch.cat(pre_nms_inds) + + post_nms_inds = batched_nms( + prop_boxes_b[pre_nms_inds], + prop_logits_b[pre_nms_inds], + level_ids[pre_nms_inds], + 0.9, + ) + keep_inds = pre_nms_inds[post_nms_inds] + + if len(keep_inds) < self.two_stage_num_proposals: + print( + f"[WARNING] nms proposals ({len(keep_inds)}) < {self.two_stage_num_proposals}, running naive topk" + ) + keep_inds = torch.topk( + proposal_logit[b], min(topk, proposal_logit[b].size(0)) + )[1] + + q_per_l = topk // len(spatial_shapes) + is_level_ordered = ( + level_ids[keep_inds][None] + == torch.arange(len(spatial_shapes), device=level_ids.device)[:, None] + ) # LS + keep_inds_mask = is_level_ordered & ( + is_level_ordered.cumsum(1) <= q_per_l + ) # LS + keep_inds_mask = keep_inds_mask.any(0) # S + + if keep_inds_mask.sum() < topk: + num_to_add = topk - keep_inds_mask.sum() + pad_inds = (~keep_inds_mask).nonzero()[:num_to_add] + keep_inds_mask[pad_inds] = True + + keep_inds_topk = keep_inds[keep_inds_mask] + topk_proposals.append(keep_inds_topk) + topk_proposals = torch.stack(topk_proposals) + else: + topk_proposals = torch.topk(proposal_logit, topk, dim=1)[1] + + topk_coords_unact = torch.gather( + enc_outputs_coord_unact, 1, topk_proposals.unsqueeze(-1).repeat(1, 1, 4) + ) + topk_coords_unact = topk_coords_unact.detach() + reference_points = topk_coords_unact.sigmoid() + init_reference_out = reference_points + pos_trans_out = self.pos_trans_norm( + self.pos_trans( + self.get_proposal_pos_embed(topk_coords_unact).to(topk_coords_unact.dtype) + ) + ) + query_pos, query = torch.split(pos_trans_out, c, dim=2) + + topk_feats = torch.stack( + [output_memory[b][topk_proposals[b]] for b in range(bs)] + ).detach() + query = query + self.pix_trans_norm(self.pix_trans(topk_feats)) + else: + query_pos, query = torch.split(query_embed, c, dim=1) + query_pos = query_pos.unsqueeze(0).expand(bs, -1, -1) + query = query.unsqueeze(0).expand(bs, -1, -1) + reference_points = self.reference_points(query_pos).sigmoid() + init_reference_out = reference_points + + inter_states, inter_references = self.decoder( + query=query, # bs, num_queries, embed_dims + key=None, # bs, num_tokens, embed_dims + value=memory, # bs, num_tokens, embed_dims + query_pos=query_pos, + key_padding_mask=mask_flatten, # bs, num_tokens + reference_points=reference_points, # num_queries, 4 + spatial_shapes=spatial_shapes, # nlvl, 2 + level_start_index=level_start_index, # nlvl + valid_ratios=valid_ratios, # bs, nlvl, 2 + **kwargs, + ) + + inter_references_out = inter_references + if self.as_two_stage: + return ( + inter_states, + init_reference_out, + inter_references_out, + enc_outputs_class, + enc_outputs_coord_unact, + output_proposals.sigmoid(), + memory, + ) + return inter_states, init_reference_out, inter_references_out, None, None, None, memory diff --git a/approach/ovod/APE/ape/modeling/deta/misc.py b/approach/ovod/APE/ape/modeling/deta/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..d697fe8e922426116ce1c1be1782d8d3b8674149 --- /dev/null +++ b/approach/ovod/APE/ape/modeling/deta/misc.py @@ -0,0 +1,469 @@ +""" +Misc functions, including distributed helpers. + +Mostly copy-paste from torchvision references. +""" +import datetime +import os +import pickle +import subprocess +import time +from collections import defaultdict, deque +from typing import List, Optional + +import torch +import torch.distributed as dist +from packaging import version +from torch import Tensor + +import torchvision + +if version.parse(torchvision.__version__) < version.parse("0.7"): + from torchvision.ops import _new_empty_tensor + from torchvision.ops.misc import _output_size + + +class SmoothedValue(object): + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], dtype=torch.float64, device="cuda") + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + return self.total / self.count + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, + avg=self.avg, + global_avg=self.global_avg, + max=self.max, + value=self.value, + ) + + +def all_gather(data): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors) + Args: + data: any picklable object + Returns: + list[data]: list of data gathered from each rank + """ + world_size = get_world_size() + if world_size == 1: + return [data] + + buffer = pickle.dumps(data) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to("cuda") + + local_size = torch.tensor([tensor.numel()], device="cuda") + size_list = [torch.tensor([0], device="cuda") for _ in range(world_size)] + dist.all_gather(size_list, local_size) + size_list = [int(size.item()) for size in size_list] + max_size = max(size_list) + + tensor_list = [] + for _ in size_list: + tensor_list.append(torch.empty((max_size,), dtype=torch.uint8, device="cuda")) + if local_size != max_size: + padding = torch.empty(size=(max_size - local_size,), dtype=torch.uint8, device="cuda") + tensor = torch.cat((tensor, padding), dim=0) + dist.all_gather(tensor_list, tensor) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + + return data_list + + +def reduce_dict(input_dict, average=True): + """ + Args: + input_dict (dict): all the values will be reduced + average (bool): whether to do average or sum + Reduce the values in the dictionary from all processes so that all processes + have the averaged results. Returns a dict with the same fields as + input_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return input_dict + with torch.no_grad(): + names = [] + values = [] + for k in sorted(input_dict.keys()): + names.append(k) + values.append(input_dict[k]) + values = torch.stack(values, dim=0) + dist.all_reduce(values) + if average: + values /= world_size + reduced_dict = {k: v for k, v in zip(names, values)} + return reduced_dict + + +class MetricLogger(object): + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError("'{}' object has no attribute '{}'".format(type(self).__name__, attr)) + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + loss_str.append("{}: {}".format(name, str(meter))) + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, print_freq, header=None): + i = 0 + if not header: + header = "" + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt="{avg:.4f}") + data_time = SmoothedValue(fmt="{avg:.4f}") + space_fmt = ":" + str(len(str(len(iterable)))) + "d" + if torch.cuda.is_available(): + log_msg = self.delimiter.join( + [ + header, + "[{0" + space_fmt + "}/{1}]", + "eta: {eta}", + "{meters}", + "time: {time}", + "data: {data}", + "max mem: {memory:.0f}", + ] + ) + else: + log_msg = self.delimiter.join( + [ + header, + "[{0" + space_fmt + "}/{1}]", + "eta: {eta}", + "{meters}", + "time: {time}", + "data: {data}", + ] + ) + MB = 1024.0 * 1024.0 + for obj in iterable: + data_time.update(time.time() - end) + yield obj + iter_time.update(time.time() - end) + if i % print_freq == 0 or i == len(iterable) - 1: + eta_seconds = iter_time.global_avg * (len(iterable) - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + if torch.cuda.is_available(): + print( + log_msg.format( + i, + len(iterable), + eta=eta_string, + meters=str(self), + time=str(iter_time), + data=str(data_time), + memory=torch.cuda.max_memory_allocated() / MB, + ) + ) + else: + print( + log_msg.format( + i, + len(iterable), + eta=eta_string, + meters=str(self), + time=str(iter_time), + data=str(data_time), + ) + ) + i += 1 + end = time.time() + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print( + "{} Total time: {} ({:.4f} s / it)".format( + header, total_time_str, total_time / len(iterable) + ) + ) + + +def get_sha(): + cwd = os.path.dirname(os.path.abspath(__file__)) + + def _run(command): + return subprocess.check_output(command, cwd=cwd).decode("ascii").strip() + + sha = "N/A" + diff = "clean" + branch = "N/A" + try: + sha = _run(["git", "rev-parse", "HEAD"]) + subprocess.check_output(["git", "diff"], cwd=cwd) + diff = _run(["git", "diff-index", "HEAD"]) + diff = "has uncommited changes" if diff else "clean" + branch = _run(["git", "rev-parse", "--abbrev-ref", "HEAD"]) + except Exception: + pass + message = f"sha: {sha}, status: {diff}, branch: {branch}" + return message + + +def collate_fn(batch): + batch = list(zip(*batch)) + batch[0] = nested_tensor_from_tensor_list(batch[0]) + return tuple(batch) + + +def _max_by_axis(the_list): + maxes = the_list[0] + for sublist in the_list[1:]: + for index, item in enumerate(sublist): + maxes[index] = max(maxes[index], item) + return maxes + + +class NestedTensor(object): + def __init__(self, tensors, mask: Optional[Tensor]): + self.tensors = tensors + self.mask = mask + + def to(self, device): + cast_tensor = self.tensors.to(device) + mask = self.mask + if mask is not None: + assert mask is not None + cast_mask = mask.to(device) + else: + cast_mask = None + return NestedTensor(cast_tensor, cast_mask) + + def decompose(self): + return self.tensors, self.mask + + def __repr__(self): + return str(self.tensors) + + +def nested_tensor_from_tensor_list(tensor_list: List[Tensor]): + if tensor_list[0].ndim == 3: + if torchvision._is_tracing(): + return _onnx_nested_tensor_from_tensor_list(tensor_list) + + max_size = _max_by_axis([list(img.shape) for img in tensor_list]) + batch_shape = [len(tensor_list)] + max_size + b, c, h, w = batch_shape + dtype = tensor_list[0].dtype + device = tensor_list[0].device + tensor = torch.zeros(batch_shape, dtype=dtype, device=device) + mask = torch.ones((b, h, w), dtype=torch.bool, device=device) + for img, pad_img, m in zip(tensor_list, tensor, mask): + pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + m[: img.shape[1], : img.shape[2]] = False + else: + raise ValueError("not supported") + return NestedTensor(tensor, mask) + + +@torch.jit.unused +def _onnx_nested_tensor_from_tensor_list(tensor_list: List[Tensor]) -> NestedTensor: + max_size = [] + for i in range(tensor_list[0].dim()): + max_size_i = torch.max( + torch.stack([img.shape[i] for img in tensor_list]).to(torch.float32) + ).to(torch.int64) + max_size.append(max_size_i) + max_size = tuple(max_size) + + padded_imgs = [] + padded_masks = [] + for img in tensor_list: + padding = [(s1 - s2) for s1, s2 in zip(max_size, tuple(img.shape))] + padded_img = torch.nn.functional.pad(img, (0, padding[2], 0, padding[1], 0, padding[0])) + padded_imgs.append(padded_img) + + m = torch.zeros_like(img[0], dtype=torch.int, device=img.device) + padded_mask = torch.nn.functional.pad(m, (0, padding[2], 0, padding[1]), "constant", 1) + padded_masks.append(padded_mask.to(torch.bool)) + + tensor = torch.stack(padded_imgs) + mask = torch.stack(padded_masks) + + return NestedTensor(tensor, mask=mask) + + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop("force", False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + if "RANK" in os.environ and "WORLD_SIZE" in os.environ: + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ["WORLD_SIZE"]) + args.gpu = int(os.environ["LOCAL_RANK"]) + elif "SLURM_PROCID" in os.environ: + args.rank = int(os.environ["SLURM_PROCID"]) + args.gpu = args.rank % torch.cuda.device_count() + else: + print("Not using distributed mode") + args.distributed = False + return + + args.distributed = True + + torch.cuda.set_device(args.gpu) + args.dist_backend = "nccl" + print("| distributed init (rank {}): {}".format(args.rank, args.dist_url), flush=True) + torch.distributed.init_process_group( + backend=args.dist_backend, + init_method=args.dist_url, + world_size=args.world_size, + rank=args.rank, + ) + torch.distributed.barrier() + setup_for_distributed(args.rank == 0) + + +@torch.no_grad() +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + if target.numel() == 0: + return [torch.zeros([], device=output.device)] + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def interpolate(input, size=None, scale_factor=None, mode="nearest", align_corners=None): + """ + Equivalent to nn.functional.interpolate, but with support for empty batch sizes. + This will eventually be supported natively by PyTorch, and this + class can go away. + """ + if version.parse(torchvision.__version__) < version.parse("0.7"): + if input.numel() > 0: + return torch.nn.functional.interpolate(input, size, scale_factor, mode, align_corners) + + output_shape = _output_size(2, input, size, scale_factor) + output_shape = list(input.shape[:-2]) + list(output_shape) + return _new_empty_tensor(input, output_shape) + else: + return torchvision.ops.misc.interpolate(input, size, scale_factor, mode, align_corners) diff --git a/approach/ovod/APE/ape/modeling/deta/segmentation.py b/approach/ovod/APE/ape/modeling/deta/segmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..95decb8b7c95d9bd990ee0ecef011a2c3899b72b --- /dev/null +++ b/approach/ovod/APE/ape/modeling/deta/segmentation.py @@ -0,0 +1,378 @@ +""" +This file provides the definition of the convolutional heads used to predict masks, as well as the losses +""" +import io +from collections import defaultdict + +import torch +import torch.nn as nn +import torch.nn.functional as F +from PIL import Image + +from detrex.layers import box_cxcywh_to_xyxy + +try: + from panopticapi.utils import id2rgb, rgb2id +except ImportError: + pass + + +class DETRsegm(nn.Module): + def __init__(self, detr, freeze_detr=False): + super().__init__() + self.detr = detr + + if freeze_detr: + for p in self.parameters(): + p.requires_grad_(False) + + hidden_dim, nheads = detr.transformer.d_model, detr.transformer.nhead + self.bbox_attention = MHAttentionMap(hidden_dim, hidden_dim, nheads, dropout=0) + self.mask_head = MaskHeadSmallConv(hidden_dim + nheads, [1024, 512, 256], hidden_dim) + + def forward(self, samples): + if not isinstance(samples, NestedTensor): + samples = nested_tensor_from_tensor_list(samples) + features, pos = self.detr.backbone(samples) + + bs = features[-1].tensors.shape[0] + + src, mask = features[-1].decompose() + src_proj = self.detr.input_proj(src) + hs, memory = self.detr.transformer(src_proj, mask, self.detr.query_embed.weight, pos[-1]) + + outputs_class = self.detr.class_embed(hs) + outputs_coord = self.detr.bbox_embed(hs).sigmoid() + out = {"pred_logits": outputs_class[-1], "pred_boxes": outputs_coord[-1]} + if self.detr.aux_loss: + out["aux_outputs"] = [ + {"pred_logits": a, "pred_boxes": b} + for a, b in zip(outputs_class[:-1], outputs_coord[:-1]) + ] + + bbox_mask = self.bbox_attention(hs[-1], memory, mask=mask) + + seg_masks = self.mask_head( + src_proj, bbox_mask, [features[2].tensors, features[1].tensors, features[0].tensors] + ) + outputs_seg_masks = seg_masks.view( + bs, self.detr.num_queries, seg_masks.shape[-2], seg_masks.shape[-1] + ) + + out["pred_masks"] = outputs_seg_masks + return out + + +class MaskHeadSmallConv(nn.Module): + """ + Simple convolutional head, using group norm. + Upsampling is done using a FPN approach + """ + + def __init__(self, dim, fpn_dims, context_dim): + super().__init__() + + inter_dims = [ + dim, + context_dim // 2, + context_dim // 4, + context_dim // 8, + context_dim // 16, + context_dim // 64, + ] + self.lay1 = torch.nn.Conv2d(dim, dim, 3, padding=1) + self.gn1 = torch.nn.GroupNorm(8, dim) + self.lay2 = torch.nn.Conv2d(dim, inter_dims[1], 3, padding=1) + self.gn2 = torch.nn.GroupNorm(8, inter_dims[1]) + self.lay3 = torch.nn.Conv2d(inter_dims[1], inter_dims[2], 3, padding=1) + self.gn3 = torch.nn.GroupNorm(8, inter_dims[2]) + self.lay4 = torch.nn.Conv2d(inter_dims[2], inter_dims[3], 3, padding=1) + self.gn4 = torch.nn.GroupNorm(8, inter_dims[3]) + self.lay5 = torch.nn.Conv2d(inter_dims[3], inter_dims[4], 3, padding=1) + self.gn5 = torch.nn.GroupNorm(8, inter_dims[4]) + self.out_lay = torch.nn.Conv2d(inter_dims[4], 1, 3, padding=1) + + self.dim = dim + + self.adapter1 = torch.nn.Conv2d(fpn_dims[0], inter_dims[1], 1) + self.adapter2 = torch.nn.Conv2d(fpn_dims[1], inter_dims[2], 1) + self.adapter3 = torch.nn.Conv2d(fpn_dims[2], inter_dims[3], 1) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_uniform_(m.weight, a=1) + nn.init.constant_(m.bias, 0) + + def forward(self, x, bbox_mask, fpns): + def expand(tensor, length): + return tensor.unsqueeze(1).repeat(1, int(length), 1, 1, 1).flatten(0, 1) + + x = torch.cat([expand(x, bbox_mask.shape[1]), bbox_mask.flatten(0, 1)], 1) + + x = self.lay1(x) + x = self.gn1(x) + x = F.relu(x) + x = self.lay2(x) + x = self.gn2(x) + x = F.relu(x) + + cur_fpn = self.adapter1(fpns[0]) + if cur_fpn.size(0) != x.size(0): + cur_fpn = expand(cur_fpn, x.size(0) / cur_fpn.size(0)) + x = cur_fpn + F.interpolate(x, size=cur_fpn.shape[-2:], mode="nearest") + x = self.lay3(x) + x = self.gn3(x) + x = F.relu(x) + + cur_fpn = self.adapter2(fpns[1]) + if cur_fpn.size(0) != x.size(0): + cur_fpn = expand(cur_fpn, x.size(0) / cur_fpn.size(0)) + x = cur_fpn + F.interpolate(x, size=cur_fpn.shape[-2:], mode="nearest") + x = self.lay4(x) + x = self.gn4(x) + x = F.relu(x) + + cur_fpn = self.adapter3(fpns[2]) + if cur_fpn.size(0) != x.size(0): + cur_fpn = expand(cur_fpn, x.size(0) / cur_fpn.size(0)) + x = cur_fpn + F.interpolate(x, size=cur_fpn.shape[-2:], mode="nearest") + x = self.lay5(x) + x = self.gn5(x) + x = F.relu(x) + + x = self.out_lay(x) + return x + + +class MHAttentionMap(nn.Module): + """This is a 2D attention module, which only returns the attention softmax (no multiplication by value)""" + + def __init__(self, query_dim, hidden_dim, num_heads, dropout=0, bias=True): + super().__init__() + self.num_heads = num_heads + self.hidden_dim = hidden_dim + self.dropout = nn.Dropout(dropout) + + self.q_linear = nn.Linear(query_dim, hidden_dim, bias=bias) + self.k_linear = nn.Linear(query_dim, hidden_dim, bias=bias) + + nn.init.zeros_(self.k_linear.bias) + nn.init.zeros_(self.q_linear.bias) + nn.init.xavier_uniform_(self.k_linear.weight) + nn.init.xavier_uniform_(self.q_linear.weight) + self.normalize_fact = float(hidden_dim / self.num_heads) ** -0.5 + + def forward(self, q, k, mask=None): + q = self.q_linear(q) + k = F.conv2d(k, self.k_linear.weight.unsqueeze(-1).unsqueeze(-1), self.k_linear.bias) + qh = q.view(q.shape[0], q.shape[1], self.num_heads, self.hidden_dim // self.num_heads) + kh = k.view( + k.shape[0], self.num_heads, self.hidden_dim // self.num_heads, k.shape[-2], k.shape[-1] + ) + weights = torch.einsum("bqnc,bnchw->bqnhw", qh * self.normalize_fact, kh) + + if mask is not None: + weights.masked_fill_(mask.unsqueeze(1).unsqueeze(1), float("-inf")) + weights = F.softmax(weights.flatten(2), dim=-1).view_as(weights) + weights = self.dropout(weights) + return weights + + +def dice_loss(inputs, targets, num_boxes): + """ + Compute the DICE loss, similar to generalized IOU for masks + Args: + inputs: A float tensor of arbitrary shape. + The predictions for each example. + targets: A float tensor with the same shape as inputs. Stores the binary + classification label for each element in inputs + (0 for the negative class and 1 for the positive class). + """ + inputs = inputs.sigmoid() + inputs = inputs.flatten(1) + numerator = 2 * (inputs * targets).sum(1) + denominator = inputs.sum(-1) + targets.sum(-1) + loss = 1 - (numerator + 1) / (denominator + 1) + return loss.sum() / num_boxes + + +def sigmoid_focal_loss(inputs, targets, num_boxes, alpha: float = 0.25, gamma: float = 2): + """ + Loss used in RetinaNet for dense detection: https://arxiv.org/abs/1708.02002. + Args: + inputs: A float tensor of arbitrary shape. + The predictions for each example. + targets: A float tensor with the same shape as inputs. Stores the binary + classification label for each element in inputs + (0 for the negative class and 1 for the positive class). + alpha: (optional) Weighting factor in range (0,1) to balance + positive vs negative examples. Default = -1 (no weighting). + gamma: Exponent of the modulating factor (1 - p_t) to + balance easy vs hard examples. + Returns: + Loss tensor + """ + prob = inputs.sigmoid() + ce_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction="none") + p_t = prob * targets + (1 - prob) * (1 - targets) + loss = ce_loss * ((1 - p_t) ** gamma) + + if alpha >= 0: + alpha_t = alpha * targets + (1 - alpha) * (1 - targets) + loss = alpha_t * loss + + return loss.mean(1).sum() / num_boxes + + +class PostProcessSegm(nn.Module): + def __init__(self, threshold=0.5): + super().__init__() + self.threshold = threshold + + @torch.no_grad() + def forward(self, results, outputs, orig_target_sizes, max_target_sizes): + assert len(orig_target_sizes) == len(max_target_sizes) + max_h, max_w = max_target_sizes.max(0)[0].tolist() + outputs_masks = outputs["pred_masks"].squeeze(2) + outputs_masks = F.interpolate( + outputs_masks, size=(max_h, max_w), mode="bilinear", align_corners=False + ) + outputs_masks = (outputs_masks.sigmoid() > self.threshold).cpu() + + for i, (cur_mask, t, tt) in enumerate( + zip(outputs_masks, max_target_sizes, orig_target_sizes) + ): + img_h, img_w = t[0], t[1] + results[i]["masks"] = cur_mask[:, :img_h, :img_w].unsqueeze(1) + results[i]["masks"] = F.interpolate( + results[i]["masks"].float(), size=tuple(tt.tolist()), mode="nearest" + ).byte() + + return results + + +class PostProcessPanoptic(nn.Module): + """This class converts the output of the model to the final panoptic result, in the format expected by the + coco panoptic API""" + + def __init__(self, is_thing_map, threshold=0.85): + """ + Parameters: + is_thing_map: This is a whose keys are the class ids, and the values a boolean indicating whether + the class is a thing (True) or a stuff (False) class + threshold: confidence threshold: segments with confidence lower than this will be deleted + """ + super().__init__() + self.threshold = threshold + self.is_thing_map = is_thing_map + + def forward(self, outputs, processed_sizes, target_sizes=None): + """This function computes the panoptic prediction from the model's predictions. + Parameters: + outputs: This is a dict coming directly from the model. See the model doc for the content. + processed_sizes: This is a list of tuples (or torch tensors) of sizes of the images that were passed to the + model, ie the size after data augmentation but before batching. + target_sizes: This is a list of tuples (or torch tensors) corresponding to the requested final size + of each prediction. If left to None, it will default to the processed_sizes + """ + if target_sizes is None: + target_sizes = processed_sizes + assert len(processed_sizes) == len(target_sizes) + out_logits, raw_masks, raw_boxes = ( + outputs["pred_logits"], + outputs["pred_masks"], + outputs["pred_boxes"], + ) + assert len(out_logits) == len(raw_masks) == len(target_sizes) + preds = [] + + def to_tuple(tup): + if isinstance(tup, tuple): + return tup + return tuple(tup.cpu().tolist()) + + for cur_logits, cur_masks, cur_boxes, size, target_size in zip( + out_logits, raw_masks, raw_boxes, processed_sizes, target_sizes + ): + scores, labels = cur_logits.softmax(-1).max(-1) + keep = labels.ne(outputs["pred_logits"].shape[-1] - 1) & (scores > self.threshold) + cur_scores, cur_classes = cur_logits.softmax(-1).max(-1) + cur_scores = cur_scores[keep] + cur_classes = cur_classes[keep] + cur_masks = cur_masks[keep] + cur_masks = F.interpolate(cur_masks[None], to_tuple(size), mode="bilinear").squeeze(0) + cur_boxes = box_cxcywh_to_xyxy(cur_boxes[keep]) + + h, w = cur_masks.shape[-2:] + assert len(cur_boxes) == len(cur_classes) + + cur_masks = cur_masks.flatten(1) + stuff_equiv_classes = defaultdict(lambda: []) + for k, label in enumerate(cur_classes): + if not self.is_thing_map[label.item()]: + stuff_equiv_classes[label.item()].append(k) + + def get_ids_area(masks, scores, dedup=False): + + m_id = masks.transpose(0, 1).softmax(-1) + + if m_id.shape[-1] == 0: + m_id = torch.zeros((h, w), dtype=torch.long, device=m_id.device) + else: + m_id = m_id.argmax(-1).view(h, w) + + if dedup: + for equiv in stuff_equiv_classes.values(): + if len(equiv) > 1: + for eq_id in equiv: + m_id.masked_fill_(m_id.eq(eq_id), equiv[0]) + + final_h, final_w = to_tuple(target_size) + + seg_img = Image.fromarray(id2rgb(m_id.view(h, w).cpu().numpy())) + seg_img = seg_img.resize(size=(final_w, final_h), resample=Image.NEAREST) + + np_seg_img = ( + torch.ByteTensor(torch.ByteStorage.from_buffer(seg_img.tobytes())) + .view(final_h, final_w, 3) + .numpy() + ) + m_id = torch.from_numpy(rgb2id(np_seg_img)) + + area = [] + for i in range(len(scores)): + area.append(m_id.eq(i).sum().item()) + return area, seg_img + + area, seg_img = get_ids_area(cur_masks, cur_scores, dedup=True) + if cur_classes.numel() > 0: + while True: + filtered_small = torch.as_tensor( + [area[i] <= 4 for i, c in enumerate(cur_classes)], + dtype=torch.bool, + device=keep.device, + ) + if filtered_small.any().item(): + cur_scores = cur_scores[~filtered_small] + cur_classes = cur_classes[~filtered_small] + cur_masks = cur_masks[~filtered_small] + area, seg_img = get_ids_area(cur_masks, cur_scores) + else: + break + + else: + cur_classes = torch.ones(1, dtype=torch.long, device=cur_classes.device) + + segments_info = [] + for i, a in enumerate(area): + cat = cur_classes[i].item() + segments_info.append( + {"id": i, "isthing": self.is_thing_map[cat], "category_id": cat, "area": a} + ) + del cur_classes + + with io.BytesIO() as out: + seg_img.save(out, format="PNG") + predictions = {"png_string": out.getvalue(), "segments_info": segments_info} + preds.append(predictions) + return preds diff --git a/approach/ovod/APE/ape/modeling/text/__init__.py b/approach/ovod/APE/ape/modeling/text/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..088d1d27fc576ca2b3fa263dca5c7650d97f3b32 --- /dev/null +++ b/approach/ovod/APE/ape/modeling/text/__init__.py @@ -0,0 +1,8 @@ +from .bert_wrapper import Bert +from .clip_wrapper import build_clip_text_encoder, get_clip_embeddings +from .clip_wrapper_eva01 import EVA01CLIP +from .clip_wrapper_eva02 import EVA02CLIP +from .clip_wrapper_open import build_openclip_text_encoder, get_openclip_embeddings +from .llama2_wrapper import Llama2 +from .t5_wrapper import T5_warpper +from .text_encoder import TextModel diff --git a/approach/ovod/APE/ape/modeling/text/clip_wrapper_eva01.py b/approach/ovod/APE/ape/modeling/text/clip_wrapper_eva01.py new file mode 100644 index 0000000000000000000000000000000000000000..e9de9a2f9f92f0a63879a245a014c1b5b656b118 --- /dev/null +++ b/approach/ovod/APE/ape/modeling/text/clip_wrapper_eva01.py @@ -0,0 +1,146 @@ +import torch +import torch.nn as nn +from torch.cuda.amp import autocast + +from clip import tokenize + +from .eva01_clip import build_eva_model_and_transforms + + +class EVA01CLIP(nn.Module): + def __init__( + self, + clip_model="EVA_CLIP_g_14", + cache_dir="eva_clip_psz14.pt", + dtype="float32", + max_batch_size=2560, + ): + super().__init__() + self.net, _ = build_eva_model_and_transforms(clip_model, pretrained=cache_dir) + + if dtype == "bfloat16": + self.dtype = torch.bfloat16 + elif dtype == "float16": + self.dtype = torch.float16 + else: + self.dtype = torch.float32 + + del self.net.visual + self.net.eval() + for name, param in self.net.named_parameters(): + param.requires_grad = False + param.data = param.data.to(self.dtype) + + self.register_buffer("unused_tensor", torch.zeros(1), False) + + self.text_list_to_feature = {} + + self.max_batch_size = max_batch_size + + @property + def device(self): + return self.unused_tensor.device + + def infer_image(self, features): + x = features["image"][0] + x = self.net.encode_image(x) + return x + + @autocast(enabled=False) + @torch.no_grad() + def encode_text(self, text_list, cache=False): + if cache and tuple(text_list) in self.text_list_to_feature: + return self.text_list_to_feature[tuple(text_list)] + + text_token = tokenize(text_list, context_length=77, truncate=True).to(self.device) + + max_batch_size = self.max_batch_size + if self.device.type == "cpu" or torch.cuda.mem_get_info(self.device)[0] / 1024**3 < 5: + max_batch_size = min(256, max_batch_size) + if len(text_token) > max_batch_size: + chunck_num = len(text_token) // max_batch_size + 1 + encoder_outputs = torch.cat( + [ + self.net.encode_text( + text_token[chunck_id * max_batch_size : (chunck_id + 1) * max_batch_size] + ) + for chunck_id in range(chunck_num) + ], + dim=0, + ) + else: + encoder_outputs = self.net.encode_text(text_token) + + ret = { + "last_hidden_state_eot": encoder_outputs, + } + + if cache: + self.text_list_to_feature[tuple(text_list)] = ret + + return ret + + @autocast(enabled=False) + @torch.no_grad() + def forward_text(self, text_list, cache=False): + if cache and tuple(text_list) in self.text_list_to_feature: + return self.text_list_to_feature[tuple(text_list)] + + text_token = tokenize(text_list, context_length=77, truncate=True).to(self.device) + + max_batch_size = self.max_batch_size + if self.device.type == "cpu" or torch.cuda.mem_get_info(self.device)[0] / 1024**3 < 5: + max_batch_size = min(256, max_batch_size) + if len(text_token) > max_batch_size: + chunck_num = len(text_token) // max_batch_size + 1 + encoder_outputs = [ + self.custom_encode_text( + text_token[chunck_id * max_batch_size : (chunck_id + 1) * max_batch_size], + self.net.text, + ) + for chunck_id in range(chunck_num) + ] + encoder_outputs_x = torch.cat([x for (x, _) in encoder_outputs], dim=0) + encoder_outputs_xx = torch.cat([xx for (_, xx) in encoder_outputs], dim=0) + else: + encoder_outputs_x, encoder_outputs_xx = self.custom_encode_text( + text_token, self.net.text + ) + + end_token_idx = text_token.argmax(dim=-1) + attention_mask = end_token_idx.new_zeros(encoder_outputs_xx.size()[:2]) + for i in range(attention_mask.size(0)): + attention_mask[i, : end_token_idx[i] + 1] = 1 + + ret = { + "end_token_idx": end_token_idx, + "attention_mask": attention_mask, + "last_hidden_state": encoder_outputs_xx, + "last_hidden_state_eot": encoder_outputs_x, + } + + if cache: + self.text_list_to_feature[tuple(text_list)] = ret + + return ret + + @autocast(enabled=False) + @torch.no_grad() + def custom_encode_text(self, text, m): + x = m.token_embedding(text) # [batch_size, n_ctx, d_model] + + x = x + m.positional_embedding + x = x.permute(1, 0, 2) # NLD -> LND + x = m.transformer(x, attn_mask=m.attn_mask) + x = x.permute(1, 0, 2) # LND -> NLD + x = m.ln_final(x) + + if m.text_projection is not None: + xx = x @ m.text_projection + + x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] + + if m.text_projection is not None: + x = x @ m.text_projection + + return x, xx diff --git a/approach/ovod/APE/ape/modeling/text/clip_wrapper_open.py b/approach/ovod/APE/ape/modeling/text/clip_wrapper_open.py new file mode 100644 index 0000000000000000000000000000000000000000..768076f7e9e99d031945e22f49d63f33757a3e34 --- /dev/null +++ b/approach/ovod/APE/ape/modeling/text/clip_wrapper_open.py @@ -0,0 +1,51 @@ +import logging +from collections import OrderedDict +from typing import List, Union + +import torch +from torch import nn + +from clip.simple_tokenizer import SimpleTokenizer as _Tokenizer + + +def build_openclip_text_encoder(open_clip_name, open_clip_model): + import open_clip + + logger = logging.getLogger(__name__) + + print(open_clip.list_pretrained()) + logger.info("Loading pretrained CLIP " + open_clip_name + " " + open_clip_model) + + model, _, preprocess = open_clip.create_model_and_transforms( + open_clip_name, pretrained=open_clip_model + ) + tokenizer = open_clip.get_tokenizer(open_clip_name) + + del model.visual + + model.eval() + + return model, tokenizer + + +def get_openclip_embeddings(model, tokenizer, vocabulary, prompt="a "): + model.eval() + + sentences = [prompt + x for x in vocabulary] + text = tokenizer(sentences).to(model.token_embedding.weight.device) + + with torch.no_grad(): + if len(text) > 10000: + text_features = torch.cat( + [ + model.encode_text(text[: len(text) // 2]), + model.encode_text(text[len(text) // 2 :]), + ], + dim=0, + ) + else: + text_features = model.encode_text(text) + + text_features = text_features.detach().contiguous() + + return text_features diff --git a/approach/ovod/APE/ape/modeling/text/llama2_wrapper.py b/approach/ovod/APE/ape/modeling/text/llama2_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..ec1e0aeb9eb577334aebecfe2b9e8bde8506192c --- /dev/null +++ b/approach/ovod/APE/ape/modeling/text/llama2_wrapper.py @@ -0,0 +1,154 @@ +import copy +import logging +import math +import time +from typing import Dict, List, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.cuda.amp import autocast +from torch.nn import CrossEntropyLoss + +import fvcore.nn.weight_init as weight_init +from detectron2.data.catalog import DatasetCatalog, MetadataCatalog +from detectron2.layers import Conv2d, ShapeSpec, get_norm, move_device_like +from detectron2.modeling import GeneralizedRCNN +from detectron2.modeling.postprocessing import detector_postprocess, sem_seg_postprocess +from detectron2.modeling.roi_heads.fast_rcnn import fast_rcnn_inference +from detectron2.structures import BitMasks, Boxes, ImageList, Instances +from detectron2.utils import comm +from detrex.layers import MLP, box_cxcywh_to_xyxy, box_xyxy_to_cxcywh +from detrex.utils import inverse_sigmoid +from torchvision.ops.boxes import batched_nms +from transformers import BitsAndBytesConfig, LlamaConfig, LlamaForCausalLM, LlamaTokenizer +from transformers.modeling_outputs import BaseModelOutput + + +class Llama2(nn.Module): + def __init__( + self, + pretrained_model_name_or_path, + bg_word="", + dtype="bfloat16", + loss_type="CE", + use_fed_loss=False, + fed_loss_num_classes=1000, + inference_text=False, + inference_prob=False, + inference_prob_fast=False, + train_positive_only=False, + test_constraint=False, + vision_port="encoder", + eval_only=False, + load_in_4bit=False, + load_in_8bit=False, + **kwargs, + ): + super().__init__(**kwargs) + + self.dtype = getattr(torch, dtype) + + self.config = LlamaConfig.from_pretrained( + pretrained_model_name_or_path=pretrained_model_name_or_path + ) + + if load_in_4bit: + quantization_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_quant_type="nf4", + bnb_4bit_compute_dtype=self.dtype, + bnb_4bit_use_double_quant=True, + ) + device_map = {"": comm.get_local_rank()} + elif load_in_8bit: + quantization_config = BitsAndBytesConfig( + load_in_8bit=True, + bnb_8bit_quant_type="nf4", + bnb_8bit_compute_dtype=self.dtype, + bnb_8bit_use_double_quant=True, + ) + device_map = {"": comm.get_local_rank()} + else: + quantization_config = None + device_map = None + self.model = LlamaForCausalLM.from_pretrained( + pretrained_model_name_or_path=pretrained_model_name_or_path, + quantization_config=quantization_config, + device_map=device_map, + ) + + if quantization_config is None: + for name, param in self.model.named_parameters(): + param.data = param.data.to(self.dtype) + + self.tokenizer = LlamaTokenizer.from_pretrained( + pretrained_model_name_or_path=pretrained_model_name_or_path + ) + + self.tokenizer.add_special_tokens({"pad_token": ""}) + self.tokenizer.padding_side = "left" + + self.model.resize_token_embeddings(len(self.tokenizer)) + self.model.config.pad_token_id = self.tokenizer.pad_token_id + + if eval_only: + self.model.eval() + for name, param in self.model.named_parameters(): + param.requires_grad = False + + logger = logging.getLogger(__name__) + logger.info("memory footprint: {}G".format(self.model.get_memory_footprint() / 1024**3)) + + self.text_list_to_feature = {} + + @autocast(enabled=False) + @torch.no_grad() + def forward_text(self, text_list, cache=False): + if cache and tuple(text_list) in self.text_list_to_feature: + return self.text_list_to_feature[tuple(text_list)] + + text_token = self.tokenizer( + text_list, + return_tensors="pt", + padding="longest", + ).to(self.device) + input_ids = text_token.input_ids + attention_mask = text_token.attention_mask + + max_batch_size = 128 + if torch.cuda.mem_get_info(self.device)[0] / 1024**3 < 5: + max_batch_size = 128 + + chunck_num = input_ids.size(0) // max_batch_size + 1 + last_hidden_state = [] + for chunck_id in range(chunck_num): + outputs = self.model( + input_ids=input_ids[chunck_id * max_batch_size : (chunck_id + 1) * max_batch_size], + attention_mask=attention_mask[ + chunck_id * max_batch_size : (chunck_id + 1) * max_batch_size + ], + inputs_embeds=None, + output_attentions=True, + output_hidden_states=True, + return_dict=True, + ) + last_hidden_state.append(outputs.hidden_states[-1].clone().detach()) + + last_hidden_state = torch.cat(last_hidden_state, dim=0) + + last_hidden_state = torch.nan_to_num(last_hidden_state, nan=0.0, posinf=0.0, neginf=0.0) + + ret = { + "attention_mask": attention_mask, + "last_hidden_state": last_hidden_state, + } + + if cache: + self.text_list_to_feature[tuple(text_list)] = ret + + return ret + + @property + def device(self): + return self.model.device diff --git a/approach/ovod/APE/ape/modeling/text/t5_wrapper.py b/approach/ovod/APE/ape/modeling/text/t5_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..2567e191dea840a83a916bdae349ba11aa883dcd --- /dev/null +++ b/approach/ovod/APE/ape/modeling/text/t5_wrapper.py @@ -0,0 +1,103 @@ +import copy +import math +import time +from typing import Dict, List, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.cuda.amp import autocast +from torch.nn import CrossEntropyLoss + +import fvcore.nn.weight_init as weight_init +from detectron2.data.catalog import DatasetCatalog, MetadataCatalog +from detectron2.layers import Conv2d, ShapeSpec, get_norm, move_device_like +from detectron2.modeling import GeneralizedRCNN +from detectron2.modeling.postprocessing import detector_postprocess, sem_seg_postprocess +from detectron2.modeling.roi_heads.fast_rcnn import fast_rcnn_inference +from detectron2.structures import BitMasks, Boxes, ImageList, Instances +from detrex.layers import MLP, box_cxcywh_to_xyxy, box_xyxy_to_cxcywh +from detrex.utils import inverse_sigmoid +from torchvision.ops.boxes import batched_nms +from transformers import AutoConfig, AutoModelForSeq2SeqLM, AutoTokenizer +from transformers.modeling_outputs import BaseModelOutput + + +class T5_warpper(nn.Module): + def __init__( + self, + pretrained_model_name_or_path, + bg_word="", + dtype="bfloat16", + loss_type="CE", + use_fed_loss=False, + fed_loss_num_classes=1000, + inference_text=False, + inference_prob=False, + inference_prob_fast=False, + train_positive_only=False, + test_constraint=False, + vision_port="encoder", + eval_only=False, + **kwargs, + ): + super().__init__(**kwargs) + + self.dtype = getattr(torch, dtype) + + self.config = AutoConfig.from_pretrained( + pretrained_model_name_or_path=pretrained_model_name_or_path + ) + self.t5_model = AutoModelForSeq2SeqLM.from_pretrained( + pretrained_model_name_or_path=pretrained_model_name_or_path + ) + self.tokenizer = AutoTokenizer.from_pretrained( + pretrained_model_name_or_path=pretrained_model_name_or_path + ) + + if eval_only: + self.t5_model.eval() + for name, param in self.t5_model.named_parameters(): + param.requires_grad = False + param.data = param.data.to(self.dtype) + + self.eos_token_id = self.tokenizer("\n", add_special_tokens=False).input_ids[0] + + self.text_list_to_feature = {} + + @autocast(enabled=False) + @torch.no_grad() + def forward_text(self, text_list, cache=False): + if cache and tuple(text_list) in self.text_list_to_feature: + return self.text_list_to_feature[tuple(text_list)] + + text_token = self.tokenizer( + text_list, + return_tensors="pt", + padding="longest", + ).to(self.device) + input_ids = text_token.input_ids + attention_mask = text_token.attention_mask + + encoder_outputs = self.t5_model.encoder( + input_ids=input_ids, + attention_mask=attention_mask, + inputs_embeds=None, + head_mask=None, + output_attentions=True, + output_hidden_states=True, + return_dict=True, + ) + + last_hidden_state = encoder_outputs.last_hidden_state + + feature = agg_lang_feat(last_hidden_state, attention_mask).clone().detach() + + if cache: + self.text_list_to_feature[tuple(text_list)] = feature + + return feature + + @property + def device(self): + return self.t5_model.device diff --git a/approach/ovod/APE/ape/modeling/text/text_encoder.py b/approach/ovod/APE/ape/modeling/text/text_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..9c9dd62c6d5de8e2faa4bbb30ee17d34b5bf403a --- /dev/null +++ b/approach/ovod/APE/ape/modeling/text/text_encoder.py @@ -0,0 +1,38 @@ +import logging +from collections import OrderedDict +from typing import List, Union + +import torch +from torch import nn + +from .clip_wrapper import build_clip_text_encoder, get_clip_embeddings +from .clip_wrapper_open import build_openclip_text_encoder, get_openclip_embeddings + + +class TextModel(nn.Module): + def __init__( + self, + model_type, + model_name, + model_path, + ): + super().__init__() + + self.model_type = model_type + self.model_name = model_name + self.model_path = model_path + + if self.model_type == "CLIP": + self.model = build_clip_text_encoder(model_path, pretrain=True) + + if self.model_type == "OPENCLIP": + self.model, self.tokenizer = build_openclip_text_encoder(model_name, model_path) + + self.model.eval() + + def forward_text(self, text, prompt="a "): + if self.model_type == "CLIP": + return get_clip_embeddings(self.model, text, prompt) + + if self.model_type == "OPENCLIP": + return get_openclip_embeddings(self.model, self.tokenizer, text, prompt) diff --git a/approach/ovod/APE/ape/modeling/text/utils.py b/approach/ovod/APE/ape/modeling/text/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..932fe05d836a16db1bb7c2a968a8ff734f147ffc --- /dev/null +++ b/approach/ovod/APE/ape/modeling/text/utils.py @@ -0,0 +1,32 @@ +import torch + + +def clean_name(name): + name = re.sub(r"\(.*\)", "", name) + name = re.sub(r"_", " ", name) + name = re.sub(r" ", " ", name) + return name + + +def reduce_language_feature(features, mask, reduce_type="average"): + """average pooling of language features""" "" + if reduce_type == "average": + embedded = ( + features * mask.unsqueeze(-1).float() + ) # use mask to zero out invalid token features + aggregate = embedded.sum(1) / (mask.sum(-1).unsqueeze(-1).float() + 1e-6) + elif reduce_type == "max": + out = [] + for i in range(len(features)): + pool_feat, _ = torch.max(features[i][mask[i]], 0) # (L, C) -> (C, ) + out.append(pool_feat) + aggregate = torch.stack(out, dim=0) # (bs, C) + elif reduce_type == "last": + out = [] + for i in range(len(features)): + pool_feat = features[i][torch.argmin(mask[i]) - 1] # (L, C) -> (C, ) + out.append(pool_feat) + aggregate = torch.stack(out, dim=0) # (bs, C) + else: + raise ValueError("reduce_type should be average or max or last.") + return aggregate diff --git a/approach/ovod/APE/ape/utils/__init__.py b/approach/ovod/APE/ape/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4ebdc90b7f3ac2ed5a085066dcf20722b90cbc77 --- /dev/null +++ b/approach/ovod/APE/ape/utils/__init__.py @@ -0,0 +1,8 @@ +# ------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Modified from DETR (https://github.com/facebookresearch/detr) +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +# ------------------------------------------------------------------------ diff --git a/approach/ovod/APE/ape/utils/box_ops.py b/approach/ovod/APE/ape/utils/box_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..bf23fb4653cc838494cce69a903c20b06c652da2 --- /dev/null +++ b/approach/ovod/APE/ape/utils/box_ops.py @@ -0,0 +1,95 @@ +# ------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Modified from DETR (https://github.com/facebookresearch/detr) +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +# ------------------------------------------------------------------------ + +""" +Utilities for bounding box manipulation and GIoU. +""" +import torch + +from torchvision.ops.boxes import box_area + + +def box_cxcywh_to_xyxy(x): + x_c, y_c, w, h = x.unbind(-1) + b = [(x_c - 0.5 * w), (y_c - 0.5 * h), (x_c + 0.5 * w), (y_c + 0.5 * h)] + return torch.stack(b, dim=-1) + + +def box_xyxy_to_cxcywh(x): + x0, y0, x1, y1 = x.unbind(-1) + b = [(x0 + x1) / 2, (y0 + y1) / 2, (x1 - x0), (y1 - y0)] + return torch.stack(b, dim=-1) + + +# modified from torchvision to also return the union +def box_iou(boxes1, boxes2): + area1 = box_area(boxes1) + area2 = box_area(boxes2) + + lt = torch.max(boxes1[:, None, :2], boxes2[:, :2]) # [N,M,2] + rb = torch.min(boxes1[:, None, 2:], boxes2[:, 2:]) # [N,M,2] + + wh = (rb - lt).clamp(min=0) # [N,M,2] + inter = wh[:, :, 0] * wh[:, :, 1] # [N,M] + + union = area1[:, None] + area2 - inter + + iou = inter / union + return iou, union + + +def generalized_box_iou(boxes1, boxes2): + """ + Generalized IoU from https://giou.stanford.edu/ + + The boxes should be in [x0, y0, x1, y1] format + + Returns a [N, M] pairwise matrix, where N = len(boxes1) + and M = len(boxes2) + """ + # degenerate boxes gives inf / nan results + # so do an early check + assert (boxes1[:, 2:] >= boxes1[:, :2]).all() + assert (boxes2[:, 2:] >= boxes2[:, :2]).all() + iou, union = box_iou(boxes1, boxes2) + + lt = torch.min(boxes1[:, None, :2], boxes2[:, :2]) + rb = torch.max(boxes1[:, None, 2:], boxes2[:, 2:]) + + wh = (rb - lt).clamp(min=0) # [N,M,2] + area = wh[:, :, 0] * wh[:, :, 1] + + return iou - (area - union) / area + + +def masks_to_boxes(masks): + """Compute the bounding boxes around the provided masks + + The masks should be in format [N, H, W] where N is the number of masks, (H, W) are the spatial dimensions. + + Returns a [N, 4] tensors, with the boxes in xyxy format + """ + if masks.numel() == 0: + return torch.zeros((0, 4), device=masks.device) + + h, w = masks.shape[-2:] + + y = torch.arange(0, h, dtype=torch.float) + x = torch.arange(0, w, dtype=torch.float) + y, x = torch.meshgrid(y, x) + + x_mask = masks * x.unsqueeze(0) + x_max = x_mask.flatten(1).max(-1)[0] + x_min = x_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + y_mask = masks * y.unsqueeze(0) + y_max = y_mask.flatten(1).max(-1)[0] + y_min = y_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + return torch.stack([x_min, y_min, x_max, y_max], 1) diff --git a/approach/ovod/APE/ape/utils/misc.py b/approach/ovod/APE/ape/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..2d24f880ea543ca8712d9fdc3239f8c42643a9a4 --- /dev/null +++ b/approach/ovod/APE/ape/utils/misc.py @@ -0,0 +1,547 @@ +# ------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Modified from DETR (https://github.com/facebookresearch/detr) +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +# ------------------------------------------------------------------------ + +""" +Misc functions, including distributed helpers. + +Mostly copy-paste from torchvision references. +""" +import datetime +import os +import pickle +import subprocess +import time +from collections import defaultdict, deque +from typing import List, Optional + +import torch +import torch.distributed as dist +import torch.nn as nn +from torch import Tensor + +# needed due to empty tensor bug in pytorch and torchvision 0.5 +import torchvision + +if ( + float(torchvision.__version__.split(".")[0]) == 0 + and float(torchvision.__version__.split(".")[1]) < 5 +): + import math + from torchvision.ops.misc import _NewEmptyTensorOp + + def _check_size_scale_factor(dim, size, scale_factor): + # type: (int, Optional[List[int]], Optional[float]) -> None + if size is None and scale_factor is None: + raise ValueError("either size or scale_factor should be defined") + if size is not None and scale_factor is not None: + raise ValueError("only one of size or scale_factor should be defined") + if not (scale_factor is not None and len(scale_factor) != dim): + raise ValueError( + "scale_factor shape must match input shape. " + "Input is {}D, scale_factor size is {}".format(dim, len(scale_factor)) + ) + + def _output_size(dim, input, size, scale_factor): + # type: (int, Tensor, Optional[List[int]], Optional[float]) -> List[int] + assert dim == 2 + _check_size_scale_factor(dim, size, scale_factor) + if size is not None: + return size + # if dim is not 2 or scale_factor is iterable use _ntuple instead of concat + assert scale_factor is not None and isinstance(scale_factor, (int, float)) + scale_factors = [scale_factor, scale_factor] + # math.floor might return float in py2.7 + return [int(math.floor(input.size(i + 2) * scale_factors[i])) for i in range(dim)] + +elif ( + float(torchvision.__version__.split(".")[0]) == 0 + and float(torchvision.__version__.split(".")[1]) < 7 +): + from torchvision.ops import _new_empty_tensor + from torchvision.ops.misc import _output_size + + +class SmoothedValue(object): + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], dtype=torch.float64, device="cuda") + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + return self.total / self.count + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, + avg=self.avg, + global_avg=self.global_avg, + max=self.max, + value=self.value, + ) + + +def all_gather(data): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors) + Args: + data: any picklable object + Returns: + list[data]: list of data gathered from each rank + """ + world_size = get_world_size() + if world_size == 1: + return [data] + + # serialized to a Tensor + buffer = pickle.dumps(data) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to("cuda") + + # obtain Tensor size of each rank + local_size = torch.tensor([tensor.numel()], device="cuda") + size_list = [torch.tensor([0], device="cuda") for _ in range(world_size)] + dist.all_gather(size_list, local_size) + size_list = [int(size.item()) for size in size_list] + max_size = max(size_list) + + # receiving Tensor from all ranks + # we pad the tensor because torch all_gather does not support + # gathering tensors of different shapes + tensor_list = [] + for _ in size_list: + tensor_list.append(torch.empty((max_size,), dtype=torch.uint8, device="cuda")) + if local_size != max_size: + padding = torch.empty(size=(max_size - local_size,), dtype=torch.uint8, device="cuda") + tensor = torch.cat((tensor, padding), dim=0) + dist.all_gather(tensor_list, tensor) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + + return data_list + + +def reduce_dict(input_dict, average=True): + """ + Args: + input_dict (dict): all the values will be reduced + average (bool): whether to do average or sum + Reduce the values in the dictionary from all processes so that all processes + have the averaged results. Returns a dict with the same fields as + input_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return input_dict + with torch.no_grad(): + names = [] + values = [] + # sort the keys so that they are consistent across processes + for k in sorted(input_dict.keys()): + names.append(k) + values.append(input_dict[k]) + values = torch.stack(values, dim=0) + dist.all_reduce(values) + if average: + values /= world_size + reduced_dict = {k: v for k, v in zip(names, values)} + return reduced_dict + + +class MetricLogger(object): + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError("'{}' object has no attribute '{}'".format(type(self).__name__, attr)) + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + loss_str.append("{}: {}".format(name, str(meter))) + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, print_freq, header=None): + i = 0 + if not header: + header = "" + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt="{avg:.4f}") + data_time = SmoothedValue(fmt="{avg:.4f}") + space_fmt = ":" + str(len(str(len(iterable)))) + "d" + if torch.cuda.is_available(): + log_msg = self.delimiter.join( + [ + header, + "[{0" + space_fmt + "}/{1}]", + "eta: {eta}", + "{meters}", + "time: {time}", + "data: {data}", + "max mem: {memory:.0f}", + ] + ) + else: + log_msg = self.delimiter.join( + [ + header, + "[{0" + space_fmt + "}/{1}]", + "eta: {eta}", + "{meters}", + "time: {time}", + "data: {data}", + ] + ) + MB = 1024.0 * 1024.0 + for obj in iterable: + data_time.update(time.time() - end) + yield obj + iter_time.update(time.time() - end) + if i % print_freq == 0 or i == len(iterable) - 1: + eta_seconds = iter_time.global_avg * (len(iterable) - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + if torch.cuda.is_available(): + print( + log_msg.format( + i, + len(iterable), + eta=eta_string, + meters=str(self), + time=str(iter_time), + data=str(data_time), + memory=torch.cuda.max_memory_allocated() / MB, + ) + ) + else: + print( + log_msg.format( + i, + len(iterable), + eta=eta_string, + meters=str(self), + time=str(iter_time), + data=str(data_time), + ) + ) + i += 1 + end = time.time() + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print( + "{} Total time: {} ({:.4f} s / it)".format( + header, total_time_str, total_time / len(iterable) + ) + ) + + +def get_sha(): + cwd = os.path.dirname(os.path.abspath(__file__)) + + def _run(command): + return subprocess.check_output(command, cwd=cwd).decode("ascii").strip() + + sha = "N/A" + diff = "clean" + branch = "N/A" + try: + sha = _run(["git", "rev-parse", "HEAD"]) + subprocess.check_output(["git", "diff"], cwd=cwd) + diff = _run(["git", "diff-index", "HEAD"]) + diff = "has uncommited changes" if diff else "clean" + branch = _run(["git", "rev-parse", "--abbrev-ref", "HEAD"]) + except Exception: + pass + message = f"sha: {sha}, status: {diff}, branch: {branch}" + return message + + +def collate_fn(batch): + batch = list(zip(*batch)) + batch[0] = nested_tensor_from_tensor_list(batch[0]) + return tuple(batch) + + +def _max_by_axis(the_list): + # type: (List[List[int]]) -> List[int] + maxes = the_list[0] + for sublist in the_list[1:]: + for index, item in enumerate(sublist): + maxes[index] = max(maxes[index], item) + return maxes + + +def nested_tensor_from_tensor_list(tensor_list: List[Tensor]): + # TODO make this more general + if tensor_list[0].ndim == 3: + # TODO make it support different-sized images + max_size = _max_by_axis([list(img.shape) for img in tensor_list]) + # min_size = tuple(min(s) for s in zip(*[img.shape for img in tensor_list])) + batch_shape = [len(tensor_list)] + max_size + b, c, h, w = batch_shape + dtype = tensor_list[0].dtype + device = tensor_list[0].device + tensor = torch.zeros(batch_shape, dtype=dtype, device=device) + mask = torch.ones((b, h, w), dtype=torch.bool, device=device) + for img, pad_img, m in zip(tensor_list, tensor, mask): + pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + m[: img.shape[1], : img.shape[2]] = False + else: + raise ValueError("not supported") + return NestedTensor(tensor, mask) + + +class NestedTensor(object): + def __init__(self, tensors, mask: Optional[Tensor]): + self.tensors = tensors + self.mask = mask + + def to(self, device, non_blocking=False): + # type: (Device) -> NestedTensor # noqa + cast_tensor = self.tensors.to(device, non_blocking=non_blocking) + mask = self.mask + if mask is not None: + assert mask is not None + cast_mask = mask.to(device, non_blocking=non_blocking) + else: + cast_mask = None + return NestedTensor(cast_tensor, cast_mask) + + def record_stream(self, *args, **kwargs): + self.tensors.record_stream(*args, **kwargs) + if self.mask is not None: + self.mask.record_stream(*args, **kwargs) + + def decompose(self): + return self.tensors, self.mask + + def __repr__(self): + return str(self.tensors) + + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop("force", False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def get_local_size(): + if not is_dist_avail_and_initialized(): + return 1 + return int(os.environ["LOCAL_SIZE"]) + + +def get_local_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return int(os.environ["LOCAL_RANK"]) + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + if "RANK" in os.environ and "WORLD_SIZE" in os.environ: + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ["WORLD_SIZE"]) + args.gpu = int(os.environ["LOCAL_RANK"]) + args.dist_url = "env://" + os.environ["LOCAL_SIZE"] = str(torch.cuda.device_count()) + elif "SLURM_PROCID" in os.environ: + proc_id = int(os.environ["SLURM_PROCID"]) + ntasks = int(os.environ["SLURM_NTASKS"]) + node_list = os.environ["SLURM_NODELIST"] + num_gpus = torch.cuda.device_count() + addr = subprocess.getoutput("scontrol show hostname {} | head -n1".format(node_list)) + os.environ["MASTER_PORT"] = os.environ.get("MASTER_PORT", "29500") + os.environ["MASTER_ADDR"] = addr + os.environ["WORLD_SIZE"] = str(ntasks) + os.environ["RANK"] = str(proc_id) + os.environ["LOCAL_RANK"] = str(proc_id % num_gpus) + os.environ["LOCAL_SIZE"] = str(num_gpus) + args.dist_url = "env://" + args.world_size = ntasks + args.rank = proc_id + args.gpu = proc_id % num_gpus + else: + print("Not using distributed mode") + args.distributed = False + return + + args.distributed = True + + torch.cuda.set_device(args.gpu) + args.dist_backend = "nccl" + print("| distributed init (rank {}): {}".format(args.rank, args.dist_url), flush=True) + torch.distributed.init_process_group( + backend=args.dist_backend, + init_method=args.dist_url, + world_size=args.world_size, + rank=args.rank, + ) + torch.distributed.barrier() + setup_for_distributed(args.rank == 0) + + +@torch.no_grad() +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + if target.numel() == 0: + return [torch.zeros([], device=output.device)] + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def interpolate(input, size=None, scale_factor=None, mode="nearest", align_corners=None): + # type: (Tensor, Optional[List[int]], Optional[float], str, Optional[bool]) -> Tensor + """ + Equivalent to nn.functional.interpolate, but with support for empty batch sizes. + This will eventually be supported natively by PyTorch, and this + class can go away. + """ + if float(torchvision.__version__[:3]) < 0.7: + if input.numel() > 0: + return torch.nn.functional.interpolate(input, size, scale_factor, mode, align_corners) + + output_shape = _output_size(2, input, size, scale_factor) + output_shape = list(input.shape[:-2]) + list(output_shape) + if float(torchvision.__version__[:3]) < 0.5: + return _NewEmptyTensorOp.apply(input, output_shape) + return _new_empty_tensor(input, output_shape) + else: + return torchvision.ops.misc.interpolate(input, size, scale_factor, mode, align_corners) + + +def get_total_grad_norm(parameters, norm_type=2): + parameters = list(filter(lambda p: p.grad is not None, parameters)) + norm_type = float(norm_type) + device = parameters[0].grad.device + total_norm = torch.norm( + torch.stack([torch.norm(p.grad.detach(), norm_type).to(device) for p in parameters]), + norm_type, + ) + return total_norm + + +def inverse_sigmoid(x, eps=1e-5): + x = x.clamp(min=0, max=1) + x1 = x.clamp(min=eps) + x2 = (1 - x).clamp(min=eps) + return torch.log(x1 / x2) diff --git a/approach/ovod/APE/ape/utils/plot_utils.py b/approach/ovod/APE/ape/utils/plot_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0a89b4e747e3150c795ba3dd542a1bc2f76c68a2 --- /dev/null +++ b/approach/ovod/APE/ape/utils/plot_utils.py @@ -0,0 +1,120 @@ +# ------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# Modified from DETR (https://github.com/facebookresearch/detr) +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +# ------------------------------------------------------------------------ + +""" +Plotting utilities to visualize training logs. +""" +from pathlib import Path, PurePath + +import matplotlib.pyplot as plt +import pandas as pd +import torch + +import seaborn as sns + + +def plot_logs( + logs, fields=("class_error", "loss_bbox_unscaled", "mAP"), ewm_col=0, log_name="log.txt" +): + """ + Function to plot specific fields from training log(s). Plots both training and test results. + + :: Inputs - logs = list containing Path objects, each pointing to individual dir with a log file + - fields = which results to plot from each log file - plots both training and test for each field. + - ewm_col = optional, which column to use as the exponential weighted smoothing of the plots + - log_name = optional, name of log file if different than default 'log.txt'. + + :: Outputs - matplotlib plots of results in fields, color coded for each log file. + - solid lines are training results, dashed lines are test results. + + """ + func_name = "plot_utils.py::plot_logs" + + # verify logs is a list of Paths (list[Paths]) or single Pathlib object Path, + # convert single Path to list to avoid 'not iterable' error + + if not isinstance(logs, list): + if isinstance(logs, PurePath): + logs = [logs] + print(f"{func_name} info: logs param expects a list argument, converted to list[Path].") + else: + raise ValueError( + f"{func_name} - invalid argument for logs parameter.\n \ + Expect list[Path] or single Path obj, received {type(logs)}" + ) + + # verify valid dir(s) and that every item in list is Path object + for i, dir in enumerate(logs): + if not isinstance(dir, PurePath): + raise ValueError( + f"{func_name} - non-Path object in logs argument of {type(dir)}: \n{dir}" + ) + if dir.exists(): + continue + raise ValueError(f"{func_name} - invalid directory in logs argument:\n{dir}") + + # load log file(s) and plot + dfs = [pd.read_json(Path(p) / log_name, lines=True) for p in logs] + + fig, axs = plt.subplots(ncols=len(fields), figsize=(16, 5)) + + for df, color in zip(dfs, sns.color_palette(n_colors=len(logs))): + for j, field in enumerate(fields): + if field == "mAP": + coco_eval = ( + pd.DataFrame(pd.np.stack(df.test_coco_eval.dropna().values)[:, 1]) + .ewm(com=ewm_col) + .mean() + ) + axs[j].plot(coco_eval, c=color) + else: + df.interpolate().ewm(com=ewm_col).mean().plot( + y=[f"train_{field}", f"test_{field}"], + ax=axs[j], + color=[color] * 2, + style=["-", "--"], + ) + for ax, field in zip(axs, fields): + ax.legend([Path(p).name for p in logs]) + ax.set_title(field) + + +def plot_precision_recall(files, naming_scheme="iter"): + if naming_scheme == "exp_id": + # name becomes exp_id + names = [f.parts[-3] for f in files] + elif naming_scheme == "iter": + names = [f.stem for f in files] + else: + raise ValueError(f"not supported {naming_scheme}") + fig, axs = plt.subplots(ncols=2, figsize=(16, 5)) + for f, color, name in zip(files, sns.color_palette("Blues", n_colors=len(files)), names): + data = torch.load(f) + # precision is n_iou, n_points, n_cat, n_area, max_det + precision = data["precision"] + recall = data["params"].recThrs + scores = data["scores"] + # take precision for all classes, all areas and 100 detections + precision = precision[0, :, :, 0, -1].mean(1) + scores = scores[0, :, :, 0, -1].mean(1) + prec = precision.mean() + rec = data["recall"][0, :, 0, -1].mean() + print( + f"{naming_scheme} {name}: mAP@50={prec * 100: 05.1f}, " + + f"score={scores.mean():0.3f}, " + + f"f1={2 * prec * rec / (prec + rec + 1e-8):0.3f}" + ) + axs[0].plot(recall, precision, c=color) + axs[1].plot(recall, scores, c=color) + + axs[0].set_title("Precision / Recall") + axs[0].legend(names) + axs[1].set_title("Scores / Recall") + axs[1].legend(names) + return fig, axs diff --git a/approach/ovod/APE/datasets/README.md b/approach/ovod/APE/datasets/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ab8b683311618929d80c34f0f5abee6623882e3d --- /dev/null +++ b/approach/ovod/APE/datasets/README.md @@ -0,0 +1,401 @@ + +# Detectron2 Builtin Datasets + +Detectron2 has builtin support for a few datasets. + +The datasets are assumed to exist in a directory specified by the environment variable +`DETECTRON2_DATASETS`. + +Under this directory, following [here](https://github.com/facebookresearch/detectron2/blob/main/datasets/README.md) to prepare COCO, LVIS, cityscapes, Pascal VOC and ADE20k. + +The expected structure is described below. +``` +$DETECTRON2_DATASETS/ + coco/ + lvis/ + cityscapes/ + VOC20{07,10,12}/ + ADEChallengeData2016/ +``` + +You can set the location for builtin datasets by `export DETECTRON2_DATASETS=/path/to/datasets`. +If left unset, the default is `./datasets` relative to your current working directory. + + +# APE Builtin Datasets + + +## Expected dataset structure for COCO and LVIS +``` +$DETECTRON2_DATASETS/ + coco/ + annotations/ + instances_{train,val}2017.json + panoptic_{train,val}2017.json + {train,val}2017/ + panoptic_{train,val}2017/ + panoptic_stuff_{train,val}2017/ + panoptic_semseg_{train,val}2017/ + lvis/ + lvis_v1_{train,val}.json + lvis_v1_{train,val}+coco_mask.json + lvis_v1_{train,val}+coco_mask_cat_info.json +``` + + +`panoptic_semseg_{train,val}2017/` are generated by runing +``` +python3 datasets/prepare_coco_semantic_annos_from_panoptic_annos.py +``` + + +`lvis_v1_{train,val}+coco_mask.json` are generated by running +``` +python3 datasets/tools/lvis/merge_lvis_coco.py +``` + + +`lvis_v1_{train,val}+coco_mask_cat_info.json` are generated by running +``` +python3 datasets/tools/lvis/add_category_info_frequence.py --json_path datasets/lvis/lvis_v1_train+coco_mask.json +python3 datasets/tools/lvis/add_category_info_frequence.py --json_path datasets/lvis/lvis_v1_val+coco_mask.json +``` + + + + +## Expected dataset structure for [Objects365](https://data.baai.ac.cn/details/Objects365_2020): +``` +$DETECTRON2_DATASETS/ + objects365/ + annotations/ + zhiyuan_objv2_{train,val}.json + objects365_{train,val,minival}_fixname.json + train/ + images/ + val/ + images/ +``` + +`objects365_train_fixname.json` and `objects365_val_fixname.json` are generated by running +```bash +python3 datasets/tools/objects3652coco/get_image_info.py --image_dir datasets/objects365/train/ --json_path datasets/objects365/annotations/zhiyuan_objv2_train.json --output_path datasets/objects365/annotations/image_info_train.txt +python3 datasets/tools/objects3652coco/get_image_info.py --image_dir datasets/objects365/val/ --json_path datasets/objects365/annotations/zhiyuan_objv2_val.json --output_path datasets/objects365/annotations/image_info_val.txt + +python3 datasets/tools/objects3652coco/convert_annotations.py --root_dir datasets/objects365/ --image_info_path datasets/objects365/annotations/image_info_train.txt --subsets train --apply_exif +python3 datasets/tools/objects3652coco/convert_annotations.py --root_dir datasets/objects365/ --image_info_path datasets/objects365/annotations/image_info_val.txt --subsets val --apply_exif +python3 datasets/tools/objects3652coco/convert_annotations.py --root_dir datasets/objects365/ --image_info_path datasets/objects365/annotations/image_info_val.txt --subsets minival --apply_exif + +python3 datasets/tools/objects3652coco/fix_o365_names.py --ann datasets/objects365/annotations/objects365_train.json +python3 datasets/tools/objects3652coco/fix_o365_names.py --ann datasets/objects365/annotations/objects365_val.json +python3 datasets/tools/objects3652coco/fix_o365_names.py --ann datasets/objects365/annotations/objects365_minival.json +``` + +As Objects365 is large, we generate annotation file for each image separetely +``` +python3 datasets/tools/generate_img_ann_pair.py --json_path datasets/objects365/annotations/objects365_train_fixname.json --image_root datasets/objects365/train/ +``` + +## Expected dataset structure for [OpenImages](https://storage.googleapis.com/openimages/web/download.html#download_manually): +``` +$DETECTRON2_DATASETS/ + openimages/ + annotations/ + openimages_v6_{train,val}_bbox.json + openimages_v6_{train,val}_bbox_nogroup.json + openimages_v6_{train,val}_bbox_cat_info.json + openimages_v6_{train,val}_bbox_nogroup_cat_info.json + train/ + validation/ +``` + +`openimages_v6_{train,val}_bbox.json` are generated by running +``` +python3 datasets/tools/openimages2coco/convert_annotations.py --path datasets/openimages/ --version v6 --subset train --task bbox --apply-exif +python3 datasets/tools/openimages2coco/convert_annotations.py --path datasets/openimages/ --version v6 --subset val --task bbox --apply-exif +``` + +`openimages_v6_{train,val}_bbox_nogroup.json` are generated by running +``` +python3 datasets/tools/openimages2coco/convert_annotations.py --path datasets/openimages/ --version v6 --subset train --task bbox --apply-exif --exclude-group +python3 datasets/tools/openimages2coco/convert_annotations.py --path datasets/openimages/ --version v6 --subset val --task bbox --apply-exif --exclude-group +``` + +`*_cat_info.json` are generated by running +``` +python3 datasets/tools/lvis/add_category_info_frequence.py --json_path datasets/openimages/annotations/openimages_v6_train_bbox.json +python3 datasets/tools/lvis/add_category_info_frequence.py --json_path datasets/openimages/annotations/openimages_v6_val_bbox.json +python3 datasets/tools/lvis/add_category_info_frequence.py --json_path datasets/openimages/annotations/openimages_v6_train_bbox_nogroup.json +python3 datasets/tools/lvis/add_category_info_frequence.py --json_path datasets/openimages/annotations/openimages_v6_val_bbox_nogroup.json +``` + +Finally, runing +``` +python3 datasets/tools/generate_img_ann_pair.py --json_path datasets/openimages/annotations/openimages_v6_train_bbox.json --image_root datasets/openimages/train/ +``` + + +## Expected dataset structure for [VisualGenome](https://homes.cs.washington.edu/~ranjay/visualgenome/api.html): +``` +$DETECTRON2_DATASETS/ + visualgenome/ + annotations/ + visualgenome_77962_box.json + visualgenome_77962_box_{train,val}.json + visualgenome_region.json + visualgenome_region_{train,val}.json + visualgenome_77962_box_and_region.json + visualgenome_77962_box_and_region__{train,val}.json + VG_100K/ + VG_100K_2/ +``` + + +`visualgenome_*.json` are generated by running +``` +python3 datasets/tools/visualgenome2coco/convert_annotations_object.py -p datasets/visualgenome/ --apply-exif --object_list "" --num_objects 99999999 --min_box_area_frac 0.0 + +python3 datasets/tools/visualgenome2coco/convert_annotations_region.py -p datasets/visualgenome/ --apply-exif --object_list "" --num_objects 99999999 --min_box_area_frac 0.0 +``` + + +## Expected dataset structure for [SA-1B](https://ai.meta.com/datasets/segment-anything-downloads/): +``` +$DETECTRON2_DATASETS/ + SA-1B/ + images/ + sam1b_instance_1000000.json + ... + sam1b_instance.json +``` + +`sam1b_instance*.json` are generated by running +``` +python tools/sa1b2coco/image+json.py --image_root datasets/SA-1B/images/ --json_path datasets/SA-1B/sam1b_instance +``` + + +## Expected dataset structure for [RefCOCO](): +``` +$DETECTRON2_DATASETS/ + SeqTR/ + mixed/ + refcocog-google/ + instances_cocofied_{train,val}.json + refcocog-umd/ + instances_cocofied_{train,val,test}.json + refcoco-unc/ + instances_cocofied_{train,val,testA,testB}.json + refcocoplus-unc/ + instances_cocofied_{train,val,testA,testB}.json + refcoco-mixed/ + instances_cocofied_train.json + refcoco-mixed_group-by-image/ + instances_cocofied_train.json +``` +Download the preprocessed json files from [here](https://github.com/seanzhuh/SeqTR#data-preparation) + +`refcoco-mixed/` and some `instances_cocofied_*.json` are generated by running +``` +python3 datasets/tools/seqtr2coco/convert_mix_ref.py +``` + +`refcoco-mixed_group-by-image//` and its `instances_cocofied_train.json` are generated by running +``` +python3 datasets/tools/seqtr2coco/convert_refcoco_mixed_group_by_image.py +``` + + + +## Expected dataset structure for [GQA](https://cs.stanford.edu/people/dorarad/gqa/download.html): +``` +$DETECTRON2_DATASETS/ + gqa/ + images/ + gqa_region_{train,val}.json + gqa_region.json +``` + +`gqa_region*.json` are generated by running +``` +python3 datasets/tools/gqa2coco/convert.py --data_path datasets/gqa/ --img_path datasets/gqa/images --sg_path datasets/gqa/ --vg_img_data_path datasets/visualgenome/annotations/ --out_path datasets/gqa/ +``` + +## Expected dataset structure for [PhraseCut](https://github.com/ChenyunWu/PhraseCutDataset): +``` +$DETECTRON2_DATASETS/ + phrasecut/ + images/ + phrasecut_{train,val,miniv,test}.json +``` + +`phrasecut_*.json` are generated by running +``` +python3 datasets/tools/phrasecut2coco/convert.py --data_path datasets/phrasecut/ --img_path datasets/phrasecut/images --out_path datasets/phrasecut/ +``` + + +## Expected dataset structure for [Flickr30k](https://shannon.cs.illinois.edu/DenotationGraph/): +``` +$DETECTRON2_DATASETS/ + flickr30k/ + flickr30k-images/ + flickr30k_separateGT_{train,val.test}.json +``` + +`flickr30k_separateGT_*.json` are generated by running +``` +python3 datasets/tools/flickr2coco/convert.py --flickr_path datasets/flickr30k/flickr30k_entities/ --out_path datasets/flickr30k/ +``` + + +## Expected dataset structure for [ODinW](https://github.com/microsoft/GLIP#the-object-detection-in-the-wild-benchmark): +``` +$DETECTRON2_DATASETS/ + odinw/ + AerialMaritimeDrone/ + AmericanSignLanguageLetters/ + ... + WildfireSmoke/ +``` + +After download, update json files by runing +``` +python3 datasets/tools/odinw/convert.py +``` + +This is because +``` +https://github.com/cocodataset/cocoapi/issues/507#issuecomment-857272753 +``` + +## Expected dataset structure for [SegInW](https://github.com/microsoft/X-Decoder/tree/seginw#download): +``` +$DETECTRON2_DATASETS/ + seginw/ + Airplane-Parts/ + Bottles/ + ... + Watermelon/ +``` + +## Expected dataset structure for [Roboflow100](https://github.com/roboflow/roboflow-100-benchmark#local-env): +``` +$DETECTRON2_DATASETS/ + rf100/ + 4-fold-defect/ + abdomen-mri/ + ... + x-ray-rheumatology/ +``` + + +## Expected dataset structure for [ADE20k-Full](https://groups.csail.mit.edu/vision/datasets/ADE20K/): +``` +ADE20K_2021_17_01/ + images/ + images_detectron2/ + annotations_detectron2/ + index_ade20k.pkl + objects.txt +``` + +The directories `images_detectron2` and `annotations_detectron2` are generated by running +``` +python datasets/prepare_ade20k_full_sem_seg.py +``` + + +## Expected dataset structure for [BDD10k](https://bdd-data.berkeley.edu/): +``` +$DETECTRON2_DATASETS/ + bdd100k/ + images/ + labels/ + pan_seg/ + coco_pano/ + meta/ + ... + ... + seg/ +``` + +`coco_pano` and `meta` is generated by running +``` +wget https://github.com/shenyunhang/APE/releases/download/0/bdd_generated.tar.gz +tar xvzf bdd_generated.tar.gz +``` + + + + +## Expected dataset structure for [PC459 and PC59](https://cs.stanford.edu/~roozbeh/pascal-context/): +``` +$DETECTRON2_DATASETS/ + VOCdevkit/ + VOC2010/ + Annotations/ + ImageSets/ + JPEGImages/ + SegmentationClass/ + SegmentationObject/ + # below are from https://www.cs.stanford.edu/~roozbeh/pascal-context/trainval.tar.gz + trainval/ + labels.txt + 59_labels.txt # https://www.cs.stanford.edu/~roozbeh/pascal-context/59_labels.txt + pascalcontext_val.txt # https://drive.google.com/file/d/1BCbiOKtLvozjVnlTJX51koIveUZHCcUh/view?usp=sharing + # below are generated + annotations_detectron2/ + pc459_val/ + pc59_val +``` + +It starts with a tar file `VOCtrainval_03-May-2010.tar`. You may want to download the 5K validation set [here](https://drive.google.com/file/d/1BCbiOKtLvozjVnlTJX51koIveUZHCcUh/view?usp=sharing). + +The directory `annotations_detectron2` is generated by running +``` +python datasets/prepare_pascal_context.py +``` + + + +## Expected dataset structure for [VOC](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/): +``` +$DETECTRON2_DATASETS/ + VOCdevkit/ + VOC2012/ + Annotations/ + ImageSets/ + JPEGImages/ + SegmentationClass/ + SegmentationObject/ + SegmentationClassAug/ # https://github.com/kazuto1011/deeplab-pytorch/blob/master/data/datasets/voc12/README.md + # below are generated + images_detectron2/ + annotations_detectron2/ + val/ +``` + +It starts with a tar file `VOCtrainval_11-May-2012.tar`. + +We use SBD augmentated training data as `SegmentationClassAug` following [Deeplab](https://github.com/kazuto1011/deeplab-pytorch/blob/master/data/datasets/voc12/README.md) + +The directories `images_detectron2` and `annotations_detectron2` are generated by running +``` +python datasets/prepare_voc_sem_seg.py +``` + + + + + +## Expected dataset structure for [D3](https://github.com/shikras/d-cube#download): +``` +$DETECTRON2_DATASETS/ + D3/ + d3_images/ + d3_json/ + d3_pkl/ +``` + + diff --git a/approach/ovod/APE/datasets/prepare_ade20k_full_sem_seg.py b/approach/ovod/APE/datasets/prepare_ade20k_full_sem_seg.py new file mode 100644 index 0000000000000000000000000000000000000000..e9ec336e1ec933f4a6f2fe2b82df40caf1fa647d --- /dev/null +++ b/approach/ovod/APE/datasets/prepare_ade20k_full_sem_seg.py @@ -0,0 +1,1007 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. +import os +import pickle as pkl +from pathlib import Path + +import cv2 +import numpy as np +import tqdm +from PIL import Image + +ADE20K_SEM_SEG_FULL_CATEGORIES = [ + {"name": "wall", "id": 2978, "trainId": 0}, + {"name": "building, edifice", "id": 312, "trainId": 1}, + {"name": "sky", "id": 2420, "trainId": 2}, + {"name": "tree", "id": 2855, "trainId": 3}, + {"name": "road, route", "id": 2131, "trainId": 4}, + {"name": "floor, flooring", "id": 976, "trainId": 5}, + {"name": "ceiling", "id": 447, "trainId": 6}, + {"name": "bed", "id": 165, "trainId": 7}, + {"name": "sidewalk, pavement", "id": 2377, "trainId": 8}, + {"name": "earth, ground", "id": 838, "trainId": 9}, + {"name": "cabinet", "id": 350, "trainId": 10}, + {"name": "person, individual, someone, somebody, mortal, soul", "id": 1831, "trainId": 11}, + {"name": "grass", "id": 1125, "trainId": 12}, + {"name": "windowpane, window", "id": 3055, "trainId": 13}, + {"name": "car, auto, automobile, machine, motorcar", "id": 401, "trainId": 14}, + {"name": "mountain, mount", "id": 1610, "trainId": 15}, + {"name": "plant, flora, plant life", "id": 1910, "trainId": 16}, + {"name": "table", "id": 2684, "trainId": 17}, + {"name": "chair", "id": 471, "trainId": 18}, + {"name": "curtain, drape, drapery, mantle, pall", "id": 687, "trainId": 19}, + {"name": "door", "id": 774, "trainId": 20}, + {"name": "sofa, couch, lounge", "id": 2473, "trainId": 21}, + {"name": "sea", "id": 2264, "trainId": 22}, + {"name": "painting, picture", "id": 1735, "trainId": 23}, + {"name": "water", "id": 2994, "trainId": 24}, + {"name": "mirror", "id": 1564, "trainId": 25}, + {"name": "house", "id": 1276, "trainId": 26}, + {"name": "rug, carpet, carpeting", "id": 2178, "trainId": 27}, + {"name": "shelf", "id": 2329, "trainId": 28}, + {"name": "armchair", "id": 57, "trainId": 29}, + {"name": "fence, fencing", "id": 907, "trainId": 30}, + {"name": "field", "id": 913, "trainId": 31}, + {"name": "lamp", "id": 1395, "trainId": 32}, + {"name": "rock, stone", "id": 2138, "trainId": 33}, + {"name": "seat", "id": 2272, "trainId": 34}, + {"name": "river", "id": 2128, "trainId": 35}, + {"name": "desk", "id": 724, "trainId": 36}, + {"name": "bathtub, bathing tub, bath, tub", "id": 155, "trainId": 37}, + {"name": "railing, rail", "id": 2053, "trainId": 38}, + {"name": "signboard, sign", "id": 2380, "trainId": 39}, + {"name": "cushion", "id": 689, "trainId": 40}, + {"name": "path", "id": 1788, "trainId": 41}, + {"name": "work surface", "id": 3087, "trainId": 42}, + {"name": "stairs, steps", "id": 2530, "trainId": 43}, + {"name": "column, pillar", "id": 581, "trainId": 44}, + {"name": "sink", "id": 2388, "trainId": 45}, + {"name": "wardrobe, closet, press", "id": 2985, "trainId": 46}, + {"name": "snow", "id": 2454, "trainId": 47}, + {"name": "refrigerator, icebox", "id": 2096, "trainId": 48}, + {"name": "base, pedestal, stand", "id": 137, "trainId": 49}, + {"name": "bridge, span", "id": 294, "trainId": 50}, + {"name": "blind, screen", "id": 212, "trainId": 51}, + {"name": "runway", "id": 2185, "trainId": 52}, + {"name": "cliff, drop, drop-off", "id": 524, "trainId": 53}, + {"name": "sand", "id": 2212, "trainId": 54}, + {"name": "fireplace, hearth, open fireplace", "id": 943, "trainId": 55}, + {"name": "pillow", "id": 1869, "trainId": 56}, + {"name": "screen door, screen", "id": 2251, "trainId": 57}, + {"name": "toilet, can, commode, crapper, pot, potty, stool, throne", "id": 2793, "trainId": 58}, + {"name": "skyscraper", "id": 2423, "trainId": 59}, + {"name": "grandstand, covered stand", "id": 1121, "trainId": 60}, + {"name": "box", "id": 266, "trainId": 61}, + {"name": "pool table, billiard table, snooker table", "id": 1948, "trainId": 62}, + {"name": "palm, palm tree", "id": 1744, "trainId": 63}, + {"name": "double door", "id": 783, "trainId": 64}, + {"name": "coffee table, cocktail table", "id": 571, "trainId": 65}, + {"name": "counter", "id": 627, "trainId": 66}, + {"name": "countertop", "id": 629, "trainId": 67}, + {"name": "chest of drawers, chest, bureau, dresser", "id": 491, "trainId": 68}, + {"name": "kitchen island", "id": 1374, "trainId": 69}, + {"name": "boat", "id": 223, "trainId": 70}, + {"name": "waterfall, falls", "id": 3016, "trainId": 71}, + { + "name": "stove, kitchen stove, range, kitchen range, cooking stove", + "id": 2598, + "trainId": 72, + }, + {"name": "flower", "id": 978, "trainId": 73}, + {"name": "bookcase", "id": 239, "trainId": 74}, + {"name": "controls", "id": 608, "trainId": 75}, + {"name": "book", "id": 236, "trainId": 76}, + {"name": "stairway, staircase", "id": 2531, "trainId": 77}, + {"name": "streetlight, street lamp", "id": 2616, "trainId": 78}, + { + "name": "computer, computing machine, computing device, data processor, electronic computer, information processing system", + "id": 591, + "trainId": 79, + }, + { + "name": "bus, autobus, coach, charabanc, double-decker, jitney, motorbus, motorcoach, omnibus, passenger vehicle", + "id": 327, + "trainId": 80, + }, + {"name": "swivel chair", "id": 2679, "trainId": 81}, + {"name": "light, light source", "id": 1451, "trainId": 82}, + {"name": "bench", "id": 181, "trainId": 83}, + {"name": "case, display case, showcase, vitrine", "id": 420, "trainId": 84}, + {"name": "towel", "id": 2821, "trainId": 85}, + {"name": "fountain", "id": 1023, "trainId": 86}, + {"name": "embankment", "id": 855, "trainId": 87}, + { + "name": "television receiver, television, television set, tv, tv set, idiot box, boob tube, telly, goggle box", + "id": 2733, + "trainId": 88, + }, + {"name": "van", "id": 2928, "trainId": 89}, + {"name": "hill", "id": 1240, "trainId": 90}, + {"name": "awning, sunshade, sunblind", "id": 77, "trainId": 91}, + {"name": "poster, posting, placard, notice, bill, card", "id": 1969, "trainId": 92}, + {"name": "truck, motortruck", "id": 2880, "trainId": 93}, + {"name": "airplane, aeroplane, plane", "id": 14, "trainId": 94}, + {"name": "pole", "id": 1936, "trainId": 95}, + {"name": "tower", "id": 2828, "trainId": 96}, + {"name": "court", "id": 631, "trainId": 97}, + {"name": "ball", "id": 103, "trainId": 98}, + { + "name": "aircraft carrier, carrier, flattop, attack aircraft carrier", + "id": 3144, + "trainId": 99, + }, + {"name": "buffet, counter, sideboard", "id": 308, "trainId": 100}, + {"name": "hovel, hut, hutch, shack, shanty", "id": 1282, "trainId": 101}, + {"name": "apparel, wearing apparel, dress, clothes", "id": 38, "trainId": 102}, + {"name": "minibike, motorbike", "id": 1563, "trainId": 103}, + {"name": "animal, animate being, beast, brute, creature, fauna", "id": 29, "trainId": 104}, + {"name": "chandelier, pendant, pendent", "id": 480, "trainId": 105}, + {"name": "step, stair", "id": 2569, "trainId": 106}, + {"name": "booth, cubicle, stall, kiosk", "id": 247, "trainId": 107}, + {"name": "bicycle, bike, wheel, cycle", "id": 187, "trainId": 108}, + {"name": "doorframe, doorcase", "id": 778, "trainId": 109}, + {"name": "sconce", "id": 2243, "trainId": 110}, + {"name": "pond", "id": 1941, "trainId": 111}, + {"name": "trade name, brand name, brand, marque", "id": 2833, "trainId": 112}, + {"name": "bannister, banister, balustrade, balusters, handrail", "id": 120, "trainId": 113}, + {"name": "bag", "id": 95, "trainId": 114}, + {"name": "traffic light, traffic signal, stoplight", "id": 2836, "trainId": 115}, + {"name": "gazebo", "id": 1087, "trainId": 116}, + {"name": "escalator, moving staircase, moving stairway", "id": 868, "trainId": 117}, + {"name": "land, ground, soil", "id": 1401, "trainId": 118}, + {"name": "board, plank", "id": 220, "trainId": 119}, + {"name": "arcade machine", "id": 47, "trainId": 120}, + {"name": "eiderdown, duvet, continental quilt", "id": 843, "trainId": 121}, + {"name": "bar", "id": 123, "trainId": 122}, + {"name": "stall, stand, sales booth", "id": 2537, "trainId": 123}, + {"name": "playground", "id": 1927, "trainId": 124}, + {"name": "ship", "id": 2337, "trainId": 125}, + {"name": "ottoman, pouf, pouffe, puff, hassock", "id": 1702, "trainId": 126}, + { + "name": "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "id": 64, + "trainId": 127, + }, + {"name": "bottle", "id": 249, "trainId": 128}, + {"name": "cradle", "id": 642, "trainId": 129}, + {"name": "pot, flowerpot", "id": 1981, "trainId": 130}, + { + "name": "conveyer belt, conveyor belt, conveyer, conveyor, transporter", + "id": 609, + "trainId": 131, + }, + {"name": "train, railroad train", "id": 2840, "trainId": 132}, + {"name": "stool", "id": 2586, "trainId": 133}, + {"name": "lake", "id": 1393, "trainId": 134}, + {"name": "tank, storage tank", "id": 2704, "trainId": 135}, + {"name": "ice, water ice", "id": 1304, "trainId": 136}, + {"name": "basket, handbasket", "id": 146, "trainId": 137}, + {"name": "manhole", "id": 1494, "trainId": 138}, + {"name": "tent, collapsible shelter", "id": 2739, "trainId": 139}, + {"name": "canopy", "id": 389, "trainId": 140}, + {"name": "microwave, microwave oven", "id": 1551, "trainId": 141}, + {"name": "barrel, cask", "id": 131, "trainId": 142}, + {"name": "dirt track", "id": 738, "trainId": 143}, + {"name": "beam", "id": 161, "trainId": 144}, + {"name": "dishwasher, dish washer, dishwashing machine", "id": 747, "trainId": 145}, + {"name": "plate", "id": 1919, "trainId": 146}, + {"name": "screen, crt screen", "id": 3109, "trainId": 147}, + {"name": "ruins", "id": 2179, "trainId": 148}, + {"name": "washer, automatic washer, washing machine", "id": 2989, "trainId": 149}, + {"name": "blanket, cover", "id": 206, "trainId": 150}, + {"name": "plaything, toy", "id": 1930, "trainId": 151}, + {"name": "food, solid food", "id": 1002, "trainId": 152}, + {"name": "screen, silver screen, projection screen", "id": 2254, "trainId": 153}, + {"name": "oven", "id": 1708, "trainId": 154}, + {"name": "stage", "id": 2526, "trainId": 155}, + {"name": "beacon, lighthouse, beacon light, pharos", "id": 160, "trainId": 156}, + {"name": "umbrella", "id": 2901, "trainId": 157}, + {"name": "sculpture", "id": 2262, "trainId": 158}, + {"name": "aqueduct", "id": 44, "trainId": 159}, + {"name": "container", "id": 597, "trainId": 160}, + {"name": "scaffolding, staging", "id": 2235, "trainId": 161}, + {"name": "hood, exhaust hood", "id": 1260, "trainId": 162}, + {"name": "curb, curbing, kerb", "id": 682, "trainId": 163}, + {"name": "roller coaster", "id": 2151, "trainId": 164}, + {"name": "horse, equus caballus", "id": 3107, "trainId": 165}, + {"name": "catwalk", "id": 432, "trainId": 166}, + {"name": "glass, drinking glass", "id": 1098, "trainId": 167}, + {"name": "vase", "id": 2932, "trainId": 168}, + {"name": "central reservation", "id": 461, "trainId": 169}, + {"name": "carousel", "id": 410, "trainId": 170}, + {"name": "radiator", "id": 2046, "trainId": 171}, + {"name": "closet", "id": 533, "trainId": 172}, + {"name": "machine", "id": 1481, "trainId": 173}, + {"name": "pier, wharf, wharfage, dock", "id": 1858, "trainId": 174}, + {"name": "fan", "id": 894, "trainId": 175}, + {"name": "inflatable bounce game", "id": 1322, "trainId": 176}, + {"name": "pitch", "id": 1891, "trainId": 177}, + {"name": "paper", "id": 1756, "trainId": 178}, + {"name": "arcade, colonnade", "id": 49, "trainId": 179}, + {"name": "hot tub", "id": 1272, "trainId": 180}, + {"name": "helicopter", "id": 1229, "trainId": 181}, + {"name": "tray", "id": 2850, "trainId": 182}, + {"name": "partition, divider", "id": 1784, "trainId": 183}, + {"name": "vineyard", "id": 2962, "trainId": 184}, + {"name": "bowl", "id": 259, "trainId": 185}, + {"name": "bullring", "id": 319, "trainId": 186}, + {"name": "flag", "id": 954, "trainId": 187}, + {"name": "pot", "id": 1974, "trainId": 188}, + {"name": "footbridge, overcrossing, pedestrian bridge", "id": 1013, "trainId": 189}, + {"name": "shower", "id": 2356, "trainId": 190}, + {"name": "bag, traveling bag, travelling bag, grip, suitcase", "id": 97, "trainId": 191}, + {"name": "bulletin board, notice board", "id": 318, "trainId": 192}, + {"name": "confessional booth", "id": 592, "trainId": 193}, + {"name": "trunk, tree trunk, bole", "id": 2885, "trainId": 194}, + {"name": "forest", "id": 1017, "trainId": 195}, + {"name": "elevator door", "id": 851, "trainId": 196}, + {"name": "laptop, laptop computer", "id": 1407, "trainId": 197}, + {"name": "instrument panel", "id": 1332, "trainId": 198}, + {"name": "bucket, pail", "id": 303, "trainId": 199}, + {"name": "tapestry, tapis", "id": 2714, "trainId": 200}, + {"name": "platform", "id": 1924, "trainId": 201}, + {"name": "jacket", "id": 1346, "trainId": 202}, + {"name": "gate", "id": 1081, "trainId": 203}, + {"name": "monitor, monitoring device", "id": 1583, "trainId": 204}, + { + "name": "telephone booth, phone booth, call box, telephone box, telephone kiosk", + "id": 2727, + "trainId": 205, + }, + {"name": "spotlight, spot", "id": 2509, "trainId": 206}, + {"name": "ring", "id": 2123, "trainId": 207}, + {"name": "control panel", "id": 602, "trainId": 208}, + {"name": "blackboard, chalkboard", "id": 202, "trainId": 209}, + {"name": "air conditioner, air conditioning", "id": 10, "trainId": 210}, + {"name": "chest", "id": 490, "trainId": 211}, + {"name": "clock", "id": 530, "trainId": 212}, + {"name": "sand dune", "id": 2213, "trainId": 213}, + {"name": "pipe, pipage, piping", "id": 1884, "trainId": 214}, + {"name": "vault", "id": 2934, "trainId": 215}, + {"name": "table football", "id": 2687, "trainId": 216}, + {"name": "cannon", "id": 387, "trainId": 217}, + {"name": "swimming pool, swimming bath, natatorium", "id": 2668, "trainId": 218}, + {"name": "fluorescent, fluorescent fixture", "id": 982, "trainId": 219}, + {"name": "statue", "id": 2547, "trainId": 220}, + { + "name": "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "id": 1474, + "trainId": 221, + }, + {"name": "exhibitor", "id": 877, "trainId": 222}, + {"name": "ladder", "id": 1391, "trainId": 223}, + {"name": "carport", "id": 414, "trainId": 224}, + {"name": "dam", "id": 698, "trainId": 225}, + {"name": "pulpit", "id": 2019, "trainId": 226}, + {"name": "skylight, fanlight", "id": 2422, "trainId": 227}, + {"name": "water tower", "id": 3010, "trainId": 228}, + {"name": "grill, grille, grillwork", "id": 1139, "trainId": 229}, + {"name": "display board", "id": 753, "trainId": 230}, + {"name": "pane, pane of glass, window glass", "id": 1747, "trainId": 231}, + {"name": "rubbish, trash, scrap", "id": 2175, "trainId": 232}, + {"name": "ice rink", "id": 1301, "trainId": 233}, + {"name": "fruit", "id": 1033, "trainId": 234}, + {"name": "patio", "id": 1789, "trainId": 235}, + {"name": "vending machine", "id": 2939, "trainId": 236}, + {"name": "telephone, phone, telephone set", "id": 2730, "trainId": 237}, + {"name": "net", "id": 1652, "trainId": 238}, + { + "name": "backpack, back pack, knapsack, packsack, rucksack, haversack", + "id": 90, + "trainId": 239, + }, + {"name": "jar", "id": 1349, "trainId": 240}, + {"name": "track", "id": 2830, "trainId": 241}, + {"name": "magazine", "id": 1485, "trainId": 242}, + {"name": "shutter", "id": 2370, "trainId": 243}, + {"name": "roof", "id": 2155, "trainId": 244}, + {"name": "banner, streamer", "id": 118, "trainId": 245}, + {"name": "landfill", "id": 1402, "trainId": 246}, + {"name": "post", "id": 1957, "trainId": 247}, + {"name": "altarpiece, reredos", "id": 3130, "trainId": 248}, + {"name": "hat, chapeau, lid", "id": 1197, "trainId": 249}, + {"name": "arch, archway", "id": 52, "trainId": 250}, + {"name": "table game", "id": 2688, "trainId": 251}, + {"name": "bag, handbag, pocketbook, purse", "id": 96, "trainId": 252}, + {"name": "document, written document, papers", "id": 762, "trainId": 253}, + {"name": "dome", "id": 772, "trainId": 254}, + {"name": "pier", "id": 1857, "trainId": 255}, + {"name": "shanties", "id": 2315, "trainId": 256}, + {"name": "forecourt", "id": 1016, "trainId": 257}, + {"name": "crane", "id": 643, "trainId": 258}, + {"name": "dog, domestic dog, canis familiaris", "id": 3105, "trainId": 259}, + {"name": "piano, pianoforte, forte-piano", "id": 1849, "trainId": 260}, + {"name": "drawing", "id": 791, "trainId": 261}, + {"name": "cabin", "id": 349, "trainId": 262}, + { + "name": "ad, advertisement, advertizement, advertising, advertizing, advert", + "id": 6, + "trainId": 263, + }, + {"name": "amphitheater, amphitheatre, coliseum", "id": 3114, "trainId": 264}, + {"name": "monument", "id": 1587, "trainId": 265}, + {"name": "henhouse", "id": 1233, "trainId": 266}, + {"name": "cockpit", "id": 559, "trainId": 267}, + {"name": "heater, warmer", "id": 1223, "trainId": 268}, + {"name": "windmill, aerogenerator, wind generator", "id": 3049, "trainId": 269}, + {"name": "pool", "id": 1943, "trainId": 270}, + {"name": "elevator, lift", "id": 853, "trainId": 271}, + {"name": "decoration, ornament, ornamentation", "id": 709, "trainId": 272}, + {"name": "labyrinth", "id": 1390, "trainId": 273}, + {"name": "text, textual matter", "id": 2748, "trainId": 274}, + {"name": "printer", "id": 2007, "trainId": 275}, + {"name": "mezzanine, first balcony", "id": 1546, "trainId": 276}, + {"name": "mattress", "id": 1513, "trainId": 277}, + {"name": "straw", "id": 2600, "trainId": 278}, + {"name": "stalls", "id": 2538, "trainId": 279}, + {"name": "patio, terrace", "id": 1790, "trainId": 280}, + {"name": "billboard, hoarding", "id": 194, "trainId": 281}, + {"name": "bus stop", "id": 326, "trainId": 282}, + {"name": "trouser, pant", "id": 2877, "trainId": 283}, + {"name": "console table, console", "id": 594, "trainId": 284}, + {"name": "rack", "id": 2036, "trainId": 285}, + {"name": "notebook", "id": 1662, "trainId": 286}, + {"name": "shrine", "id": 2366, "trainId": 287}, + {"name": "pantry", "id": 1754, "trainId": 288}, + {"name": "cart", "id": 418, "trainId": 289}, + {"name": "steam shovel", "id": 2553, "trainId": 290}, + {"name": "porch", "id": 1951, "trainId": 291}, + {"name": "postbox, mailbox, letter box", "id": 1963, "trainId": 292}, + {"name": "figurine, statuette", "id": 918, "trainId": 293}, + {"name": "recycling bin", "id": 2086, "trainId": 294}, + {"name": "folding screen", "id": 997, "trainId": 295}, + {"name": "telescope", "id": 2731, "trainId": 296}, + {"name": "deck chair, beach chair", "id": 704, "trainId": 297}, + {"name": "kennel", "id": 1365, "trainId": 298}, + {"name": "coffee maker", "id": 569, "trainId": 299}, + {"name": "altar, communion table, lord's table", "id": 3108, "trainId": 300}, + {"name": "fish", "id": 948, "trainId": 301}, + {"name": "easel", "id": 839, "trainId": 302}, + {"name": "artificial golf green", "id": 63, "trainId": 303}, + {"name": "iceberg", "id": 1305, "trainId": 304}, + {"name": "candlestick, candle holder", "id": 378, "trainId": 305}, + {"name": "shower stall, shower bath", "id": 2362, "trainId": 306}, + {"name": "television stand", "id": 2734, "trainId": 307}, + { + "name": "wall socket, wall plug, electric outlet, electrical outlet, outlet, electric receptacle", + "id": 2982, + "trainId": 308, + }, + {"name": "skeleton", "id": 2398, "trainId": 309}, + {"name": "grand piano, grand", "id": 1119, "trainId": 310}, + {"name": "candy, confect", "id": 382, "trainId": 311}, + {"name": "grille door", "id": 1141, "trainId": 312}, + {"name": "pedestal, plinth, footstall", "id": 1805, "trainId": 313}, + {"name": "jersey, t-shirt, tee shirt", "id": 3102, "trainId": 314}, + {"name": "shoe", "id": 2341, "trainId": 315}, + {"name": "gravestone, headstone, tombstone", "id": 1131, "trainId": 316}, + {"name": "shanty", "id": 2316, "trainId": 317}, + {"name": "structure", "id": 2626, "trainId": 318}, + {"name": "rocking chair, rocker", "id": 3104, "trainId": 319}, + {"name": "bird", "id": 198, "trainId": 320}, + {"name": "place mat", "id": 1896, "trainId": 321}, + {"name": "tomb", "id": 2800, "trainId": 322}, + {"name": "big top", "id": 190, "trainId": 323}, + {"name": "gas pump, gasoline pump, petrol pump, island dispenser", "id": 3131, "trainId": 324}, + {"name": "lockers", "id": 1463, "trainId": 325}, + {"name": "cage", "id": 357, "trainId": 326}, + {"name": "finger", "id": 929, "trainId": 327}, + {"name": "bleachers", "id": 209, "trainId": 328}, + {"name": "ferris wheel", "id": 912, "trainId": 329}, + {"name": "hairdresser chair", "id": 1164, "trainId": 330}, + {"name": "mat", "id": 1509, "trainId": 331}, + {"name": "stands", "id": 2539, "trainId": 332}, + {"name": "aquarium, fish tank, marine museum", "id": 3116, "trainId": 333}, + {"name": "streetcar, tram, tramcar, trolley, trolley car", "id": 2615, "trainId": 334}, + {"name": "napkin, table napkin, serviette", "id": 1644, "trainId": 335}, + {"name": "dummy", "id": 818, "trainId": 336}, + {"name": "booklet, brochure, folder, leaflet, pamphlet", "id": 242, "trainId": 337}, + {"name": "sand trap", "id": 2217, "trainId": 338}, + {"name": "shop, store", "id": 2347, "trainId": 339}, + {"name": "table cloth", "id": 2686, "trainId": 340}, + {"name": "service station", "id": 2300, "trainId": 341}, + {"name": "coffin", "id": 572, "trainId": 342}, + {"name": "drawer", "id": 789, "trainId": 343}, + {"name": "cages", "id": 358, "trainId": 344}, + {"name": "slot machine, coin machine", "id": 2443, "trainId": 345}, + {"name": "balcony", "id": 101, "trainId": 346}, + {"name": "volleyball court", "id": 2969, "trainId": 347}, + {"name": "table tennis", "id": 2692, "trainId": 348}, + {"name": "control table", "id": 606, "trainId": 349}, + {"name": "shirt", "id": 2339, "trainId": 350}, + {"name": "merchandise, ware, product", "id": 1533, "trainId": 351}, + {"name": "railway", "id": 2060, "trainId": 352}, + {"name": "parterre", "id": 1782, "trainId": 353}, + {"name": "chimney", "id": 495, "trainId": 354}, + {"name": "can, tin, tin can", "id": 371, "trainId": 355}, + {"name": "tanks", "id": 2707, "trainId": 356}, + {"name": "fabric, cloth, material, textile", "id": 889, "trainId": 357}, + {"name": "alga, algae", "id": 3156, "trainId": 358}, + {"name": "system", "id": 2683, "trainId": 359}, + {"name": "map", "id": 1499, "trainId": 360}, + {"name": "greenhouse", "id": 1135, "trainId": 361}, + {"name": "mug", "id": 1619, "trainId": 362}, + {"name": "barbecue", "id": 125, "trainId": 363}, + {"name": "trailer", "id": 2838, "trainId": 364}, + {"name": "toilet tissue, toilet paper, bathroom tissue", "id": 2792, "trainId": 365}, + {"name": "organ", "id": 1695, "trainId": 366}, + {"name": "dishrag, dishcloth", "id": 746, "trainId": 367}, + {"name": "island", "id": 1343, "trainId": 368}, + {"name": "keyboard", "id": 1370, "trainId": 369}, + {"name": "trench", "id": 2858, "trainId": 370}, + {"name": "basket, basketball hoop, hoop", "id": 145, "trainId": 371}, + {"name": "steering wheel, wheel", "id": 2565, "trainId": 372}, + {"name": "pitcher, ewer", "id": 1892, "trainId": 373}, + {"name": "goal", "id": 1103, "trainId": 374}, + {"name": "bread, breadstuff, staff of life", "id": 286, "trainId": 375}, + {"name": "beds", "id": 170, "trainId": 376}, + {"name": "wood", "id": 3073, "trainId": 377}, + {"name": "file cabinet", "id": 922, "trainId": 378}, + {"name": "newspaper, paper", "id": 1655, "trainId": 379}, + {"name": "motorboat", "id": 1602, "trainId": 380}, + {"name": "rope", "id": 2160, "trainId": 381}, + {"name": "guitar", "id": 1151, "trainId": 382}, + {"name": "rubble", "id": 2176, "trainId": 383}, + {"name": "scarf", "id": 2239, "trainId": 384}, + {"name": "barrels", "id": 132, "trainId": 385}, + {"name": "cap", "id": 394, "trainId": 386}, + {"name": "leaves", "id": 1424, "trainId": 387}, + {"name": "control tower", "id": 607, "trainId": 388}, + {"name": "dashboard", "id": 700, "trainId": 389}, + {"name": "bandstand", "id": 116, "trainId": 390}, + {"name": "lectern", "id": 1425, "trainId": 391}, + {"name": "switch, electric switch, electrical switch", "id": 2676, "trainId": 392}, + {"name": "baseboard, mopboard, skirting board", "id": 141, "trainId": 393}, + {"name": "shower room", "id": 2360, "trainId": 394}, + {"name": "smoke", "id": 2449, "trainId": 395}, + {"name": "faucet, spigot", "id": 897, "trainId": 396}, + {"name": "bulldozer", "id": 317, "trainId": 397}, + {"name": "saucepan", "id": 2228, "trainId": 398}, + {"name": "shops", "id": 2351, "trainId": 399}, + {"name": "meter", "id": 1543, "trainId": 400}, + {"name": "crevasse", "id": 656, "trainId": 401}, + {"name": "gear", "id": 1088, "trainId": 402}, + {"name": "candelabrum, candelabra", "id": 373, "trainId": 403}, + {"name": "sofa bed", "id": 2472, "trainId": 404}, + {"name": "tunnel", "id": 2892, "trainId": 405}, + {"name": "pallet", "id": 1740, "trainId": 406}, + {"name": "wire, conducting wire", "id": 3067, "trainId": 407}, + {"name": "kettle, boiler", "id": 1367, "trainId": 408}, + {"name": "bidet", "id": 188, "trainId": 409}, + { + "name": "baby buggy, baby carriage, carriage, perambulator, pram, stroller, go-cart, pushchair, pusher", + "id": 79, + "trainId": 410, + }, + {"name": "music stand", "id": 1633, "trainId": 411}, + {"name": "pipe, tube", "id": 1885, "trainId": 412}, + {"name": "cup", "id": 677, "trainId": 413}, + {"name": "parking meter", "id": 1779, "trainId": 414}, + {"name": "ice hockey rink", "id": 1297, "trainId": 415}, + {"name": "shelter", "id": 2334, "trainId": 416}, + {"name": "weeds", "id": 3027, "trainId": 417}, + {"name": "temple", "id": 2735, "trainId": 418}, + {"name": "patty, cake", "id": 1791, "trainId": 419}, + {"name": "ski slope", "id": 2405, "trainId": 420}, + {"name": "panel", "id": 1748, "trainId": 421}, + {"name": "wallet", "id": 2983, "trainId": 422}, + {"name": "wheel", "id": 3035, "trainId": 423}, + {"name": "towel rack, towel horse", "id": 2824, "trainId": 424}, + {"name": "roundabout", "id": 2168, "trainId": 425}, + {"name": "canister, cannister, tin", "id": 385, "trainId": 426}, + {"name": "rod", "id": 2148, "trainId": 427}, + {"name": "soap dispenser", "id": 2465, "trainId": 428}, + {"name": "bell", "id": 175, "trainId": 429}, + {"name": "canvas", "id": 390, "trainId": 430}, + {"name": "box office, ticket office, ticket booth", "id": 268, "trainId": 431}, + {"name": "teacup", "id": 2722, "trainId": 432}, + {"name": "trellis", "id": 2857, "trainId": 433}, + {"name": "workbench", "id": 3088, "trainId": 434}, + {"name": "valley, vale", "id": 2926, "trainId": 435}, + {"name": "toaster", "id": 2782, "trainId": 436}, + {"name": "knife", "id": 1378, "trainId": 437}, + {"name": "podium", "id": 1934, "trainId": 438}, + {"name": "ramp", "id": 2072, "trainId": 439}, + {"name": "tumble dryer", "id": 2889, "trainId": 440}, + {"name": "fireplug, fire hydrant, plug", "id": 944, "trainId": 441}, + {"name": "gym shoe, sneaker, tennis shoe", "id": 1158, "trainId": 442}, + {"name": "lab bench", "id": 1383, "trainId": 443}, + {"name": "equipment", "id": 867, "trainId": 444}, + {"name": "rocky formation", "id": 2145, "trainId": 445}, + {"name": "plastic", "id": 1915, "trainId": 446}, + {"name": "calendar", "id": 361, "trainId": 447}, + {"name": "caravan", "id": 402, "trainId": 448}, + {"name": "check-in-desk", "id": 482, "trainId": 449}, + {"name": "ticket counter", "id": 2761, "trainId": 450}, + {"name": "brush", "id": 300, "trainId": 451}, + {"name": "mill", "id": 1554, "trainId": 452}, + {"name": "covered bridge", "id": 636, "trainId": 453}, + {"name": "bowling alley", "id": 260, "trainId": 454}, + {"name": "hanger", "id": 1186, "trainId": 455}, + {"name": "excavator", "id": 871, "trainId": 456}, + {"name": "trestle", "id": 2859, "trainId": 457}, + {"name": "revolving door", "id": 2103, "trainId": 458}, + {"name": "blast furnace", "id": 208, "trainId": 459}, + {"name": "scale, weighing machine", "id": 2236, "trainId": 460}, + {"name": "projector", "id": 2012, "trainId": 461}, + {"name": "soap", "id": 2462, "trainId": 462}, + {"name": "locker", "id": 1462, "trainId": 463}, + {"name": "tractor", "id": 2832, "trainId": 464}, + {"name": "stretcher", "id": 2617, "trainId": 465}, + {"name": "frame", "id": 1024, "trainId": 466}, + {"name": "grating", "id": 1129, "trainId": 467}, + {"name": "alembic", "id": 18, "trainId": 468}, + {"name": "candle, taper, wax light", "id": 376, "trainId": 469}, + {"name": "barrier", "id": 134, "trainId": 470}, + {"name": "cardboard", "id": 407, "trainId": 471}, + {"name": "cave", "id": 434, "trainId": 472}, + {"name": "puddle", "id": 2017, "trainId": 473}, + {"name": "tarp", "id": 2717, "trainId": 474}, + {"name": "price tag", "id": 2005, "trainId": 475}, + {"name": "watchtower", "id": 2993, "trainId": 476}, + {"name": "meters", "id": 1545, "trainId": 477}, + { + "name": "light bulb, lightbulb, bulb, incandescent lamp, electric light, electric-light bulb", + "id": 1445, + "trainId": 478, + }, + {"name": "tracks", "id": 2831, "trainId": 479}, + {"name": "hair dryer", "id": 1161, "trainId": 480}, + {"name": "skirt", "id": 2411, "trainId": 481}, + {"name": "viaduct", "id": 2949, "trainId": 482}, + {"name": "paper towel", "id": 1769, "trainId": 483}, + {"name": "coat", "id": 552, "trainId": 484}, + {"name": "sheet", "id": 2327, "trainId": 485}, + {"name": "fire extinguisher, extinguisher, asphyxiator", "id": 939, "trainId": 486}, + {"name": "water wheel", "id": 3013, "trainId": 487}, + {"name": "pottery, clayware", "id": 1986, "trainId": 488}, + {"name": "magazine rack", "id": 1486, "trainId": 489}, + {"name": "teapot", "id": 2723, "trainId": 490}, + {"name": "microphone, mike", "id": 1549, "trainId": 491}, + {"name": "support", "id": 2649, "trainId": 492}, + {"name": "forklift", "id": 1020, "trainId": 493}, + {"name": "canyon", "id": 392, "trainId": 494}, + {"name": "cash register, register", "id": 422, "trainId": 495}, + {"name": "leaf, leafage, foliage", "id": 1419, "trainId": 496}, + {"name": "remote control, remote", "id": 2099, "trainId": 497}, + {"name": "soap dish", "id": 2464, "trainId": 498}, + {"name": "windshield, windscreen", "id": 3058, "trainId": 499}, + {"name": "cat", "id": 430, "trainId": 500}, + {"name": "cue, cue stick, pool cue, pool stick", "id": 675, "trainId": 501}, + {"name": "vent, venthole, vent-hole, blowhole", "id": 2941, "trainId": 502}, + {"name": "videos", "id": 2955, "trainId": 503}, + {"name": "shovel", "id": 2355, "trainId": 504}, + {"name": "eaves", "id": 840, "trainId": 505}, + {"name": "antenna, aerial, transmitting aerial", "id": 32, "trainId": 506}, + {"name": "shipyard", "id": 2338, "trainId": 507}, + {"name": "hen, biddy", "id": 1232, "trainId": 508}, + {"name": "traffic cone", "id": 2834, "trainId": 509}, + {"name": "washing machines", "id": 2991, "trainId": 510}, + {"name": "truck crane", "id": 2879, "trainId": 511}, + {"name": "cds", "id": 444, "trainId": 512}, + {"name": "niche", "id": 1657, "trainId": 513}, + {"name": "scoreboard", "id": 2246, "trainId": 514}, + {"name": "briefcase", "id": 296, "trainId": 515}, + {"name": "boot", "id": 245, "trainId": 516}, + {"name": "sweater, jumper", "id": 2661, "trainId": 517}, + {"name": "hay", "id": 1202, "trainId": 518}, + {"name": "pack", "id": 1714, "trainId": 519}, + {"name": "bottle rack", "id": 251, "trainId": 520}, + {"name": "glacier", "id": 1095, "trainId": 521}, + {"name": "pergola", "id": 1828, "trainId": 522}, + {"name": "building materials", "id": 311, "trainId": 523}, + {"name": "television camera", "id": 2732, "trainId": 524}, + {"name": "first floor", "id": 947, "trainId": 525}, + {"name": "rifle", "id": 2115, "trainId": 526}, + {"name": "tennis table", "id": 2738, "trainId": 527}, + {"name": "stadium", "id": 2525, "trainId": 528}, + {"name": "safety belt", "id": 2194, "trainId": 529}, + {"name": "cover", "id": 634, "trainId": 530}, + {"name": "dish rack", "id": 740, "trainId": 531}, + {"name": "synthesizer", "id": 2682, "trainId": 532}, + {"name": "pumpkin", "id": 2020, "trainId": 533}, + {"name": "gutter", "id": 1156, "trainId": 534}, + {"name": "fruit stand", "id": 1036, "trainId": 535}, + {"name": "ice floe, floe", "id": 1295, "trainId": 536}, + {"name": "handle, grip, handgrip, hold", "id": 1181, "trainId": 537}, + {"name": "wheelchair", "id": 3037, "trainId": 538}, + {"name": "mousepad, mouse mat", "id": 1614, "trainId": 539}, + {"name": "diploma", "id": 736, "trainId": 540}, + {"name": "fairground ride", "id": 893, "trainId": 541}, + {"name": "radio", "id": 2047, "trainId": 542}, + {"name": "hotplate", "id": 1274, "trainId": 543}, + {"name": "junk", "id": 1361, "trainId": 544}, + {"name": "wheelbarrow", "id": 3036, "trainId": 545}, + {"name": "stream", "id": 2606, "trainId": 546}, + {"name": "toll plaza", "id": 2797, "trainId": 547}, + {"name": "punching bag", "id": 2022, "trainId": 548}, + {"name": "trough", "id": 2876, "trainId": 549}, + {"name": "throne", "id": 2758, "trainId": 550}, + {"name": "chair desk", "id": 472, "trainId": 551}, + {"name": "weighbridge", "id": 3028, "trainId": 552}, + {"name": "extractor fan", "id": 882, "trainId": 553}, + {"name": "hanging clothes", "id": 1189, "trainId": 554}, + {"name": "dish, dish aerial, dish antenna, saucer", "id": 743, "trainId": 555}, + {"name": "alarm clock, alarm", "id": 3122, "trainId": 556}, + {"name": "ski lift", "id": 2401, "trainId": 557}, + {"name": "chain", "id": 468, "trainId": 558}, + {"name": "garage", "id": 1061, "trainId": 559}, + {"name": "mechanical shovel", "id": 1523, "trainId": 560}, + {"name": "wine rack", "id": 3059, "trainId": 561}, + {"name": "tramway", "id": 2843, "trainId": 562}, + {"name": "treadmill", "id": 2853, "trainId": 563}, + {"name": "menu", "id": 1529, "trainId": 564}, + {"name": "block", "id": 214, "trainId": 565}, + {"name": "well", "id": 3032, "trainId": 566}, + {"name": "witness stand", "id": 3071, "trainId": 567}, + {"name": "branch", "id": 277, "trainId": 568}, + {"name": "duck", "id": 813, "trainId": 569}, + {"name": "casserole", "id": 426, "trainId": 570}, + {"name": "frying pan", "id": 1039, "trainId": 571}, + {"name": "desk organizer", "id": 727, "trainId": 572}, + {"name": "mast", "id": 1508, "trainId": 573}, + {"name": "spectacles, specs, eyeglasses, glasses", "id": 2490, "trainId": 574}, + {"name": "service elevator", "id": 2299, "trainId": 575}, + {"name": "dollhouse", "id": 768, "trainId": 576}, + {"name": "hammock", "id": 1172, "trainId": 577}, + {"name": "clothes hanging", "id": 537, "trainId": 578}, + {"name": "photocopier", "id": 1847, "trainId": 579}, + {"name": "notepad", "id": 1664, "trainId": 580}, + {"name": "golf cart", "id": 1110, "trainId": 581}, + {"name": "footpath", "id": 1014, "trainId": 582}, + {"name": "cross", "id": 662, "trainId": 583}, + {"name": "baptismal font", "id": 121, "trainId": 584}, + {"name": "boiler", "id": 227, "trainId": 585}, + {"name": "skip", "id": 2410, "trainId": 586}, + {"name": "rotisserie", "id": 2165, "trainId": 587}, + {"name": "tables", "id": 2696, "trainId": 588}, + {"name": "water mill", "id": 3005, "trainId": 589}, + {"name": "helmet", "id": 1231, "trainId": 590}, + {"name": "cover curtain", "id": 635, "trainId": 591}, + {"name": "brick", "id": 292, "trainId": 592}, + {"name": "table runner", "id": 2690, "trainId": 593}, + {"name": "ashtray", "id": 65, "trainId": 594}, + {"name": "street box", "id": 2607, "trainId": 595}, + {"name": "stick", "id": 2574, "trainId": 596}, + {"name": "hangers", "id": 1188, "trainId": 597}, + {"name": "cells", "id": 456, "trainId": 598}, + {"name": "urinal", "id": 2913, "trainId": 599}, + {"name": "centerpiece", "id": 459, "trainId": 600}, + {"name": "portable fridge", "id": 1955, "trainId": 601}, + {"name": "dvds", "id": 827, "trainId": 602}, + {"name": "golf club", "id": 1111, "trainId": 603}, + {"name": "skirting board", "id": 2412, "trainId": 604}, + {"name": "water cooler", "id": 2997, "trainId": 605}, + {"name": "clipboard", "id": 528, "trainId": 606}, + {"name": "camera, photographic camera", "id": 366, "trainId": 607}, + {"name": "pigeonhole", "id": 1863, "trainId": 608}, + {"name": "chips", "id": 500, "trainId": 609}, + {"name": "food processor", "id": 1001, "trainId": 610}, + {"name": "post box", "id": 1958, "trainId": 611}, + {"name": "lid", "id": 1441, "trainId": 612}, + {"name": "drum", "id": 809, "trainId": 613}, + {"name": "blender", "id": 210, "trainId": 614}, + {"name": "cave entrance", "id": 435, "trainId": 615}, + {"name": "dental chair", "id": 718, "trainId": 616}, + {"name": "obelisk", "id": 1674, "trainId": 617}, + {"name": "canoe", "id": 388, "trainId": 618}, + {"name": "mobile", "id": 1572, "trainId": 619}, + {"name": "monitors", "id": 1584, "trainId": 620}, + {"name": "pool ball", "id": 1944, "trainId": 621}, + {"name": "cue rack", "id": 674, "trainId": 622}, + {"name": "baggage carts", "id": 99, "trainId": 623}, + {"name": "shore", "id": 2352, "trainId": 624}, + {"name": "fork", "id": 1019, "trainId": 625}, + {"name": "paper filer", "id": 1763, "trainId": 626}, + {"name": "bicycle rack", "id": 185, "trainId": 627}, + {"name": "coat rack", "id": 554, "trainId": 628}, + {"name": "garland", "id": 1066, "trainId": 629}, + {"name": "sports bag", "id": 2508, "trainId": 630}, + {"name": "fish tank", "id": 951, "trainId": 631}, + {"name": "towel dispenser", "id": 2822, "trainId": 632}, + {"name": "carriage", "id": 415, "trainId": 633}, + {"name": "brochure", "id": 297, "trainId": 634}, + {"name": "plaque", "id": 1914, "trainId": 635}, + {"name": "stringer", "id": 2619, "trainId": 636}, + {"name": "iron", "id": 1338, "trainId": 637}, + {"name": "spoon", "id": 2505, "trainId": 638}, + {"name": "flag pole", "id": 955, "trainId": 639}, + {"name": "toilet brush", "id": 2786, "trainId": 640}, + {"name": "book stand", "id": 238, "trainId": 641}, + {"name": "water faucet, water tap, tap, hydrant", "id": 3000, "trainId": 642}, + {"name": "ticket office", "id": 2763, "trainId": 643}, + {"name": "broom", "id": 299, "trainId": 644}, + {"name": "dvd", "id": 822, "trainId": 645}, + {"name": "ice bucket", "id": 1288, "trainId": 646}, + {"name": "carapace, shell, cuticle, shield", "id": 3101, "trainId": 647}, + {"name": "tureen", "id": 2894, "trainId": 648}, + {"name": "folders", "id": 992, "trainId": 649}, + {"name": "chess", "id": 489, "trainId": 650}, + {"name": "root", "id": 2157, "trainId": 651}, + {"name": "sewing machine", "id": 2309, "trainId": 652}, + {"name": "model", "id": 1576, "trainId": 653}, + {"name": "pen", "id": 1810, "trainId": 654}, + {"name": "violin", "id": 2964, "trainId": 655}, + {"name": "sweatshirt", "id": 2662, "trainId": 656}, + {"name": "recycling materials", "id": 2087, "trainId": 657}, + {"name": "mitten", "id": 1569, "trainId": 658}, + {"name": "chopping board, cutting board", "id": 503, "trainId": 659}, + {"name": "mask", "id": 1505, "trainId": 660}, + {"name": "log", "id": 1468, "trainId": 661}, + {"name": "mouse, computer mouse", "id": 1613, "trainId": 662}, + {"name": "grill", "id": 1138, "trainId": 663}, + {"name": "hole", "id": 1256, "trainId": 664}, + {"name": "target", "id": 2715, "trainId": 665}, + {"name": "trash bag", "id": 2846, "trainId": 666}, + {"name": "chalk", "id": 477, "trainId": 667}, + {"name": "sticks", "id": 2576, "trainId": 668}, + {"name": "balloon", "id": 108, "trainId": 669}, + {"name": "score", "id": 2245, "trainId": 670}, + {"name": "hair spray", "id": 1162, "trainId": 671}, + {"name": "roll", "id": 2149, "trainId": 672}, + {"name": "runner", "id": 2183, "trainId": 673}, + {"name": "engine", "id": 858, "trainId": 674}, + {"name": "inflatable glove", "id": 1324, "trainId": 675}, + {"name": "games", "id": 1055, "trainId": 676}, + {"name": "pallets", "id": 1741, "trainId": 677}, + {"name": "baskets", "id": 149, "trainId": 678}, + {"name": "coop", "id": 615, "trainId": 679}, + {"name": "dvd player", "id": 825, "trainId": 680}, + {"name": "rocking horse", "id": 2143, "trainId": 681}, + {"name": "buckets", "id": 304, "trainId": 682}, + {"name": "bread rolls", "id": 283, "trainId": 683}, + {"name": "shawl", "id": 2322, "trainId": 684}, + {"name": "watering can", "id": 3017, "trainId": 685}, + {"name": "spotlights", "id": 2510, "trainId": 686}, + {"name": "post-it", "id": 1960, "trainId": 687}, + {"name": "bowls", "id": 265, "trainId": 688}, + {"name": "security camera", "id": 2282, "trainId": 689}, + {"name": "runner cloth", "id": 2184, "trainId": 690}, + {"name": "lock", "id": 1461, "trainId": 691}, + {"name": "alarm, warning device, alarm system", "id": 3113, "trainId": 692}, + {"name": "side", "id": 2372, "trainId": 693}, + {"name": "roulette", "id": 2166, "trainId": 694}, + {"name": "bone", "id": 232, "trainId": 695}, + {"name": "cutlery", "id": 693, "trainId": 696}, + {"name": "pool balls", "id": 1945, "trainId": 697}, + {"name": "wheels", "id": 3039, "trainId": 698}, + {"name": "spice rack", "id": 2494, "trainId": 699}, + {"name": "plant pots", "id": 1908, "trainId": 700}, + {"name": "towel ring", "id": 2827, "trainId": 701}, + {"name": "bread box", "id": 280, "trainId": 702}, + {"name": "video", "id": 2950, "trainId": 703}, + {"name": "funfair", "id": 1044, "trainId": 704}, + {"name": "breads", "id": 288, "trainId": 705}, + {"name": "tripod", "id": 2863, "trainId": 706}, + {"name": "ironing board", "id": 1342, "trainId": 707}, + {"name": "skimmer", "id": 2409, "trainId": 708}, + {"name": "hollow", "id": 1258, "trainId": 709}, + {"name": "scratching post", "id": 2249, "trainId": 710}, + {"name": "tricycle", "id": 2862, "trainId": 711}, + {"name": "file box", "id": 920, "trainId": 712}, + {"name": "mountain pass", "id": 1607, "trainId": 713}, + {"name": "tombstones", "id": 2802, "trainId": 714}, + {"name": "cooker", "id": 610, "trainId": 715}, + {"name": "card game, cards", "id": 3129, "trainId": 716}, + {"name": "golf bag", "id": 1108, "trainId": 717}, + {"name": "towel paper", "id": 2823, "trainId": 718}, + {"name": "chaise lounge", "id": 476, "trainId": 719}, + {"name": "sun", "id": 2641, "trainId": 720}, + {"name": "toilet paper holder", "id": 2788, "trainId": 721}, + {"name": "rake", "id": 2070, "trainId": 722}, + {"name": "key", "id": 1368, "trainId": 723}, + {"name": "umbrella stand", "id": 2903, "trainId": 724}, + {"name": "dartboard", "id": 699, "trainId": 725}, + {"name": "transformer", "id": 2844, "trainId": 726}, + {"name": "fireplace utensils", "id": 942, "trainId": 727}, + {"name": "sweatshirts", "id": 2663, "trainId": 728}, + { + "name": "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "id": 457, + "trainId": 729, + }, + {"name": "tallboy", "id": 2701, "trainId": 730}, + {"name": "stapler", "id": 2540, "trainId": 731}, + {"name": "sauna", "id": 2231, "trainId": 732}, + {"name": "test tube", "id": 2746, "trainId": 733}, + {"name": "palette", "id": 1738, "trainId": 734}, + {"name": "shopping carts", "id": 2350, "trainId": 735}, + {"name": "tools", "id": 2808, "trainId": 736}, + {"name": "push button, push, button", "id": 2025, "trainId": 737}, + {"name": "star", "id": 2541, "trainId": 738}, + {"name": "roof rack", "id": 2156, "trainId": 739}, + {"name": "barbed wire", "id": 126, "trainId": 740}, + {"name": "spray", "id": 2512, "trainId": 741}, + {"name": "ear", "id": 831, "trainId": 742}, + {"name": "sponge", "id": 2503, "trainId": 743}, + {"name": "racket", "id": 2039, "trainId": 744}, + {"name": "tins", "id": 2774, "trainId": 745}, + {"name": "eyeglasses", "id": 886, "trainId": 746}, + {"name": "file", "id": 919, "trainId": 747}, + {"name": "scarfs", "id": 2240, "trainId": 748}, + {"name": "sugar bowl", "id": 2636, "trainId": 749}, + {"name": "flip flop", "id": 963, "trainId": 750}, + {"name": "headstones", "id": 1218, "trainId": 751}, + {"name": "laptop bag", "id": 1406, "trainId": 752}, + {"name": "leash", "id": 1420, "trainId": 753}, + {"name": "climbing frame", "id": 526, "trainId": 754}, + {"name": "suit hanger", "id": 2639, "trainId": 755}, + {"name": "floor spotlight", "id": 975, "trainId": 756}, + {"name": "plate rack", "id": 1921, "trainId": 757}, + {"name": "sewer", "id": 2305, "trainId": 758}, + {"name": "hard drive", "id": 1193, "trainId": 759}, + {"name": "sprinkler", "id": 2517, "trainId": 760}, + {"name": "tools box", "id": 2809, "trainId": 761}, + {"name": "necklace", "id": 1647, "trainId": 762}, + {"name": "bulbs", "id": 314, "trainId": 763}, + {"name": "steel industry", "id": 2560, "trainId": 764}, + {"name": "club", "id": 545, "trainId": 765}, + {"name": "jack", "id": 1345, "trainId": 766}, + {"name": "door bars", "id": 775, "trainId": 767}, + { + "name": "control panel, instrument panel, control board, board, panel", + "id": 603, + "trainId": 768, + }, + {"name": "hairbrush", "id": 1163, "trainId": 769}, + {"name": "napkin holder", "id": 1641, "trainId": 770}, + {"name": "office", "id": 1678, "trainId": 771}, + {"name": "smoke detector", "id": 2450, "trainId": 772}, + {"name": "utensils", "id": 2915, "trainId": 773}, + {"name": "apron", "id": 42, "trainId": 774}, + {"name": "scissors", "id": 2242, "trainId": 775}, + {"name": "terminal", "id": 2741, "trainId": 776}, + {"name": "grinder", "id": 1143, "trainId": 777}, + {"name": "entry phone", "id": 862, "trainId": 778}, + {"name": "newspaper stand", "id": 1654, "trainId": 779}, + {"name": "pepper shaker", "id": 1826, "trainId": 780}, + {"name": "onions", "id": 1689, "trainId": 781}, + { + "name": "central processing unit, cpu, c p u , central processor, processor, mainframe", + "id": 3124, + "trainId": 782, + }, + {"name": "tape", "id": 2710, "trainId": 783}, + {"name": "bat", "id": 152, "trainId": 784}, + {"name": "coaster", "id": 549, "trainId": 785}, + {"name": "calculator", "id": 360, "trainId": 786}, + {"name": "potatoes", "id": 1982, "trainId": 787}, + {"name": "luggage rack", "id": 1478, "trainId": 788}, + {"name": "salt", "id": 2203, "trainId": 789}, + {"name": "street number", "id": 2612, "trainId": 790}, + {"name": "viewpoint", "id": 2956, "trainId": 791}, + {"name": "sword", "id": 2681, "trainId": 792}, + {"name": "cd", "id": 437, "trainId": 793}, + {"name": "rowing machine", "id": 2171, "trainId": 794}, + {"name": "plug", "id": 1933, "trainId": 795}, + {"name": "andiron, firedog, dog, dog-iron", "id": 3110, "trainId": 796}, + {"name": "pepper", "id": 1824, "trainId": 797}, + {"name": "tongs", "id": 2803, "trainId": 798}, + {"name": "bonfire", "id": 234, "trainId": 799}, + {"name": "dog dish", "id": 764, "trainId": 800}, + {"name": "belt", "id": 177, "trainId": 801}, + {"name": "dumbbells", "id": 817, "trainId": 802}, + {"name": "videocassette recorder, vcr", "id": 3145, "trainId": 803}, + {"name": "hook", "id": 1262, "trainId": 804}, + {"name": "envelopes", "id": 864, "trainId": 805}, + {"name": "shower faucet", "id": 2359, "trainId": 806}, + {"name": "watch", "id": 2992, "trainId": 807}, + {"name": "padlock", "id": 1725, "trainId": 808}, + {"name": "swimming pool ladder", "id": 2667, "trainId": 809}, + {"name": "spanners", "id": 2484, "trainId": 810}, + {"name": "gravy boat", "id": 1133, "trainId": 811}, + {"name": "notice board", "id": 1667, "trainId": 812}, + {"name": "trash bags", "id": 2847, "trainId": 813}, + {"name": "fire alarm", "id": 932, "trainId": 814}, + {"name": "ladle", "id": 1392, "trainId": 815}, + {"name": "stethoscope", "id": 2573, "trainId": 816}, + {"name": "rocket", "id": 2140, "trainId": 817}, + {"name": "funnel", "id": 1046, "trainId": 818}, + {"name": "bowling pins", "id": 264, "trainId": 819}, + {"name": "valve", "id": 2927, "trainId": 820}, + {"name": "thermometer", "id": 2752, "trainId": 821}, + {"name": "cups", "id": 679, "trainId": 822}, + {"name": "spice jar", "id": 2493, "trainId": 823}, + {"name": "night light", "id": 1658, "trainId": 824}, + {"name": "soaps", "id": 2466, "trainId": 825}, + {"name": "games table", "id": 1057, "trainId": 826}, + {"name": "slotted spoon", "id": 2444, "trainId": 827}, + {"name": "reel", "id": 2093, "trainId": 828}, + {"name": "scourer", "id": 2248, "trainId": 829}, + {"name": "sleeping robe", "id": 2432, "trainId": 830}, + {"name": "desk mat", "id": 726, "trainId": 831}, + {"name": "dumbbell", "id": 816, "trainId": 832}, + {"name": "hammer", "id": 1171, "trainId": 833}, + {"name": "tie", "id": 2766, "trainId": 834}, + {"name": "typewriter", "id": 2900, "trainId": 835}, + {"name": "shaker", "id": 2313, "trainId": 836}, + {"name": "cheese dish", "id": 488, "trainId": 837}, + {"name": "sea star", "id": 2265, "trainId": 838}, + {"name": "racquet", "id": 2043, "trainId": 839}, + {"name": "butane gas cylinder", "id": 332, "trainId": 840}, + {"name": "paper weight", "id": 1771, "trainId": 841}, + {"name": "shaving brush", "id": 2320, "trainId": 842}, + {"name": "sunglasses", "id": 2646, "trainId": 843}, + {"name": "gear shift", "id": 1089, "trainId": 844}, + {"name": "towel rail", "id": 2826, "trainId": 845}, + {"name": "adding machine, totalizer, totaliser", "id": 3148, "trainId": 846}, +] + + +def loadAde20K(file): + fileseg = file.replace(".jpg", "_seg.png") + with Image.open(fileseg) as io: + seg = np.array(io) + + R = seg[:, :, 0] + G = seg[:, :, 1] + ObjectClassMasks = (R / 10).astype(np.int32) * 256 + (G.astype(np.int32)) + + return {"img_name": file, "segm_name": fileseg, "class_mask": ObjectClassMasks} + + +if __name__ == "__main__": + dataset_dir = Path(os.getenv("DETECTRON2_DATASETS", "datasets")) + index_file = dataset_dir / "ADE20K_2021_17_01" / "index_ade20k.pkl" + with open(index_file, "rb") as f: + index_ade20k = pkl.load(f) + + id_map = {} + for cat in ADE20K_SEM_SEG_FULL_CATEGORIES: + id_map[cat["id"]] = cat["trainId"] + + # make output dir + for name in ["training", "validation"]: + image_dir = dataset_dir / "ADE20K_2021_17_01" / "images_detectron2" / name + image_dir.mkdir(parents=True, exist_ok=True) + annotation_dir = dataset_dir / "ADE20K_2021_17_01" / "annotations_detectron2" / name + annotation_dir.mkdir(parents=True, exist_ok=True) + + # process image and gt + for i, (folder_name, file_name) in tqdm.tqdm( + enumerate(zip(index_ade20k["folder"], index_ade20k["filename"])), + total=len(index_ade20k["filename"]), + ): + split = "validation" if file_name.split("_")[1] == "val" else "training" + info = loadAde20K(str(dataset_dir / folder_name / file_name)) + + # resize image and label + img = np.asarray(Image.open(info["img_name"])) + lab = np.asarray(info["class_mask"]) + + h, w = img.shape[0], img.shape[1] + max_size = 512 + resize = True + if w >= h > max_size: + h_new, w_new = max_size, round(w / float(h) * max_size) + elif h >= w > max_size: + h_new, w_new = round(h / float(w) * max_size), max_size + else: + resize = False + + if resize: + img = cv2.resize(img, (w_new, h_new), interpolation=cv2.INTER_LINEAR) + lab = cv2.resize(lab, (w_new, h_new), interpolation=cv2.INTER_NEAREST) + + assert img.dtype == np.uint8 + assert lab.dtype == np.int32 + + # apply label conversion and save into uint16 images + output = np.zeros_like(lab, dtype=np.uint16) + 65535 + for obj_id in np.unique(lab): + if obj_id in id_map: + output[lab == obj_id] = id_map[obj_id] + + output_img = dataset_dir / "ADE20K_2021_17_01" / "images_detectron2" / split / file_name + output_lab = ( + dataset_dir + / "ADE20K_2021_17_01" + / "annotations_detectron2" + / split + / file_name.replace(".jpg", ".tif") + ) + Image.fromarray(img).save(output_img) + + assert output.dtype == np.uint16 + Image.fromarray(output).save(output_lab) diff --git a/approach/ovod/APE/datasets/prepare_coco_semantic_annos_from_panoptic_annos.py b/approach/ovod/APE/datasets/prepare_coco_semantic_annos_from_panoptic_annos.py new file mode 100644 index 0000000000000000000000000000000000000000..3090c9bc2f9a63156a4132e89c635613691eb350 --- /dev/null +++ b/approach/ovod/APE/datasets/prepare_coco_semantic_annos_from_panoptic_annos.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import functools +import json +import multiprocessing as mp +import numpy as np +import os +import time +from fvcore.common.download import download +from panopticapi.utils import rgb2id +from PIL import Image + +from detectron2.data.datasets.builtin_meta import COCO_CATEGORIES + + +def _process_panoptic_to_semantic(input_panoptic, output_semantic, segments, id_map): + panoptic = np.asarray(Image.open(input_panoptic), dtype=np.uint32) + panoptic = rgb2id(panoptic) + output = np.zeros_like(panoptic, dtype=np.uint8) + 255 + for seg in segments: + cat_id = seg["category_id"] + new_cat_id = id_map[cat_id] + output[panoptic == seg["id"]] = new_cat_id + Image.fromarray(output).save(output_semantic) + + +def separate_coco_semantic_from_panoptic(panoptic_json, panoptic_root, sem_seg_root, categories): + """ + Create semantic segmentation annotations from panoptic segmentation + annotations, to be used by PanopticFPN. + It maps all thing categories to class 0, and maps all unlabeled pixels to class 255. + It maps all stuff categories to contiguous ids starting from 1. + Args: + panoptic_json (str): path to the panoptic json file, in COCO's format. + panoptic_root (str): a directory with panoptic annotation files, in COCO's format. + sem_seg_root (str): a directory to output semantic annotation files + categories (list[dict]): category metadata. Each dict needs to have: + "id": corresponds to the "category_id" in the json annotations + "isthing": 0 or 1 + """ + os.makedirs(sem_seg_root, exist_ok=True) + + id_map = {} # map from category id to id in the output semantic annotation + assert len(categories) <= 254 + for i, k in enumerate(categories): + id_map[k["id"]] = i + # what is id = 0? + # id_map[0] = 255 + print(id_map) + + with open(panoptic_json) as f: + obj = json.load(f) + + pool = mp.Pool(processes=max(mp.cpu_count() // 2, 4)) + + def iter_annotations(): + for anno in obj["annotations"]: + file_name = anno["file_name"] + segments = anno["segments_info"] + input = os.path.join(panoptic_root, file_name) + output = os.path.join(sem_seg_root, file_name) + yield input, output, segments + + print("Start writing to {} ...".format(sem_seg_root)) + start = time.time() + pool.starmap( + functools.partial(_process_panoptic_to_semantic, id_map=id_map), + iter_annotations(), + chunksize=100, + ) + print("Finished. time: {:.2f}s".format(time.time() - start)) + + +if __name__ == "__main__": + dataset_dir = os.path.join(os.getenv("DETECTRON2_DATASETS", "datasets"), "coco") + for s in ["val2017", "train2017"]: + separate_coco_semantic_from_panoptic( + os.path.join(dataset_dir, "annotations/panoptic_{}.json".format(s)), + os.path.join(dataset_dir, "panoptic_{}".format(s)), + os.path.join(dataset_dir, "panoptic_semseg_{}".format(s)), + COCO_CATEGORIES, + ) diff --git a/approach/ovod/APE/datasets/prepare_pascal_context.py b/approach/ovod/APE/datasets/prepare_pascal_context.py new file mode 100644 index 0000000000000000000000000000000000000000..25d38469242affc188617cbd23eaaf33219bd317 --- /dev/null +++ b/approach/ovod/APE/datasets/prepare_pascal_context.py @@ -0,0 +1,69 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# Copyright (c) Meta Platforms, Inc. All Rights Reserved + +import tqdm +import os +import os.path as osp +from pathlib import Path + +import numpy as np +from PIL import Image +import scipy.io + +def convert_pc59(mask_path, new_mask_path, pc59_dict): + mat = scipy.io.loadmat(mask_path) + mask = mat['LabelMap'] + + mask_copy = np.ones_like(mask, dtype=np.uint8) * 255 + for trID, clsID in pc59_dict.items(): + mask_copy[mask == clsID] = trID + + min_value = np.amin(mask_copy) + assert min_value >= 0, print(min_value) + Image.fromarray(mask_copy).save(new_mask_path, "PNG") + +def convert_pc459(mask_path, new_mask_path): + mat = scipy.io.loadmat(mask_path) + mask = mat['LabelMap'] + mask = mask - 1 + min_value = np.amin(mask) + assert min_value >= 0, print(min_value) + Image.fromarray(mask).save(new_mask_path, "TIFF") + + +if __name__ == "__main__": + dataset_dir = Path(os.getenv("DETECTRON2_DATASETS", "datasets")) + print('Caution: we only generate the validation set!') + pc_path = dataset_dir / "VOCdevkit/VOC2010" + + val_list = open(pc_path / "pascalcontext_val.txt", "r") + pc459_labels = open(pc_path / "labels.txt", "r") + pc59_labels = open(pc_path / "59_labels.txt", "r") + + pc459_dict = {} + for line in pc459_labels.readlines(): + if ':' in line: + idx, name = line.split(':') + idx = int(idx.strip()) + name = name.strip() + pc459_dict[name] = idx + + pc59_dict = {} + for i, line in enumerate(pc59_labels.readlines()): + name = line.split(':')[-1].strip() + if name is not '': + pc59_dict[i] = pc459_dict[name] + + pc459_dir = pc_path / "annotations_detectron2" / "pc459_val" + pc459_dir.mkdir(parents=True, exist_ok=True) + pc59_dir = pc_path / "annotations_detectron2" / "pc59_val" + pc59_dir.mkdir(parents=True, exist_ok=True) + + for line in tqdm.tqdm(val_list.readlines()): + fileid = line.strip() + ori_mask = f'{pc_path}/trainval/{fileid}.mat' + pc459_dst = f'{pc459_dir}/{fileid}.tif' + pc59_dst = f'{pc59_dir}/{fileid}.png' + if osp.exists(ori_mask): + convert_pc459(ori_mask, pc459_dst) + convert_pc59(ori_mask, pc59_dst, pc59_dict) diff --git a/approach/ovod/APE/datasets/prepare_voc_sem_seg.py b/approach/ovod/APE/datasets/prepare_voc_sem_seg.py new file mode 100644 index 0000000000000000000000000000000000000000..1dbe80a5b8ae53627998214ec6a1f9a7fc30fad9 --- /dev/null +++ b/approach/ovod/APE/datasets/prepare_voc_sem_seg.py @@ -0,0 +1,71 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# Copyright (c) Meta Platforms, Inc. All Rights Reserved +# Modified by Feng Liang from https://github.com/MendelXu/zsseg.baseline/blob/master/datasets/prepare_voc_sem_seg.py + +import os +import os.path as osp +from pathlib import Path +import tqdm + +import numpy as np +from PIL import Image + + +clsID_to_trID = { + 0: 255, + 1: 0, + 2: 1, + 3: 2, + 4: 3, + 5: 4, + 6: 5, + 7: 6, + 8: 7, + 9: 8, + 10: 9, + 11: 10, + 12: 11, + 13: 12, + 14: 13, + 15: 14, + 16: 15, + 17: 16, + 18: 17, + 19: 18, + 20: 19, + 255: 255, +} + +def convert_to_trainID( + maskpath, out_mask_dir, is_train, clsID_to_trID=clsID_to_trID, suffix="" +): + mask = np.array(Image.open(maskpath)) + mask_copy = np.ones_like(mask, dtype=np.uint8) * 255 + for clsID, trID in clsID_to_trID.items(): + mask_copy[mask == clsID] = trID + seg_filename = ( + osp.join(out_mask_dir, "train" + suffix, osp.basename(maskpath)) + if is_train + else osp.join(out_mask_dir, "val" + suffix, osp.basename(maskpath)) + ) + if len(np.unique(mask_copy)) == 1 and np.unique(mask_copy)[0] == 255: + return + Image.fromarray(mask_copy).save(seg_filename, "PNG") + + + +if __name__ == "__main__": + dataset_dir = Path(os.getenv("DETECTRON2_DATASETS", "datasets")) + print('Caution: we only generate the validation set!') + voc_path = dataset_dir / "VOCdevkit" / "VOC2012" + out_mask_dir = voc_path / "annotations_detectron2" + out_image_dir = voc_path / "images_detectron2" + for name in ["val"]: + os.makedirs((out_mask_dir / name), exist_ok=True) + os.makedirs((out_image_dir / name), exist_ok=True) + val_list = [ + osp.join(voc_path, "SegmentationClassAug", f + ".png") + for f in np.loadtxt(osp.join(voc_path, "ImageSets/Segmentation/val.txt"), dtype=np.str).tolist() + ] + for file in tqdm.tqdm(val_list): + convert_to_trainID(file, out_mask_dir, is_train=False) diff --git a/approach/ovod/APE/demo/README.md b/approach/ovod/APE/demo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..967853a1705340731efa7cb8dd639bc76b65e87b --- /dev/null +++ b/approach/ovod/APE/demo/README.md @@ -0,0 +1,13 @@ +--- +title: APE +emoji: 🌍 +colorFrom: blue +colorTo: indigo +sdk: gradio +sdk_version: 4.7.1 +app_file: app.py +pinned: false +license: apache-2.0 +--- + +Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference diff --git a/approach/ovod/APE/demo/ape_inference.py b/approach/ovod/APE/demo/ape_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..a09b927d1e6c1a55fa401ad6dc8e2d946a93cd28 --- /dev/null +++ b/approach/ovod/APE/demo/ape_inference.py @@ -0,0 +1,149 @@ +import torch +import glob +import json +import os +import time +from collections import abc +import cv2 +import numpy as np +import tqdm +from detectron2.config import LazyConfig, get_cfg +from detectron2.data.detection_utils import read_image +from detectron2.evaluation.coco_evaluation import instances_to_coco_json +from detectron2.utils.logger import setup_logger +# from predictor_lazy import VisualizationDemo +from approach.ovod.APE.demo.predictor_lazy import VisualizationDemo + +# Constants +WINDOW_NAME = "APE" + +def setup_cfg(custom_args): + cfg = LazyConfig.load(custom_args['config_file']) + cfg = LazyConfig.apply_overrides(cfg, custom_args['opts']) + + if "output_dir" in cfg.model: + cfg.model.output_dir = cfg.train.output_dir + if "model_vision" in cfg.model and "output_dir" in cfg.model.model_vision: + cfg.model.model_vision.output_dir = cfg.train.output_dir + if "train" in cfg.dataloader: + if isinstance(cfg.dataloader.train, abc.MutableSequence): + for i in range(len(cfg.dataloader.train)): + if "output_dir" in cfg.dataloader.train[i].mapper: + cfg.dataloader.train[i].mapper.output_dir = cfg.train.output_dir + else: + if "output_dir" in cfg.dataloader.train.mapper: + cfg.dataloader.train.mapper.output_dir = cfg.train.output_dir + + if "model_vision" in cfg.model: + cfg.model.model_vision.test_score_thresh = custom_args['confidence_threshold'] + else: + cfg.model.test_score_thresh = custom_args['confidence_threshold'] + + setup_logger(name="ape") + setup_logger(name="timm") + + return cfg + +def run_ape_model_inference(config_file, input_path, output_path, confidence_threshold, text_prompt, + with_box, with_mask, with_sseg, opts): + custom_args = { + 'config_file': config_file, + 'input': input_path, + 'output': output_path, + 'confidence_threshold': confidence_threshold, + 'text_prompt': text_prompt, + 'with_box': with_box, + 'with_mask': with_mask, + 'with_sseg': with_sseg, + 'opts': opts, + } + + setup_logger(name="fvcore") + setup_logger(name="ape") + logger = setup_logger() + logger.info("Arguments: " + str(custom_args)) + + cfg = setup_cfg(custom_args) + + demo = VisualizationDemo(cfg) + + if custom_args['input']: + input_paths = glob.glob(os.path.expanduser(custom_args['input']), recursive=True) + assert input_paths, "The input path(s) was not found" + for path in tqdm.tqdm(input_paths, disable=not custom_args['output']): + try: + img = read_image(path, format="BGR") + except Exception as e: + print("*" * 60) + print("fail to open image: ", e) + print("*" * 60) + continue + start_time = time.time() + predictions, visualized_output, visualized_outputs, metadata = demo.run_on_image( + img, + text_prompt=custom_args['text_prompt'], + with_box=custom_args['with_box'], + with_mask=custom_args['with_mask'], + with_sseg=custom_args['with_sseg'], + ) + logger.info( + "{}: {} in {:.2f}s".format( + path, + "detected {} instances".format(len(predictions["instances"])) + if "instances" in predictions + else "finished", + time.time() - start_time, + ) + ) + + if custom_args['output']: + if os.path.isdir(custom_args['output']): + assert os.path.isdir(custom_args['output']), custom_args['output'] + out_filename = os.path.join(custom_args['output'], os.path.basename(path)) + else: + assert len(custom_args['input']) == 1, "Please specify a directory with args.output" + out_filename = custom_args['output'] + out_filename = out_filename.replace(".webp", ".png") + out_filename = out_filename.replace(".crdownload", ".png") + out_filename = out_filename.replace(".jfif", ".png") + visualized_output.save(out_filename) + + for i in range(len(visualized_outputs)): + out_filename = ( + os.path.join(custom_args['output'], os.path.basename(path)) + "." + str(i) + ".png" + ) + visualized_outputs[i].save(out_filename) + + if "instances" in predictions: + results = instances_to_coco_json( + predictions["instances"].to(demo.cpu_device), path + ) + for result in results: + result["category_name"] = metadata.thing_classes[result["category_id"]] + result["image_name"] = result["image_id"] + + with open(out_filename + ".json", "w") as outp: + json.dump(results, outp, indent=4) + + return results + +if __name__ == "__main__": + run_ape_model_inference( + config_file='configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k.py', + input_path='demo/examples/Pisa.jpg', + output_path='./test_output', + confidence_threshold=0.1, + text_prompt='sky', + with_box=True, + with_mask=False, + with_sseg=False, + opts=[ + "train.init_checkpoint='./ape_d_model_final.pth'", + "model.model_language.cache_dir=''", + "model.model_vision.select_box_nums_for_evaluation=500", + "model.model_vision.text_feature_bank_reset=True", + "model.model_vision.backbone.net.xattn=False", + "model.model_vision.transformer.encoder.pytorch_attn=True", + "model.model_vision.transformer.decoder.pytorch_attn=True" + ] + ) diff --git a/approach/ovod/APE/demo/app.py b/approach/ovod/APE/demo/app.py new file mode 100644 index 0000000000000000000000000000000000000000..b1264dd338a664e45d4dc5981a75947f4c5ffd6d --- /dev/null +++ b/approach/ovod/APE/demo/app.py @@ -0,0 +1,1034 @@ +import gc +import multiprocessing as mp +import os +import shutil +import sys +import time +from os import path + +import cv2 +import torch +from huggingface_hub import hf_hub_download +from PIL import Image + +import ape +import detectron2.data.transforms as T +import gradio as gr +from ape.model_zoo import get_config_file +from demo_lazy import get_parser, setup_cfg +from detectron2.config import CfgNode +from detectron2.data.detection_utils import read_image +from detectron2.evaluation.coco_evaluation import instances_to_coco_json +from detectron2.utils.logger import setup_logger +from predictor_lazy import VisualizationDemo + +this_dir = path.dirname(path.abspath(__file__)) + +# os.system("git clone https://github.com/shenyunhang/APE.git") +# os.system("python3.10 -m pip install -e APE/") + +example_list = [ + [ + this_dir + "/examples/Totoro01.png", + # "Sky, Water, Tree, The biggest Chinchilla, The older girl wearing skirt on branch, Grass", + "Girl with hat", + # 0.05, + 0.25, + ["object detection", "instance segmentation"], + ], + [ + this_dir + "/examples/Totoro01.png", + "Sky, Water, Tree, Chinchilla, Grass, Girl", + 0.15, + ["semantic segmentation"], + ], + [ + this_dir + "/examples/199_3946193540.jpg", + "chess piece of horse head", + 0.30, + ["object detection", "instance segmentation"], + ], + [ + this_dir + "/examples/TheGreatWall.jpg", + "The Great Wall", + 0.1, + ["semantic segmentation"], + ], + [ + this_dir + "/examples/Pisa.jpg", + "Pisa", + 0.01, + ["object detection", "instance segmentation"], + ], + [ + this_dir + "/examples/SolvayConference1927.jpg", + # "Albert Einstein, Madame Curie", + "Madame Curie", + # 0.01, + 0.03, + ["object detection", "instance segmentation"], + ], + [ + this_dir + "/examples/Transformers.webp", + "Optimus Prime", + 0.11, + ["object detection", "instance segmentation"], + ], + [ + this_dir + "/examples/Terminator3.jpg", + "Humanoid Robot", + 0.10, + ["object detection", "instance segmentation"], + ], + [ + this_dir + "/examples/MatrixRevolutionForZion.jpg", + """machine killer with gun in fighting, +donut with colored granules on the surface, +railings being crossed by horses, +a horse running or jumping, +equestrian rider's helmet, +outdoor dog led by rope, +a dog being touched, +clothed dog, +basketball in hand, +a basketball player with both feet off the ground, +player with basketball in the hand, +spoon on the plate, +coffee cup with coffee, +the nearest dessert to the coffee cup, +the bartender who is mixing wine, +a bartender in a suit, +wine glass with wine, +a person in aprons, +pot with food, +a knife being used to cut vegetables, +striped sofa in the room, +a sofa with pillows on it in the room, +lights on in the room, +an indoor lying pet, +a cat on the sofa, +one pet looking directly at the camera indoors, +a bed with patterns in the room, +the lamp on the table beside the bed, +pillow placed at the head of the bed, +a blackboard full of words in the classroom, +child sitting at desks in the classroom, +a person standing in front of bookshelves in the library, +the table someone is using in the library, +a person who touches books in the library, +a person standing in front of the cake counter, +a square plate full of cakes, +a cake decorated with cream, +hot dog with vegetables, +hot dog with sauce on the surface, +red sausage, +flowerpot with flowers potted inside, +monochrome flowerpot, +a flowerpot filled with black soil, +apple growing on trees, +red complete apple, +apple with a stalk, +a woman brushing her teeth, +toothbrush held by someone, +toilet brush with colored bristles, +a customer whose hair is being cut by barber, +a barber at work, +cloth covering the barber, +shopping cart pushed by people in the supermarket, +shopping cart with people in the supermarket, +shopping cart full of goods, +a child wearing a mask, +refrigerator with fruit, +a drink bottle in the refrigerator, +refrigerator with more than two doors, +a watch placed on a table or cloth, +a watch with three or more watch hands can be seen, +a watch with one or more small dials, +clothes hanger, +a piece of clothing hanging on the hanger, +a piece of clothing worn on plastic models, +leather bag with glossy surface, +backpack, +open package, +a fish held by people, +a person who is fishing with a fishing rod, +a fisherman standing on the shore with his body soaked in water, camera hold on someone's shoulder, +a person being interviewed, +a person with microphone hold in hand, + """, + 0.20, + ["object detection", "instance segmentation"], + ], + [ + this_dir + "/examples/094_56726435.jpg", + # "donut with colored granules on the surface", + """donut with colored granules on the surface, +railings being crossed by horses, +a horse running or jumping, +equestrian rider's helmet, +outdoor dog led by rope, +a dog being touched, +clothed dog, +basketball in hand, +a basketball player with both feet off the ground, +player with basketball in the hand, +spoon on the plate, +coffee cup with coffee, +the nearest dessert to the coffee cup, +the bartender who is mixing wine, +a bartender in a suit, +wine glass with wine, +a person in aprons, +pot with food, +a knife being used to cut vegetables, +striped sofa in the room, +a sofa with pillows on it in the room, +lights on in the room, +an indoor lying pet, +a cat on the sofa, +one pet looking directly at the camera indoors, +a bed with patterns in the room, +the lamp on the table beside the bed, +pillow placed at the head of the bed, +a blackboard full of words in the classroom, +a blackboard or whiteboard with something pasted, +child sitting at desks in the classroom, +a person standing in front of bookshelves in the library, +the table someone is using in the library, +a person who touches books in the library, +a person standing in front of the cake counter, +a square plate full of cakes, +a cake decorated with cream, +hot dog with vegetables, +hot dog with sauce on the surface, +red sausage, +flowerpot with flowers potted inside, +monochrome flowerpot, +a flowerpot filled with black soil, +apple growing on trees, +red complete apple, +apple with a stalk, +a woman brushing her teeth, +toothbrush held by someone, +toilet brush with colored bristles, +a customer whose hair is being cut by barber, +a barber at work, +cloth covering the barber, +a plastic toy, +a plush toy, +a humanoid toy, +shopping cart pushed by people in the supermarket, +shopping cart with people in the supermarket, +shopping cart full of goods, +a child wearing a mask, +a mask on face with half a face exposed, +a mask on face with only eyes exposed, +refrigerator with fruit, +a drink bottle in the refrigerator, +refrigerator with more than two doors, +a watch placed on a table or cloth, +a watch with three or more watch hands can be seen, +a watch with one or more small dials, +clothes hanger, +a piece of clothing hanging on the hanger, +a piece of clothing worn on plastic models, +leather bag with glossy surface, +backpack, +open package, +a fish held by people, +a person who is fishing with a fishing rod, +a fisherman standing on the shore with his body soaked in water, camera hold on someone's shoulder, +a person being interviewed, +a person with microphone hold in hand, + """, + 0.50, + ["object detection", "instance segmentation"], + ], + [ + this_dir + "/examples/013_438973263.jpg", + # "a male lion with a mane", + """a male lion with a mane, +railings being crossed by horses, +a horse running or jumping, +equestrian rider's helmet, +outdoor dog led by rope, +a dog being touched, +clothed dog, +basketball in hand, +a basketball player with both feet off the ground, +player with basketball in the hand, +spoon on the plate, +coffee cup with coffee, +the nearest dessert to the coffee cup, +the bartender who is mixing wine, +a bartender in a suit, +wine glass with wine, +a person in aprons, +pot with food, +a knife being used to cut vegetables, +striped sofa in the room, +a sofa with pillows on it in the room, +lights on in the room, +an indoor lying pet, +a cat on the sofa, +one pet looking directly at the camera indoors, +a bed with patterns in the room, +the lamp on the table beside the bed, +pillow placed at the head of the bed, +a blackboard full of words in the classroom, +a blackboard or whiteboard with something pasted, +child sitting at desks in the classroom, +a person standing in front of bookshelves in the library, +the table someone is using in the library, +a person who touches books in the library, +a person standing in front of the cake counter, +a square plate full of cakes, +a cake decorated with cream, +hot dog with vegetables, +hot dog with sauce on the surface, +red sausage, +flowerpot with flowers potted inside, +monochrome flowerpot, +a flowerpot filled with black soil, +apple growing on trees, +red complete apple, +apple with a stalk, +a woman brushing her teeth, +toothbrush held by someone, +toilet brush with colored bristles, +a customer whose hair is being cut by barber, +a barber at work, +cloth covering the barber, +a plastic toy, +a plush toy, +a humanoid toy, +shopping cart pushed by people in the supermarket, +shopping cart with people in the supermarket, +shopping cart full of goods, +a child wearing a mask, +a mask on face with half a face exposed, +a mask on face with only eyes exposed, +refrigerator with fruit, +a drink bottle in the refrigerator, +refrigerator with more than two doors, +a watch placed on a table or cloth, +a watch with three or more watch hands can be seen, +a watch with one or more small dials, +clothes hanger, +a piece of clothing hanging on the hanger, +a piece of clothing worn on plastic models, +leather bag with glossy surface, +backpack, +open package, +a fish held by people, +a person who is fishing with a fishing rod, +a fisherman standing on the shore with his body soaked in water, camera hold on someone's shoulder, +a person being interviewed, +a person with microphone hold in hand, + """, + # 0.25, + 0.50, + ["object detection", "instance segmentation"], + ], +] + +ckpt_repo_id = "shenyunhang/APE" + + +def setup_model(name): + gc.collect() + torch.cuda.empty_cache() + + if save_memory: + pass + else: + return + + for key, demo in all_demo.items(): + if key == name: + demo.predictor.model.to(running_device) + else: + demo.predictor.model.to("cpu") + + gc.collect() + torch.cuda.empty_cache() + + +def run_on_image_A(input_image_path, input_text, score_threshold, output_type): + logger.info("run_on_image") + + setup_model("APE_A") + demo = all_demo["APE_A"] + cfg = all_cfg["APE_A"] + demo.predictor.model.model_vision.test_score_thresh = score_threshold + + return run_on_image( + input_image_path, + input_text, + output_type, + demo, + cfg, + ) + + +def run_on_image_C(input_image_path, input_text, score_threshold, output_type): + logger.info("run_on_image_C") + + setup_model("APE_C") + demo = all_demo["APE_C"] + cfg = all_cfg["APE_C"] + demo.predictor.model.model_vision.test_score_thresh = score_threshold + + return run_on_image( + input_image_path, + input_text, + output_type, + demo, + cfg, + ) + + +def run_on_image_D(input_image_path, input_text, score_threshold, output_type): + logger.info("run_on_image_D") + + setup_model("APE_D") + demo = all_demo["APE_D"] + cfg = all_cfg["APE_D"] + demo.predictor.model.model_vision.test_score_thresh = score_threshold + + return run_on_image( + input_image_path, + input_text, + output_type, + demo, + cfg, + ) + + +def run_on_image_comparison(input_image_path, input_text, score_threshold, output_type): + logger.info("run_on_image_comparison") + + r = [] + for key in all_demo.keys(): + logger.info("run_on_image_comparison {}".format(key)) + setup_model(key) + demo = all_demo[key] + cfg = all_cfg[key] + demo.predictor.model.model_vision.test_score_thresh = score_threshold + + img, _ = run_on_image( + input_image_path, + input_text, + output_type, + demo, + cfg, + ) + r.append(img) + + return r + + +def run_on_image( + input_image_path, + input_text, + output_type, + demo, + cfg, +): + with_box = False + with_mask = False + with_sseg = False + if "object detection" in output_type: + with_box = True + if "instance segmentation" in output_type: + with_mask = True + if "semantic segmentation" in output_type: + with_sseg = True + + if isinstance(input_image_path, dict): + input_mask_path = input_image_path["mask"] + input_image_path = input_image_path["image"] + print("input_image_path", input_image_path) + print("input_mask_path", input_mask_path) + else: + input_mask_path = None + + print("input_text", input_text) + + if isinstance(cfg, CfgNode): + input_format = cfg.INPUT.FORMAT + else: + if "model_vision" in cfg.model: + input_format = cfg.model.model_vision.input_format + else: + input_format = cfg.model.input_format + + input_image = read_image(input_image_path, format="BGR") + # img = cv2.imread(input_image_path) + # cv2.imwrite("tmp.jpg", img) + # # input_image = read_image("tmp.jpg", format=input_format) + # input_image = read_image("tmp.jpg", format="BGR") + + if input_mask_path is not None: + input_mask = read_image(input_mask_path, "L").squeeze(2) + print("input_mask", input_mask) + print("input_mask", input_mask.shape) + else: + input_mask = None + + if not with_box and not with_mask and not with_sseg: + return input_image[:, :, ::-1] + + if input_image.shape[0] > 1024 or input_image.shape[1] > 1024: + transform = aug.get_transform(input_image) + input_image = transform.apply_image(input_image) + else: + transform = None + + start_time = time.time() + predictions, visualized_output, _, metadata = demo.run_on_image( + input_image, + text_prompt=input_text, + mask_prompt=input_mask, + with_box=with_box, + with_mask=with_mask, + with_sseg=with_sseg, + ) + + logger.info( + "{} in {:.2f}s".format( + "detected {} instances".format(len(predictions["instances"])) + if "instances" in predictions + else "finished", + time.time() - start_time, + ) + ) + + output_image = visualized_output.get_image() + print("output_image", output_image.shape) + # if input_format == "RGB": + # output_image = output_image[:, :, ::-1] + if transform: + output_image = transform.inverse().apply_image(output_image) + print("output_image", output_image.shape) + + output_image = Image.fromarray(output_image) + + gc.collect() + torch.cuda.empty_cache() + + json_results = instances_to_coco_json(predictions["instances"].to(demo.cpu_device), 0) + for json_result in json_results: + json_result["category_name"] = metadata.thing_classes[json_result["category_id"]] + del json_result["image_id"] + + return output_image, json_results + + +def load_APE_A(): + # init_checkpoint= "output2/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VG/ape_deta/ape_deta_vitl_eva02_lsj_cp_720k_20230504_002019/model_final.pth" + init_checkpoint = "configs/LVISCOCOCOCOSTUFF_O365_OID_VG/ape_deta/ape_deta_vitl_eva02_lsj_cp_720k_20230504_002019/model_final.pth" + init_checkpoint = hf_hub_download(repo_id=ckpt_repo_id, filename=init_checkpoint) + + args = get_parser().parse_args() + args.config_file = get_config_file( + "LVISCOCOCOCOSTUFF_O365_OID_VG/ape_deta/ape_deta_vitl_eva02_lsj1024_cp_720k.py" + ) + args.confidence_threshold = 0.01 + args.opts = [ + "train.init_checkpoint='{}'".format(init_checkpoint), + "model.model_language.cache_dir=''", + "model.model_vision.select_box_nums_for_evaluation=500", + "model.model_vision.backbone.net.xattn=False", + "model.model_vision.transformer.encoder.pytorch_attn=True", + "model.model_vision.transformer.decoder.pytorch_attn=True", + ] + if running_device == "cpu": + args.opts += [ + "model.model_language.dtype='float32'", + ] + logger.info("Arguments: " + str(args)) + cfg = setup_cfg(args) + + cfg.model.model_vision.criterion[0].use_fed_loss = False + cfg.model.model_vision.criterion[2].use_fed_loss = False + cfg.train.device = running_device + + ape.modeling.text.eva01_clip.eva_clip._MODEL_CONFIGS[cfg.model.model_language.clip_model][ + "vision_cfg" + ]["layers"] = 1 + ape.modeling.text.eva01_clip.eva_clip._MODEL_CONFIGS[cfg.model.model_language.clip_model][ + "vision_cfg" + ]["fusedLN"] = False + + demo = VisualizationDemo(cfg, args=args) + if save_memory: + demo.predictor.model.to("cpu") + # demo.predictor.model.half() + else: + demo.predictor.model.to(running_device) + + all_demo["APE_A"] = demo + all_cfg["APE_A"] = cfg + + +def load_APE_B(): + # init_checkpoint= "output2/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj_cp_1080k_20230702_225418/model_final.pth" + init_checkpoint = "configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj_cp_1080k_20230702_225418/model_final.pth" + init_checkpoint = hf_hub_download(repo_id=ckpt_repo_id, filename=init_checkpoint) + + args = get_parser().parse_args() + args.config_file = get_config_file( + "LVISCOCOCOCOSTUFF_O365_OID_VGR_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_1080k.py" + ) + args.confidence_threshold = 0.01 + args.opts = [ + "train.init_checkpoint='{}'".format(init_checkpoint), + "model.model_language.cache_dir=''", + "model.model_vision.select_box_nums_for_evaluation=500", + "model.model_vision.text_feature_bank_reset=True", + "model.model_vision.backbone.net.xattn=False", + "model.model_vision.transformer.encoder.pytorch_attn=True", + "model.model_vision.transformer.decoder.pytorch_attn=True", + ] + if running_device == "cpu": + args.opts += [ + "model.model_language.dtype='float32'", + ] + logger.info("Arguments: " + str(args)) + cfg = setup_cfg(args) + + cfg.model.model_vision.criterion[0].use_fed_loss = False + cfg.model.model_vision.criterion[2].use_fed_loss = False + cfg.train.device = running_device + + ape.modeling.text.eva01_clip.eva_clip._MODEL_CONFIGS[cfg.model.model_language.clip_model][ + "vision_cfg" + ]["layers"] = 1 + ape.modeling.text.eva01_clip.eva_clip._MODEL_CONFIGS[cfg.model.model_language.clip_model][ + "vision_cfg" + ]["fusedLN"] = False + + demo = VisualizationDemo(cfg, args=args) + if save_memory: + demo.predictor.model.to("cpu") + # demo.predictor.model.half() + else: + demo.predictor.model.to(running_device) + + all_demo["APE_B"] = demo + all_cfg["APE_B"] = cfg + + +def load_APE_C(): + # init_checkpoint= "output2/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj_cp_1080k_20230702_210950/model_final.pth" + init_checkpoint = "configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj_cp_1080k_20230702_210950/model_final.pth" + init_checkpoint = hf_hub_download(repo_id=ckpt_repo_id, filename=init_checkpoint) + + args = get_parser().parse_args() + args.config_file = get_config_file( + "LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_1080k.py" + ) + args.confidence_threshold = 0.01 + args.opts = [ + "train.init_checkpoint='{}'".format(init_checkpoint), + "model.model_language.cache_dir=''", + "model.model_vision.select_box_nums_for_evaluation=500", + "model.model_vision.text_feature_bank_reset=True", + "model.model_vision.backbone.net.xattn=False", + "model.model_vision.transformer.encoder.pytorch_attn=True", + "model.model_vision.transformer.decoder.pytorch_attn=True", + ] + if running_device == "cpu": + args.opts += [ + "model.model_language.dtype='float32'", + ] + logger.info("Arguments: " + str(args)) + cfg = setup_cfg(args) + + cfg.model.model_vision.criterion[0].use_fed_loss = False + cfg.model.model_vision.criterion[2].use_fed_loss = False + cfg.train.device = running_device + + ape.modeling.text.eva01_clip.eva_clip._MODEL_CONFIGS[cfg.model.model_language.clip_model][ + "vision_cfg" + ]["layers"] = 1 + ape.modeling.text.eva01_clip.eva_clip._MODEL_CONFIGS[cfg.model.model_language.clip_model][ + "vision_cfg" + ]["fusedLN"] = False + + demo = VisualizationDemo(cfg, args=args) + if save_memory: + demo.predictor.model.to("cpu") + # demo.predictor.model.half() + else: + demo.predictor.model.to(running_device) + + all_demo["APE_C"] = demo + all_cfg["APE_C"] = cfg + + +def load_APE_D(): + # init_checkpoint= "output2/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k_mdl_20230829_162438/model_final.pth" + init_checkpoint = "configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k_mdl_20230829_162438/model_final.pth" + init_checkpoint = hf_hub_download(repo_id=ckpt_repo_id, filename=init_checkpoint) + + args = get_parser().parse_args() + args.config_file = get_config_file( + "LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k.py" + ) + args.confidence_threshold = 0.01 + args.opts = [ + "train.init_checkpoint='{}'".format(init_checkpoint), + "model.model_language.cache_dir=''", + "model.model_vision.select_box_nums_for_evaluation=500", + "model.model_vision.text_feature_bank_reset=True", + "model.model_vision.backbone.net.xattn=False", + "model.model_vision.transformer.encoder.pytorch_attn=True", + "model.model_vision.transformer.decoder.pytorch_attn=True", + ] + if running_device == "cpu": + args.opts += [ + "model.model_language.dtype='float32'", + ] + logger.info("Arguments: " + str(args)) + cfg = setup_cfg(args) + + cfg.model.model_vision.criterion[0].use_fed_loss = False + cfg.model.model_vision.criterion[2].use_fed_loss = False + cfg.train.device = running_device + + ape.modeling.text.eva02_clip.factory._MODEL_CONFIGS[cfg.model.model_language.clip_model][ + "vision_cfg" + ]["layers"] = 1 + + demo = VisualizationDemo(cfg, args=args) + if save_memory: + demo.predictor.model.to("cpu") + # demo.predictor.model.half() + else: + demo.predictor.model.to(running_device) + + all_demo["APE_D"] = demo + all_cfg["APE_D"] = cfg + + +def APE_A_tab(): + with gr.Tab("APE A"): + with gr.Row(equal_height=False): + with gr.Column(scale=1): + input_image = gr.Image( + sources=["upload"], + type="filepath", + # tool="sketch", + # brush_radius=50, + ) + input_text = gr.Textbox( + label="Object Prompt (optional, if not provided, will only find COCO object.)", + info="格式: word1,word2,word3,...", + ) + + score_threshold = gr.Slider( + label="Score Threshold", minimum=0.01, maximum=1.0, value=0.3, step=0.01 + ) + + output_type = gr.CheckboxGroup( + ["object detection", "instance segmentation"], + value=["object detection", "instance segmentation"], + label="Output Type", + info="Which kind of output is displayed?", + ).style(item_container=True, container=True) + + run_button = gr.Button("Run") + + with gr.Column(scale=2): + gallery = gr.Image( + type="pil", + ) + + example_data = gr.Dataset( + components=[input_image, input_text, score_threshold], + samples=examples, + samples_per_page=5, + ) + example_data.click(fn=set_example, inputs=example_data, outputs=example_data.components) + + # add_tail_info() + output_json = gr.JSON(label="json results") + + run_button.click( + fn=run_on_image, + inputs=[input_image, input_text, score_threshold, output_type], + outputs=[gallery, output_json], + ) + + +def APE_C_tab(): + with gr.Tab("APE C"): + with gr.Row(equal_height=False): + with gr.Column(scale=1): + input_image = gr.Image( + sources=["upload"], + type="filepath", + # tool="sketch", + # brush_radius=50, + ) + input_text = gr.Textbox( + label="Object Prompt (optional, if not provided, will only find COCO object.)", + info="格式: word1,word2,sentence1,sentence2,...", + ) + + score_threshold = gr.Slider( + label="Score Threshold", minimum=0.01, maximum=1.0, value=0.3, step=0.01 + ) + + output_type = gr.CheckboxGroup( + ["object detection", "instance segmentation", "semantic segmentation"], + value=["object detection", "instance segmentation"], + label="Output Type", + info="Which kind of output is displayed?", + ).style(item_container=True, container=True) + + run_button = gr.Button("Run") + + with gr.Column(scale=2): + gallery = gr.Image( + type="pil", + ) + + example_data = gr.Dataset( + components=[input_image, input_text, score_threshold], + samples=example_list, + samples_per_page=5, + ) + example_data.click(fn=set_example, inputs=example_data, outputs=example_data.components) + + # add_tail_info() + output_json = gr.JSON(label="json results") + + run_button.click( + fn=run_on_image_C, + inputs=[input_image, input_text, score_threshold, output_type], + outputs=[gallery, output_json], + ) + + +def APE_D_tab(): + with gr.Tab("APE D"): + with gr.Row(equal_height=False): + with gr.Column(scale=1): + input_image = gr.Image( + sources=["upload"], + type="filepath", + # tool="sketch", + # brush_radius=50, + ) + input_text = gr.Textbox( + label="Object Prompt (optional, if not provided, will only find COCO object.)", + info="格式: word1,word2,sentence1,sentence2,...", + ) + + score_threshold = gr.Slider( + label="Score Threshold", minimum=0.01, maximum=1.0, value=0.1, step=0.01 + ) + + output_type = gr.CheckboxGroup( + ["object detection", "instance segmentation", "semantic segmentation"], + value=["object detection", "instance segmentation"], + label="Output Type", + info="Which kind of output is displayed?", + ) + + run_button = gr.Button("Run") + + with gr.Column(scale=2): + gallery = gr.Image( + type="pil", + ) + + gr.Examples( + examples=example_list, + inputs=[input_image, input_text, score_threshold, output_type], + examples_per_page=20, + ) + + # add_tail_info() + output_json = gr.JSON(label="json results") + + run_button.click( + fn=run_on_image_D, + inputs=[input_image, input_text, score_threshold, output_type], + outputs=[gallery, output_json], + ) + + +def comparison_tab(): + with gr.Tab("APE all"): + with gr.Row(equal_height=False): + with gr.Column(scale=1): + input_image = gr.Image( + sources=["upload"], + type="filepath", + # tool="sketch", + # brush_radius=50, + ) + input_text = gr.Textbox( + label="Object Prompt (optional, if not provided, will only find COCO object.)", + info="格式: word1,word2,sentence1,sentence2,...", + ) + + score_threshold = gr.Slider( + label="Score Threshold", minimum=0.01, maximum=1.0, value=0.1, step=0.01 + ) + + output_type = gr.CheckboxGroup( + ["object detection", "instance segmentation", "semantic segmentation"], + value=["object detection", "instance segmentation"], + label="Output Type", + info="Which kind of output is displayed?", + ) + + run_button = gr.Button("Run") + + gallery_all = [] + with gr.Column(scale=2): + for key in all_demo.keys(): + gallery = gr.Image( + label=key, + type="pil", + ) + gallery_all.append(gallery) + + gr.Examples( + examples=example_list, + inputs=[input_image, input_text, score_threshold, output_type], + examples_per_page=20, + ) + + # add_tail_info() + + run_button.click( + fn=run_on_image_comparison, + inputs=[input_image, input_text, score_threshold, output_type], + outputs=gallery_all, + ) + + +def is_port_in_use(port: int) -> bool: + import socket + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return s.connect_ex(("localhost", port)) == 0 + + +def add_head_info(max_available_memory): + gr.Markdown( + "# APE: Aligning and Prompting Everything All at Once for Universal Visual Perception" + ) + if max_available_memory: + gr.Markdown( + "Note multiple models are deployed on single GPU, so it may take several minutes to run the models and visualize the results." + ) + else: + gr.Markdown( + "Note multiple models are deployed on CPU, so it may take a while to run the models and visualize the results." + ) + gr.Markdown( + "Noted results computed by CPU are slightly different to results computed by GPU, and some libraries are disabled on CPU." + ) + gr.Markdown( + "If the demo is out of memory, try to ***decrease*** the number of object prompt and ***increase*** score threshold." + ) + + gr.Markdown("---") + + +def add_tail_info(): + gr.Markdown("---") + gr.Markdown("### We also support Prompt") + gr.Markdown( + """ + | Location prompt | result | Location prompt | result | + | ---- | ---- | ---- | ---- | + | ![Location prompt](/file=examples/prompt/20230627-131346_11.176.20.67_mask.PNG) | ![结果](/file=examples/prompt/20230627-131346_11.176.20.67_pred.png) | ![Location prompt](/file=examples/prompt/20230627-131530_11.176.20.67_mask.PNG) | ![结果](/file=examples/prompt/20230627-131530_11.176.20.67_pred.png) | + | ![Location prompt](/file=examples/prompt/20230627-131520_11.176.20.67_mask.PNG) | ![结果](/file=examples/prompt/20230627-131520_11.176.20.67_pred.png) | ![Location prompt](/file=examples/prompt/20230627-114219_11.176.20.67_mask.PNG) | ![结果](/file=examples/prompt/20230627-114219_11.176.20.67_pred.png) | + """ + ) + gr.Markdown("---") + + +if __name__ == "__main__": + available_port = [80, 8080] + for port in available_port: + if is_port_in_use(port): + continue + else: + server_port = port + break + print("server_port", server_port) + + available_memory = [ + torch.cuda.mem_get_info(i)[0] / 1024**3 for i in range(torch.cuda.device_count()) + ] + + global running_device + if len(available_memory) > 0: + max_available_memory = max(available_memory) + device_id = available_memory.index(max_available_memory) + + running_device = "cuda:" + str(device_id) + else: + max_available_memory = 0 + running_device = "cpu" + + global save_memory + save_memory = False + if max_available_memory > 0 and max_available_memory < 40: + save_memory = True + + print("available_memory", available_memory) + print("max_available_memory", max_available_memory) + print("running_device", running_device) + print("save_memory", save_memory) + + # ========================================================================================== + + mp.set_start_method("spawn", force=True) + setup_logger(name="fvcore") + setup_logger(name="ape") + global logger + logger = setup_logger() + + global aug + aug = T.ResizeShortestEdge([1024, 1024], 1024) + + global all_demo + all_demo = {} + all_cfg = {} + + # load_APE_A() + # load_APE_B() + # load_APE_C() + save_memory = False + load_APE_D() + + title = "APE: Aligning and Prompting Everything All at Once for Universal Visual Perception" + block = gr.Blocks(title=title).queue() + with block: + add_head_info(max_available_memory) + + # APE_A_tab() + # APE_C_tab() + APE_D_tab() + + comparison_tab() + + # add_tail_info() + + block.launch( + share=False, + # server_name="0.0.0.0", + # server_port=server_port, + show_api=False, + show_error=True, + ) diff --git a/approach/ovod/APE/demo/demo_lazy.py b/approach/ovod/APE/demo/demo_lazy.py new file mode 100644 index 0000000000000000000000000000000000000000..b858b87e83e79e5c6a2a02927585ce49abab9841 --- /dev/null +++ b/approach/ovod/APE/demo/demo_lazy.py @@ -0,0 +1,263 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import argparse +import glob +import json +import multiprocessing as mp +import os +import tempfile +import time +import warnings +from collections import abc + +import cv2 +import numpy as np +import tqdm + +from detectron2.config import LazyConfig, get_cfg +from detectron2.data.detection_utils import read_image +from detectron2.evaluation.coco_evaluation import instances_to_coco_json + +# from detectron2.projects.deeplab import add_deeplab_config +# from detectron2.projects.panoptic_deeplab import add_panoptic_deeplab_config +from detectron2.utils.logger import setup_logger +from predictor_lazy import VisualizationDemo + +# constants +WINDOW_NAME = "APE" + + +def setup_cfg(args): + # load config from file and command-line arguments + cfg = LazyConfig.load(args.config_file) + cfg = LazyConfig.apply_overrides(cfg, args.opts) + + if "output_dir" in cfg.model: + cfg.model.output_dir = cfg.train.output_dir + if "model_vision" in cfg.model and "output_dir" in cfg.model.model_vision: + cfg.model.model_vision.output_dir = cfg.train.output_dir + if "train" in cfg.dataloader: + if isinstance(cfg.dataloader.train, abc.MutableSequence): + for i in range(len(cfg.dataloader.train)): + if "output_dir" in cfg.dataloader.train[i].mapper: + cfg.dataloader.train[i].mapper.output_dir = cfg.train.output_dir + else: + if "output_dir" in cfg.dataloader.train.mapper: + cfg.dataloader.train.mapper.output_dir = cfg.train.output_dir + + if "model_vision" in cfg.model: + cfg.model.model_vision.test_score_thresh = args.confidence_threshold + else: + cfg.model.test_score_thresh = args.confidence_threshold + + # default_setup(cfg, args) + + setup_logger(name="ape") + setup_logger(name="timm") + + return cfg + + +def get_parser(): + parser = argparse.ArgumentParser(description="Detectron2 demo for builtin configs") + parser.add_argument( + "--config-file", + default="configs/quick_schedules/mask_rcnn_R_50_FPN_inference_acc_test.yaml", + metavar="FILE", + help="path to config file", + ) + parser.add_argument("--webcam", action="store_true", help="Take inputs from webcam.") + parser.add_argument("--video-input", help="Path to video file.") + parser.add_argument( + "--input", + nargs="+", + help="A list of space separated input images; " + "or a single glob pattern such as 'directory/*.jpg'", + ) + parser.add_argument( + "--output", + help="A file or directory to save output visualizations. " + "If not given, will show output in an OpenCV window.", + ) + + parser.add_argument( + "--confidence-threshold", + type=float, + default=0.5, + help="Minimum score for instance predictions to be shown", + ) + parser.add_argument( + "--opts", + help="Modify config options using the command-line 'KEY VALUE' pairs", + default=[], + nargs=argparse.REMAINDER, + ) + + parser.add_argument("--text-prompt", default=None) + + parser.add_argument("--with-box", action="store_true", help="show box of instance") + parser.add_argument("--with-mask", action="store_true", help="show mask of instance") + parser.add_argument("--with-sseg", action="store_true", help="show mask of class") + + return parser + + +def test_opencv_video_format(codec, file_ext): + with tempfile.TemporaryDirectory(prefix="video_format_test") as dir: + filename = os.path.join(dir, "test_file" + file_ext) + writer = cv2.VideoWriter( + filename=filename, + fourcc=cv2.VideoWriter_fourcc(*codec), + fps=float(30), + frameSize=(10, 10), + isColor=True, + ) + [writer.write(np.zeros((10, 10, 3), np.uint8)) for _ in range(30)] + writer.release() + if os.path.isfile(filename): + return True + return False + + +if __name__ == "__main__": + mp.set_start_method("spawn", force=True) + args = get_parser().parse_args() + setup_logger(name="fvcore") + setup_logger(name="ape") + logger = setup_logger() + logger.info("Arguments: " + str(args)) + + cfg = setup_cfg(args) + + if args.video_input: + demo = VisualizationDemo(cfg, parallel=True, args=args) + else: + demo = VisualizationDemo(cfg, args=args) + + if args.input: + if len(args.input) == 1: + args.input = glob.glob(os.path.expanduser(args.input[0]), recursive=True) + assert args.input, "The input path(s) was not found" + for path in tqdm.tqdm(args.input, disable=not args.output): + # use PIL, to be consistent with evaluation + try: + img = read_image(path, format="BGR") + except Exception as e: + print("*" * 60) + print("fail to open image: ", e) + print("*" * 60) + continue + start_time = time.time() + predictions, visualized_output, visualized_outputs, metadata = demo.run_on_image( + img, + text_prompt=args.text_prompt, + with_box=args.with_box, + with_mask=args.with_mask, + with_sseg=args.with_sseg, + ) + logger.info( + "{}: {} in {:.2f}s".format( + path, + "detected {} instances".format(len(predictions["instances"])) + if "instances" in predictions + else "finished", + time.time() - start_time, + ) + ) + + if args.output: + if os.path.isdir(args.output): + assert os.path.isdir(args.output), args.output + out_filename = os.path.join(args.output, os.path.basename(path)) + else: + assert len(args.input) == 1, "Please specify a directory with args.output" + out_filename = args.output + out_filename = out_filename.replace(".webp", ".png") + out_filename = out_filename.replace(".crdownload", ".png") + out_filename = out_filename.replace(".jfif", ".png") + visualized_output.save(out_filename) + + for i in range(len(visualized_outputs)): + out_filename = ( + os.path.join(args.output, os.path.basename(path)) + "." + str(i) + ".png" + ) + visualized_outputs[i].save(out_filename) + + # import pickle + # with open(out_filename + ".pkl", "wb") as outp: + # pickle.dump(predictions, outp, pickle.HIGHEST_PROTOCOL) + + if "instances" in predictions: + results = instances_to_coco_json( + predictions["instances"].to(demo.cpu_device), path + ) + for result in results: + result["category_name"] = metadata.thing_classes[result["category_id"]] + result["image_name"] = result["image_id"] + + with open(out_filename + ".json", "w") as outp: + json.dump(results, outp) + else: + cv2.namedWindow(WINDOW_NAME, cv2.WINDOW_NORMAL) + cv2.imshow(WINDOW_NAME, visualized_output.get_image()[:, :, ::-1]) + if cv2.waitKey(0) == 27: + break # esc to quit + elif args.webcam: + assert args.input is None, "Cannot have both --input and --webcam!" + assert args.output is None, "output not yet supported with --webcam!" + cam = cv2.VideoCapture(0) + for vis in tqdm.tqdm(demo.run_on_video(cam)): + cv2.namedWindow(WINDOW_NAME, cv2.WINDOW_NORMAL) + cv2.imshow(WINDOW_NAME, vis) + if cv2.waitKey(1) == 27: + break # esc to quit + cam.release() + cv2.destroyAllWindows() + elif args.video_input: + video = cv2.VideoCapture(args.video_input) + width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH)) + height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT)) + frames_per_second = video.get(cv2.CAP_PROP_FPS) + num_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) + basename = os.path.basename(args.video_input) + codec, file_ext = ( + ("x264", ".mkv") if test_opencv_video_format("x264", ".mkv") else ("mp4v", ".mp4") + ) + codec, file_ext = "mp4v", ".mp4" + if codec == ".mp4v": + warnings.warn("x264 codec not available, switching to mp4v") + if args.output: + if os.path.isdir(args.output): + output_fname = os.path.join(args.output, basename) + output_fname = os.path.splitext(output_fname)[0] + file_ext + else: + output_fname = args.output + assert not os.path.isfile(output_fname), output_fname + output_file = cv2.VideoWriter( + filename=output_fname, + # some installation of opencv may not support x264 (due to its license), + # you can try other format (e.g. MPEG) + fourcc=cv2.VideoWriter_fourcc(*codec), + fps=float(frames_per_second), + frameSize=(width, height), + isColor=True, + ) + # i = 0 + assert os.path.isfile(args.video_input) + for vis_frame, predictions in tqdm.tqdm(demo.run_on_video(video), total=num_frames): + if args.output: + output_file.write(vis_frame) + + # import pickle + # with open(output_fname + "." + str(i) + ".pkl", "wb") as outp: + # pickle.dump(predictions, outp, pickle.HIGHEST_PROTOCOL) + # i += 1 + else: + cv2.namedWindow(basename, cv2.WINDOW_NORMAL) + cv2.imshow(basename, vis_frame) + if cv2.waitKey(1) == 27: + break # esc to quit + video.release() + if args.output: + output_file.release() + else: + cv2.destroyAllWindows() diff --git a/approach/ovod/APE/demo/examples/013_438973263.jpg b/approach/ovod/APE/demo/examples/013_438973263.jpg new file mode 100644 index 0000000000000000000000000000000000000000..730a1d7d439979d9e038ba4a2f35ebaa330aeb6a Binary files /dev/null and b/approach/ovod/APE/demo/examples/013_438973263.jpg differ diff --git a/approach/ovod/APE/demo/examples/094_56726435.jpg b/approach/ovod/APE/demo/examples/094_56726435.jpg new file mode 100644 index 0000000000000000000000000000000000000000..62ba4abd7d4f4d301a02de3b89394611b9066c55 Binary files /dev/null and b/approach/ovod/APE/demo/examples/094_56726435.jpg differ diff --git a/approach/ovod/APE/demo/examples/199_3946193540.jpg b/approach/ovod/APE/demo/examples/199_3946193540.jpg new file mode 100644 index 0000000000000000000000000000000000000000..56bcb87ef1ae33f7c29233be03a677787d9126cc Binary files /dev/null and b/approach/ovod/APE/demo/examples/199_3946193540.jpg differ diff --git a/approach/ovod/APE/demo/examples/Pisa.jpg b/approach/ovod/APE/demo/examples/Pisa.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bf6822bd77e1a578d9bcb0eb8771b53670ec269d Binary files /dev/null and b/approach/ovod/APE/demo/examples/Pisa.jpg differ diff --git a/approach/ovod/APE/demo/examples/TheGreatWall.jpg b/approach/ovod/APE/demo/examples/TheGreatWall.jpg new file mode 100644 index 0000000000000000000000000000000000000000..10adbbdce575bdd21fc23ede19585eac3d044034 Binary files /dev/null and b/approach/ovod/APE/demo/examples/TheGreatWall.jpg differ diff --git a/approach/ovod/APE/scripts/eval_all_A.sh b/approach/ovod/APE/scripts/eval_all_A.sh new file mode 100644 index 0000000000000000000000000000000000000000..142da36b39348b3a2cdc17943ccef74d863a6c8f --- /dev/null +++ b/approach/ovod/APE/scripts/eval_all_A.sh @@ -0,0 +1,38 @@ +#!/bin/bash -e + +set -x +set -e + + +init_checkpoint="output2/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VG/ape_deta/ape_deta_vitl_eva02_lsj_cp_720k_20230504_002019/model_final.pth" + +num_gpus=7 +output_dir="./output2/eval_all/A/" + + +config_files=( + "configs/LVISCOCOCOCOSTUFF_O365_OID_VG/ape_deta/ape_deta_vitl_eva02_lsj1024_cp_720k.py" + "configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024_cp_12ep.py" + "configs/COCO_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py" + "configs/ODinW_Detection/ape_deta/ape_deta_vitl_eva02_lsj1024_13.py" + "configs/ODinW_Detection/ape_deta/ape_deta_vitl_eva02_lsj1024_35.py" + "configs/SegInW_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py" + "configs/Roboflow_Detection/ape_deta/ape_deta_vitl_eva02_lsj1024.py" + "configs/ADE20k_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py" + "configs/ADE20k_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py" + "configs/ADE20kFull_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py" + "configs/BDD10k_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py" + "configs/BDD10k_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py" + "configs/Cityscapes_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py" + "configs/PascalContext459_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py" + "configs/PascalContext59_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py" + "configs/PascalVOC20_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py" + "configs/D3_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py" +) + +for config_file in ${config_files[@]} +do + echo "==============================================================================================" + echo ${config_file} + python3.9 tools/train_net.py --eval-only --dist-url=tcp://127.0.0.1:49193 --config-file ${config_file} --num-gpus ${num_gpus} train.output_dir=${output_dir}/${config_file}/"`date +'%Y%m%d_%H%M%S'`" train.init_checkpoint=${init_checkpoint} +done diff --git a/approach/ovod/APE/scripts/eval_all_B.sh b/approach/ovod/APE/scripts/eval_all_B.sh new file mode 100644 index 0000000000000000000000000000000000000000..e19b1c9e9842145f67e6e57831f6cde96a821fad --- /dev/null +++ b/approach/ovod/APE/scripts/eval_all_B.sh @@ -0,0 +1,40 @@ +#!/bin/bash -e + +set -x +set -e + + +init_checkpoint="output2/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj_cp_1080k_20230702_225418/model_final.pth" + +num_gpus=7 +output_dir="./output2/eval_all/B/" + +kwargs="" + + +config_files=( + "configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_1080k.py" + "configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_12ep.py" + "configs/COCO_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/ODinW_Detection/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_13.py" + "configs/ODinW_Detection/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_35.py" + "configs/SegInW_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/Roboflow_Detection/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/ADE20k_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/ADE20k_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/ADE20kFull_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/BDD10k_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/BDD10k_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/Cityscapes_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/PascalContext459_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/PascalContext59_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/PascalVOC20_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/D3_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" +) + +for config_file in ${config_files[@]} +do + echo "==============================================================================================" + echo ${config_file} + python3.9 tools/train_net.py --eval-only --dist-url=tcp://127.0.0.1:49193 --config-file ${config_file} --num-gpus ${num_gpus} train.output_dir=${output_dir}/${config_file}/"`date +'%Y%m%d_%H%M%S'`" train.init_checkpoint=${init_checkpoint} ${kwargs} +done diff --git a/approach/ovod/APE/scripts/eval_all_C.sh b/approach/ovod/APE/scripts/eval_all_C.sh new file mode 100644 index 0000000000000000000000000000000000000000..32e4f1de781e39345255b81d4ed2b3547769bb10 --- /dev/null +++ b/approach/ovod/APE/scripts/eval_all_C.sh @@ -0,0 +1,38 @@ +#!/bin/bash -e + +set -x +set -e + + +init_checkpoint="output2/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj_cp_1080k_20230702_210950/model_final.pth" + +num_gpus=7 +output_dir="output2/eval_all/C/" + + +config_files=( + "configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_1080k.py" + "configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_12ep.py" + "configs/COCO_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/ODinW_Detection/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_13.py" + "configs/ODinW_Detection/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_35.py" + "configs/SegInW_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/Roboflow_Detection/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/ADE20k_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/ADE20k_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/ADE20kFull_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/BDD10k_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/BDD10k_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/Cityscapes_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/PascalContext459_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/PascalContext59_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/PascalVOC20_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" + "configs/D3_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py" +) + +for config_file in ${config_files[@]} +do + echo "==============================================================================================" + echo ${config_file} + python3.9 tools/train_net.py --eval-only --dist-url=tcp://127.0.0.1:49194 --config-file ${config_file} --num-gpus ${num_gpus} train.output_dir=${output_dir}/${config_file}/"`date +'%Y%m%d_%H%M%S'`" train.init_checkpoint=${init_checkpoint} +done diff --git a/approach/ovod/APE/scripts/eval_all_D.sh b/approach/ovod/APE/scripts/eval_all_D.sh new file mode 100644 index 0000000000000000000000000000000000000000..3c5d5718a95b9cba72050e3ea057579444d731c4 --- /dev/null +++ b/approach/ovod/APE/scripts/eval_all_D.sh @@ -0,0 +1,39 @@ +#!/bin/bash -e + +set -x +set -e + +kwargs="model.model_vision.transformer.proposal_ambiguous=1" +init_checkpoint="output2/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k_mdl_20230829_162438/model_final.pth" +output_dir="output2/eval_all/D_20230829_162438/" + + +num_gpus=7 + + +config_files=( + "configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k.py" + "configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_12ep.py" + "configs/COCO_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py" + "configs/ODinW_Detection/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_13.py" + "configs/ODinW_Detection/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_35.py" + "configs/SegInW_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py" + "configs/Roboflow_Detection/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py" + "configs/ADE20k_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py" + "configs/ADE20k_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py" + "configs/ADE20kFull_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py" + "configs/BDD10k_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py" + "configs/BDD10k_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py" + "configs/Cityscapes_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py" + "configs/PascalContext459_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py" + "configs/PascalContext59_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py" + "configs/PascalVOC20_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py" + "configs/D3_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py" +) + +for config_file in ${config_files[@]} +do + echo "==============================================================================================" + echo ${config_file} + python3.9 tools/train_net.py --eval-only --dist-url=tcp://127.0.0.1:49193 --config-file ${config_file} --num-gpus ${num_gpus} train.output_dir=${output_dir}/${config_file}/"`date +'%Y%m%d_%H%M%S'`" train.init_checkpoint=${init_checkpoint} ${kwargs} +done diff --git a/approach/ovod/APE/scripts/eval_computational_cost.sh b/approach/ovod/APE/scripts/eval_computational_cost.sh new file mode 100644 index 0000000000000000000000000000000000000000..aeb6141a152823ff56f95fc8e3ac1f280478d624 --- /dev/null +++ b/approach/ovod/APE/scripts/eval_computational_cost.sh @@ -0,0 +1,78 @@ +#!/bin/bash -e + +set -x +set -e + + +num_gpus=8 +output_dir="./output2/eval_computational_cost/" + + +# REC R50 +config_files=( + #"configs/REFCOCO_VisualGrounding/something/something_r50_12ep.py" # bs=16 for training + #"configs/REFCOCO_VisualGrounding/something/something_r50_vlf_12ep.py" # bs=16 for training +) + +for config_file in ${config_files[@]} +do + echo "==============================================================================================" + echo ${config_file} + #python3.9 tools/train_net.py --dist-url=tcp://127.0.0.1:49193 --config-file ${config_file} --num-gpus ${num_gpus} train.output_dir=${output_dir}/${config_file}/"`date +'%Y%m%d_%H%M%S'`" model.model_vision.segm_type="" + #python3.9 tools/train_net.py --eval-only --dist-url=tcp://127.0.0.1:49193 --config-file ${config_file} --num-gpus ${num_gpus} train.output_dir=${output_dir}/${config_file}/"`date +'%Y%m%d_%H%M%S'`" model.model_vision.segm_type="" model.model_vision.num_classes=1 model.model_vision.select_box_nums_for_evaluation=1 model.model_vision.test_score_thresh=0.5 + #python3.9 tools/train_net.py --eval-only --dist-url=tcp://127.0.0.1:49193 --config-file ${config_file} --num-gpus ${num_gpus} train.output_dir=${output_dir}/${config_file}/"`date +'%Y%m%d_%H%M%S'`" model.model_vision.segm_type="" model.model_vision.num_classes=128 model.model_vision.select_box_nums_for_evaluation=128 model.model_vision.test_score_thresh=0.5 + #python3.9 tools/train_net.py --eval-only --dist-url=tcp://127.0.0.1:49193 --config-file ${config_file} --num-gpus ${num_gpus} train.output_dir=${output_dir}/${config_file}/"`date +'%Y%m%d_%H%M%S'`" model.model_vision.segm_type="" model.model_vision.num_classes=1280 model.model_vision.select_box_nums_for_evaluation=1280 model.model_vision.test_score_thresh=0.5 +done + + +# REC ViT-L +config_files=( + #"configs/REFCOCO_VisualGrounding/something/something_vitl_eva02_clip_lsj1024_12ep.py" # bs=8 for training + #"configs/REFCOCO_VisualGrounding/something/something_vitl_eva02_clip_vlf_lsj1024_12ep.py" # bs=8 for training +) + +kwargs="dataloader.train.total_batch_size=8 model.model_vision.segm_type=\"\" model.model_vision.test_score_thresh=0.5 model.model_language.max_batch_size=128 model.model_vision.neck.in_features=[\"p3\",\"p4\",\"p5\",\"p6\"] model.model_vision.neck.num_outs=5 model.model_vision.transformer.num_feature_levels=5 model.model_vision.backbone.scale_factors=[2.0,1.0,0.5]" +for config_file in ${config_files[@]} +do + echo "==============================================================================================" + echo ${config_file} + #python3.9 tools/train_net.py --dist-url=tcp://127.0.0.1:49193 --config-file ${config_file} --num-gpus ${num_gpus} train.output_dir=${output_dir}/${config_file}/"`date +'%Y%m%d_%H%M%S'`" ${kwargs} + #python3.9 tools/train_net.py --eval-only --dist-url=tcp://127.0.0.1:49193 --config-file ${config_file} --num-gpus ${num_gpus} train.output_dir=${output_dir}/${config_file}/"`date +'%Y%m%d_%H%M%S'`" ${kwargs} model.model_vision.num_classes=1 model.model_vision.select_box_nums_for_evaluation=1 + #python3.9 tools/train_net.py --eval-only --dist-url=tcp://127.0.0.1:49193 --config-file ${config_file} --num-gpus ${num_gpus} train.output_dir=${output_dir}/${config_file}/"`date +'%Y%m%d_%H%M%S'`" ${kwargs} model.model_vision.num_classes=128 model.model_vision.select_box_nums_for_evaluation=128 + #python3.9 tools/train_net.py --eval-only --dist-url=tcp://127.0.0.1:49193 --config-file ${config_file} --num-gpus ${num_gpus} train.output_dir=${output_dir}/${config_file}/"`date +'%Y%m%d_%H%M%S'`" ${kwargs} model.model_vision.num_classes=1280 model.model_vision.select_box_nums_for_evaluation=1280 +done + + +# OVD R50 +config_files=( + #"configs/COCO_InstanceSegmentation/something/something_r50_12ep.py" # bs=16 for training + #"configs/LVIS_InstanceSegmentation/something/something_r50_24ep.py" # bs=16 for training + #"configs/COCO_InstanceSegmentation/something/something_r50_vlf_12ep.py" # bs=16 for training + #"configs/LVIS_InstanceSegmentation/something/something_r50_vlf_24ep.py" # bs=16 for training +) + +for config_file in ${config_files[@]} +do + echo "==============================================================================================" + echo ${config_file} + #python3.9 tools/train_net.py --dist-url=tcp://127.0.0.1:49193 --config-file ${config_file} --num-gpus ${num_gpus} train.output_dir=${output_dir}/${config_file}/"`date +'%Y%m%d_%H%M%S'`" model.model_vision.segm_type="" + #python3.9 tools/train_net.py --eval-only --dist-url=tcp://127.0.0.1:49193 --config-file ${config_file} --num-gpus ${num_gpus} train.output_dir=${output_dir}/${config_file}/"`date +'%Y%m%d_%H%M%S'`" model.model_vision.segm_type="" model.model_vision.test_score_thresh=0.5 +done + +# OVD ViT-L +config_files=( + #"configs/COCO_InstanceSegmentation/something/something_vitl_eva02_clip_lsj1024_cp_12ep.py" # bs=8 for training + #"configs/LVIS_InstanceSegmentation/something/something_vitl_eva02_clip_lsj1024_cp_24ep.py" # bs=8 for training + #"configs/COCO_InstanceSegmentation/something/something_vitl_eva02_clip_vlf_lsj1024_cp_12ep.py" # bs=8 for training + "configs/LVIS_InstanceSegmentation/something/something_vitl_eva02_clip_vlf_lsj1024_cp_24ep.py" # bs=8 for training +) + +kwargs="dataloader.train.total_batch_size=8 model.model_vision.segm_type=\"\" model.model_vision.test_score_thresh=0.5 model.model_language.max_batch_size=128 model.model_vision.neck.in_features=[\"p3\",\"p4\",\"p5\",\"p6\"] model.model_vision.neck.num_outs=5 model.model_vision.transformer.num_feature_levels=5 model.model_vision.backbone.scale_factors=[2.0,1.0,0.5]" + +for config_file in ${config_files[@]} +do + echo "==============================================================================================" + echo ${config_file} + #python3.9 tools/train_net.py --dist-url=tcp://127.0.0.1:49193 --config-file ${config_file} --num-gpus ${num_gpus} train.output_dir=${output_dir}/${config_file}/"`date +'%Y%m%d_%H%M%S'`" ${kwargs} + python3.9 tools/train_net.py --eval-only --dist-url=tcp://127.0.0.1:49193 --config-file ${config_file} --num-gpus ${num_gpus} train.output_dir=${output_dir}/${config_file}/"`date +'%Y%m%d_%H%M%S'`" ${kwargs} +done diff --git a/approach/ovod/APE/tools/analyze_model.py b/approach/ovod/APE/tools/analyze_model.py new file mode 100644 index 0000000000000000000000000000000000000000..4e7e31e02ba2ec2c1ff5fc657b660e9ec1b0b504 --- /dev/null +++ b/approach/ovod/APE/tools/analyze_model.py @@ -0,0 +1,161 @@ +import logging +from collections import Counter + +import numpy as np +import tqdm + +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import CfgNode, LazyConfig, get_cfg, instantiate +from detectron2.data import build_detection_test_loader +from detectron2.engine import default_argument_parser +from detectron2.modeling import build_model +from detectron2.projects.deeplab import add_deeplab_config +from detectron2.projects.panoptic_deeplab import add_panoptic_deeplab_config +from detectron2.utils.analysis import ( + FlopCountAnalysis, + activation_count_operators, + parameter_count_table, +) +from detectron2.utils.logger import setup_logger +from fvcore.nn import flop_count_table # can also try flop_count_str + +logger = logging.getLogger("detectron2") + + +def setup(args): + if args.config_file.endswith(".yaml"): + cfg = get_cfg() + add_deeplab_config(cfg) + add_panoptic_deeplab_config(cfg) + cfg.merge_from_file(args.config_file) + cfg.DATALOADER.NUM_WORKERS = 0 + cfg.merge_from_list(args.opts) + cfg.freeze() + else: + cfg = LazyConfig.load(args.config_file) + cfg = LazyConfig.apply_overrides(cfg, args.opts) + setup_logger(name="fvcore") + setup_logger() + return cfg + + +def do_flop(cfg): + if isinstance(cfg, CfgNode): + data_loader = build_detection_test_loader(cfg, cfg.DATASETS.TEST[0]) + model = build_model(cfg) + DetectionCheckpointer(model).load(cfg.MODEL.WEIGHTS) + else: + data_loader = instantiate(cfg.dataloader.test) + model = instantiate(cfg.model) + model.to(cfg.train.device) + DetectionCheckpointer(model).load(cfg.train.init_checkpoint) + model.eval() + + counts = Counter() + total_flops = [] + for idx, data in zip(tqdm.trange(args.num_inputs), data_loader): # noqa + flops = FlopCountAnalysis(model, data) + if idx > 0: + flops.unsupported_ops_warnings(False).uncalled_modules_warnings(False) + counts += flops.by_operator() + total_flops.append(flops.total()) + + logger.info("Flops table computed from only one input sample:\n" + flop_count_table(flops)) + logger.info( + "Average GFlops for each type of operators:\n" + + str([(k, v / (idx + 1) / 1e9) for k, v in counts.items()]) + ) + logger.info( + "Total GFlops: {:.1f}±{:.1f}".format(np.mean(total_flops) / 1e9, np.std(total_flops) / 1e9) + ) + + +def do_activation(cfg): + if isinstance(cfg, CfgNode): + data_loader = build_detection_test_loader(cfg, cfg.DATASETS.TEST[0]) + model = build_model(cfg) + DetectionCheckpointer(model).load(cfg.MODEL.WEIGHTS) + else: + data_loader = instantiate(cfg.dataloader.test) + model = instantiate(cfg.model) + model.to(cfg.train.device) + DetectionCheckpointer(model).load(cfg.train.init_checkpoint) + model.eval() + + counts = Counter() + total_activations = [] + for idx, data in zip(tqdm.trange(args.num_inputs), data_loader): # noqa + count = activation_count_operators(model, data) + counts += count + total_activations.append(sum(count.values())) + logger.info( + "(Million) Activations for Each Type of Operators:\n" + + str([(k, v / idx) for k, v in counts.items()]) + ) + logger.info( + "Total (Million) Activations: {}±{}".format( + np.mean(total_activations), np.std(total_activations) + ) + ) + + +def do_parameter(cfg): + if isinstance(cfg, CfgNode): + model = build_model(cfg) + else: + model = instantiate(cfg.model) + logger.info("Parameter Count:\n" + parameter_count_table(model, max_depth=5)) + + +def do_structure(cfg): + if isinstance(cfg, CfgNode): + model = build_model(cfg) + else: + model = instantiate(cfg.model) + logger.info("Model Structure:\n" + str(model)) + + +if __name__ == "__main__": + parser = default_argument_parser( + epilog=""" +Examples: + +To show parameters of a model: +$ ./analyze_model.py --tasks parameter \\ + --config-file ../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml + +Flops and activations are data-dependent, therefore inputs and model weights +are needed to count them: + +$ ./analyze_model.py --num-inputs 100 --tasks flop \\ + --config-file ../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \\ + MODEL.WEIGHTS /path/to/model.pkl +""" + ) + parser.add_argument( + "--tasks", + choices=["flop", "activation", "parameter", "structure"], + required=True, + nargs="+", + ) + parser.add_argument( + "-n", + "--num-inputs", + default=100, + type=int, + help="number of inputs used to compute statistics for flops/activations, " + "both are data dependent.", + ) + args = parser.parse_args() + assert not args.eval_only + assert args.num_gpus == 1 + + cfg = setup(args) + + for task in args.tasks: + { + "flop": do_flop, + "activation": do_activation, + "parameter": do_parameter, + "structure": do_structure, + }[task](cfg) diff --git a/approach/ovod/APE/tools/eva_interpolate_patch_14to16.py b/approach/ovod/APE/tools/eva_interpolate_patch_14to16.py new file mode 100644 index 0000000000000000000000000000000000000000..07c915c66ffdc7cb10e27ad5b99b2e82939d12cf --- /dev/null +++ b/approach/ovod/APE/tools/eva_interpolate_patch_14to16.py @@ -0,0 +1,113 @@ +# -------------------------------------------------------- +# EVA: Exploring the Limits of Masked Visual Representation Learning at Scale (https://arxiv.org/abs/2211.07636) +# Github source: https://github.com/baaivision/EVA +# Copyright (c) 2022 Beijing Academy of Artificial Intelligence (BAAI) +# Licensed under The MIT License [see LICENSE for details] +# By Yuxin Fang +# Based on timm, DINO, DeiT and BEiT codebases +# https://github.com/rwightman/pytorch-image-models/tree/master/timm +# https://github.com/facebookresearch/deit +# https://github.com/facebookresearch/dino +# https://github.com/microsoft/unilm/tree/master/beit +# --------------------------------------------------------' + +import argparse + +import torch + + +def interpolate_pos_embed(checkpoint_model, new_size=16, image_size=224): + if "pos_embed" in checkpoint_model: + pos_embed_checkpoint = checkpoint_model["pos_embed"] + print("pos_embed_checkpoint", pos_embed_checkpoint.size(), pos_embed_checkpoint.dtype) + embedding_size = pos_embed_checkpoint.shape[-1] + num_patches = int(image_size / new_size) ** 2 + num_extra_tokens = 1 + # height (== width) for the checkpoint position embedding + orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5) + # height (== width) for the new position embedding + new_size = int(num_patches**0.5) + # class_token and dist_token are kept unchanged + if orig_size != new_size: + print( + "Position interpolate from %dx%d to %dx%d" + % (orig_size, orig_size, new_size, new_size) + ) + extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens] + # only the position tokens are interpolated + pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:] + pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute( + 0, 3, 1, 2 + ) + ori_dtype = pos_tokens.dtype + pos_tokens = pos_tokens.to(torch.float32) + pos_tokens = torch.nn.functional.interpolate( + pos_tokens, size=(new_size, new_size), mode="bicubic", align_corners=False + ) + pos_tokens = pos_tokens.to(ori_dtype) + pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2) + new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1) + checkpoint_model["pos_embed"] = new_pos_embed + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="interpolate patch_embed kernel") + parser.add_argument( + "--input", + default="/path/to/eva_psz14.pt", + type=str, + metavar="PATH", + required=True, + help="path to input EVA checkpoint with patch_embed kernel_size=14x14", + ) + parser.add_argument( + "--output", + default="/path/to/eva_psz14to16.pt", + type=str, + metavar="PATH", + required=True, + help="path to output EVA checkpoint with patch_embed kernel_size=16x16", + ) + parser.add_argument("--image_size", type=int, required=True) + args = parser.parse_args() + + checkpoint = torch.load(args.input, map_location=torch.device("cpu")) + + # interpolate patch_embed + if "model" in checkpoint: + patch_embed = checkpoint["model"]["patch_embed.proj.weight"] + else: + patch_embed = checkpoint["visual.patch_embed.proj.weight"] + C_o, C_in, H, W = patch_embed.shape + patch_embed = torch.nn.functional.interpolate( + patch_embed.float(), size=(16, 16), mode="bicubic", align_corners=False + ) + if "model" in checkpoint: + checkpoint["model"]["patch_embed.proj.weight"] = patch_embed + else: + checkpoint["visual.patch_embed.proj.weight"] = patch_embed + + # interpolate pos_embed too + if "model" in checkpoint: + interpolate_pos_embed(checkpoint["model"], new_size=16, image_size=args.image_size) + else: + checkpoint["pos_embed"] = checkpoint["visual.pos_embed"] + interpolate_pos_embed(checkpoint, new_size=16, image_size=args.image_size) + checkpoint["visual.pos_embed"] = checkpoint.pop("pos_embed") + + print("======== new state_dict ========") + if "model" in checkpoint: + for k, v in list(checkpoint["model"].items()): + print(k, " ", v.shape) + else: + for k, v in list(checkpoint.items()): + if k.startswith("text.") or k == "logit_scale": + checkpoint.pop(k) + print("pop", k, " ", v.shape) + if k.startswith("visual."): + checkpoint["backbone.net." + k[7:]] = checkpoint.pop(k) + print("rename", k, " ", "backbone.net." + k[7:]) + for k, v in list(checkpoint.items()): + print(k, " ", v.shape) + + torch.save(checkpoint, args.output) diff --git a/approach/ovod/APE/tools/train_net.py b/approach/ovod/APE/tools/train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..3fc0a49f3a3336471934638fc5ecfa40a44b8189 --- /dev/null +++ b/approach/ovod/APE/tools/train_net.py @@ -0,0 +1,663 @@ +""" +Training script using the new "LazyConfig" python config files. + +This scripts reads a given python config file and runs the training or evaluation. +It can be used to train any models or dataset as long as they can be +instantiated by the recursive construction defined in the given config file. + +Besides lazy construction of models, dataloader, etc., this scripts expects a +few common configuration parameters currently defined in "configs/common/train.py". +To add more complicated training logic, you can easily add other configs +in the config file and implement a new train_net.py to handle them. +""" +import logging +import os +import random +import sys +import time +from collections import abc +from contextlib import nullcontext + +import torch +from torch.nn.parallel import DataParallel, DistributedDataParallel + +import ape +from ape.checkpoint import DetectionCheckpointer +from ape.engine import SimpleTrainer +from ape.evaluation import inference_on_dataset +from detectron2.config import LazyConfig, instantiate +from detectron2.engine import default_argument_parser # SimpleTrainer, +from detectron2.engine import default_setup, hooks, launch +from detectron2.engine.defaults import create_ddp_model +from detectron2.evaluation import print_csv_format +from detectron2.utils import comm +from detectron2.utils.events import ( + CommonMetricPrinter, + JSONWriter, + TensorboardXWriter, + get_event_storage, +) +from detectron2.utils.file_io import PathManager +from detectron2.utils.logger import setup_logger +from detrex.modeling import ema +from detrex.utils import WandbWriter + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))) + +logger = logging.getLogger("ape") + + +class Trainer(SimpleTrainer): + """ + We've combine Simple and AMP Trainer together. + """ + + def __init__( + self, + model, + dataloader, + optimizer, + amp=False, + clip_grad_params=None, + grad_scaler=None, + iter_size=1, + iter_loop=True, + dataset_ratio=None, + save_memory=False, + ): + super().__init__(model=model, data_loader=dataloader, optimizer=optimizer) + + unsupported = "AMPTrainer does not support single-process multi-device training!" + if isinstance(model, DistributedDataParallel): + assert not (model.device_ids and len(model.device_ids) > 1), unsupported + assert not isinstance(model, DataParallel), unsupported + + if amp: + if grad_scaler is None: + from torch.cuda.amp import GradScaler + + grad_scaler = GradScaler() + self.grad_scaler = grad_scaler + + self.amp = amp + + self.clip_grad_params = clip_grad_params + + if isinstance(model, DistributedDataParallel): + if hasattr(model.module, "model_vision"): + self.dataset_names = model.module.model_vision.dataset_names + else: + self.dataset_names = ["unknown"] + else: + if hasattr(model, "model_vision"): + self.dataset_names = model.model_vision.dataset_names + else: + self.dataset_names = ["unknown"] + self.dataset_image_counts = { + k: torch.tensor(0, dtype=torch.float).to(comm.get_local_rank()) + for k in self.dataset_names + } + self.dataset_object_counts = { + k: torch.tensor(0, dtype=torch.float).to(comm.get_local_rank()) + for k in self.dataset_names + } + + self.iter_size = iter_size + self.iter_loop = iter_loop + self.dataset_ratio = dataset_ratio + self.save_memory = save_memory + + def run_step(self): + if self.iter_size > 1: + if self.iter_loop: + return self.run_step_accumulate_iter_loop() + else: + return self.run_step_accumulate() + """ + Implement the standard training logic described above. + """ + assert self.model.training, "[Trainer] model was changed to eval mode!" + assert torch.cuda.is_available(), "[Trainer] CUDA is required for AMP training!" + from torch.cuda.amp import autocast + + start = time.perf_counter() + """ + If you want to do something with the data, you can wrap the dataloader. + """ + while True: + data = next(self._data_loader_iter) + if all([len(x["instances"]) > 0 for x in data]): + break + data_time = time.perf_counter() - start + + for d in data: + if d.get("dataloader_id", None) is not None: + d["dataset_id"] = d["dataloader_id"] + self.dataset_image_counts[self.dataset_names[d.get("dataset_id", 0)]] += 1 + self.dataset_object_counts[self.dataset_names[d.get("dataset_id", 0)]] += len( + d.get("instances", []) + ) + dataset_image_counts = {f"count_image/{k}": v for k, v in self.dataset_image_counts.items()} + dataset_object_counts = { + f"count_object/{k}": v for k, v in self.dataset_object_counts.items() + } + if self.async_write_metrics: + self.concurrent_executor.submit( + self._write_metrics_common, dataset_image_counts, iter=self.iter + ) + self.concurrent_executor.submit( + self._write_metrics_common, dataset_object_counts, iter=self.iter + ) + else: + self._write_metrics_common(dataset_image_counts) + self._write_metrics_common(dataset_object_counts) + + """ + If you want to do something with the losses, you can wrap the model. + """ + with autocast(enabled=self.amp): + loss_dict = self.model(data) + if isinstance(loss_dict, torch.Tensor): + losses = loss_dict + loss_dict = {"total_loss": loss_dict} + else: + losses = sum(loss_dict.values()) + + """ + If you need to accumulate gradients or do something similar, you can + wrap the optimizer with your custom `zero_grad()` method. + """ + self.optimizer.zero_grad() + + if self.amp: + self.grad_scaler.scale(losses).backward() + if self.clip_grad_params is not None: + self.grad_scaler.unscale_(self.optimizer) + self.clip_grads(self.model.parameters()) + self.grad_scaler.step(self.optimizer) + self.grad_scaler.update() + else: + losses.backward() + if self.clip_grad_params is not None: + self.clip_grads(self.model.parameters()) + self.optimizer.step() + + if self.async_write_metrics: + self.concurrent_executor.submit( + self._write_metrics, loss_dict, data_time, iter=self.iter + ) + else: + self._write_metrics(loss_dict, data_time) + + if self.save_memory: + del losses + del loss_dict + torch.cuda.empty_cache() + + def run_step_accumulate(self): + """ + Implement the standard training logic described above. + """ + assert self.model.training, "[Trainer] model was changed to eval mode!" + assert torch.cuda.is_available(), "[Trainer] CUDA is required for AMP training!" + from torch.cuda.amp import autocast + + start = time.perf_counter() + """ + If you want to do something with the data, you can wrap the dataloader. + """ + while True: + data = next(self._data_loader_iter) + if all([len(x["instances"]) > 0 for x in data]): + break + data_time = time.perf_counter() - start + + for d in data: + if d.get("dataloader_id", None) is not None: + d["dataset_id"] = d["dataloader_id"] + self.dataset_image_counts[self.dataset_names[d.get("dataset_id", 0)]] += 1 + self.dataset_object_counts[self.dataset_names[d.get("dataset_id", 0)]] += len( + d.get("instances", []) + ) + dataset_image_counts = {f"count_image/{k}": v for k, v in self.dataset_image_counts.items()} + dataset_object_counts = { + f"count_object/{k}": v for k, v in self.dataset_object_counts.items() + } + if self.async_write_metrics: + self.concurrent_executor.submit( + self._write_metrics_common, dataset_image_counts, iter=self.iter + ) + self.concurrent_executor.submit( + self._write_metrics_common, dataset_object_counts, iter=self.iter + ) + else: + self._write_metrics_common(dataset_image_counts) + self._write_metrics_common(dataset_object_counts) + + sync_context = self.model.no_sync if (self.iter + 1) % self.iter_size != 0 else nullcontext + """ + If you want to do something with the losses, you can wrap the model. + """ + with sync_context(): + with autocast(enabled=self.amp): + loss_dict = self.model(data) + + if isinstance(loss_dict, torch.Tensor): + losses = loss_dict + loss_dict = {"total_loss": loss_dict} + else: + losses = sum(loss_dict.values()) + + """ + If you need to accumulate gradients or do something similar, you can + wrap the optimizer with your custom `zero_grad()` method. + """ + if self.iter == self.start_iter: + self.optimizer.zero_grad() + + if self.iter_size > 1: + losses = losses / self.iter_size + + if self.amp: + self.grad_scaler.scale(losses).backward() + if (self.iter + 1) % self.iter_size == 0: + if self.clip_grad_params is not None: + self.grad_scaler.unscale_(self.optimizer) + self.clip_grads(self.model.parameters()) + self.grad_scaler.step(self.optimizer) + self.grad_scaler.update() + self.optimizer.zero_grad() + else: + losses.backward() + if (self.iter + 1) % self.iter_size == 0: + if self.clip_grad_params is not None: + self.clip_grads(self.model.parameters()) + self.optimizer.step() + self.optimizer.zero_grad() + + if self.async_write_metrics: + self.concurrent_executor.submit( + self._write_metrics, loss_dict, data_time, iter=self.iter + ) + else: + self._write_metrics(loss_dict, data_time) + + if self.save_memory: + del losses + del loss_dict + torch.cuda.empty_cache() + + def run_step_accumulate_iter_loop(self): + """ + Implement the standard training logic described above. + """ + assert self.model.training, "[Trainer] model was changed to eval mode!" + assert torch.cuda.is_available(), "[Trainer] CUDA is required for AMP training!" + from torch.cuda.amp import autocast + + self.optimizer.zero_grad() + for inner_iter in range(self.iter_size): + start = time.perf_counter() + """ + If you want to do something with the data, you can wrap the dataloader. + """ + while True: + data = next(self._data_loader_iter) + if all([len(x["instances"]) > 0 for x in data]): + break + data_time = time.perf_counter() - start + + for d in data: + if d.get("dataloader_id", None) is not None: + d["dataset_id"] = d["dataloader_id"] + self.dataset_image_counts[self.dataset_names[d.get("dataset_id", 0)]] += 1 + self.dataset_object_counts[self.dataset_names[d.get("dataset_id", 0)]] += len( + d.get("instances", []) + ) + dataset_image_counts = { + f"count_image/{k}": v for k, v in self.dataset_image_counts.items() + } + dataset_object_counts = { + f"count_object/{k}": v for k, v in self.dataset_object_counts.items() + } + if self.async_write_metrics: + self.concurrent_executor.submit( + self._write_metrics_common, dataset_image_counts, iter=self.iter + ) + self.concurrent_executor.submit( + self._write_metrics_common, dataset_object_counts, iter=self.iter + ) + else: + self._write_metrics_common(dataset_image_counts) + self._write_metrics_common(dataset_object_counts) + + sync_context = self.model.no_sync if inner_iter != self.iter_size - 1 else nullcontext + """ + If you want to do something with the losses, you can wrap the model. + """ + with sync_context(): + with autocast(enabled=self.amp): + loss_dict = self.model(data) + + if isinstance(loss_dict, torch.Tensor): + losses = loss_dict + loss_dict = {"total_loss": loss_dict} + else: + losses = sum(loss_dict.values()) + + """ + If you need to accumulate gradients or do something similar, you can + wrap the optimizer with your custom `zero_grad()` method. + """ + + losses = losses / self.iter_size + + if self.amp: + self.grad_scaler.scale(losses).backward() + else: + losses.backward() + + if self.async_write_metrics: + self.concurrent_executor.submit( + self._write_metrics, loss_dict, data_time, iter=self.iter + ) + else: + self._write_metrics(loss_dict, data_time) + + if self.save_memory: + del losses + del loss_dict + torch.cuda.empty_cache() + + if self.amp: + if self.clip_grad_params is not None: + self.grad_scaler.unscale_(self.optimizer) + self.clip_grads(self.model.parameters()) + self.grad_scaler.step(self.optimizer) + self.grad_scaler.update() + else: + if self.clip_grad_params is not None: + self.clip_grads(self.model.parameters()) + self.optimizer.step() + + def clip_grads(self, params): + params = list(filter(lambda p: p.requires_grad and p.grad is not None, params)) + if len(params) > 0: + return torch.nn.utils.clip_grad_norm_( + parameters=params, + **self.clip_grad_params, + ) + + def state_dict(self): + ret = super().state_dict() + if self.grad_scaler and self.amp: + ret["grad_scaler"] = self.grad_scaler.state_dict() + return ret + + def load_state_dict(self, state_dict): + super().load_state_dict(state_dict) + if self.grad_scaler and self.amp: + self.grad_scaler.load_state_dict(state_dict["grad_scaler"]) + + @property + def _data_loader_iter(self): + if isinstance(self.data_loader, abc.MutableSequence): + if self._data_loader_iter_obj is None: + self._data_loader_iter_obj = [iter(x) for x in self.data_loader] + self._data_loader_indices = [] + + if len(self._data_loader_indices) == 0: + self._data_loader_indices = random.choices( + list(range(len(self.data_loader))), weights=self.dataset_ratio, k=10000 + ) + idx = self._data_loader_indices.pop() + return self._data_loader_iter_obj[idx] + + if self._data_loader_iter_obj is None: + self._data_loader_iter_obj = iter(self.data_loader) + return self._data_loader_iter_obj + + +def do_test(cfg, model, eval_only=False): + logger = logging.getLogger("ape") + if "evaluator" in cfg.dataloader: + if isinstance(model, DistributedDataParallel): + if hasattr(model.module, "set_eval_dataset"): + model.module.set_eval_dataset(cfg.dataloader.test.dataset.names) + else: + if hasattr(model, "set_eval_dataset"): + model.set_eval_dataset(cfg.dataloader.test.dataset.names) + output_dir = os.path.join( + cfg.train.output_dir, "inference_{}".format(cfg.dataloader.test.dataset.names) + ) + if "cityscapes" in cfg.dataloader.test.dataset.names: + pass + else: + if isinstance(cfg.dataloader.evaluator, abc.MutableSequence): + for evaluator in cfg.dataloader.evaluator: + evaluator.output_dir = output_dir + else: + cfg.dataloader.evaluator.output_dir = output_dir + + ret = inference_on_dataset( + model, instantiate(cfg.dataloader.test), instantiate(cfg.dataloader.evaluator) + ) + logger.info( + "Evaluation results for {} in csv format:".format(cfg.dataloader.test.dataset.names) + ) + print_csv_format(ret) + ret = {f"{k}_{cfg.dataloader.test.dataset.names}": v for k, v in ret.items()} + else: + ret = {} + + if "evaluators" in cfg.dataloader: + for test, evaluator in zip(cfg.dataloader.tests, cfg.dataloader.evaluators): + if isinstance(model, DistributedDataParallel): + model.module.set_eval_dataset(test.dataset.names) + else: + model.set_eval_dataset(test.dataset.names) + output_dir = os.path.join( + cfg.train.output_dir, "inference_{}".format(test.dataset.names) + ) + if isinstance(evaluator, abc.MutableSequence): + for eva in evaluator: + eva.output_dir = output_dir + else: + evaluator.output_dir = output_dir + ret_ = inference_on_dataset(model, instantiate(test), instantiate(evaluator)) + logger.info("Evaluation results for {} in csv format:".format(test.dataset.names)) + print_csv_format(ret_) + ret.update({f"{k}_{test.dataset.names}": v for k, v in ret_.items()}) + + bbox_odinw_AP = {"AP": [], "AP50": [], "AP75": [], "APs": [], "APm": [], "APl": []} + segm_seginw_AP = {"AP": [], "AP50": [], "AP75": [], "APs": [], "APm": [], "APl": []} + bbox_rf100_AP = {"AP": [], "AP50": [], "AP75": [], "APs": [], "APm": [], "APl": []} + for k, v in ret.items(): + for kk, vv in v.items(): + if k.startswith("bbox_odinw") and kk in bbox_odinw_AP and vv == vv: + bbox_odinw_AP[kk].append(vv) + if k.startswith("segm_seginw") and kk in segm_seginw_AP and vv == vv: + segm_seginw_AP[kk].append(vv) + if k.startswith("bbox_rf100") and kk in bbox_rf100_AP and vv == vv: + bbox_rf100_AP[kk].append(vv) + + from statistics import median, mean + + logger.info("Evaluation results: {}".format(ret)) + for k, v in bbox_odinw_AP.items(): + if len(v) > 0: + logger.info( + "Evaluation results for odinw bbox {}: mean {} median {}".format( + k, mean(v), median(v) + ) + ) + for k, v in segm_seginw_AP.items(): + if len(v) > 0: + logger.info( + "Evaluation results for seginw segm {}: mean {} median {}".format( + k, mean(v), median(v) + ) + ) + for k, v in bbox_rf100_AP.items(): + if len(v) > 0: + logger.info( + "Evaluation results for rf100 bbox {}: mean {} median {}".format( + k, mean(v), median(v) + ) + ) + + return ret + + +def do_train(args, cfg): + """ + Args: + cfg: an object with the following attributes: + model: instantiate to a module + dataloader.{train,test}: instantiate to dataloaders + dataloader.evaluator: instantiate to evaluator for test set + optimizer: instantaite to an optimizer + lr_multiplier: instantiate to a fvcore scheduler + train: other misc config defined in `configs/common/train.py`, including: + output_dir (str) + init_checkpoint (str) + amp.enabled (bool) + max_iter (int) + eval_period, log_period (int) + device (str) + checkpointer (dict) + ddp (dict) + """ + model = instantiate(cfg.model) + logger = logging.getLogger("ape") + logger.info("Model:\n{}".format(model)) + model.to(cfg.train.device) + + cfg.optimizer.params.model = model + optim = instantiate(cfg.optimizer) + + if "wait_group" in cfg.dataloader: + wait = comm.get_local_rank() % cfg.dataloader.wait_group * cfg.dataloader.wait_time + logger.info("rank {} sleep {}".format(comm.get_local_rank(), wait)) + time.sleep(wait) + if isinstance(cfg.dataloader.train, abc.MutableSequence): + train_loader = [instantiate(x) for x in cfg.dataloader.train] + else: + train_loader = instantiate(cfg.dataloader.train) + + model = create_ddp_model(model, **cfg.train.ddp) + + ema.may_build_model_ema(cfg, model) + + trainer = Trainer( + model=model, + dataloader=train_loader, + optimizer=optim, + amp=cfg.train.amp.enabled, + clip_grad_params=cfg.train.clip_grad.params if cfg.train.clip_grad.enabled else None, + iter_size=cfg.train.iter_size if "iter_size" in cfg.train else 1, + iter_loop=cfg.train.iter_loop if "iter_loop" in cfg.train else True, + dataset_ratio=cfg.train.dataset_ratio if "dataset_ratio" in cfg.train else None, + ) + + checkpointer = DetectionCheckpointer( + model, + cfg.train.output_dir, + trainer=trainer, + **ema.may_get_ema_checkpointer(cfg, model), + ) + + if comm.is_main_process(): + output_dir = cfg.train.output_dir + PathManager.mkdirs(output_dir) + writers = [ + CommonMetricPrinter(cfg.train.max_iter), + JSONWriter(os.path.join(output_dir, "metrics.json")), + TensorboardXWriter(output_dir), + ] + if cfg.train.wandb.enabled: + PathManager.mkdirs(cfg.train.wandb.params.dir) + writers.append(WandbWriter(cfg)) + + trainer.register_hooks( + [ + hooks.IterationTimer(), + ema.EMAHook(cfg, model) if cfg.train.model_ema.enabled else None, + hooks.LRScheduler(scheduler=instantiate(cfg.lr_multiplier)), + hooks.PeriodicCheckpointer(checkpointer, **cfg.train.checkpointer) + if comm.is_main_process() + else None, + hooks.EvalHook(cfg.train.eval_period, lambda: do_test(cfg, model)), + hooks.PeriodicWriter( + writers, + period=cfg.train.log_period, + ) + if comm.is_main_process() + else None, + ] + ) + + checkpointer.resume_or_load(cfg.train.init_checkpoint, resume=args.resume) + if args.resume and checkpointer.has_checkpoint(): + start_iter = trainer.iter + 1 + else: + start_iter = 0 + trainer.train(start_iter, cfg.train.max_iter) + + +def main(args): + cfg = LazyConfig.load(args.config_file) + cfg = LazyConfig.apply_overrides(cfg, args.opts) + + if "output_dir" in cfg.model: + cfg.model.output_dir = cfg.train.output_dir + if "model_vision" in cfg.model and "output_dir" in cfg.model.model_vision: + cfg.model.model_vision.output_dir = cfg.train.output_dir + if "train" in cfg.dataloader: + if isinstance(cfg.dataloader.train, abc.MutableSequence): + for i in range(len(cfg.dataloader.train)): + if "output_dir" in cfg.dataloader.train[i].mapper: + cfg.dataloader.train[i].mapper.output_dir = cfg.train.output_dir + else: + if "output_dir" in cfg.dataloader.train.mapper: + cfg.dataloader.train.mapper.output_dir = cfg.train.output_dir + + default_setup(cfg, args) + + setup_logger(cfg.train.output_dir, distributed_rank=comm.get_rank(), name="sota") + setup_logger(cfg.train.output_dir, distributed_rank=comm.get_rank(), name="ape") + setup_logger(cfg.train.output_dir, distributed_rank=comm.get_rank(), name="timm") + + if cfg.train.fast_dev_run.enabled: + cfg.train.max_iter = 20 + cfg.train.eval_period = 10 + cfg.train.log_period = 1 + + if args.eval_only: + model = instantiate(cfg.model) + logger = logging.getLogger("ape") + logger.info("Model:\n{}".format(model)) + model.to(cfg.train.device) + model = create_ddp_model(model) + + ema.may_build_model_ema(cfg, model) + DetectionCheckpointer(model, **ema.may_get_ema_checkpointer(cfg, model)).load( + cfg.train.init_checkpoint + ) + if cfg.train.model_ema.enabled and cfg.train.model_ema.use_ema_weights_for_eval_only: + ema.apply_model_ema(model) + print(do_test(cfg, model, eval_only=True)) + else: + do_train(args, cfg) + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + launch( + main, + args.num_gpus, + num_machines=args.num_machines, + machine_rank=args.machine_rank, + dist_url=args.dist_url, + args=(args,), + ) diff --git a/approach/ovod/APE/tools/visualize_json_results.py b/approach/ovod/APE/tools/visualize_json_results.py new file mode 100644 index 0000000000000000000000000000000000000000..0c4959b1b4171c754ea1e3362cf3448da8bf007b --- /dev/null +++ b/approach/ovod/APE/tools/visualize_json_results.py @@ -0,0 +1,98 @@ +import argparse +import json +import os +from collections import defaultdict + +import cv2 +import numpy as np +import tqdm + +import ape +from detectron2.data import DatasetCatalog, MetadataCatalog +from detectron2.structures import Boxes, BoxMode, Instances +from detectron2.utils.file_io import PathManager +from detectron2.utils.logger import setup_logger +from detectron2.utils.visualizer import Visualizer + + +def create_instances(predictions, image_size): + ret = Instances(image_size) + + score = np.asarray([x["score"] for x in predictions]) + chosen = (score > args.conf_threshold).nonzero()[0] + score = score[chosen] + bbox = np.asarray([predictions[i]["bbox"] for i in chosen]).reshape(-1, 4) + bbox = BoxMode.convert(bbox, BoxMode.XYWH_ABS, BoxMode.XYXY_ABS) + + labels = np.asarray([dataset_id_map(predictions[i]["category_id"]) for i in chosen]) + + ret.scores = score + ret.pred_boxes = Boxes(bbox) + ret.pred_classes = labels + + try: + ret.pred_masks = [predictions[i]["segmentation"] for i in chosen] + except KeyError: + pass + return ret + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="A script that visualizes the json predictions from COCO or LVIS dataset." + ) + parser.add_argument("--input", required=True, help="JSON file produced by the model") + parser.add_argument("--output", required=True, help="output directory") + parser.add_argument("--dataset", help="name of the dataset", default="coco_2017_val") + parser.add_argument("--conf-threshold", default=0.5, type=float, help="confidence threshold") + args = parser.parse_args() + + logger = setup_logger() + + with PathManager.open(args.input, "r") as f: + predictions = json.load(f) + + pred_by_image = defaultdict(list) + for p in predictions: + pred_by_image[p["image_id"]].append(p) + + dicts = list(DatasetCatalog.get(args.dataset)) + metadata = MetadataCatalog.get(args.dataset) + if hasattr(metadata, "thing_dataset_id_to_contiguous_id"): + + def dataset_id_map(ds_id): + return metadata.thing_dataset_id_to_contiguous_id[ds_id] + + elif "lvis" in args.dataset: + + def dataset_id_map(ds_id): + return ds_id - 1 + + else: + raise ValueError("Unsupported dataset: {}".format(args.dataset)) + + os.makedirs(args.output, exist_ok=True) + + for dic in tqdm.tqdm(dicts): + + img = cv2.imread(dic["file_name"], cv2.IMREAD_COLOR)[:, :, ::-1] + basename = os.path.basename(dic["file_name"]) + + predictions = create_instances(pred_by_image[dic["image_id"]], img.shape[:2]) + vis = Visualizer(img, metadata) + vis_pred = vis.draw_instance_predictions(predictions).get_image() + + vis = Visualizer(img, metadata) + vis_gt = vis.draw_dataset_dict(dic).get_image() + + concat = np.concatenate((vis_pred, vis_gt), axis=1) + cv2.imwrite(os.path.join(args.output, basename), concat[:, :, ::-1]) + + if True and False: + for i, ann in enumerate(dic.pop("annotations")): + dic["annotations"] = [ann] + vis = Visualizer(img, metadata) + vis_gt = vis.draw_dataset_dict(dic).get_image() + cv2.imwrite( + os.path.join(args.output, basename + "_{}.png".format(i)), vis_gt[:, :, ::-1] + )