Add files using upload-large-folder tool

groundingLMM/eval/gcg/evaluate.py

@@ -0,0 +1,277 @@
+import os
+import json
+import argparse
+from tqdm import tqdm
+from pycocotools.coco import COCO
+from pycocotools.cocoeval import COCOeval
+from pycocotools import mask as maskUtils
+from pycocoevalcap.eval import COCOEvalCap
+from transformers import AutoTokenizer, AutoModel
+from sklearn.metrics.pairwise import cosine_similarity
+
+from eval.utils import *
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Training")
+
+    parser.add_argument("--split", required=True, help="Evaluation split, options are 'val', 'test'")
+    parser.add_argument("--prediction_dir_path", required=True, help="The path where the inference results are stored.")
+    parser.add_argument("--gt_dir_path", required=False, default="./data/GranDf/annotations/val_test",
+                        help="The path containing GranD-f evaluation annotations.")
+
+    args = parser.parse_args()
+
+    return args
+
+
+# Load pre-trained model tokenizer and model for evaluation
+tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
+model = AutoModel.from_pretrained("bert-base-uncased")
+
+
+def get_bert_embedding(text):
+    inputs = tokenizer(text, return_tensors="pt", max_length=512, truncation=True)
+    outputs = model(**inputs)
+    # Use the mean of the last hidden states as sentence embedding
+    sentence_embedding = torch.mean(outputs.last_hidden_state[0], dim=0).detach().numpy()
+
+    return sentence_embedding
+
+
+def compute_miou(pred_masks, gt_masks):
+    # Computing mIoU between predicted masks and ground truth masks
+    iou_matrix = np.zeros((len(pred_masks), len(gt_masks)))
+    for i, pred_mask in enumerate(pred_masks):
+        for j, gt_mask in enumerate(gt_masks):
+            iou_matrix[i, j] = compute_iou(pred_mask, gt_mask)
+
+    # One-to-one pairing and mean IoU calculation
+    paired_iou = []
+    while iou_matrix.size > 0 and np.max(iou_matrix) > 0:
+        max_iou_idx = np.unravel_index(np.argmax(iou_matrix, axis=None), iou_matrix.shape)
+        paired_iou.append(iou_matrix[max_iou_idx])
+        iou_matrix = np.delete(iou_matrix, max_iou_idx[0], axis=0)
+        iou_matrix = np.delete(iou_matrix, max_iou_idx[1], axis=1)
+
+    return np.mean(paired_iou) if paired_iou else 0.0
+
+
+def evaluate_mask_miou(coco_gt, image_ids, pred_save_path):
+    # Load predictions
+    coco_dt = coco_gt.loadRes(pred_save_path)
+
+    mious = []
+    for image_id in tqdm(image_ids):
+        # Getting ground truth masks
+        matching_anns = [ann for ann in coco_gt.anns.values() if ann['image_id'] == image_id]
+        ann_ids = [ann['id'] for ann in matching_anns]
+
+        gt_anns = coco_gt.loadAnns(ann_ids)
+        gt_masks = [maskUtils.decode(ann['segmentation']) for ann in gt_anns if 'segmentation' in ann]
+
+        # Getting predicted masks
+        matching_anns = [ann for ann in coco_dt.anns.values() if ann['image_id'] == image_id]
+        dt_ann_ids = [ann['id'] for ann in matching_anns]
+        pred_anns = coco_dt.loadAnns(dt_ann_ids)
+        pred_masks = [maskUtils.decode(ann['segmentation']) for ann in pred_anns if 'segmentation' in ann]
+
+        # Compute and save the mIoU for the current image
+        mious.append(compute_miou(pred_masks, gt_masks))
+
+    # Report mean IoU across all images
+    mean_miou = np.mean(mious) if mious else 0.0  # If list is empty, return 0.0
+
+    print(f"Mean IoU (mIoU) across all images: {mean_miou:.3f}")
+
+
+def compute_iou_matrix(pred_masks, gt_masks):
+    iou_matrix = np.zeros((len(pred_masks), len(gt_masks)))
+    for i, pred_mask in enumerate(pred_masks):
+        for j, gt_mask in enumerate(gt_masks):
+            iou_matrix[i, j] = compute_iou(pred_mask, gt_mask)
+
+    return iou_matrix
+
+
+def text_similarity_bert(str1, str2):
+    emb1 = get_bert_embedding(str1)
+    emb2 = get_bert_embedding(str2)
+
+    return cosine_similarity([emb1], [emb2])[0, 0]
+
+
+def find_best_matches(gt_anns, gt_labels, dt_anns, dt_labels, iou_threshold, text_sim_threshold, vectorizer=None):
+    best_matches = []
+
+    # Compute pair - wise IoU
+    pred_masks = [maskUtils.decode(ann['segmentation']) for ann in dt_anns]
+    gt_masks = [maskUtils.decode(ann['segmentation']) for ann in gt_anns]
+    ious = compute_iou_matrix(gt_masks, pred_masks)
+
+    text_sims = np.zeros((len(gt_labels), len(dt_labels)))
+
+    for i, gt_label in enumerate(gt_labels):
+        for j, dt_label in enumerate(dt_labels):
+            text_sims[i, j] = text_similarity_bert(gt_label, dt_label)
+
+    # Find one-to-one matches satisfying both IoU and text similarity thresholds
+    while ious.size > 0:
+        max_iou_idx = np.unravel_index(np.argmax(ious), ious.shape)
+        if ious[max_iou_idx] < iou_threshold or text_sims[max_iou_idx] < text_sim_threshold:
+            break  # No admissible pair found
+
+        best_matches.append(max_iou_idx)
+
+        # Remove selected annotations from consideration
+        ious[max_iou_idx[0], :] = 0
+        ious[:, max_iou_idx[1]] = 0
+        text_sims[max_iou_idx[0], :] = 0
+        text_sims[:, max_iou_idx[1]] = 0
+
+    return best_matches  # List of index pairs [(gt_idx, dt_idx), ...]
+
+
+def evaluate_recall_with_mapping(coco_gt, coco_cap_gt, image_ids, pred_save_path, cap_pred_save_path, iou_threshold=0.5,
+                                 text_sim_threshold=0.5):
+    coco_dt = coco_gt.loadRes(pred_save_path)
+    coco_cap_dt = coco_cap_gt.loadRes(cap_pred_save_path)
+
+    true_positives = 0
+    actual_positives = 0
+
+    for image_id in tqdm(image_ids):
+        try:
+            # gt_ann_ids = coco_gt.getAnnIds(imgIds=image_id, iscrowd=None)
+            matching_anns = [ann for ann in coco_gt.anns.values() if ann['image_id'] == image_id]
+            gt_ann_ids = [ann['id'] for ann in matching_anns]
+            gt_anns = coco_gt.loadAnns(gt_ann_ids)
+
+            # dt_ann_ids = coco_dt.getAnnIds(imgIds=image_id, iscrowd=None)
+            matching_anns = [ann for ann in coco_dt.anns.values() if ann['image_id'] == image_id]
+            dt_ann_ids = [ann['id'] for ann in matching_anns]
+            dt_anns = coco_dt.loadAnns(dt_ann_ids)
+
+            # gt_cap_ann_ids = coco_cap_gt.getAnnIds(imgIds=image_id)
+            matching_anns = [ann for ann in coco_cap_gt.anns.values() if ann['image_id'] == image_id]
+            gt_cap_ann_ids = [ann['id'] for ann in matching_anns]
+            gt_cap_ann = coco_cap_gt.loadAnns(gt_cap_ann_ids)[0]
+
+            # dt_cap_ann_ids = coco_cap_dt.getAnnIds(imgIds=image_id)
+            matching_anns = [ann for ann in coco_cap_dt.anns.values() if ann['image_id'] == image_id]
+            dt_cap_ann_ids = [ann['id'] for ann in matching_anns]
+            dt_cap_ann = coco_cap_dt.loadAnns(dt_cap_ann_ids)[0]
+
+            gt_labels = gt_cap_ann['labels']
+            dt_labels = dt_cap_ann['labels']
+
+            actual_positives += len(gt_labels)
+
+            # Find best matching pairs
+            best_matches = find_best_matches(gt_anns, gt_labels, dt_anns, dt_labels, iou_threshold, text_sim_threshold)
+
+            true_positives += len(best_matches)
+        except Exception as e:
+            print(e)
+
+    recall = true_positives / actual_positives if actual_positives > 0 else 0
+
+    print(f"Recall: {recall:.3f}")
+
+
+def main():
+    args = parse_args()
+
+    # Set the correct split
+    split = args.split
+    assert split == "val" or split == "test"  # GCG Evaluation has only val and test splits
+    gt_mask_path = f"{args.gt_dir_path}/{split}_gcg_coco_mask_gt.json"
+    gt_cap_path = f"{args.gt_dir_path}/{split}_gcg_coco_caption_gt.json"
+
+    print(f"Starting evalution on {split} split.")
+
+    # Get the image names of the split
+    all_images_ids = []
+    with open(gt_cap_path, 'r') as f:
+        contents = json.load(f)
+        for image in contents['images']:
+            all_images_ids.append(image['id'])
+
+    # The directory is used to store intermediate files
+    tmp_dir_path = f"tmp/{os.path.basename(args.prediction_dir_path)}_{split}"
+    os.makedirs(tmp_dir_path, exist_ok=True)  # Create directory if not exists already
+
+    # Create predictions
+    pred_save_path = f"{tmp_dir_path}/mask_pred_tmp_save.json"
+    cap_pred_save_path = f"{tmp_dir_path}/cap_pred_tmp_save.json"
+    coco_pred_file = []
+    caption_pred_dict = {}
+    for image_id in all_images_ids:
+        prediction_path = f"{args.prediction_dir_path}/{image_id}.json"
+        with open(prediction_path, 'r') as f:
+            pred = json.load(f)
+            bu = pred
+            key = list(pred.keys())[0]
+            pred = pred[key]
+            try:
+                caption_pred_dict[image_id] = {'caption': pred['caption'], 'labels': pred['phrases']}
+            except Exception as e:
+                pred = bu
+                caption_pred_dict[image_id] = {'caption': pred['caption'], 'labels': pred['phrases']}
+            for rle_mask in pred['pred_masks']:
+                coco_pred_file.append({"image_id": image_id, "category_id": 1, "segmentation": rle_mask, "score": 1.0})
+
+    # Save gcg_coco_predictions
+    with open(pred_save_path, 'w') as f:
+        json.dump(coco_pred_file, f)
+
+    # Prepare the CAPTION predictions in COCO format
+    cap_image_ids = []
+    coco_cap_pred_file = []
+    for image_id, values in caption_pred_dict.items():
+        cap_image_ids.append(image_id)
+        coco_cap_pred_file.append({"image_id": image_id, "caption": values['caption'], "labels": values['labels']})
+
+    # Save gcg_caption_coco_predictions
+    with open(cap_pred_save_path, 'w') as f:
+        json.dump(coco_cap_pred_file, f)
+
+    # # -------------------------------#
+    # 1. Evaluate AP
+    # Calculate mask mAP
+    # Load the ground truth and predictions in COCO format
+    coco_gt = COCO(gt_mask_path)
+    coco_dt = coco_gt.loadRes(pred_save_path)  # load predictions
+    # Initialize COCOEval and specify the metric you want to use
+    coco_eval = COCOeval(coco_gt, coco_dt, "segm")  # "segm" for segmentation
+    # Evaluate on a specific category
+    coco_eval.params.catIds = [1]  # your category ID
+    # Evaluate
+    coco_eval.evaluate()
+    coco_eval.accumulate()
+    coco_eval.summarize()
+
+    # # -------------------------------#
+    # # 2. Evaluate Caption Quality
+    coco_cap_gt = COCO(gt_cap_path)
+    coco_cap_result = coco_cap_gt.loadRes(cap_pred_save_path)
+    # create coco_eval object by taking coco and coco_result
+    coco_eval = COCOEvalCap(coco_cap_gt, coco_cap_result)
+    coco_eval.params['image_id'] = coco_cap_result.getImgIds()
+    coco_eval.evaluate()
+    for metric, score in coco_eval.eval.items():
+        print(f'{metric}: {score:.3f}')
+
+    # # -------------------------------#
+    # 3. Evaluate Mask Mean MIoU
+    coco_gt = COCO(gt_mask_path)  # Load ground truth annotations
+    evaluate_mask_miou(coco_gt, all_images_ids, pred_save_path)
+
+    # # -------------------------------#
+    # 4. Evaluate Recall
+    evaluate_recall_with_mapping(coco_gt, coco_cap_gt, all_images_ids, pred_save_path, cap_pred_save_path,
+                                 iou_threshold=0.5, text_sim_threshold=0.5)
+
+
+if __name__ == "__main__":
+    main()

groundingLMM/eval/gcg/infer.py

@@ -0,0 +1,184 @@
+import re
+import cv2
+import json
+import bleach
+import argparse
+from tqdm import tqdm
+from torch.utils.data import DataLoader, DistributedSampler
+from transformers import AutoTokenizer, CLIPImageProcessor
+
+from eval.utils import *
+from eval.ddp import *
+from model.GLaMM import GLaMMForCausalLM
+from model.llava import conversation as conversation_lib
+from model.llava.mm_utils import tokenizer_image_token
+from model.SAM.utils.transforms import ResizeLongestSide
+from tools.utils import DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="GLaMM Inference - GCG")
+
+    parser.add_argument("--hf_model_path", required=True, help="The model path in huggingface format.")
+    parser.add_argument("--img_dir", required=False, default="./data/GranDf/GranDf_HA_images/val_test",
+                        help="The directory containing images to run inference.")
+    parser.add_argument("--output_dir", required=True, help="The directory to store the response in json format.")
+
+    parser.add_argument("--image_size", default=1024, type=int, help="image size")
+    parser.add_argument("--model_max_length", default=512, type=int)
+    parser.add_argument("--use_mm_start_end", action="store_true", default=True)
+    parser.add_argument("--conv_type", default="llava_v1", type=str, choices=["llava_v1", "llava_llama_2"])
+
+    # DDP Related parameters
+    parser.add_argument("--batch_size_per_gpu", required=False, default=1)
+    parser.add_argument('--world_size', default=1, type=int, help='number of distributed processes')
+    parser.add_argument('--local_rank', default=-1, type=int)
+    parser.add_argument('--dist_url', default='env://', help='url used to set up distributed training')
+
+    return parser.parse_args()
+
+
+def inference(instructions, image_path):
+    # Filter out special chars
+    instructions = bleach.clean(instructions)
+    instructions = instructions.replace('&lt;', '<').replace('&gt;', '>')
+
+    # Prepare prompt for model Inference
+    conv = conversation_lib.conv_templates[args.conv_type].copy()
+    conv.messages = []
+    begin_str = f"""The {DEFAULT_IMAGE_TOKEN} provides an overview of the picture.\n"""
+    prompt = begin_str + instructions
+    if args.use_mm_start_end:
+        replace_token = (DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN)
+        prompt = prompt.replace(DEFAULT_IMAGE_TOKEN, replace_token)
+    conv.append_message(conv.roles[0], prompt)
+    conv.append_message(conv.roles[1], "")
+    prompt = conv.get_prompt()
+
+    # Read and preprocess the image (Global image encoder - CLIP)
+    image_np = cv2.imread(image_path)
+    image_np = cv2.cvtColor(image_np, cv2.COLOR_BGR2RGB)
+    original_size_list = [image_np.shape[:2]]
+    image_clip = (clip_image_processor.preprocess(image_np, return_tensors="pt")["pixel_values"][0].unsqueeze(0).cuda())
+    image_clip = image_clip.bfloat16()  # Precision is bf16 by default
+
+    # Preprocess the image (Grounding image encoder)
+    image = transform.apply_image(image_np)
+    resize_list = [image.shape[:2]]
+    image = (
+        grounding_image_ecoder_preprocess(torch.from_numpy(image).permute(2, 0, 1).contiguous()).unsqueeze(0).cuda())
+    image = image.bfloat16()  # Precision is bf16 by default
+
+    # Prepare inputs for inference
+    input_ids = tokenizer_image_token(prompt, tokenizer, return_tensors="pt")
+    input_ids = input_ids.unsqueeze(0).cuda()
+    bboxes = None  # No box/region is input in GCG task
+
+    # Generate output
+    output_ids, pred_masks = model.evaluate(image_clip, image, input_ids, resize_list, original_size_list,
+                                            max_tokens_new=512, bboxes=bboxes)
+    output_ids = output_ids[0][output_ids[0] != IMAGE_TOKEN_INDEX]
+
+    # Post-processing
+    text_output = tokenizer.decode(output_ids, skip_special_tokens=False)
+    text_output = text_output.replace("\n", "").replace("  ", " ")
+    text_output = text_output.split("ASSISTANT: ")[-1]
+
+    cleaned_str = re.sub(r'<.*?>', '', text_output)
+
+    pattern = re.compile(r'<p>(.*?)<\/p>')
+    phrases = pattern.findall(text_output)
+    phrases = [p.strip() for p in phrases]
+
+    # Remove the [SEG] token
+    cleaned_str = cleaned_str.replace('[SEG]', '')
+
+    # Strip unnecessary spaces
+    cleaned_str = ' '.join(cleaned_str.split()).strip("'")
+    cleaned_str = cleaned_str.strip()
+
+    return cleaned_str, pred_masks, phrases
+
+
+def custom_collate_fn(batch):
+    image_id = [item[0] for item in batch]
+    image_path = [item[1] for item in batch]
+
+    return image_id, image_path
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    init_distributed_mode(args)
+
+    # Initialize tokenizer and model
+    tokenizer = AutoTokenizer.from_pretrained(args.hf_model_path, cache_dir=None,
+                                              model_max_length=args.model_max_length, padding_side="right",
+                                              use_fast=False)
+    tokenizer.pad_token = tokenizer.unk_token
+    seg_token_idx = tokenizer("[SEG]", add_special_tokens=False).input_ids[0]
+    torch_dtype = torch.bfloat16  # By default, using bf16
+    kwargs = {"torch_dtype": torch_dtype}
+    model = GLaMMForCausalLM.from_pretrained(args.hf_model_path, low_cpu_mem_usage=True,
+                                             seg_token_idx=seg_token_idx, **kwargs)
+    # Update model config
+    model.config.eos_token_id = tokenizer.eos_token_id
+    model.config.bos_token_id = tokenizer.bos_token_id
+    model.config.pad_token_id = tokenizer.pad_token_id
+
+    # Initialize Global Image Encoder (CLIP)
+    model.get_model().initialize_vision_modules(model.get_model().config)
+    vision_tower = model.get_model().get_vision_tower()
+    vision_tower.to(dtype=torch_dtype)
+
+    # Transfer the model to GPU
+    model = model.bfloat16().cuda()  # Replace with model = model.float().cuda() for 32 bit inference
+    vision_tower = model.get_model().get_vision_tower()
+    vision_tower.to(device="cuda")
+
+    # Initialize Image Processor for GLobal Image Encoder (CLIP)
+    clip_image_processor = CLIPImageProcessor.from_pretrained(model.config.vision_tower)
+    transform = ResizeLongestSide(args.image_size)
+
+    model.eval()  # Model should be in evaluation mode for inference
+
+    # Prompt model to return grounded conversations
+    instruction = "Could you please give me a detailed description of the image? Please respond with interleaved \
+    segmentation masks for the corresponding parts of the answer."
+
+    # Create output directory if not exists already
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    # Create DDP Dataset
+    dataset = GCGEvalDDP(args.img_dir)
+    distributed_sampler = DistributedSampler(dataset, rank=args.rank, shuffle=False)
+    dataloader = DataLoader(dataset, batch_size=args.batch_size_per_gpu, num_workers=2,
+                            sampler=distributed_sampler, collate_fn=custom_collate_fn)
+
+    # Iterate over all the images, run inference and save results
+    for (image_id, image_path) in tqdm(dataloader):
+        image_id, image_path = image_id[0], image_path[0]
+
+        output_path = f"{args.output_dir}/{image_id[:-4]}.json"
+
+        result_caption, pred_masks, phrases = inference(instruction, image_path)  # GLaMM Inference
+
+        # Convert the predicted masks into RLE format
+        pred_masks_tensor = pred_masks[0].cpu()
+        binary_pred_masks = pred_masks_tensor > 0
+        uncompressed_mask_rles = mask_to_rle_pytorch(binary_pred_masks)
+        rle_masks = []
+        for m in uncompressed_mask_rles:
+            rle_masks.append(coco_encode_rle(m))
+
+        # Create results dictionary
+        result_dict = {
+            "image_id": image_id[:-4],
+            "caption": result_caption,
+            "phrases": phrases,
+            "pred_masks": rle_masks
+        }
+
+        # Save the inference results
+        with open(output_path, 'w') as f:
+            json.dump(result_dict, f)

groundingLMM/eval/gcg/run_evaluation.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+## USAGE
+
+## bash eval/gcg/run_evaluation.sh <path to the HF checkpoints path> <path to the directory to save the evaluation results>
+
+## USAGE
+
+
+export PYTHONPATH="./:$PYTHONPATH"
+MASTER_PORT=24999
+NUM_GPUS=1  # Adjust it as per the available #GPU
+
+# Positional arguments for the bash scripts
+CKPT_PATH=$1
+RESULT_PATH=$2
+
+# Path to the GranD-f evaluation dataset images directory
+IMAGE_DIR=./data/GranDf/GranDf_HA_images/val_test
+
+# Run Inference
+torchrun --nnodes=1 --nproc_per_node="$NUM_GPUS" --master_port="$MASTER_PORT" eval/gcg/infer.py --hf_model_path "$CKPT_PATH" --img_dir "$IMAGE_DIR" --output_dir "$RESULT_PATH"
+
+# Path to the GranD-f evaluation dataset ground-truths directory
+GT_DIR=./data/GranDf/annotations/val_test
+
+# Evaluate
+python eval/gcg/evaluate.py --prediction_dir_path "$RESULT_PATH" --gt_dir_path "$GT_DIR" --split "val"
+python eval/gcg/evaluate.py --prediction_dir_path "$RESULT_PATH" --gt_dir_path "$GT_DIR" --split "test"

groundingLMM/eval/referring_seg/infer_and_evaluate.py

@@ -0,0 +1,372 @@
+import os
+import sys
+import json
+import tqdm
+import torch
+import argparse
+import deepspeed
+import transformers
+from functools import partial
+from torch.utils.data import ConcatDataset
+from peft import LoraConfig, get_peft_model
+from torch.utils.tensorboard import SummaryWriter
+
+from model.GLaMM import GLaMMForCausalLM
+from model.llava import conversation as conversation_lib
+from dataset.dataset import custom_collate_fn
+from dataset.segm_datasets.RefCOCO_Segm_ds import ReferSegmDataset
+from tools.utils import (DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN, AverageMeter, Summary,
+                         intersectionAndUnionGPU, dict_to_cuda)
+
+
+def parse_args(args):
+    parser = argparse.ArgumentParser(description="GLaMM Model Evaluation")
+
+    # Model-specific settings
+    parser.add_argument("--version", required=True, help="Path to the pretrained model for evaluation.")
+    parser.add_argument("--pretrained", action="store_true", default=False)
+    parser.add_argument("--vision_pretrained", default="./checkpoints/sam_vit_h_4b8939.pth", type=str)
+    parser.add_argument("--vision-tower", default="openai/clip-vit-large-patch14-336", type=str)
+    parser.add_argument("--conv_type", default="llava_v1", type=str, choices=["llava_v1", "llava_llama_2"])
+    parser.add_argument("--tune_mm_mlp_adapter", action="store_true", default=False)
+    parser.add_argument("--freeze_mm_mlp_adapter", action="store_true", default=False)
+    parser.add_argument("--mm_use_im_start_end", action="store_true", default=True)
+    parser.add_argument("--out_dim", default=256, type=int)
+    parser.add_argument("--lora_target_modules", default="q_proj,v_proj", type=str)
+    parser.add_argument("--with_region", action="store_true", default=True)
+    parser.add_argument("--mm_vision_select_layer", default=-2, type=int)
+    parser.add_argument("--pretrain_mm_mlp_adapter", default="", type=str)
+    parser.add_argument("--precision", default='bf16', type=str)
+
+    # Training settings
+    parser.add_argument("--lr", default=0.0003, type=float)
+    parser.add_argument("--epochs", default=10, type=int)
+    parser.add_argument("--steps_per_epoch", default=500, type=int)
+    parser.add_argument("--batch_size", default=2, type=int, help="batch size per device per step")
+    parser.add_argument("--grad_accumulation_steps", default=10, type=int)
+    parser.add_argument("--lora_r", default=8, type=int)
+    parser.add_argument("--lora_alpha", default=16, type=int)
+    parser.add_argument("--lora_dropout", default=0.05, type=float)
+    parser.add_argument("--ce_loss_weight", default=1.0, type=float)
+    parser.add_argument("--dice_loss_weight", default=0.5, type=float)
+    parser.add_argument("--bce_loss_weight", default=2.0, type=float)
+    parser.add_argument("--beta1", default=0.9, type=float)
+    parser.add_argument("--beta2", default=0.95, type=float)
+    parser.add_argument("--gradient_checkpointing", action="store_true", default=True)
+    parser.add_argument("--train_mask_decoder", action="store_true", default=True)
+    parser.add_argument("--use_mm_start_end", action="store_true", default=True)
+    parser.add_argument("--print_freq", default=1, type=int)
+    parser.add_argument("--start_epoch", default=0, type=int)
+
+    # Dataset settings
+    parser.add_argument("--dataset_dir", default="./data", type=str)
+    parser.add_argument("--image_size", default=1024, type=int, help="Image size for grounding image encoder")
+    parser.add_argument("--model_max_length", default=1536, type=int)
+    parser.add_argument("--refer_seg_data", default="refcocog|val", type=str)
+    parser.add_argument("--results_path", default="referring_seg_eval.json", type=str)
+
+    # Evaluation settings
+    parser.add_argument("--val_batch_size", default=1, type=int)
+    parser.add_argument("--workers", default=2, type=int)
+    parser.add_argument("--local_rank", default=0, type=int, help="node rank")
+
+    # Experiment settings
+    parser.add_argument("--log_base_dir", default="./runs", type=str)
+    parser.add_argument("--exp_name", default="glam_eval_referseg", type=str)
+
+    return parser.parse_args(args)
+
+
+def initialize_environment(args):
+    """ Set up logging and model directories. """
+    args.log_dir = os.path.join(args.log_base_dir, args.exp_name)
+    if args.local_rank == 0:
+        os.makedirs(args.log_dir, exist_ok=True)
+        return SummaryWriter(args.log_dir)
+    return None
+
+
+def setup_tokenizer_and_special_tokens(args):
+    """ Load tokenizer and add special tokens. """
+    tokenizer = transformers.AutoTokenizer.from_pretrained(
+        args.version, model_max_length=args.model_max_length, padding_side="right", use_fast=False
+    )
+    print('\033[92m' + "---- Initialized tokenizer from: {} ----".format(args.version) + '\033[0m')
+    tokenizer.pad_token = tokenizer.unk_token
+
+    if not args.pretrained:
+        if args.use_mm_start_end:
+            tokenizer.add_tokens(
+                [DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True
+            )
+        # modifications specific for regions
+        reg_tokens = ['<bbox>', '<point>']
+        # Adding special tokens for pixel grounding
+        segmentation_tokens = ['[SEG]']
+        # Adding tokens for GCG
+        phrase_tokens = ['<p>', '</p>']
+        special_tokens = reg_tokens + segmentation_tokens + phrase_tokens
+        tokenizer.add_tokens(special_tokens, special_tokens=True)
+
+    args.bbox_token_idx = tokenizer("<bbox>", add_special_tokens=False).input_ids[0]
+    args.seg_token_idx = tokenizer("[SEG]", add_special_tokens=False).input_ids[0]
+    args.bop_token_idx = tokenizer("<p>", add_special_tokens=False).input_ids[0]
+    args.eop_token_idx = tokenizer("</p>", add_special_tokens=False).input_ids[0]
+
+    return tokenizer
+
+
+def initialize_model(args, tokenizer):
+    """ Initialize the GLaMM model. """
+    model_args = {k: getattr(args, k) for k in
+                  ["train_mask_decoder", "out_dim", "ce_loss_weight", "dice_loss_weight", "bce_loss_weight",
+                   "seg_token_idx", "vision_pretrained", "vision_tower", "use_mm_start_end", "mm_vision_select_layer",
+                   "pretrain_mm_mlp_adapter", "tune_mm_mlp_adapter", "freeze_mm_mlp_adapter", "mm_use_im_start_end",
+                   "with_region", "bbox_token_idx", "eop_token_idx", "bop_token_idx"]}
+    model_args["num_level_reg_features"] = 4
+
+    model = GLaMMForCausalLM.from_pretrained(
+        args.version, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, **model_args
+    )
+    print('\033[92m' + "---- Initialized model from: {} ----".format(args.version) + '\033[0m')
+
+    # Configure model tokens
+    model.config.eos_token_id = tokenizer.eos_token_id
+    model.config.bos_token_id = tokenizer.bos_token_id
+    model.config.pad_token_id = tokenizer.pad_token_id
+
+    return model
+
+
+def prepare_model_for_training(model, tokenizer, args):
+    # Enable input gradients
+    model.enable_input_require_grads()
+    model.gradient_checkpointing_enable()
+
+    # Initialize vision tower
+    print(
+        '\033[92m' + "---- Initialized Global Image Encoder (vision tower) from: {} ----".format(
+            args.vision_tower
+        ) + '\033[0m'
+    )
+    model.get_model().initialize_vision_modules(model.get_model().config)
+    vision_tower = model.get_model().get_vision_tower()
+    vision_tower.to(dtype=torch.bfloat16, device=args.local_rank)
+
+    # Initialize GLaMM model and adjust requires_grad
+    if not args.pretrained:
+        model.get_model().initialize_glamm_model(model.get_model().config)
+    else:
+        for param in model.get_model().grounding_encoder.parameters():
+            param.requires_grad = False
+        if model.get_model().config.train_mask_decoder:
+            model.get_model().grounding_encoder.mask_decoder.train()
+            for param in model.get_model().grounding_encoder.mask_decoder.parameters():
+                param.requires_grad = True
+
+        # Projection layer
+        model.get_model().text_hidden_fcs.train()
+        for param in model.get_model().text_hidden_fcs.parameters():
+            param.requires_grad = True
+
+    # Set requires_grad for vision tower and mm projector
+    for p in vision_tower.parameters():
+        p.requires_grad = False
+    for p in model.get_model().mm_projector.parameters():
+        p.requires_grad = False
+
+    # Set requires_grad based on LoRA training
+    lora_r = args.lora_r
+    if lora_r == 0:
+        for p in model.get_model().layers.parameters():
+            p.requires_grad = True
+        for p in model.get_model().mm_projector.parameters():
+            p.requires_grad = True
+
+    # Configure conversation library
+    conversation_lib.default_conversation = conversation_lib.conv_templates[args.conv_type]
+
+    # Configure LoRA if applicable
+    if lora_r > 0:
+        lora_config = setup_lora_config(model, args)
+        model = get_peft_model(model, lora_config)
+
+    # Resize token embeddings
+    model.resize_token_embeddings(len(tokenizer))
+
+
+def setup_lora_config(model, args):
+    """ Configure LoRA settings for the model. """
+
+    def find_proj_layers(model, target_modules):
+        """ Identify projection layers in the model for LoRA adaptation. """
+        linear_cls = torch.nn.Linear
+        lora_module_names = set()
+        for name, module in model.named_modules():
+            if (isinstance(module, linear_cls) and all(
+                    x not in name for x in ["grounding_encoder", "vision_tower", "mm_projector", "text_hidden_fcs"]
+            ) and any(x in name for x in target_modules)):
+                lora_module_names.add(name)
+        return sorted(list(lora_module_names))
+
+    # Extracting LoRA target modules
+    lora_target_modules = args.lora_target_modules.split(",")
+    lora_module_names = find_proj_layers(model, lora_target_modules)
+
+    # Configuring LoRA
+    lora_config = LoraConfig(
+        r=args.lora_r, lora_alpha=args.lora_alpha, target_modules=lora_module_names, lora_dropout=args.lora_dropout,
+        bias="none", task_type="CAUSAL_LM"
+    )
+    return lora_config
+
+
+def initialize_datasets_and_loaders(args, tokenizer):
+    # Dataset settings for ReferSegDataset
+    common_ds_args = {
+        "dataset_dir": args.dataset_dir,
+        "tokenizer": tokenizer,
+        "global_image_encoder": args.vision_tower,
+        "precision": args.precision,
+        "image_size": args.image_size
+    }
+
+    # Validation datasets
+    dataset, split = args.refer_seg_data.split('|')
+    val_datasets = [ReferSegmDataset(**common_ds_args, validation=True, refer_segm_data=dataset, split=split,
+                                     inference=True)]
+    _ = [d._set_len(len(d.refer_segm_data[dataset]['images'])) for d in val_datasets]
+
+    return val_datasets
+
+
+def setup_data_loaders(args, val_datasets, tokenizer):
+    sampler_args = {"shuffle": False, "drop_last": False}
+    val_loader_args = {"batch_size": args.val_batch_size, "shuffle": False, "num_workers": args.workers,
+                       "pin_memory": False}
+    collate_fn_args_val = partial(
+        custom_collate_fn, tokenizer=tokenizer, use_mm_start_end=args.use_mm_start_end, local_rank=args.local_rank,
+        inference=True
+    )
+
+    # Validation loader
+    combined_val_datasets = ConcatDataset(val_datasets)
+    val_loader = torch.utils.data.DataLoader(
+        combined_val_datasets, **val_loader_args, collate_fn=collate_fn_args_val,
+        sampler=torch.utils.data.distributed.DistributedSampler(combined_val_datasets, **sampler_args), )
+
+    return val_loader
+
+
+def initialize_deepspeed(model, tokenizer, args):
+    ds_config = {"train_micro_batch_size_per_gpu": args.batch_size,
+                 "gradient_accumulation_steps": args.grad_accumulation_steps, "optimizer": {"type": "AdamW",
+                                                                                            "params": {"lr": args.lr,
+                                                                                                       "weight_decay": 0.0,
+                                                                                                       "betas": (
+                                                                                                           args.beta1,
+                                                                                                           args.beta2)}},
+                 "scheduler": {"type": "WarmupDecayLR",
+                               "params": {"total_num_steps": args.epochs * args.steps_per_epoch, "warmup_min_lr": 0,
+                                          "warmup_max_lr": args.lr, "warmup_num_steps": 100, "warmup_type": "linear"}},
+                 "fp16": {"enabled": args.precision == "fp16"}, "bf16": {"enabled": args.precision == "bf16"},
+                 "gradient_clipping": 1.0,
+                 "zero_optimization": {"stage": 2, "contiguous_gradients": True, "overlap_comm": True,
+                                       "reduce_scatter": True, "reduce_bucket_size": 5e8,
+                                       "allgather_bucket_size": 5e8}, }
+
+    model_engine, optimizer, _, scheduler = deepspeed.initialize(
+        model=model, model_parameters=model.parameters(), collate_fn=partial(
+            custom_collate_fn, tokenizer=tokenizer, use_mm_start_end=args.use_mm_start_end, local_rank=args.local_rank
+        ), config=ds_config
+    )
+
+    return model_engine, optimizer, scheduler
+
+
+def evaluate_model_performance(validation_loader, model, args):
+    # Trackers for metrics
+    trackers = {
+        "intersection": AverageMeter("Intersec", ":6.3f", Summary.SUM),
+        "union": AverageMeter("Union", ":6.3f", Summary.SUM),
+        "gIoU": AverageMeter("gIoU", ":6.3f", Summary.SUM)
+    }
+
+    model.eval()
+    for data_batch in tqdm.tqdm(validation_loader):
+        # Prepare data and convert relevant tensors to the appropriate type
+        data_batch = dict_to_cuda(data_batch)
+        for key in ["global_enc_images", "grounding_enc_images"]:
+            data_batch[key] = data_batch[key].to(dtype=torch.bfloat16, device=args.local_rank)
+
+        torch.cuda.empty_cache()
+
+        # Model inference without gradient tracking
+        with torch.no_grad():
+            results = model(**data_batch)
+
+        predictions = results["pred_masks"]
+        gt_masks = results["gt_masks"][0].int()
+        predicted_masks = (predictions[0] > 0).int()  # Thresholding to get binary masks
+        assert len(predictions) == 1
+
+        intersection, union, accuracy_iou = 0.0, 0.0, 0.0
+        for target, prediction in zip(gt_masks, predicted_masks):
+            intersect, union_, _ = intersectionAndUnionGPU(
+                prediction.contiguous().clone(), target.contiguous(), 2, ignore_index=255
+            )
+            intersection += intersect
+            union += union_
+            accuracy_iou += intersect / (union_ + 1e-5)
+            # handles no-object targets
+            accuracy_iou[union_ == 0] += 1.0
+
+        intersection, union = intersection.cpu().numpy(), union.cpu().numpy()
+        accuracy_iou = accuracy_iou.cpu().numpy() / gt_masks.shape[0]
+        trackers["intersection"].update(intersection)
+        trackers["union"].update(union)
+        trackers["gIoU"].update(accuracy_iou, n=gt_masks.shape[0])
+
+    for meter in trackers.values():
+        meter.all_reduce()
+
+    iou_per_class = trackers["intersection"].sum / (trackers["union"].sum + 1e-10)
+    class_iou = iou_per_class[1]
+    global_iou = trackers["gIoU"].avg[1]
+
+    return global_iou, class_iou
+
+
+def main(args):
+    tokenizer = setup_tokenizer_and_special_tokens(args)
+    model = initialize_model(args, tokenizer)
+    prepare_model_for_training(model, tokenizer, args)
+
+    model_engine, _, _ = initialize_deepspeed(model, tokenizer, args)
+
+    val_datasets = initialize_datasets_and_loaders(args, tokenizer)
+    val_loader = setup_data_loaders(args, val_datasets, tokenizer)
+
+    giou, ciou = evaluate_model_performance(val_loader, model_engine, args)
+
+    torch.distributed.barrier()
+    if args.local_rank == 0:
+        # Update and save the results
+        os.makedirs(args.results_path, exist_ok=True)
+        if os.path.exists(f"{args.results_path}/stats.json"):
+            with open(f"{args.results_path}/stats.json", 'r') as json_file:
+                result_list = json.load(json_file)
+        else:
+            result_list = []
+        result_dict = {"model": args.results_path, "dataset": args.refer_seg_data, "giou": str(giou), "ciou": str(ciou)}
+        result_list.append(result_dict)
+
+        with open(f"{args.results_path}/stats.json", 'w') as json_file:
+            json.dump(result_list, json_file, indent=2)
+
+        print(result_list)  # Print all the results
+
+
+if __name__ == "__main__":
+    args = parse_args(sys.argv[1:])
+    main(args)

groundingLMM/eval/referring_seg/run_evaluation.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+## USAGE
+
+## bash eval/referring_seg/run_evaluation.sh <path to the HF checkpoints path> <path to the directory to save the evaluation results>
+
+## USAGE
+
+
+# Adjust the environment variable if you have multiple gpus available, e.g. CUDA_VISIBLE_DEVICES=0,1,2,3 if you have 4 GPUs available
+export CUDA_VISIBLE_DEVICES=0
+export PYTHONPATH="./:$PYTHONPATH"
+MASTER_PORT=24999
+
+# Positional arguments for the bash scripts
+CKPT_PATH=$1
+RESULT_PATH=$2
+
+# RefCOCO
+deepspeed --master_port="$MASTER_PORT" eval/referring_seg/infer_and_evaluate.py --version "$CKPT_PATH" --refer_seg_data "refcoco|val" --results_path "$RESULT_PATH" --pretrained
+deepspeed --master_port="$MASTER_PORT" eval/referring_seg/infer_and_evaluate.py --version "$CKPT_PATH" --refer_seg_data "refcoco|testA" --results_path "$RESULT_PATH" --pretrained
+deepspeed --master_port="$MASTER_PORT" eval/referring_seg/infer_and_evaluate.py --version "$CKPT_PATH" --refer_seg_data "refcoco|testB" --results_path "$RESULT_PATH" --pretrained
+
+# RefCOCO+
+deepspeed --master_port="$MASTER_PORT" eval/referring_seg/infer_and_evaluate.py --version "$CKPT_PATH" --refer_seg_data "refcoco+|val" --results_path "$RESULT_PATH" --pretrained
+deepspeed --master_port="$MASTER_PORT" eval/referring_seg/infer_and_evaluate.py --version "$CKPT_PATH" --refer_seg_data "refcoco+|testA" --results_path "$RESULT_PATH" --pretrained
+deepspeed --master_port="$MASTER_PORT" eval/referring_seg/infer_and_evaluate.py --version "$CKPT_PATH" --refer_seg_data "refcoco+|testB" --results_path "$RESULT_PATH" --pretrained
+
+# RefCOCOg
+deepspeed --master_port="$MASTER_PORT" eval/referring_seg/infer_and_evaluate.py --version "$CKPT_PATH" --refer_seg_data "refcocog|val" --results_path "$RESULT_PATH" --pretrained
+deepspeed --master_port="$MASTER_PORT" eval/referring_seg/infer_and_evaluate.py --version "$CKPT_PATH" --refer_seg_data "refcocog|test" --results_path "$RESULT_PATH" --pretrained

groundingLMM/eval/region_captioning/run_evaluation_RefCOCOg.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+## USAGE
+
+## bash eval/region_captioning/run_evaluation.sh <path to the HF checkpoints path> <path to the directory to save the evaluation results>
+
+## USAGE
+
+
+export PYTHONPATH="./:$PYTHONPATH"
+MASTER_PORT=24999
+NUM_GPUS=1  # Adjust it as per the available #GPU
+
+# Positional arguments for the bash scripts
+CKPT_PATH=$1
+RESULT_PATH=$2
+
+# Adjust if needed
+ANNOTATION_FILE=./data/RefCoco_Reg/mdetr_annotations/finetune_refcocog_val_captions.json
+IMAGE_DIR=./data/coco_2014/train2014
+DATASET=refcocog
+
+# Run Inference
+torchrun --nnodes=1 --nproc_per_node="$NUM_GPUS" --master_port="$MASTER_PORT" eval/region_captioning/infer.py --hf_model_path "$CKPT_PATH" --annotation_file "$ANNOTATION_FILE" --image_dir "$IMAGE_DIR" --dataset "$DATASET" --results_dir "$RESULT_PATH"
+
+
+# Evaluate
+python eval/region_captioning/evaluate.py --annotation_file "$ANNOTATION_FILE" --results_dir "$RESULT_PATH"

groundingLMM/mmcv/.circleci/config.yml

@@ -0,0 +1,175 @@
+version: 2.1
+jobs:
+  lint:
+    docker:
+      - image: cimg/python:3.7.4
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt-add-repository ppa:brightbox/ruby-ng -y
+            sudo apt-get update
+            sudo apt-get install -y ruby2.7
+      - run:
+          name: Install pre-commit hook
+          command: |
+            pip install pre-commit
+            pre-commit install
+      - run:
+          name: Linting
+          command: pre-commit run --all-files
+
+  build_cpu:
+    parameters:
+      # The python version must match available image tags in
+      # https://circleci.com/developer/images/image/cimg/python
+      python:
+        type: string
+        default: "3.7.0"
+      torch:
+        type: string
+      torchvision:
+        type: string
+    machine:
+      image: ubuntu-2004:202010-01
+    resource_class: large
+    steps:
+      - checkout
+      - run:
+          name: Install system dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y ffmpeg libturbojpeg ninja-build
+            ffmpeg -version
+      - run:
+          # https://github.com/pytorch/vision/issues/2921
+          name: Install dependency of torchvision when using pyenv
+          command: sudo apt-get install -y liblzma-dev
+      - run:
+          # python3.7 should be re-installed due to the issue https://github.com/pytorch/vision/issues/2921
+          name: Select Python
+          command: |
+            pyenv uninstall -f << parameters.python >>
+            pyenv install << parameters.python >>
+            pyenv global << parameters.python >>
+      - run:
+          name: Upgrade pip
+          command: |
+            python -m pip install pip --upgrade
+      - run:
+          name: Install PyTorch
+          command: python -m pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html
+      - run:
+          name: Install psutil
+          command: python -m pip install psutil
+      - run:
+          name: Build and install
+          command: |
+            rm -rf .eggs
+            python setup.py check -m -s
+            python -m pip install -e .
+          no_output_timeout: 20m
+          environment:
+            MMCV_WITH_OPS: 1
+      - run:
+          name: Install dependencies of unit test
+          command: |
+            python -m pip install -r requirements/test.txt
+      - run:
+          name: Run unittests and generate coverage report
+          command: |
+            python -m coverage run --branch --source mmcv -m pytest tests/
+            python -m coverage xml
+            python -m coverage report -m
+
+  build_cu102:
+    machine:
+      image: ubuntu-1604-cuda-10.1:201909-23  # the actual version of cuda is 10.2
+    resource_class: gpu.nvidia.small
+    steps:
+      - checkout
+      - run:
+          name: Set CUDA environment
+          command: |
+            echo 'export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH' >> $BASH_ENV
+            echo 'export PATH=/usr/local/cuda/bin:$PATH' >> $BASH_ENV
+            echo 'export CUDA_HOME=/usr/local/cuda' >> $BASH_ENV
+            source $BASH_ENV
+            nvidia-smi
+            nvcc --version
+            gcc --version
+      - run:
+          name: Install system dependencies
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y libturbojpeg ninja-build
+            # the default version of ffmpeg is 2.8.7, which should be upgraded to 4+
+            sudo add-apt-repository -y ppa:jonathonf/ffmpeg-4
+            sudo apt-get update
+            sudo apt-get install -y ffmpeg
+            ffmpeg -version
+            sudo add-apt-repository --remove ppa:jonathonf/ffmpeg-4 -y
+      - run:
+          # https://github.com/pytorch/vision/issues/2921
+          name: Install dependency of torchvision when using pyenv
+          command: sudo apt-get install -y liblzma-dev
+      - run:
+          # python3.7 should be re-installed due to the issue https://github.com/pytorch/vision/issues/2921
+          name: Select python3.7
+          command: |
+            pyenv uninstall -f 3.7.0
+            pyenv install 3.7.0
+            pyenv global 3.7.0
+      - run:
+          name: Upgrade pip
+          command: |
+            python -m pip install pip --upgrade
+      - run:
+          name: Install PyTorch
+          command: python -m pip install torch==1.8.1+cu102 torchvision==0.9.1+cu102 -f https://download.pytorch.org/whl/torch_stable.html
+      - run:
+          name: Install psutil
+          command: python -m pip install psutil
+      - run:
+          name: Download onnxruntime library and install onnxruntime
+          command: |
+            wget https://github.com/microsoft/onnxruntime/releases/download/v1.8.1/onnxruntime-linux-x64-1.8.1.tgz
+            tar -zxvf onnxruntime-linux-x64-1.8.1.tgz
+            echo 'export ONNXRUNTIME_DIR=$(pwd)/onnxruntime-linux-x64-1.8.1' >> $BASH_ENV
+            echo 'export LD_LIBRARY_PATH=$ONNXRUNTIME_DIR/lib:$LD_LIBRARY_PATH' >> $BASH_ENV
+            source $BASH_ENV
+            python -m pip install onnxruntime==1.8.1
+      - run:
+          name: Build and install
+          command: |
+            rm -rf .eggs
+            python setup.py check -m -s
+            python -m pip install -e .
+          environment:
+            MMCV_WITH_OPS: 1
+            MMCV_WITH_ORT: 1
+      - run:
+          name: Install dependencies for unit test
+          command: |
+            python -m pip install -r requirements/test.txt
+      - run:
+          name: Run unittests and generate coverage report
+          command: |
+            python -m coverage run --branch --source mmcv -m pytest tests/
+            python -m coverage xml
+            python -m coverage report -m
+workflows:
+  unit_tests:
+    jobs:
+      - lint
+      - build_cpu:
+          name: build_py3.8_pt1.9_cpu
+          torch: 1.9.0
+          torchvision: 0.10.0
+          python: "3.8.0"
+          requires:
+            - lint
+      - build_cu102:
+          requires:
+            - build_py3.8_pt1.9_cpu

groundingLMM/mmcv/.dev_scripts/check_installation.py

@@ -0,0 +1,44 @@
+import numpy as np
+import torch
+
+from mmcv.ops import box_iou_rotated
+from mmcv.utils import collect_env
+
+
+def check_installation():
+    """Check whether mmcv-full has been installed successfully."""
+    np_boxes1 = np.asarray(
+        [[1.0, 1.0, 3.0, 4.0, 0.5], [2.0, 2.0, 3.0, 4.0, 0.6],
+         [7.0, 7.0, 8.0, 8.0, 0.4]],
+        dtype=np.float32)
+    np_boxes2 = np.asarray(
+        [[0.0, 2.0, 2.0, 5.0, 0.3], [2.0, 1.0, 3.0, 3.0, 0.5],
+         [5.0, 5.0, 6.0, 7.0, 0.4]],
+        dtype=np.float32)
+    boxes1 = torch.from_numpy(np_boxes1)
+    boxes2 = torch.from_numpy(np_boxes2)
+
+    # test mmcv-full with CPU ops
+    box_iou_rotated(boxes1, boxes2)
+    print('CPU ops were compiled successfully.')
+
+    # test mmcv-full with both CPU and CUDA ops
+    if torch.cuda.is_available():
+        boxes1 = boxes1.cuda()
+        boxes2 = boxes2.cuda()
+        box_iou_rotated(boxes1, boxes2)
+        print('CUDA ops were compiled successfully.')
+    else:
+        print('No CUDA runtime is found, skipping the checking of CUDA ops.')
+
+
+if __name__ == '__main__':
+    print('Start checking the installation of mmcv-full ...')
+    check_installation()
+    print('mmcv-full has been installed successfully.\n')
+
+    env_info_dict = collect_env()
+    env_info = '\n'.join([(f'{k}: {v}') for k, v in env_info_dict.items()])
+    dash_line = '-' * 60 + '\n'
+    print('Environment information:')
+    print(dash_line + env_info + '\n' + dash_line)

groundingLMM/mmcv/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,9 @@
+blank_issues_enabled: false
+
+contact_links:
+  - name: Common Issues
+    url: https://mmcv.readthedocs.io/en/latest/trouble_shooting.html
+    about: Check if your issue already has solutions
+  - name: MMCV Documentation
+    url: https://mmcv.readthedocs.io/en/latest/
+    about: Check if your question is answered in docs

groundingLMM/mmcv/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,22 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the feature**
+
+**Motivation**
+A clear and concise description of the motivation of the feature.
+Ex1. It is inconvenient when [....].
+Ex2. There is a recent paper [....], which is very helpful for [....].
+
+**Related resources**
+If there is an official code release or third-party implementations, please also provide the information here, which would be very helpful.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.
+If you would like to implement the feature and create a PR, please leave a comment here and that would be much appreciated.

groundingLMM/mmcv/.github/ISSUE_TEMPLATE/general_questions.md

@@ -0,0 +1,13 @@
+---
+name: General questions
+about: Ask general questions to get help
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Checklist**
+
+1. I have searched related issues but cannot get the expected help.
+2. I have read the FAQ documentation but cannot get the expected help.

groundingLMM/mmcv/.github/ISSUE_TEMPLATE/unexpected_report.md

@@ -0,0 +1,46 @@
+---
+name: Unexpected Results
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+Thanks for reporting the unexpected results and we appreciate it a lot.
+
+**Checklist**
+
+1. I have searched related issues but cannot get the expected help.
+2. I have read the [FAQ documentation](https://mmcv.readthedocs.io/en/latest/trouble_shooting.html) but cannot get the expected help.
+3. The unexpected results still exist in the latest version.
+
+**Describe the Issue**
+A clear and concise description of what the bug is, including what results are expected and what the real results you got.
+
+**Reproduction**
+
+1. What command, code, or script did you run?
+
+```bash
+A placeholder for the command.
+```
+
+2. Did you make any modifications on the code? Did you understand what you have modified?
+
+**Environment**
+
+1. Please run `python -c "from mmcv.utils import collect_env; print(collect_env())"` to collect necessary environment information and paste it here.
+2. You may add addition that may be helpful for locating the problem, such as
+    - How you installed PyTorch [e.g., pip, conda, source]
+    - Other environment variables that may be related (such as `$PATH`, `$LD_LIBRARY_PATH`, `$PYTHONPATH`, etc.)
+
+**Error traceback**
+If applicable, paste the error traceback here.
+
+```none
+A placeholder for traceback.
+```
+
+**Bug fix**
+If you have already identified the reason, you can provide the information here. If you are willing to create a PR to fix it, please also leave a comment here and that would be much appreciated!

groundingLMM/mmcv/.github/pull_request_template.md

@@ -0,0 +1,33 @@
+Thanks for your contribution and we appreciate it a lot. The following instructions would make your pull request more healthy and more easily get feedback. If you do not understand some items, don't worry, just make the pull request and seek help from maintainers.
+
+## Motivation
+
+Please describe the motivation of this PR and the goal you want to achieve through this PR.
+
+## Modification
+
+Please briefly describe what modification is made in this PR.
+
+## BC-breaking (Optional)
+
+Does the modification introduce changes that break the backward-compatibility of the downstream repositories?
+If so, please describe how it breaks the compatibility and how the downstream projects should modify their code to keep compatibility with this PR.
+
+## Use cases (Optional)
+
+If this PR introduces a new feature, it is better to list some use cases here, and update the documentation.
+
+## Checklist
+
+**Before PR**:
+
+- [ ] I have read and followed the workflow indicated in the [CONTRIBUTING.md](https://github.com/open-mmlab/mmcv/blob/master/CONTRIBUTING.md) to create this PR.
+- [ ] Pre-commit or linting tools indicated in [CONTRIBUTING.md](https://github.com/open-mmlab/mmcv/blob/master/CONTRIBUTING.md) are used to fix the potential lint issues.
+- [ ] Bug fixes are covered by unit tests, the case that causes the bug should be added in the unit tests.
+- [ ] New functionalities are covered by complete unit tests. If not, please add more unit test to ensure the correctness.
+- [ ] The documentation has been modified accordingly, including docstring or example tutorials.
+
+**After PR**:
+
+- [ ] If the modification has potential influence on downstream or other related projects, this PR should be tested with some of those projects, like MMDet or MMCls.
+- [ ] CLA has been signed and all committers have signed the CLA in this PR.

groundingLMM/mmcv/.github/workflows/build.yml

@@ -0,0 +1,362 @@
+name: build
+
+on:
+  push:
+    paths-ignore:
+      - 'README.md'
+      - 'README_zh-CN.md'
+      - 'docs/**'
+      - 'examples/**'
+      - '.dev_scripts/**'
+
+  pull_request:
+    paths-ignore:
+      - 'README.md'
+      - 'README_zh-CN.md'
+      - 'docs/**'
+      - 'examples/**'
+      - '.dev_scripts/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  MMCV_WITH_OPS: 1
+
+jobs:
+  build_without_torch:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python-version: [3.7]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install system dependencies
+        run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg
+      - name: Build and install
+        run: rm -rf .eggs && pip install -e .
+      - name: Validate the installation
+        run: python -c "import mmcv"
+      - name: Run unittests and generate coverage report
+        run: |
+          pip install -r requirements/test.txt
+          pytest tests/ --ignore=tests/test_runner --ignore=tests/test_optimizer.py --ignore=tests/test_cnn --ignore=tests/test_parallel.py --ignore=tests/test_ops --ignore=tests/test_load_model_zoo.py --ignore=tests/test_utils/test_logging.py --ignore=tests/test_image/test_io.py --ignore=tests/test_utils/test_registry.py --ignore=tests/test_utils/test_parrots_jit.py --ignore=tests/test_utils/test_trace.py --ignore=tests/test_utils/test_hub.py
+
+  build_without_ops:
+    runs-on: ubuntu-18.04
+    env:
+      MMCV_WITH_OPS: 0
+    strategy:
+      matrix:
+        python-version: [3.7]
+        torch: [1.7.0, 1.8.0, 1.9.0]
+        include:
+          - torch: 1.7.0
+            torchvision: 0.8.1
+          - torch: 1.8.0
+            torchvision: 0.9.0
+          - torch: 1.9.0
+            torchvision: 0.10.0
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install system dependencies
+        run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg
+      - name: Install PyTorch
+        run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
+      - name: Build and install
+        run: rm -rf .eggs && pip install -e .
+      - name: Validate the installation
+        run: python -c "import mmcv"
+      - name: Run unittests
+        run: |
+          pip install -r requirements/test.txt
+          pytest tests/ --ignore=tests/test_ops
+
+  build_cpu:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python-version: [3.7]
+        torch: [1.5.1, 1.6.0, 1.7.0, 1.8.0, 1.9.0]
+        include:
+          - torch: 1.5.1
+            torchvision: 0.6.1
+          - torch: 1.6.0
+            torchvision: 0.7.0
+          - torch: 1.7.0
+            torchvision: 0.8.1
+          - torch: 1.8.0
+            torchvision: 0.9.0
+          - torch: 1.9.0
+            torchvision: 0.10.0
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install system dependencies
+        run: sudo apt-get update && sudo apt-get install -y ffmpeg libturbojpeg
+      - name: Install PyTorch
+        run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
+      # pstuil is an optional package to detect the number of CPU for compiling mmcv
+      - name: Install psutil
+        run: pip install psutil
+      - name: Create sdist and untar
+        run: |
+          MMCV_WITH_OPS=1 python setup.py sdist
+          tar zxvf dist/mmcv-full* -C /tmp
+          rm -r mmcv
+      - name: Build and install from sdist
+        run: |
+          pushd /tmp/mmcv-full*
+          pip install -e .
+          popd
+      - name: Validate the installation
+        run: python -c "import mmcv"
+      - name: Run unittests and generate coverage report
+        run: |
+          pip install -r requirements/test.txt
+          coverage run --branch --source=mmcv -m pytest tests/
+          coverage xml
+          coverage report -m
+
+  build_cu101:
+    runs-on: ubuntu-18.04
+    container:
+      image: pytorch/pytorch:1.6.0-cuda10.1-cudnn7-devel
+    env:
+      FORCE_CUDA: 1
+      MMCV_CUDA_ARGS: -gencode=arch=compute_61,code=sm_61
+    strategy:
+      matrix:
+        python-version: [3.7]
+        torch: [1.3.1, 1.5.1+cu101, 1.6.0+cu101, 1.7.0+cu101, 1.8.0+cu101]
+        include:
+          - torch: 1.3.1
+            torchvision: 0.4.2
+          - torch: 1.5.1+cu101
+            torchvision: 0.6.1+cu101
+          - torch: 1.6.0+cu101
+            torchvision: 0.7.0+cu101
+          - torch: 1.7.0+cu101
+            torchvision: 0.8.1+cu101
+          - torch: 1.8.0+cu101
+            torchvision: 0.9.0+cu101
+          - python-version: 3.6
+            torch: 1.8.0+cu101
+            torchvision: 0.9.0+cu101
+          - python-version: 3.8
+            torch: 1.8.0+cu101
+            torchvision: 0.9.0+cu101
+          - python-version: 3.9
+            torch: 1.8.0+cu101
+            torchvision: 0.9.0+cu101
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install python-dev
+        run: apt-get update && apt-get install -y python${{matrix.python-version}}-dev
+        if: ${{matrix.python-version != '3.9'}}
+      - name: Install Pillow
+        run: python -m pip install Pillow==6.2.2
+        if: ${{matrix.torchvision == '0.4.2'}}
+      # When we use a third-party container, we need to add python -m to call
+      # the user-installed pip when we use the pip command, otherwise it will
+      # call the system pip
+      - name: Install PyTorch
+        run: python -m pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html
+      - name: Install system dependencies
+        run: apt-get update && apt-get install -y ffmpeg libturbojpeg ninja-build
+      - name: Install dependencies for compiling onnx when python=3.9
+        run: python -m pip install protobuf && apt-get -y install libprotobuf-dev protobuf-compiler cmake
+        if: ${{matrix.python-version == '3.9'}}
+      # pstuil is an optional package to detect the number of CPU for compiling mmcv
+      - name: Install psutil
+        run: python -m pip install psutil
+      - name: Build and install
+        run: rm -rf .eggs && python -m pip install -e .
+      - name: Validate the installation
+        run: python -c "import mmcv"
+      - name: Run unittests and generate coverage report
+        run: |
+          python -m pip install -r requirements/test.txt
+          coverage run --branch --source=mmcv -m pytest tests/
+          coverage xml
+          coverage report -m
+      # Only upload coverage report for python3.7 && pytorch1.6
+      - name: Upload coverage to Codecov
+        if: ${{matrix.torch == '1.6.0+cu101' && matrix.python-version == '3.7'}}
+        uses: codecov/codecov-action@v1.0.14
+        with:
+          file: ./coverage.xml
+          flags: unittests
+          env_vars: OS,PYTHON
+          name: codecov-umbrella
+          fail_ci_if_error: false
+
+  build_cu102:
+    runs-on: ubuntu-18.04
+    container:
+      image: pytorch/pytorch:1.9.0-cuda10.2-cudnn7-devel
+    env:
+      FORCE_CUDA: 1
+      MMCV_CUDA_ARGS: -gencode=arch=compute_61,code=sm_61
+    strategy:
+      matrix:
+        python-version: [3.7]
+        torch: [1.9.0+cu102, 1.10.0+cu102, 1.11.0+cu102]
+        include:
+          - torch: 1.9.0+cu102
+            torchvision: 0.10.0+cu102
+          - torch: 1.10.0+cu102
+            torchvision: 0.11.0+cu102
+          - torch: 1.11.0+cu102
+            torchvision: 0.12.0+cu102
+          - python-version: 3.6
+            torch: 1.9.0+cu102
+            torchvision: 0.10.0+cu102
+          - python-version: 3.8
+            torch: 1.9.0+cu102
+            torchvision: 0.10.0+cu102
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install python-dev
+        run: apt-get update && apt-get install -y python${{matrix.python-version}}-dev
+      - name: python -m Install PyTorch
+        run: python -m pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html
+      - name: Install system dependencies
+        run: apt-get update && apt-get install -y ffmpeg libturbojpeg ninja-build
+      # pstuil is an optional package to detect the number of CPU for compiling mmcv
+      - name: Install psutil
+        run: python -m pip install psutil
+      - name: Build and install
+        run: rm -rf .eggs && python -m pip install -e .
+      - name: Validate the installation
+        run: python -c "import mmcv"
+      - name: Run unittests and generate coverage report
+        run: |
+          python -m pip install -r requirements/test.txt
+          coverage run --branch --source=mmcv -m pytest tests/
+          coverage xml
+
+  build_windows_without_ops:
+    runs-on: windows-latest
+    env:
+      MMCV_WITH_OPS: 0
+    strategy:
+      matrix:
+        torch: [1.7.1, 1.8.0, 1.9.0]
+        include:
+          - torch: 1.7.1
+            torchvision: 0.8.2
+          - torch: 1.8.0
+            torchvision: 0.9.0
+          - torch: 1.9.0
+            torchvision: 0.10.0
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Install PyTorch
+        run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu --no-cache-dir -f https://download.pytorch.org/whl/torch_stable.html
+      - name: Build and install
+        run: pip install -e .
+      - name: Validate the installation
+        run: python -c "import mmcv"
+      - name: Run unittests
+        run: |
+          pip install -r requirements/test.txt
+          pytest tests/ --ignore=tests/test_ops --ignore tests/test_utils/test_progressbar.py --ignore tests/test_utils/test_timer.py --ignore tests/test_image/test_io.py
+
+  build_windows:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        torch: [1.7.1, 1.8.0, 1.9.0]
+        include:
+          - torch: 1.7.1
+            torchvision: 0.8.2
+          - torch: 1.8.0
+            torchvision: 0.9.0
+          - torch: 1.9.0
+            torchvision: 0.10.0
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Install PyTorch
+        run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu --no-cache-dir -f https://download.pytorch.org/whl/torch_stable.html
+      - name: Build and install
+        run: pip install -e .
+      - name: Validate the installation
+        run: python -c "import mmcv"
+      - name: Run unittests
+        run: |
+          pip install -r requirements/test.txt
+          pytest tests/ --ignore tests/test_utils/test_progressbar.py --ignore tests/test_utils/test_timer.py --ignore tests/test_image/test_io.py
+
+  build_macos:
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        torch: [1.3.1, 1.5.1, 1.6.0, 1.7.0, 1.8.0, 1.9.0]
+        include:
+          - torch: 1.3.1
+            torchvision: 0.4.2
+          - torch: 1.5.1
+            torchvision: 0.6.1
+          - torch: 1.6.0
+            torchvision: 0.7.0
+          - torch: 1.7.0
+            torchvision: 0.8.1
+          - torch: 1.8.0
+            torchvision: 0.9.0
+          - torch: 1.9.0
+            torchvision: 0.10.0
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Install system dependencies
+        run: brew install ffmpeg jpeg-turbo
+      - name: Install utils
+        run: pip install psutil
+      - name: Install Pillow
+        run: pip install Pillow==6.2.2
+        if: ${{matrix.torchvision == '0.4.2'}}
+      - name: Install PyTorch
+        run: pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} --no-cache-dir
+      - name: Build and install
+        run: |
+          rm -rf .eggs
+          CC=clang CXX=clang++ CFLAGS='-stdlib=libc++' pip install -e .
+      - name: Validate the installation
+        run: python -c "import mmcv"
+      - name: Run unittests
+        run: |
+          pip install -r requirements/test.txt
+          # The timing on macos VMs is not precise, so we skip the progressbar tests
+          pytest tests/ --ignore tests/test_utils/test_progressbar.py --ignore tests/test_utils/test_timer.py

groundingLMM/mmcv/.github/workflows/build_pat.yml

@@ -0,0 +1,26 @@
+name: build_pat
+
+on: push
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  MMCV_WITH_OPS: 1
+
+jobs:
+  build_parrots:
+    runs-on: ubuntu-18.04
+    container:
+      image: ghcr.io/zhouzaida/parrots-mmcv:1.3.4
+      credentials:
+        username: zhouzaida
+        password: ${{ secrets.CR_PAT }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install unittest dependencies
+        run: pip install -r requirements/test.txt
+      - name: Build and install
+        run: rm -rf .eggs && MMCV_WITH_OPS=1 pip install -e .

groundingLMM/mmcv/.github/workflows/lint.yml

@@ -0,0 +1,33 @@
+name: lint
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Install pre-commit hook
+        run: |
+          # markdownlint requires ruby >= 2.7
+          sudo apt-add-repository ppa:brightbox/ruby-ng -y
+          sudo apt-get update
+          sudo apt-get install -y ruby2.7
+          pip install pre-commit
+          pre-commit install
+      - name: Linting
+        run: pre-commit run --all-files
+      - name: Format c/cuda codes with clang-format
+        uses: DoozyX/clang-format-lint-action@v0.11
+        with:
+          source: mmcv/ops/csrc
+          extensions: h,c,cpp,hpp,cu,cuh
+          style: google

groundingLMM/mmcv/.github/workflows/publish-to-pypi.yml

@@ -0,0 +1,46 @@
+name: deploy
+
+on: push
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-n-publish:
+    runs-on: ubuntu-18.04
+    if: startsWith(github.event.ref, 'refs/tags')
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Upgrade Setuptools
+        run: pip install setuptools --upgrade
+      - name: Build MMCV
+        run: python setup.py sdist
+      - name: Publish distribution to PyPI
+        run: |
+          pip install twine
+          twine upload dist/* -u __token__ -p ${{ secrets.pypi_password }}
+
+  build-n-publish_with_ops:
+    runs-on: ubuntu-18.04
+    if: startsWith(github.event.ref, 'refs/tags')
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Upgrade Setuptools
+        run: pip install setuptools --upgrade
+      - name: Build MMCV with ops
+        run: |
+          sed -i "s/os.getenv('MMCV_WITH_OPS', '0')/os.getenv('MMCV_WITH_OPS', '1')/g" setup.py
+          python setup.py sdist
+      - name: Publish distribution to PyPI
+        run: |
+          pip install twine
+          twine upload dist/* -u __token__ -p ${{ secrets.pypi_password }}

groundingLMM/mmcv/examples/train.py

@@ -0,0 +1,84 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+import torchvision.transforms as transforms
+from torch.utils.data import DataLoader
+from torchvision.datasets import CIFAR10
+
+from mmcv.parallel import MMDataParallel
+from mmcv.runner import EpochBasedRunner
+from mmcv.utils import get_logger
+
+
+class Model(nn.Module):
+
+    def __init__(self):
+        super(Model, self).__init__()
+        self.conv1 = nn.Conv2d(3, 6, 5)
+        self.pool = nn.MaxPool2d(2, 2)
+        self.conv2 = nn.Conv2d(6, 16, 5)
+        self.fc1 = nn.Linear(16 * 5 * 5, 120)
+        self.fc2 = nn.Linear(120, 84)
+        self.fc3 = nn.Linear(84, 10)
+        self.loss_fn = nn.CrossEntropyLoss()
+
+    def forward(self, x):
+        x = self.pool(F.relu(self.conv1(x)))
+        x = self.pool(F.relu(self.conv2(x)))
+        x = x.view(-1, 16 * 5 * 5)
+        x = F.relu(self.fc1(x))
+        x = F.relu(self.fc2(x))
+        x = self.fc3(x)
+        return x
+
+    def train_step(self, data, optimizer):
+        images, labels = data
+        predicts = self(images)  # -> self.__call__() -> self.forward()
+        loss = self.loss_fn(predicts, labels)
+        return {'loss': loss}
+
+
+if __name__ == '__main__':
+    model = Model()
+    if torch.cuda.is_available():
+        # only use gpu:0 to train
+        # Solved issue https://github.com/open-mmlab/mmcv/issues/1470
+        model = MMDataParallel(model.cuda(), device_ids=[0])
+
+    # dataset and dataloader
+    transform = transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
+    ])
+    trainset = CIFAR10(
+        root='data', train=True, download=True, transform=transform)
+    trainloader = DataLoader(
+        trainset, batch_size=128, shuffle=True, num_workers=2)
+
+    optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
+    logger = get_logger('mmcv')
+    # runner is a scheduler to manage the training
+    runner = EpochBasedRunner(
+        model,
+        optimizer=optimizer,
+        work_dir='./work_dir',
+        logger=logger,
+        max_epochs=4)
+
+    # learning rate scheduler config
+    lr_config = dict(policy='step', step=[2, 3])
+    # configuration of optimizer
+    optimizer_config = dict(grad_clip=None)
+    # configuration of saving checkpoints periodically
+    checkpoint_config = dict(interval=1)
+    # save log periodically and multiple hooks can be used simultaneously
+    log_config = dict(interval=100, hooks=[dict(type='TextLoggerHook')])
+    # register hooks to runner and those hooks will be invoked automatically
+    runner.register_training_hooks(
+        lr_config=lr_config,
+        optimizer_config=optimizer_config,
+        checkpoint_config=checkpoint_config,
+        log_config=log_config)
+
+    runner.run([trainloader], [('train', 1)])

groundingLMM/mmcv/mmcv.egg-info/PKG-INFO

@@ -0,0 +1,23 @@
+Metadata-Version: 2.1
+Name: mmcv
+Version: 1.4.7
+Summary: OpenMMLab Computer Vision Foundation
+Home-page: https://github.com/open-mmlab/mmcv
+Author: MMCV Contributors
+Author-email: openmmlab@gmail.com
+Keywords: computer vision
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Topic :: Utilities
+Provides-Extra: all
+Provides-Extra: tests
+Provides-Extra: build
+Provides-Extra: optional
+License-File: LICENSE
+License-File: LICENSES.md

groundingLMM/mmcv/mmcv.egg-info/SOURCES.txt

@@ -0,0 +1,877 @@
+.dockerignore
+.gitignore
+.owners.yml
+.pre-commit-config.yaml
+.readthedocs.yml
+CITATION.cff
+CONTRIBUTING.md
+Dockerfile
+Jenkinsfile
+LICENSE
+LICENSES.md
+MANIFEST.in
+README.md
+README_zh-CN.md
+TERMINOLOGY.md
+requirements.txt
+setup.cfg
+setup.py
+.circleci/config.yml
+.dev_scripts/check_installation.py
+.github/pull_request_template.md
+.github/ISSUE_TEMPLATE/config.yml
+.github/ISSUE_TEMPLATE/feature_request.md
+.github/ISSUE_TEMPLATE/general_questions.md
+.github/ISSUE_TEMPLATE/unexpected_report.md
+.github/workflows/build.yml
+.github/workflows/build_pat.yml
+.github/workflows/lint.yml
+.github/workflows/publish-to-pypi.yml
+docs/en/Makefile
+docs/en/api.rst
+docs/en/compatibility.md
+docs/en/conf.py
+docs/en/faq.md
+docs/en/index.rst
+docs/en/make.bat
+docs/en/mmcv-logo.png
+docs/en/_static/flow_img2toimg1.png
+docs/en/_static/flow_raw_images.png
+docs/en/_static/flow_visualization.png
+docs/en/_static/flow_warp.png
+docs/en/_static/flow_warp_diff.png
+docs/en/_static/parallel_progress.gif
+docs/en/_static/parallel_progress.png
+docs/en/_static/progress.gif
+docs/en/_static/progress.png
+docs/en/_static/qq_group_qrcode.jpg
+docs/en/_static/wechat_qrcode.jpg
+docs/en/_static/zhihu_qrcode.jpg
+docs/en/_static/community/1.png
+docs/en/_static/community/2.png
+docs/en/_static/community/3.png
+docs/en/_static/css/readthedocs.css
+docs/en/_static/image/mmcv-logo.png
+docs/en/community/contributing.md
+docs/en/community/pr.md
+docs/en/deployment/mmcv_ops_definition.md
+docs/en/deployment/onnx.md
+docs/en/deployment/onnxruntime_custom_ops.md
+docs/en/deployment/onnxruntime_op.md
+docs/en/deployment/tensorrt_custom_ops.md
+docs/en/deployment/tensorrt_plugin.md
+docs/en/get_started/build.md
+docs/en/get_started/installation.md
+docs/en/get_started/introduction.md
+docs/en/get_started/previous_versions.md
+docs/en/understand_mmcv/cnn.md
+docs/en/understand_mmcv/config.md
+docs/en/understand_mmcv/data_process.md
+docs/en/understand_mmcv/io.md
+docs/en/understand_mmcv/ops.md
+docs/en/understand_mmcv/registry.md
+docs/en/understand_mmcv/runner.md
+docs/en/understand_mmcv/utils.md
+docs/en/understand_mmcv/visualization.md
+docs/zh_cn/Makefile
+docs/zh_cn/api.rst
+docs/zh_cn/compatibility.md
+docs/zh_cn/conf.py
+docs/zh_cn/faq.md
+docs/zh_cn/index.rst
+docs/zh_cn/make.bat
+docs/zh_cn/_static/css/readthedocs.css
+docs/zh_cn/_static/image/mmcv-logo.png
+docs/zh_cn/community/contributing.md
+docs/zh_cn/community/pr.md
+docs/zh_cn/deployment/onnx.md
+docs/zh_cn/deployment/onnxruntime_custom_ops.md
+docs/zh_cn/deployment/onnxruntime_op.md
+docs/zh_cn/deployment/tensorrt_custom_ops.md
+docs/zh_cn/deployment/tensorrt_plugin.md
+docs/zh_cn/get_started/build.md
+docs/zh_cn/get_started/installation.md
+docs/zh_cn/get_started/introduction.md
+docs/zh_cn/get_started/previous_versions.md
+docs/zh_cn/understand_mmcv/cnn.md
+docs/zh_cn/understand_mmcv/config.md
+docs/zh_cn/understand_mmcv/data_process.md
+docs/zh_cn/understand_mmcv/io.md
+docs/zh_cn/understand_mmcv/ops.md
+docs/zh_cn/understand_mmcv/registry.md
+docs/zh_cn/understand_mmcv/runner.md
+docs/zh_cn/understand_mmcv/utils.md
+docs/zh_cn/understand_mmcv/visualization.md
+examples/train.py
+mmcv/__init__.py
+mmcv/version.py
+mmcv.egg-info/PKG-INFO
+mmcv.egg-info/SOURCES.txt
+mmcv.egg-info/dependency_links.txt
+mmcv.egg-info/not-zip-safe
+mmcv.egg-info/requires.txt
+mmcv.egg-info/top_level.txt
+mmcv/arraymisc/__init__.py
+mmcv/arraymisc/quantization.py
+mmcv/cnn/__init__.py
+mmcv/cnn/alexnet.py
+mmcv/cnn/builder.py
+mmcv/cnn/resnet.py
+mmcv/cnn/vgg.py
+mmcv/cnn/bricks/__init__.py
+mmcv/cnn/bricks/activation.py
+mmcv/cnn/bricks/context_block.py
+mmcv/cnn/bricks/conv.py
+mmcv/cnn/bricks/conv2d_adaptive_padding.py
+mmcv/cnn/bricks/conv_module.py
+mmcv/cnn/bricks/conv_ws.py
+mmcv/cnn/bricks/depthwise_separable_conv_module.py
+mmcv/cnn/bricks/drop.py
+mmcv/cnn/bricks/generalized_attention.py
+mmcv/cnn/bricks/hsigmoid.py
+mmcv/cnn/bricks/hswish.py
+mmcv/cnn/bricks/non_local.py
+mmcv/cnn/bricks/norm.py
+mmcv/cnn/bricks/padding.py
+mmcv/cnn/bricks/plugin.py
+mmcv/cnn/bricks/registry.py
+mmcv/cnn/bricks/scale.py
+mmcv/cnn/bricks/swish.py
+mmcv/cnn/bricks/transformer.py
+mmcv/cnn/bricks/upsample.py
+mmcv/cnn/bricks/wrappers.py
+mmcv/cnn/utils/__init__.py
+mmcv/cnn/utils/flops_counter.py
+mmcv/cnn/utils/fuse_conv_bn.py
+mmcv/cnn/utils/sync_bn.py
+mmcv/cnn/utils/weight_init.py
+mmcv/engine/__init__.py
+mmcv/engine/test.py
+mmcv/fileio/__init__.py
+mmcv/fileio/file_client.py
+mmcv/fileio/io.py
+mmcv/fileio/parse.py
+mmcv/fileio/handlers/__init__.py
+mmcv/fileio/handlers/base.py
+mmcv/fileio/handlers/json_handler.py
+mmcv/fileio/handlers/pickle_handler.py
+mmcv/fileio/handlers/yaml_handler.py
+mmcv/image/__init__.py
+mmcv/image/colorspace.py
+mmcv/image/geometric.py
+mmcv/image/io.py
+mmcv/image/misc.py
+mmcv/image/photometric.py
+mmcv/model_zoo/deprecated.json
+mmcv/model_zoo/mmcls.json
+mmcv/model_zoo/open_mmlab.json
+mmcv/onnx/__init__.py
+mmcv/onnx/info.py
+mmcv/onnx/symbolic.py
+mmcv/onnx/onnx_utils/__init__.py
+mmcv/onnx/onnx_utils/symbolic_helper.py
+mmcv/ops/__init__.py
+mmcv/ops/active_rotated_filter.py
+mmcv/ops/assign_score_withk.py
+mmcv/ops/ball_query.py
+mmcv/ops/bbox.py
+mmcv/ops/border_align.py
+mmcv/ops/box_iou_rotated.py
+mmcv/ops/carafe.py
+mmcv/ops/cc_attention.py
+mmcv/ops/contour_expand.py
+mmcv/ops/convex_iou.py
+mmcv/ops/corner_pool.py
+mmcv/ops/correlation.py
+mmcv/ops/deform_conv.py
+mmcv/ops/deform_roi_pool.py
+mmcv/ops/deprecated_wrappers.py
+mmcv/ops/focal_loss.py
+mmcv/ops/furthest_point_sample.py
+mmcv/ops/fused_bias_leakyrelu.py
+mmcv/ops/gather_points.py
+mmcv/ops/group_points.py
+mmcv/ops/info.py
+mmcv/ops/iou3d.py
+mmcv/ops/knn.py
+mmcv/ops/masked_conv.py
+mmcv/ops/merge_cells.py
+mmcv/ops/min_area_polygons.py
+mmcv/ops/modulated_deform_conv.py
+mmcv/ops/multi_scale_deform_attn.py
+mmcv/ops/nms.py
+mmcv/ops/pixel_group.py
+mmcv/ops/point_sample.py
+mmcv/ops/points_in_boxes.py
+mmcv/ops/points_in_polygons.py
+mmcv/ops/points_sampler.py
+mmcv/ops/psa_mask.py
+mmcv/ops/riroi_align_rotated.py
+mmcv/ops/roi_align.py
+mmcv/ops/roi_align_rotated.py
+mmcv/ops/roi_pool.py
+mmcv/ops/roiaware_pool3d.py
+mmcv/ops/roipoint_pool3d.py
+mmcv/ops/rotated_feature_align.py
+mmcv/ops/saconv.py
+mmcv/ops/scatter_points.py
+mmcv/ops/sparse_conv.py
+mmcv/ops/sparse_functional.py
+mmcv/ops/sparse_modules.py
+mmcv/ops/sparse_ops.py
+mmcv/ops/sparse_pool.py
+mmcv/ops/sparse_structure.py
+mmcv/ops/sync_bn.py
+mmcv/ops/three_interpolate.py
+mmcv/ops/three_nn.py
+mmcv/ops/tin_shift.py
+mmcv/ops/upfirdn2d.py
+mmcv/ops/voxelize.py
+mmcv/ops/csrc/README.md
+mmcv/ops/csrc/common/box_iou_rotated_utils.hpp
+mmcv/ops/csrc/common/parrots_cpp_helper.hpp
+mmcv/ops/csrc/common/parrots_cuda_helper.hpp
+mmcv/ops/csrc/common/pytorch_cpp_helper.hpp
+mmcv/ops/csrc/common/pytorch_cuda_helper.hpp
+mmcv/ops/csrc/common/pytorch_device_registry.hpp
+mmcv/ops/csrc/common/cuda/active_rotated_filter_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/assign_score_withk_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/ball_query_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/bbox_overlaps_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/border_align_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/box_iou_rotated_cuda.cuh
+mmcv/ops/csrc/common/cuda/carafe_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/carafe_naive_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/common_cuda_helper.hpp
+mmcv/ops/csrc/common/cuda/convex_iou_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/correlation_cuda.cuh
+mmcv/ops/csrc/common/cuda/deform_conv_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/deform_roi_pool_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/furthest_point_sample_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/gather_points_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/group_points_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/iou3d_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/knn_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/masked_conv2d_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/min_area_polygons_cuda.cuh
+mmcv/ops/csrc/common/cuda/modulated_deform_conv_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/ms_deform_attn_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/nms_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/nms_rotated_cuda.cuh
+mmcv/ops/csrc/common/cuda/parrots_cudawarpfunction.cuh
+mmcv/ops/csrc/common/cuda/points_in_boxes_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/points_in_polygons_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/psamask_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/riroi_align_rotated_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/roi_align_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/roi_align_rotated_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/roi_pool_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/roiaware_pool3d_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/roipoint_pool3d_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/rotated_feature_align_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/scatter_points_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/sigmoid_focal_loss_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/softmax_focal_loss_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/sync_bn_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/three_interpolate_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/three_nn_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/tin_shift_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/voxelization_cuda_kernel.cuh
+mmcv/ops/csrc/common/cuda/spconv/indice.cuh
+mmcv/ops/csrc/common/cuda/spconv/reordering.cuh
+mmcv/ops/csrc/common/utils/spconv/paramsgrid.h
+mmcv/ops/csrc/common/utils/spconv/prettyprint.h
+mmcv/ops/csrc/common/utils/spconv/pybind11_utils.h
+mmcv/ops/csrc/common/utils/spconv/spconv/geometry.h
+mmcv/ops/csrc/common/utils/spconv/spconv/indice.h
+mmcv/ops/csrc/common/utils/spconv/spconv/maxpool.h
+mmcv/ops/csrc/common/utils/spconv/spconv/mp_helper.h
+mmcv/ops/csrc/common/utils/spconv/spconv/point2voxel.h
+mmcv/ops/csrc/common/utils/spconv/spconv/reordering.h
+mmcv/ops/csrc/common/utils/spconv/tensorview/helper_kernel.cuh
+mmcv/ops/csrc/common/utils/spconv/tensorview/helper_launch.h
+mmcv/ops/csrc/common/utils/spconv/tensorview/tensorview.h
+mmcv/ops/csrc/onnxruntime/corner_pool.h
+mmcv/ops/csrc/onnxruntime/deform_conv.h
+mmcv/ops/csrc/onnxruntime/grid_sample.h
+mmcv/ops/csrc/onnxruntime/modulated_deform_conv.h
+mmcv/ops/csrc/onnxruntime/nms.h
+mmcv/ops/csrc/onnxruntime/onnxruntime_register.h
+mmcv/ops/csrc/onnxruntime/onnxruntime_session_options_config_keys.h
+mmcv/ops/csrc/onnxruntime/ort_mmcv_utils.h
+mmcv/ops/csrc/onnxruntime/reduce_ops.h
+mmcv/ops/csrc/onnxruntime/roi_align.h
+mmcv/ops/csrc/onnxruntime/roi_align_rotated.h
+mmcv/ops/csrc/onnxruntime/soft_nms.h
+mmcv/ops/csrc/onnxruntime/cpu/corner_pool.cpp
+mmcv/ops/csrc/onnxruntime/cpu/deform_conv.cpp
+mmcv/ops/csrc/onnxruntime/cpu/gridSample.cpp
+mmcv/ops/csrc/onnxruntime/cpu/modulated_deform_conv.cpp
+mmcv/ops/csrc/onnxruntime/cpu/nms.cpp
+mmcv/ops/csrc/onnxruntime/cpu/onnxruntime_register.cpp
+mmcv/ops/csrc/onnxruntime/cpu/reduce_ops.cpp
+mmcv/ops/csrc/onnxruntime/cpu/roi_align.cpp
+mmcv/ops/csrc/onnxruntime/cpu/roi_align_rotated.cpp
+mmcv/ops/csrc/onnxruntime/cpu/soft_nms.cpp
+mmcv/ops/csrc/parrots/active_rotated_filter.cpp
+mmcv/ops/csrc/parrots/active_rotated_filter_parrots.cpp
+mmcv/ops/csrc/parrots/active_rotated_filter_pytorch.h
+mmcv/ops/csrc/parrots/assign_score_withk.cpp
+mmcv/ops/csrc/parrots/assign_score_withk_parrots.cpp
+mmcv/ops/csrc/parrots/assign_score_withk_pytorch.h
+mmcv/ops/csrc/parrots/ball_query._parrots.cpp
+mmcv/ops/csrc/parrots/ball_query.cpp
+mmcv/ops/csrc/parrots/ball_query_pytorch.h
+mmcv/ops/csrc/parrots/bbox_overlaps.cpp
+mmcv/ops/csrc/parrots/bbox_overlaps_parrots.cpp
+mmcv/ops/csrc/parrots/bbox_overlaps_pytorch.h
+mmcv/ops/csrc/parrots/border_align.cpp
+mmcv/ops/csrc/parrots/border_align_parrots.cpp
+mmcv/ops/csrc/parrots/border_align_pytorch.h
+mmcv/ops/csrc/parrots/box_iou_rotated.cpp
+mmcv/ops/csrc/parrots/box_iou_rotated_parrots.cpp
+mmcv/ops/csrc/parrots/box_iou_rotated_pytorch.h
+mmcv/ops/csrc/parrots/carafe.cpp
+mmcv/ops/csrc/parrots/carafe_naive.cpp
+mmcv/ops/csrc/parrots/carafe_naive_parrots.cpp
+mmcv/ops/csrc/parrots/carafe_naive_pytorch.h
+mmcv/ops/csrc/parrots/carafe_parrots.cpp
+mmcv/ops/csrc/parrots/carafe_pytorch.h
+mmcv/ops/csrc/parrots/contour_expand.cpp
+mmcv/ops/csrc/parrots/contour_expand_parrots.cpp
+mmcv/ops/csrc/parrots/contour_expand_pytorch.h
+mmcv/ops/csrc/parrots/convex_iou.cpp
+mmcv/ops/csrc/parrots/convex_iou_parrots.cpp
+mmcv/ops/csrc/parrots/convex_iou_pytorch.h
+mmcv/ops/csrc/parrots/corner_pool.cpp
+mmcv/ops/csrc/parrots/corner_pool_parrots.cpp
+mmcv/ops/csrc/parrots/corner_pool_pytorch.h
+mmcv/ops/csrc/parrots/correlation.cpp
+mmcv/ops/csrc/parrots/correlation_parrots.cpp
+mmcv/ops/csrc/parrots/correlation_pytorch.h
+mmcv/ops/csrc/parrots/cudabind.cpp
+mmcv/ops/csrc/parrots/deform_conv.cpp
+mmcv/ops/csrc/parrots/deform_conv_parrots.cpp
+mmcv/ops/csrc/parrots/deform_conv_pytorch.h
+mmcv/ops/csrc/parrots/deform_roi_pool.cpp
+mmcv/ops/csrc/parrots/deform_roi_pool_parrots.cpp
+mmcv/ops/csrc/parrots/deform_roi_pool_pytorch.h
+mmcv/ops/csrc/parrots/focal_loss.cpp
+mmcv/ops/csrc/parrots/focal_loss_parrots.cpp
+mmcv/ops/csrc/parrots/focal_loss_pytorch.h
+mmcv/ops/csrc/parrots/furthest_point_sample.cpp
+mmcv/ops/csrc/parrots/furthest_point_sample_parrots.cpp
+mmcv/ops/csrc/parrots/furthest_point_sample_pytorch.h
+mmcv/ops/csrc/parrots/fused_bias_leakyrelu.cpp
+mmcv/ops/csrc/parrots/fused_bias_parrots.cpp
+mmcv/ops/csrc/parrots/gather_points.cpp
+mmcv/ops/csrc/parrots/gather_points_parrots.cpp
+mmcv/ops/csrc/parrots/gather_points_pytorch.h
+mmcv/ops/csrc/parrots/group_points.cpp
+mmcv/ops/csrc/parrots/group_points_parrots.cpp
+mmcv/ops/csrc/parrots/group_points_pytorch.h
+mmcv/ops/csrc/parrots/info.cpp
+mmcv/ops/csrc/parrots/iou3d.cpp
+mmcv/ops/csrc/parrots/iou3d_parrots.cpp
+mmcv/ops/csrc/parrots/iou3d_pytorch.h
+mmcv/ops/csrc/parrots/knn.cpp
+mmcv/ops/csrc/parrots/knn_parrots.cpp
+mmcv/ops/csrc/parrots/knn_pytorch.h
+mmcv/ops/csrc/parrots/masked_conv2d.cpp
+mmcv/ops/csrc/parrots/masked_conv2d_parrots.cpp
+mmcv/ops/csrc/parrots/masked_conv2d_pytorch.h
+mmcv/ops/csrc/parrots/min_area_polygons.cpp
+mmcv/ops/csrc/parrots/min_area_polygons_parrots.cpp
+mmcv/ops/csrc/parrots/min_area_polygons_pytorch.h
+mmcv/ops/csrc/parrots/modulated_deform_conv.cpp
+mmcv/ops/csrc/parrots/modulated_deform_conv_parrots.cpp
+mmcv/ops/csrc/parrots/modulated_deform_conv_pytorch.h
+mmcv/ops/csrc/parrots/ms_deform_attn.cpp
+mmcv/ops/csrc/parrots/ms_deform_attn_parrots.cpp
+mmcv/ops/csrc/parrots/nms.cpp
+mmcv/ops/csrc/parrots/nms_parrots.cpp
+mmcv/ops/csrc/parrots/nms_pytorch.h
+mmcv/ops/csrc/parrots/nms_rotated.cpp
+mmcv/ops/csrc/parrots/pixel_group.cpp
+mmcv/ops/csrc/parrots/pixel_group_parrots.cpp
+mmcv/ops/csrc/parrots/pixel_group_pytorch.h
+mmcv/ops/csrc/parrots/points_in_boxes.cpp
+mmcv/ops/csrc/parrots/points_in_boxes_parrots.cpp
+mmcv/ops/csrc/parrots/points_in_boxes_pytorch.h
+mmcv/ops/csrc/parrots/points_in_polygons.cpp
+mmcv/ops/csrc/parrots/points_in_polygons_parrots.cpp
+mmcv/ops/csrc/parrots/points_in_polygons_pytorch.h
+mmcv/ops/csrc/parrots/psamask.cpp
+mmcv/ops/csrc/parrots/psamask_parrots.cpp
+mmcv/ops/csrc/parrots/psamask_pytorch.h
+mmcv/ops/csrc/parrots/riroi_align_rotated.cpp
+mmcv/ops/csrc/parrots/riroi_align_rotated_parrots.cpp
+mmcv/ops/csrc/parrots/riroi_align_rotated_pytorch.h
+mmcv/ops/csrc/parrots/roi_align.cpp
+mmcv/ops/csrc/parrots/roi_align_parrots.cpp
+mmcv/ops/csrc/parrots/roi_align_pytorch.h
+mmcv/ops/csrc/parrots/roi_align_rotated.cpp
+mmcv/ops/csrc/parrots/roi_align_rotated_parrots.cpp
+mmcv/ops/csrc/parrots/roi_align_rotated_pytorch.h
+mmcv/ops/csrc/parrots/roi_pool.cpp
+mmcv/ops/csrc/parrots/roi_pool_parrots.cpp
+mmcv/ops/csrc/parrots/roi_pool_pytorch.h
+mmcv/ops/csrc/parrots/roiaware_pool3d.cpp
+mmcv/ops/csrc/parrots/roiaware_pool3d_parrots.cpp
+mmcv/ops/csrc/parrots/roiaware_pool3d_pytorch.h
+mmcv/ops/csrc/parrots/roipoint_pool3d.cpp
+mmcv/ops/csrc/parrots/roipoint_pool3d_parrots.cpp
+mmcv/ops/csrc/parrots/roipoint_pool3d_pytorch.h
+mmcv/ops/csrc/parrots/rotated_feature_align.cpp
+mmcv/ops/csrc/parrots/rotated_feature_align_parrots.cpp
+mmcv/ops/csrc/parrots/rotated_feature_align_pytorch.h
+mmcv/ops/csrc/parrots/sync_bn.cpp
+mmcv/ops/csrc/parrots/sync_bn_parrots.cpp
+mmcv/ops/csrc/parrots/sync_bn_pytorch.h
+mmcv/ops/csrc/parrots/three_interpolate.cpp
+mmcv/ops/csrc/parrots/three_interpolate_parrots.cpp
+mmcv/ops/csrc/parrots/three_interpolate_pytorch.h
+mmcv/ops/csrc/parrots/three_nn.cpp
+mmcv/ops/csrc/parrots/three_nn_parrots.cpp
+mmcv/ops/csrc/parrots/three_nn_pytorch.h
+mmcv/ops/csrc/parrots/tin_shift.cpp
+mmcv/ops/csrc/parrots/tin_shift_parrots.cpp
+mmcv/ops/csrc/parrots/tin_shift_pytorch.h
+mmcv/ops/csrc/parrots/upfirdn2d.cpp
+mmcv/ops/csrc/parrots/upfirdn2d_parrots.cpp
+mmcv/ops/csrc/parrots/voxelization.cpp
+mmcv/ops/csrc/parrots/voxelization_parrots.cpp
+mmcv/ops/csrc/parrots/voxelization_pytorch.h
+mmcv/ops/csrc/pytorch/active_rotated_filter.cpp
+mmcv/ops/csrc/pytorch/assign_score_withk.cpp
+mmcv/ops/csrc/pytorch/ball_query.cpp
+mmcv/ops/csrc/pytorch/bbox_overlaps.cpp
+mmcv/ops/csrc/pytorch/border_align.cpp
+mmcv/ops/csrc/pytorch/box_iou_rotated.cpp
+mmcv/ops/csrc/pytorch/carafe.cpp
+mmcv/ops/csrc/pytorch/carafe_naive.cpp
+mmcv/ops/csrc/pytorch/contour_expand.cpp
+mmcv/ops/csrc/pytorch/convex_iou.cpp
+mmcv/ops/csrc/pytorch/corner_pool.cpp
+mmcv/ops/csrc/pytorch/correlation.cpp
+mmcv/ops/csrc/pytorch/deform_conv.cpp
+mmcv/ops/csrc/pytorch/deform_roi_pool.cpp
+mmcv/ops/csrc/pytorch/focal_loss.cpp
+mmcv/ops/csrc/pytorch/furthest_point_sample.cpp
+mmcv/ops/csrc/pytorch/fused_bias_leakyrelu.cpp
+mmcv/ops/csrc/pytorch/fused_spconv_ops.cpp
+mmcv/ops/csrc/pytorch/gather_points.cpp
+mmcv/ops/csrc/pytorch/group_points.cpp
+mmcv/ops/csrc/pytorch/info.cpp
+mmcv/ops/csrc/pytorch/iou3d.cpp
+mmcv/ops/csrc/pytorch/knn.cpp
+mmcv/ops/csrc/pytorch/masked_conv2d.cpp
+mmcv/ops/csrc/pytorch/min_area_polygons.cpp
+mmcv/ops/csrc/pytorch/modulated_deform_conv.cpp
+mmcv/ops/csrc/pytorch/ms_deform_attn.cpp
+mmcv/ops/csrc/pytorch/nms.cpp
+mmcv/ops/csrc/pytorch/nms_rotated.cpp
+mmcv/ops/csrc/pytorch/pixel_group.cpp
+mmcv/ops/csrc/pytorch/points_in_boxes.cpp
+mmcv/ops/csrc/pytorch/points_in_polygons.cpp
+mmcv/ops/csrc/pytorch/psamask.cpp
+mmcv/ops/csrc/pytorch/pybind.cpp
+mmcv/ops/csrc/pytorch/riroi_align_rotated.cpp
+mmcv/ops/csrc/pytorch/roi_align.cpp
+mmcv/ops/csrc/pytorch/roi_align_rotated.cpp
+mmcv/ops/csrc/pytorch/roi_pool.cpp
+mmcv/ops/csrc/pytorch/roiaware_pool3d.cpp
+mmcv/ops/csrc/pytorch/roipoint_pool3d.cpp
+mmcv/ops/csrc/pytorch/rotated_feature_align.cpp
+mmcv/ops/csrc/pytorch/scatter_points.cpp
+mmcv/ops/csrc/pytorch/sparse_pool_ops.cpp
+mmcv/ops/csrc/pytorch/spconv_ops.cpp
+mmcv/ops/csrc/pytorch/spconv_utils.h
+mmcv/ops/csrc/pytorch/sync_bn.cpp
+mmcv/ops/csrc/pytorch/three_interpolate.cpp
+mmcv/ops/csrc/pytorch/three_nn.cpp
+mmcv/ops/csrc/pytorch/tin_shift.cpp
+mmcv/ops/csrc/pytorch/upfirdn2d.cpp
+mmcv/ops/csrc/pytorch/voxelization.cpp
+mmcv/ops/csrc/pytorch/cpu/active_rotated_filter.cpp
+mmcv/ops/csrc/pytorch/cpu/box_iou_rotated.cpp
+mmcv/ops/csrc/pytorch/cpu/deform_conv.cpp
+mmcv/ops/csrc/pytorch/cpu/modulated_deform_conv.cpp
+mmcv/ops/csrc/pytorch/cpu/nms.cpp
+mmcv/ops/csrc/pytorch/cpu/nms_rotated.cpp
+mmcv/ops/csrc/pytorch/cpu/pixel_group.cpp
+mmcv/ops/csrc/pytorch/cpu/points_in_boxes.cpp
+mmcv/ops/csrc/pytorch/cpu/psamask.cpp
+mmcv/ops/csrc/pytorch/cpu/roi_align.cpp
+mmcv/ops/csrc/pytorch/cpu/roi_align_rotated.cpp
+mmcv/ops/csrc/pytorch/cpu/sparse_indice.cpp
+mmcv/ops/csrc/pytorch/cpu/sparse_maxpool.cpp
+mmcv/ops/csrc/pytorch/cpu/sparse_reordering.cpp
+mmcv/ops/csrc/pytorch/cpu/voxelization.cpp
+mmcv/ops/csrc/pytorch/cuda/active_rotated_filter_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/assign_score_withk_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/ball_query_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/bbox_overlaps_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/border_align_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/box_iou_rotated_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/carafe_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/carafe_naive_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/convex_iou.cu
+mmcv/ops/csrc/pytorch/cuda/correlation_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/cudabind.cpp
+mmcv/ops/csrc/pytorch/cuda/deform_conv_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/deform_roi_pool_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/focal_loss_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/furthest_point_sample_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/fused_bias_leakyrelu_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/fused_spconv_ops_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/gather_points_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/group_points_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/iou3d_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/knn_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/masked_conv2d_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/min_area_polygons.cu
+mmcv/ops/csrc/pytorch/cuda/modulated_deform_conv_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/ms_deform_attn_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/nms_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/nms_rotated_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/points_in_boxes_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/points_in_polygons_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/psamask_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/riroi_align_rotated_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/roi_align_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/roi_align_rotated_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/roi_pool_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/roiaware_pool3d_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/roipoint_pool3d_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/rotated_feature_align_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/scatter_points_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/sparse_indice.cu
+mmcv/ops/csrc/pytorch/cuda/sparse_maxpool.cu
+mmcv/ops/csrc/pytorch/cuda/sparse_pool_ops_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/sparse_reordering.cu
+mmcv/ops/csrc/pytorch/cuda/spconv_ops_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/sync_bn_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/three_interpolate_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/three_nn_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/tin_shift_cuda.cu
+mmcv/ops/csrc/pytorch/cuda/upfirdn2d_kernel.cu
+mmcv/ops/csrc/pytorch/cuda/voxelization_cuda.cu
+mmcv/ops/csrc/tensorrt/trt_corner_pool.hpp
+mmcv/ops/csrc/tensorrt/trt_cuda_helper.cuh
+mmcv/ops/csrc/tensorrt/trt_cummaxmin.hpp
+mmcv/ops/csrc/tensorrt/trt_deform_conv.hpp
+mmcv/ops/csrc/tensorrt/trt_grid_sampler.hpp
+mmcv/ops/csrc/tensorrt/trt_instance_norm.hpp
+mmcv/ops/csrc/tensorrt/trt_modulated_deform_conv.hpp
+mmcv/ops/csrc/tensorrt/trt_nms.hpp
+mmcv/ops/csrc/tensorrt/trt_plugin.hpp
+mmcv/ops/csrc/tensorrt/trt_plugin_helper.hpp
+mmcv/ops/csrc/tensorrt/trt_roi_align.hpp
+mmcv/ops/csrc/tensorrt/trt_scatternd.hpp
+mmcv/ops/csrc/tensorrt/trt_serialize.hpp
+mmcv/ops/csrc/tensorrt/plugins/trt_corner_pool.cpp
+mmcv/ops/csrc/tensorrt/plugins/trt_corner_pool_kernel.cu
+mmcv/ops/csrc/tensorrt/plugins/trt_cuda_helper.cu
+mmcv/ops/csrc/tensorrt/plugins/trt_cummaxmin.cpp
+mmcv/ops/csrc/tensorrt/plugins/trt_cummaxmin_kernel.cu
+mmcv/ops/csrc/tensorrt/plugins/trt_deform_conv.cpp
+mmcv/ops/csrc/tensorrt/plugins/trt_deform_conv_kernel.cu
+mmcv/ops/csrc/tensorrt/plugins/trt_grid_sampler.cpp
+mmcv/ops/csrc/tensorrt/plugins/trt_grid_sampler_kernel.cu
+mmcv/ops/csrc/tensorrt/plugins/trt_instance_norm.cpp
+mmcv/ops/csrc/tensorrt/plugins/trt_modulated_deform_conv.cpp
+mmcv/ops/csrc/tensorrt/plugins/trt_modulated_deform_conv_kernel.cu
+mmcv/ops/csrc/tensorrt/plugins/trt_nms.cpp
+mmcv/ops/csrc/tensorrt/plugins/trt_nms_kernel.cu
+mmcv/ops/csrc/tensorrt/plugins/trt_plugin.cpp
+mmcv/ops/csrc/tensorrt/plugins/trt_roi_align.cpp
+mmcv/ops/csrc/tensorrt/plugins/trt_roi_align_kernel.cu
+mmcv/ops/csrc/tensorrt/plugins/trt_scatternd.cpp
+mmcv/ops/csrc/tensorrt/plugins/trt_scatternd_kernel.cu
+mmcv/parallel/__init__.py
+mmcv/parallel/_functions.py
+mmcv/parallel/collate.py
+mmcv/parallel/data_container.py
+mmcv/parallel/data_parallel.py
+mmcv/parallel/distributed.py
+mmcv/parallel/distributed_deprecated.py
+mmcv/parallel/registry.py
+mmcv/parallel/scatter_gather.py
+mmcv/parallel/utils.py
+mmcv/runner/__init__.py
+mmcv/runner/base_module.py
+mmcv/runner/base_runner.py
+mmcv/runner/builder.py
+mmcv/runner/checkpoint.py
+mmcv/runner/default_constructor.py
+mmcv/runner/dist_utils.py
+mmcv/runner/epoch_based_runner.py
+mmcv/runner/fp16_utils.py
+mmcv/runner/iter_based_runner.py
+mmcv/runner/log_buffer.py
+mmcv/runner/priority.py
+mmcv/runner/utils.py
+mmcv/runner/hooks/__init__.py
+mmcv/runner/hooks/checkpoint.py
+mmcv/runner/hooks/closure.py
+mmcv/runner/hooks/ema.py
+mmcv/runner/hooks/evaluation.py
+mmcv/runner/hooks/hook.py
+mmcv/runner/hooks/iter_timer.py
+mmcv/runner/hooks/lr_updater.py
+mmcv/runner/hooks/memory.py
+mmcv/runner/hooks/momentum_updater.py
+mmcv/runner/hooks/optimizer.py
+mmcv/runner/hooks/profiler.py
+mmcv/runner/hooks/sampler_seed.py
+mmcv/runner/hooks/sync_buffer.py
+mmcv/runner/hooks/logger/__init__.py
+mmcv/runner/hooks/logger/base.py
+mmcv/runner/hooks/logger/dvclive.py
+mmcv/runner/hooks/logger/mlflow.py
+mmcv/runner/hooks/logger/neptune.py
+mmcv/runner/hooks/logger/pavi.py
+mmcv/runner/hooks/logger/segmind.py
+mmcv/runner/hooks/logger/tensorboard.py
+mmcv/runner/hooks/logger/text.py
+mmcv/runner/hooks/logger/wandb.py
+mmcv/runner/optimizer/__init__.py
+mmcv/runner/optimizer/builder.py
+mmcv/runner/optimizer/default_constructor.py
+mmcv/tensorrt/__init__.py
+mmcv/tensorrt/init_plugins.py
+mmcv/tensorrt/preprocess.py
+mmcv/tensorrt/tensorrt_utils.py
+mmcv/utils/__init__.py
+mmcv/utils/config.py
+mmcv/utils/env.py
+mmcv/utils/ext_loader.py
+mmcv/utils/hub.py
+mmcv/utils/logging.py
+mmcv/utils/misc.py
+mmcv/utils/parrots_jit.py
+mmcv/utils/parrots_wrapper.py
+mmcv/utils/path.py
+mmcv/utils/progressbar.py
+mmcv/utils/registry.py
+mmcv/utils/seed.py
+mmcv/utils/testing.py
+mmcv/utils/timer.py
+mmcv/utils/trace.py
+mmcv/utils/version_utils.py
+mmcv/video/__init__.py
+mmcv/video/io.py
+mmcv/video/optflow.py
+mmcv/video/processing.py
+mmcv/visualization/__init__.py
+mmcv/visualization/color.py
+mmcv/visualization/image.py
+mmcv/visualization/optflow.py
+requirements/build.txt
+requirements/docs.txt
+requirements/optional.txt
+requirements/runtime.txt
+requirements/test.txt
+tests/test_arraymisc.py
+tests/test_fileclient.py
+tests/test_fileio.py
+tests/test_load_model_zoo.py
+tests/test_parallel.py
+tests/test_visualization.py
+tests/data/batched_nms_data.pkl
+tests/data/color.jpg
+tests/data/color_exif.jpg
+tests/data/filelist.txt
+tests/data/gray_alpha.png
+tests/data/grayscale.jpg
+tests/data/grayscale_dim3.jpg
+tests/data/mapping.txt
+tests/data/optflow.flo
+tests/data/optflow_concat0.jpg
+tests/data/optflow_concat1.jpg
+tests/data/palette.gif
+tests/data/sparse_flow.png
+tests/data/test.mp4
+tests/data/uint16-5channel.tif
+tests/data/config/a.b.py
+tests/data/config/a.py
+tests/data/config/b.json
+tests/data/config/base.py
+tests/data/config/c.yaml
+tests/data/config/code.py
+tests/data/config/d.py
+tests/data/config/delete.py
+tests/data/config/deprecated.py
+tests/data/config/deprecated_as_base.py
+tests/data/config/e.py
+tests/data/config/expected.py
+tests/data/config/f.py
+tests/data/config/g.py
+tests/data/config/h.py
+tests/data/config/i_base.py
+tests/data/config/i_child.py
+tests/data/config/l.py
+tests/data/config/l1.py
+tests/data/config/l2.yaml
+tests/data/config/l3.json
+tests/data/config/l4.py
+tests/data/config/m.py
+tests/data/config/n.py
+tests/data/config/o.json
+tests/data/config/p.yaml
+tests/data/config/q.py
+tests/data/config/r.py
+tests/data/config/s.py
+tests/data/config/t.json
+tests/data/config/t.py
+tests/data/config/t.yaml
+tests/data/config/u.json
+tests/data/config/u.py
+tests/data/config/u.yaml
+tests/data/config/v.py
+tests/data/demo.lmdb/data.mdb
+tests/data/demo.lmdb/lock.mdb
+tests/data/for_3d_ops/features_for_fps_distance.npy
+tests/data/for_3d_ops/fps_idx.npy
+tests/data/for_3d_ops/test_voxel.npy
+tests/data/for_ccattention/ccattention_input.bin
+tests/data/for_ccattention/ccattention_output.bin
+tests/data/for_psa_mask/psa_input.bin
+tests/data/for_psa_mask/psa_output_collect.bin
+tests/data/for_psa_mask/psa_output_distribute.bin
+tests/data/for_scan/.file
+tests/data/for_scan/1.json
+tests/data/for_scan/1.txt
+tests/data/for_scan/2.json
+tests/data/for_scan/2.txt
+tests/data/for_scan/3.TXT
+tests/data/for_scan/a.bin
+tests/data/for_scan/sub/1.json
+tests/data/for_scan/sub/1.txt
+tests/data/model_zoo/deprecated.json
+tests/data/model_zoo/open_mmlab.json
+tests/data/model_zoo/mmcv_home/open_mmlab.json
+tests/data/model_zoo/mmcv_home/test.pth
+tests/data/model_zoo/mmcv_home/val.pth
+tests/data/patches/0.npy
+tests/data/patches/1.npy
+tests/data/patches/2.npy
+tests/data/patches/3.npy
+tests/data/patches/4.npy
+tests/data/patches/pad0_0.npy
+tests/data/patches/pad0_1.npy
+tests/data/patches/pad0_2.npy
+tests/data/patches/pad0_3.npy
+tests/data/patches/pad0_4.npy
+tests/data/patches/pad_0.npy
+tests/data/patches/pad_1.npy
+tests/data/patches/pad_2.npy
+tests/data/patches/pad_3.npy
+tests/data/patches/pad_4.npy
+tests/data/patches/scale_0.npy
+tests/data/patches/scale_1.npy
+tests/data/patches/scale_2.npy
+tests/data/patches/scale_3.npy
+tests/data/patches/scale_4.npy
+tests/data/scripts/hello.py
+tests/test_cnn/test_build_layers.py
+tests/test_cnn/test_context_block.py
+tests/test_cnn/test_conv2d_adaptive_padding.py
+tests/test_cnn/test_conv_module.py
+tests/test_cnn/test_depthwise_seperable_conv_module.py
+tests/test_cnn/test_flops_counter.py
+tests/test_cnn/test_fuse_conv_bn.py
+tests/test_cnn/test_generalized_attention.py
+tests/test_cnn/test_hsigmoid.py
+tests/test_cnn/test_hswish.py
+tests/test_cnn/test_model_registry.py
+tests/test_cnn/test_non_local.py
+tests/test_cnn/test_revert_syncbn.py
+tests/test_cnn/test_scale.py
+tests/test_cnn/test_swish.py
+tests/test_cnn/test_transformer.py
+tests/test_cnn/test_weight_init.py
+tests/test_cnn/test_wrappers.py
+tests/test_image/test_colorspace.py
+tests/test_image/test_geometric.py
+tests/test_image/test_image_misc.py
+tests/test_image/test_io.py
+tests/test_image/test_photometric.py
+tests/test_ops/test_active_rotated_filter.py
+tests/test_ops/test_assign_score_withk.py
+tests/test_ops/test_ball_query.py
+tests/test_ops/test_bbox.py
+tests/test_ops/test_bilinear_grid_sample.py
+tests/test_ops/test_border_align.py
+tests/test_ops/test_box_iou_rotated.py
+tests/test_ops/test_carafe.py
+tests/test_ops/test_cc_attention.py
+tests/test_ops/test_contour_expand.py
+tests/test_ops/test_convex_iou.py
+tests/test_ops/test_corner_pool.py
+tests/test_ops/test_correlation.py
+tests/test_ops/test_deform_conv.py
+tests/test_ops/test_deform_roi_pool.py
+tests/test_ops/test_focal_loss.py
+tests/test_ops/test_furthest_point_sample.py
+tests/test_ops/test_fused_bias_leakyrelu.py
+tests/test_ops/test_gather_points.py
+tests/test_ops/test_group_points.py
+tests/test_ops/test_info.py
+tests/test_ops/test_iou3d.py
+tests/test_ops/test_knn.py
+tests/test_ops/test_masked_conv2d.py
+tests/test_ops/test_merge_cells.py
+tests/test_ops/test_min_area_polygons.py
+tests/test_ops/test_modulated_deform_conv.py
+tests/test_ops/test_ms_deformable_attn.py
+tests/test_ops/test_nms.py
+tests/test_ops/test_nms_rotated.py
+tests/test_ops/test_onnx.py
+tests/test_ops/test_pixel_group.py
+tests/test_ops/test_points_in_polygons.py
+tests/test_ops/test_psa_mask.py
+tests/test_ops/test_riroi_align_rotated.py
+tests/test_ops/test_roi_align.py
+tests/test_ops/test_roi_align_rotated.py
+tests/test_ops/test_roi_pool.py
+tests/test_ops/test_roiaware_pool3d.py
+tests/test_ops/test_roipoint_pool3d.py
+tests/test_ops/test_rotated_feature_align.py
+tests/test_ops/test_saconv.py
+tests/test_ops/test_scatter_points.py
+tests/test_ops/test_spconv.py
+tests/test_ops/test_syncbn.py
+tests/test_ops/test_tensorrt.py
+tests/test_ops/test_tensorrt_preprocess.py
+tests/test_ops/test_three_interpolate.py
+tests/test_ops/test_three_nn.py
+tests/test_ops/test_tin_shift.py
+tests/test_ops/test_upfirdn2d.py
+tests/test_ops/test_voxelization.py
+tests/test_runner/test_basemodule.py
+tests/test_runner/test_checkpoint.py
+tests/test_runner/test_dist_utils.py
+tests/test_runner/test_eval_hook.py
+tests/test_runner/test_fp16.py
+tests/test_runner/test_hooks.py
+tests/test_runner/test_optimizer.py
+tests/test_runner/test_runner.py
+tests/test_runner/test_utils.py
+tests/test_utils/test_config.py
+tests/test_utils/test_env.py
+tests/test_utils/test_hub.py
+tests/test_utils/test_logging.py
+tests/test_utils/test_misc.py
+tests/test_utils/test_parrots_jit.py
+tests/test_utils/test_path.py
+tests/test_utils/test_progressbar.py
+tests/test_utils/test_registry.py
+tests/test_utils/test_testing.py
+tests/test_utils/test_timer.py
+tests/test_utils/test_trace.py
+tests/test_utils/test_version_utils.py
+tests/test_video/test_optflow.py
+tests/test_video/test_processing.py
+tests/test_video/test_reader.py

groundingLMM/mmcv/mmcv.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

groundingLMM/mmcv/mmcv.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

groundingLMM/mmcv/mmcv.egg-info/requires.txt

@@ -0,0 +1,48 @@
+addict
+numpy
+packaging
+Pillow
+pyyaml
+yapf
+
+[:sys_platform == "win32"]
+regex
+
+[all]
+pytest-runner
+ninja
+addict
+numpy
+packaging
+Pillow
+pyyaml
+yapf
+coverage
+lmdb
+onnx==1.7.0
+onnxoptimizer
+onnxruntime>=1.8.0
+pytest
+PyTurboJPEG
+scipy
+tifffile
+
+[all:sys_platform == "win32"]
+regex
+
+[build]
+pytest-runner
+
+[optional]
+ninja
+
+[tests]
+coverage
+lmdb
+onnx==1.7.0
+onnxoptimizer
+onnxruntime>=1.8.0
+pytest
+PyTurboJPEG
+scipy
+tifffile

groundingLMM/mmcv/mmcv.egg-info/top_level.txt

@@ -0,0 +1 @@
+mmcv

groundingLMM/mmcv/mmcv/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+# flake8: noqa
+from .arraymisc import *
+from .fileio import *
+from .image import *
+from .utils import *
+from .version import *
+from .video import *
+from .visualization import *
+
+# The following modules are not imported to this level, so mmcv may be used
+# without PyTorch.
+# - runner
+# - parallel
+# - op

groundingLMM/mmcv/mmcv/arraymisc/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from .quantization import dequantize, quantize
+
+__all__ = ['quantize', 'dequantize']

groundingLMM/mmcv/mmcv/arraymisc/quantization.py

@@ -0,0 +1,55 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import numpy as np
+
+
+def quantize(arr, min_val, max_val, levels, dtype=np.int64):
+    """Quantize an array of (-inf, inf) to [0, levels-1].
+
+    Args:
+        arr (ndarray): Input array.
+        min_val (scalar): Minimum value to be clipped.
+        max_val (scalar): Maximum value to be clipped.
+        levels (int): Quantization levels.
+        dtype (np.type): The type of the quantized array.
+
+    Returns:
+        tuple: Quantized array.
+    """
+    if not (isinstance(levels, int) and levels > 1):
+        raise ValueError(
+            f'levels must be a positive integer, but got {levels}')
+    if min_val >= max_val:
+        raise ValueError(
+            f'min_val ({min_val}) must be smaller than max_val ({max_val})')
+
+    arr = np.clip(arr, min_val, max_val) - min_val
+    quantized_arr = np.minimum(
+        np.floor(levels * arr / (max_val - min_val)).astype(dtype), levels - 1)
+
+    return quantized_arr
+
+
+def dequantize(arr, min_val, max_val, levels, dtype=np.float64):
+    """Dequantize an array.
+
+    Args:
+        arr (ndarray): Input array.
+        min_val (scalar): Minimum value to be clipped.
+        max_val (scalar): Maximum value to be clipped.
+        levels (int): Quantization levels.
+        dtype (np.type): The type of the dequantized array.
+
+    Returns:
+        tuple: Dequantized array.
+    """
+    if not (isinstance(levels, int) and levels > 1):
+        raise ValueError(
+            f'levels must be a positive integer, but got {levels}')
+    if min_val >= max_val:
+        raise ValueError(
+            f'min_val ({min_val}) must be smaller than max_val ({max_val})')
+
+    dequantized_arr = (arr + 0.5).astype(dtype) * (max_val -
+                                                   min_val) / levels + min_val
+
+    return dequantized_arr

groundingLMM/mmcv/mmcv/fileio/__init__.py

@@ -0,0 +1,11 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from .file_client import BaseStorageBackend, FileClient
+from .handlers import BaseFileHandler, JsonHandler, PickleHandler, YamlHandler
+from .io import dump, load, register_handler
+from .parse import dict_from_file, list_from_file
+
+__all__ = [
+    'BaseStorageBackend', 'FileClient', 'load', 'dump', 'register_handler',
+    'BaseFileHandler', 'JsonHandler', 'PickleHandler', 'YamlHandler',
+    'list_from_file', 'dict_from_file'
+]

groundingLMM/mmcv/mmcv/fileio/file_client.py

@@ -0,0 +1,1149 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import inspect
+import os
+import os.path as osp
+import re
+import tempfile
+import warnings
+from abc import ABCMeta, abstractmethod
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Iterable, Iterator, Optional, Tuple, Union
+from urllib.request import urlopen
+
+import mmcv
+from mmcv.utils.misc import has_method
+from mmcv.utils.path import is_filepath
+
+
+class BaseStorageBackend(metaclass=ABCMeta):
+    """Abstract class of storage backends.
+
+    All backends need to implement two apis: ``get()`` and ``get_text()``.
+    ``get()`` reads the file as a byte stream and ``get_text()`` reads the file
+    as texts.
+    """
+
+    # a flag to indicate whether the backend can create a symlink for a file
+    _allow_symlink = False
+
+    @property
+    def name(self):
+        return self.__class__.__name__
+
+    @property
+    def allow_symlink(self):
+        return self._allow_symlink
+
+    @abstractmethod
+    def get(self, filepath):
+        pass
+
+    @abstractmethod
+    def get_text(self, filepath):
+        pass
+
+
+class CephBackend(BaseStorageBackend):
+    """Ceph storage backend (for internal use).
+
+    Args:
+        path_mapping (dict|None): path mapping dict from local path to Petrel
+            path. When ``path_mapping={'src': 'dst'}``, ``src`` in ``filepath``
+            will be replaced by ``dst``. Default: None.
+
+    .. warning::
+        :class:`mmcv.fileio.file_client.CephBackend` will be deprecated,
+        please use :class:`mmcv.fileio.file_client.PetrelBackend` instead.
+    """
+
+    def __init__(self, path_mapping=None):
+        try:
+            import ceph
+        except ImportError:
+            raise ImportError('Please install ceph to enable CephBackend.')
+
+        warnings.warn(
+            'CephBackend will be deprecated, please use PetrelBackend instead',
+            DeprecationWarning)
+        self._client = ceph.S3Client()
+        assert isinstance(path_mapping, dict) or path_mapping is None
+        self.path_mapping = path_mapping
+
+    def get(self, filepath):
+        filepath = str(filepath)
+        if self.path_mapping is not None:
+            for k, v in self.path_mapping.items():
+                filepath = filepath.replace(k, v)
+        value = self._client.Get(filepath)
+        value_buf = memoryview(value)
+        return value_buf
+
+    def get_text(self, filepath, encoding=None):
+        raise NotImplementedError
+
+
+class PetrelBackend(BaseStorageBackend):
+    """Petrel storage backend (for internal use).
+
+    PetrelBackend supports reading and writing data to multiple clusters.
+    If the file path contains the cluster name, PetrelBackend will read data
+    from specified cluster or write data to it. Otherwise, PetrelBackend will
+    access the default cluster.
+
+    Args:
+        path_mapping (dict, optional): Path mapping dict from local path to
+            Petrel path. When ``path_mapping={'src': 'dst'}``, ``src`` in
+            ``filepath`` will be replaced by ``dst``. Default: None.
+        enable_mc (bool, optional): Whether to enable memcached support.
+            Default: True.
+
+    Examples:
+        >>> filepath1 = 's3://path/of/file'
+        >>> filepath2 = 'cluster-name:s3://path/of/file'
+        >>> client = PetrelBackend()
+        >>> client.get(filepath1)  # get data from default cluster
+        >>> client.get(filepath2)  # get data from 'cluster-name' cluster
+    """
+
+    def __init__(self,
+                 path_mapping: Optional[dict] = None,
+                 enable_mc: bool = True):
+        try:
+            from petrel_client import client
+        except ImportError:
+            raise ImportError('Please install petrel_client to enable '
+                              'PetrelBackend.')
+
+        self._client = client.Client(enable_mc=enable_mc)
+        assert isinstance(path_mapping, dict) or path_mapping is None
+        self.path_mapping = path_mapping
+
+    def _map_path(self, filepath: Union[str, Path]) -> str:
+        """Map ``filepath`` to a string path whose prefix will be replaced by
+        :attr:`self.path_mapping`.
+
+        Args:
+            filepath (str): Path to be mapped.
+        """
+        filepath = str(filepath)
+        if self.path_mapping is not None:
+            for k, v in self.path_mapping.items():
+                filepath = filepath.replace(k, v)
+        return filepath
+
+    def _format_path(self, filepath: str) -> str:
+        """Convert a ``filepath`` to standard format of petrel oss.
+
+        If the ``filepath`` is concatenated by ``os.path.join``, in a Windows
+        environment, the ``filepath`` will be the format of
+        's3://bucket_name\\image.jpg'. By invoking :meth:`_format_path`, the
+        above ``filepath`` will be converted to 's3://bucket_name/image.jpg'.
+
+        Args:
+            filepath (str): Path to be formatted.
+        """
+        return re.sub(r'\\+', '/', filepath)
+
+    def get(self, filepath: Union[str, Path]) -> memoryview:
+        """Read data from a given ``filepath`` with 'rb' mode.
+
+        Args:
+            filepath (str or Path): Path to read data.
+
+        Returns:
+            memoryview: A memory view of expected bytes object to avoid
+                copying. The memoryview object can be converted to bytes by
+                ``value_buf.tobytes()``.
+        """
+        filepath = self._map_path(filepath)
+        filepath = self._format_path(filepath)
+        value = self._client.Get(filepath)
+        value_buf = memoryview(value)
+        return value_buf
+
+    def get_text(self,
+                 filepath: Union[str, Path],
+                 encoding: str = 'utf-8') -> str:
+        """Read data from a given ``filepath`` with 'r' mode.
+
+        Args:
+            filepath (str or Path): Path to read data.
+            encoding (str): The encoding format used to open the ``filepath``.
+                Default: 'utf-8'.
+
+        Returns:
+            str: Expected text reading from ``filepath``.
+        """
+        return str(self.get(filepath), encoding=encoding)
+
+    def put(self, obj: bytes, filepath: Union[str, Path]) -> None:
+        """Save data to a given ``filepath``.
+
+        Args:
+            obj (bytes): Data to be saved.
+            filepath (str or Path): Path to write data.
+        """
+        filepath = self._map_path(filepath)
+        filepath = self._format_path(filepath)
+        self._client.put(filepath, obj)
+
+    def put_text(self,
+                 obj: str,
+                 filepath: Union[str, Path],
+                 encoding: str = 'utf-8') -> None:
+        """Save data to a given ``filepath``.
+
+        Args:
+            obj (str): Data to be written.
+            filepath (str or Path): Path to write data.
+            encoding (str): The encoding format used to encode the ``obj``.
+                Default: 'utf-8'.
+        """
+        self.put(bytes(obj, encoding=encoding), filepath)
+
+    def remove(self, filepath: Union[str, Path]) -> None:
+        """Remove a file.
+
+        Args:
+            filepath (str or Path): Path to be removed.
+        """
+        if not has_method(self._client, 'delete'):
+            raise NotImplementedError(
+                ('Current version of Petrel Python SDK has not supported '
+                 'the `delete` method, please use a higher version or dev'
+                 ' branch instead.'))
+
+        filepath = self._map_path(filepath)
+        filepath = self._format_path(filepath)
+        self._client.delete(filepath)
+
+    def exists(self, filepath: Union[str, Path]) -> bool:
+        """Check whether a file path exists.
+
+        Args:
+            filepath (str or Path): Path to be checked whether exists.
+
+        Returns:
+            bool: Return ``True`` if ``filepath`` exists, ``False`` otherwise.
+        """
+        if not (has_method(self._client, 'contains')
+                and has_method(self._client, 'isdir')):
+            raise NotImplementedError(
+                ('Current version of Petrel Python SDK has not supported '
+                 'the `contains` and `isdir` methods, please use a higher'
+                 'version or dev branch instead.'))
+
+        filepath = self._map_path(filepath)
+        filepath = self._format_path(filepath)
+        return self._client.contains(filepath) or self._client.isdir(filepath)
+
+    def isdir(self, filepath: Union[str, Path]) -> bool:
+        """Check whether a file path is a directory.
+
+        Args:
+            filepath (str or Path): Path to be checked whether it is a
+                directory.
+
+        Returns:
+            bool: Return ``True`` if ``filepath`` points to a directory,
+            ``False`` otherwise.
+        """
+        if not has_method(self._client, 'isdir'):
+            raise NotImplementedError(
+                ('Current version of Petrel Python SDK has not supported '
+                 'the `isdir` method, please use a higher version or dev'
+                 ' branch instead.'))
+
+        filepath = self._map_path(filepath)
+        filepath = self._format_path(filepath)
+        return self._client.isdir(filepath)
+
+    def isfile(self, filepath: Union[str, Path]) -> bool:
+        """Check whether a file path is a file.
+
+        Args:
+            filepath (str or Path): Path to be checked whether it is a file.
+
+        Returns:
+            bool: Return ``True`` if ``filepath`` points to a file, ``False``
+            otherwise.
+        """
+        if not has_method(self._client, 'contains'):
+            raise NotImplementedError(
+                ('Current version of Petrel Python SDK has not supported '
+                 'the `contains` method, please use a higher version or '
+                 'dev branch instead.'))
+
+        filepath = self._map_path(filepath)
+        filepath = self._format_path(filepath)
+        return self._client.contains(filepath)
+
+    def join_path(self, filepath: Union[str, Path],
+                  *filepaths: Union[str, Path]) -> str:
+        """Concatenate all file paths.
+
+        Args:
+            filepath (str or Path): Path to be concatenated.
+
+        Returns:
+            str: The result after concatenation.
+        """
+        filepath = self._format_path(self._map_path(filepath))
+        if filepath.endswith('/'):
+            filepath = filepath[:-1]
+        formatted_paths = [filepath]
+        for path in filepaths:
+            formatted_paths.append(self._format_path(self._map_path(path)))
+        return '/'.join(formatted_paths)
+
+    @contextmanager
+    def get_local_path(self, filepath: Union[str, Path]) -> Iterable[str]:
+        """Download a file from ``filepath`` and return a temporary path.
+
+        ``get_local_path`` is decorated by :meth:`contxtlib.contextmanager`. It
+        can be called with ``with`` statement, and when exists from the
+        ``with`` statement, the temporary path will be released.
+
+        Args:
+            filepath (str | Path): Download a file from ``filepath``.
+
+        Examples:
+            >>> client = PetrelBackend()
+            >>> # After existing from the ``with`` clause,
+            >>> # the path will be removed
+            >>> with client.get_local_path('s3://path/of/your/file') as path:
+            ...     # do something here
+
+        Yields:
+            Iterable[str]: Only yield one temporary path.
+        """
+        filepath = self._map_path(filepath)
+        filepath = self._format_path(filepath)
+        assert self.isfile(filepath)
+        try:
+            f = tempfile.NamedTemporaryFile(delete=False)
+            f.write(self.get(filepath))
+            f.close()
+            yield f.name
+        finally:
+            os.remove(f.name)
+
+    def list_dir_or_file(self,
+                         dir_path: Union[str, Path],
+                         list_dir: bool = True,
+                         list_file: bool = True,
+                         suffix: Optional[Union[str, Tuple[str]]] = None,
+                         recursive: bool = False) -> Iterator[str]:
+        """Scan a directory to find the interested directories or files in
+        arbitrary order.
+
+        Note:
+            Petrel has no concept of directories but it simulates the directory
+            hierarchy in the filesystem through public prefixes. In addition,
+            if the returned path ends with '/', it means the path is a public
+            prefix which is a logical directory.
+
+        Note:
+            :meth:`list_dir_or_file` returns the path relative to ``dir_path``.
+            In addition, the returned path of directory will not contains the
+            suffix '/' which is consistent with other backends.
+
+        Args:
+            dir_path (str | Path): Path of the directory.
+            list_dir (bool): List the directories. Default: True.
+            list_file (bool): List the path of files. Default: True.
+            suffix (str or tuple[str], optional):  File suffix
+                that we are interested in. Default: None.
+            recursive (bool): If set to True, recursively scan the
+                directory. Default: False.
+
+        Yields:
+            Iterable[str]: A relative path to ``dir_path``.
+        """
+        if not has_method(self._client, 'list'):
+            raise NotImplementedError(
+                ('Current version of Petrel Python SDK has not supported '
+                 'the `list` method, please use a higher version or dev'
+                 ' branch instead.'))
+
+        dir_path = self._map_path(dir_path)
+        dir_path = self._format_path(dir_path)
+        if list_dir and suffix is not None:
+            raise TypeError(
+                '`list_dir` should be False when `suffix` is not None')
+
+        if (suffix is not None) and not isinstance(suffix, (str, tuple)):
+            raise TypeError('`suffix` must be a string or tuple of strings')
+
+        # Petrel's simulated directory hierarchy assumes that directory paths
+        # should end with `/`
+        if not dir_path.endswith('/'):
+            dir_path += '/'
+
+        root = dir_path
+
+        def _list_dir_or_file(dir_path, list_dir, list_file, suffix,
+                              recursive):
+            for path in self._client.list(dir_path):
+                # the `self.isdir` is not used here to determine whether path
+                # is a directory, because `self.isdir` relies on
+                # `self._client.list`
+                if path.endswith('/'):  # a directory path
+                    next_dir_path = self.join_path(dir_path, path)
+                    if list_dir:
+                        # get the relative path and exclude the last
+                        # character '/'
+                        rel_dir = next_dir_path[len(root):-1]
+                        yield rel_dir
+                    if recursive:
+                        yield from _list_dir_or_file(next_dir_path, list_dir,
+                                                     list_file, suffix,
+                                                     recursive)
+                else:  # a file path
+                    absolute_path = self.join_path(dir_path, path)
+                    rel_path = absolute_path[len(root):]
+                    if (suffix is None
+                            or rel_path.endswith(suffix)) and list_file:
+                        yield rel_path
+
+        return _list_dir_or_file(dir_path, list_dir, list_file, suffix,
+                                 recursive)
+
+
+class MemcachedBackend(BaseStorageBackend):
+    """Memcached storage backend.
+
+    Attributes:
+        server_list_cfg (str): Config file for memcached server list.
+        client_cfg (str): Config file for memcached client.
+        sys_path (str | None): Additional path to be appended to `sys.path`.
+            Default: None.
+    """
+
+    def __init__(self, server_list_cfg, client_cfg, sys_path=None):
+        if sys_path is not None:
+            import sys
+            sys.path.append(sys_path)
+        try:
+            import mc
+        except ImportError:
+            raise ImportError(
+                'Please install memcached to enable MemcachedBackend.')
+
+        self.server_list_cfg = server_list_cfg
+        self.client_cfg = client_cfg
+        self._client = mc.MemcachedClient.GetInstance(self.server_list_cfg,
+                                                      self.client_cfg)
+        # mc.pyvector servers as a point which points to a memory cache
+        self._mc_buffer = mc.pyvector()
+
+    def get(self, filepath):
+        filepath = str(filepath)
+        import mc
+        self._client.Get(filepath, self._mc_buffer)
+        value_buf = mc.ConvertBuffer(self._mc_buffer)
+        return value_buf
+
+    def get_text(self, filepath, encoding=None):
+        raise NotImplementedError
+
+
+class LmdbBackend(BaseStorageBackend):
+    """Lmdb storage backend.
+
+    Args:
+        db_path (str): Lmdb database path.
+        readonly (bool, optional): Lmdb environment parameter. If True,
+            disallow any write operations. Default: True.
+        lock (bool, optional): Lmdb environment parameter. If False, when
+            concurrent access occurs, do not lock the database. Default: False.
+        readahead (bool, optional): Lmdb environment parameter. If False,
+            disable the OS filesystem readahead mechanism, which may improve
+            random read performance when a database is larger than RAM.
+            Default: False.
+
+    Attributes:
+        db_path (str): Lmdb database path.
+    """
+
+    def __init__(self,
+                 db_path,
+                 readonly=True,
+                 lock=False,
+                 readahead=False,
+                 **kwargs):
+        try:
+            import lmdb
+        except ImportError:
+            raise ImportError('Please install lmdb to enable LmdbBackend.')
+
+        self.db_path = str(db_path)
+        self._client = lmdb.open(
+            self.db_path,
+            readonly=readonly,
+            lock=lock,
+            readahead=readahead,
+            **kwargs)
+
+    def get(self, filepath):
+        """Get values according to the filepath.
+
+        Args:
+            filepath (str | obj:`Path`): Here, filepath is the lmdb key.
+        """
+        filepath = str(filepath)
+        with self._client.begin(write=False) as txn:
+            value_buf = txn.get(filepath.encode('ascii'))
+        return value_buf
+
+    def get_text(self, filepath, encoding=None):
+        raise NotImplementedError
+
+
+class HardDiskBackend(BaseStorageBackend):
+    """Raw hard disks storage backend."""
+
+    _allow_symlink = True
+
+    def get(self, filepath: Union[str, Path]) -> bytes:
+        """Read data from a given ``filepath`` with 'rb' mode.
+
+        Args:
+            filepath (str or Path): Path to read data.
+
+        Returns:
+            bytes: Expected bytes object.
+        """
+        with open(filepath, 'rb') as f:
+            value_buf = f.read()
+        return value_buf
+
+    def get_text(self,
+                 filepath: Union[str, Path],
+                 encoding: str = 'utf-8') -> str:
+        """Read data from a given ``filepath`` with 'r' mode.
+
+        Args:
+            filepath (str or Path): Path to read data.
+            encoding (str): The encoding format used to open the ``filepath``.
+                Default: 'utf-8'.
+
+        Returns:
+            str: Expected text reading from ``filepath``.
+        """
+        with open(filepath, 'r', encoding=encoding) as f:
+            value_buf = f.read()
+        return value_buf
+
+    def put(self, obj: bytes, filepath: Union[str, Path]) -> None:
+        """Write data to a given ``filepath`` with 'wb' mode.
+
+        Note:
+            ``put`` will create a directory if the directory of ``filepath``
+            does not exist.
+
+        Args:
+            obj (bytes): Data to be written.
+            filepath (str or Path): Path to write data.
+        """
+        mmcv.mkdir_or_exist(osp.dirname(filepath))
+        with open(filepath, 'wb') as f:
+            f.write(obj)
+
+    def put_text(self,
+                 obj: str,
+                 filepath: Union[str, Path],
+                 encoding: str = 'utf-8') -> None:
+        """Write data to a given ``filepath`` with 'w' mode.
+
+        Note:
+            ``put_text`` will create a directory if the directory of
+            ``filepath`` does not exist.
+
+        Args:
+            obj (str): Data to be written.
+            filepath (str or Path): Path to write data.
+            encoding (str): The encoding format used to open the ``filepath``.
+                Default: 'utf-8'.
+        """
+        mmcv.mkdir_or_exist(osp.dirname(filepath))
+        with open(filepath, 'w', encoding=encoding) as f:
+            f.write(obj)
+
+    def remove(self, filepath: Union[str, Path]) -> None:
+        """Remove a file.
+
+        Args:
+            filepath (str or Path): Path to be removed.
+        """
+        os.remove(filepath)
+
+    def exists(self, filepath: Union[str, Path]) -> bool:
+        """Check whether a file path exists.
+
+        Args:
+            filepath (str or Path): Path to be checked whether exists.
+
+        Returns:
+            bool: Return ``True`` if ``filepath`` exists, ``False`` otherwise.
+        """
+        return osp.exists(filepath)
+
+    def isdir(self, filepath: Union[str, Path]) -> bool:
+        """Check whether a file path is a directory.
+
+        Args:
+            filepath (str or Path): Path to be checked whether it is a
+                directory.
+
+        Returns:
+            bool: Return ``True`` if ``filepath`` points to a directory,
+            ``False`` otherwise.
+        """
+        return osp.isdir(filepath)
+
+    def isfile(self, filepath: Union[str, Path]) -> bool:
+        """Check whether a file path is a file.
+
+        Args:
+            filepath (str or Path): Path to be checked whether it is a file.
+
+        Returns:
+            bool: Return ``True`` if ``filepath`` points to a file, ``False``
+            otherwise.
+        """
+        return osp.isfile(filepath)
+
+    def join_path(self, filepath: Union[str, Path],
+                  *filepaths: Union[str, Path]) -> str:
+        """Concatenate all file paths.
+
+        Join one or more filepath components intelligently. The return value
+        is the concatenation of filepath and any members of *filepaths.
+
+        Args:
+            filepath (str or Path): Path to be concatenated.
+
+        Returns:
+            str: The result of concatenation.
+        """
+        return osp.join(filepath, *filepaths)
+
+    @contextmanager
+    def get_local_path(
+            self, filepath: Union[str, Path]) -> Iterable[Union[str, Path]]:
+        """Only for unified API and do nothing."""
+        yield filepath
+
+    def list_dir_or_file(self,
+                         dir_path: Union[str, Path],
+                         list_dir: bool = True,
+                         list_file: bool = True,
+                         suffix: Optional[Union[str, Tuple[str]]] = None,
+                         recursive: bool = False) -> Iterator[str]:
+        """Scan a directory to find the interested directories or files in
+        arbitrary order.
+
+        Note:
+            :meth:`list_dir_or_file` returns the path relative to ``dir_path``.
+
+        Args:
+            dir_path (str | Path): Path of the directory.
+            list_dir (bool): List the directories. Default: True.
+            list_file (bool): List the path of files. Default: True.
+            suffix (str or tuple[str], optional):  File suffix
+                that we are interested in. Default: None.
+            recursive (bool): If set to True, recursively scan the
+                directory. Default: False.
+
+        Yields:
+            Iterable[str]: A relative path to ``dir_path``.
+        """
+        if list_dir and suffix is not None:
+            raise TypeError('`suffix` should be None when `list_dir` is True')
+
+        if (suffix is not None) and not isinstance(suffix, (str, tuple)):
+            raise TypeError('`suffix` must be a string or tuple of strings')
+
+        root = dir_path
+
+        def _list_dir_or_file(dir_path, list_dir, list_file, suffix,
+                              recursive):
+            for entry in os.scandir(dir_path):
+                if not entry.name.startswith('.') and entry.is_file():
+                    rel_path = osp.relpath(entry.path, root)
+                    if (suffix is None
+                            or rel_path.endswith(suffix)) and list_file:
+                        yield rel_path
+                elif osp.isdir(entry.path):
+                    if list_dir:
+                        rel_dir = osp.relpath(entry.path, root)
+                        yield rel_dir
+                    if recursive:
+                        yield from _list_dir_or_file(entry.path, list_dir,
+                                                     list_file, suffix,
+                                                     recursive)
+
+        return _list_dir_or_file(dir_path, list_dir, list_file, suffix,
+                                 recursive)
+
+
+class HTTPBackend(BaseStorageBackend):
+    """HTTP and HTTPS storage bachend."""
+
+    def get(self, filepath):
+        value_buf = urlopen(filepath).read()
+        return value_buf
+
+    def get_text(self, filepath, encoding='utf-8'):
+        value_buf = urlopen(filepath).read()
+        return value_buf.decode(encoding)
+
+    @contextmanager
+    def get_local_path(self, filepath: str) -> Iterable[str]:
+        """Download a file from ``filepath``.
+
+        ``get_local_path`` is decorated by :meth:`contxtlib.contextmanager`. It
+        can be called with ``with`` statement, and when exists from the
+        ``with`` statement, the temporary path will be released.
+
+        Args:
+            filepath (str): Download a file from ``filepath``.
+
+        Examples:
+            >>> client = HTTPBackend()
+            >>> # After existing from the ``with`` clause,
+            >>> # the path will be removed
+            >>> with client.get_local_path('http://path/of/your/file') as path:
+            ...     # do something here
+        """
+        try:
+            f = tempfile.NamedTemporaryFile(delete=False)
+            f.write(self.get(filepath))
+            f.close()
+            yield f.name
+        finally:
+            os.remove(f.name)
+
+
+class FileClient:
+    """A general file client to access files in different backends.
+
+    The client loads a file or text in a specified backend from its path
+    and returns it as a binary or text file. There are two ways to choose a
+    backend, the name of backend and the prefix of path. Although both of them
+    can be used to choose a storage backend, ``backend`` has a higher priority
+    that is if they are all set, the storage backend will be chosen by the
+    backend argument. If they are all `None`, the disk backend will be chosen.
+    Note that It can also register other backend accessor with a given name,
+    prefixes, and backend class. In addition, We use the singleton pattern to
+    avoid repeated object creation. If the arguments are the same, the same
+    object will be returned.
+
+    Args:
+        backend (str, optional): The storage backend type. Options are "disk",
+            "ceph", "memcached", "lmdb", "http" and "petrel". Default: None.
+        prefix (str, optional): The prefix of the registered storage backend.
+            Options are "s3", "http", "https". Default: None.
+
+    Examples:
+        >>> # only set backend
+        >>> file_client = FileClient(backend='petrel')
+        >>> # only set prefix
+        >>> file_client = FileClient(prefix='s3')
+        >>> # set both backend and prefix but use backend to choose client
+        >>> file_client = FileClient(backend='petrel', prefix='s3')
+        >>> # if the arguments are the same, the same object is returned
+        >>> file_client1 = FileClient(backend='petrel')
+        >>> file_client1 is file_client
+        True
+
+    Attributes:
+        client (:obj:`BaseStorageBackend`): The backend object.
+    """
+
+    _backends = {
+        'disk': HardDiskBackend,
+        'ceph': CephBackend,
+        'memcached': MemcachedBackend,
+        'lmdb': LmdbBackend,
+        'petrel': PetrelBackend,
+        'http': HTTPBackend,
+    }
+    # This collection is used to record the overridden backends, and when a
+    # backend appears in the collection, the singleton pattern is disabled for
+    # that backend, because if the singleton pattern is used, then the object
+    # returned will be the backend before overwriting
+    _overridden_backends = set()
+    _prefix_to_backends = {
+        's3': PetrelBackend,
+        'http': HTTPBackend,
+        'https': HTTPBackend,
+    }
+    _overridden_prefixes = set()
+
+    _instances = {}
+
+    def __new__(cls, backend=None, prefix=None, **kwargs):
+        if backend is None and prefix is None:
+            backend = 'disk'
+        if backend is not None and backend not in cls._backends:
+            raise ValueError(
+                f'Backend {backend} is not supported. Currently supported ones'
+                f' are {list(cls._backends.keys())}')
+        if prefix is not None and prefix not in cls._prefix_to_backends:
+            raise ValueError(
+                f'prefix {prefix} is not supported. Currently supported ones '
+                f'are {list(cls._prefix_to_backends.keys())}')
+
+        # concatenate the arguments to a unique key for determining whether
+        # objects with the same arguments were created
+        arg_key = f'{backend}:{prefix}'
+        for key, value in kwargs.items():
+            arg_key += f':{key}:{value}'
+
+        # if a backend was overridden, it will create a new object
+        if (arg_key in cls._instances
+                and backend not in cls._overridden_backends
+                and prefix not in cls._overridden_prefixes):
+            _instance = cls._instances[arg_key]
+        else:
+            # create a new object and put it to _instance
+            _instance = super().__new__(cls)
+            if backend is not None:
+                _instance.client = cls._backends[backend](**kwargs)
+            else:
+                _instance.client = cls._prefix_to_backends[prefix](**kwargs)
+
+            cls._instances[arg_key] = _instance
+
+        return _instance
+
+    @property
+    def name(self):
+        return self.client.name
+
+    @property
+    def allow_symlink(self):
+        return self.client.allow_symlink
+
+    @staticmethod
+    def parse_uri_prefix(uri: Union[str, Path]) -> Optional[str]:
+        """Parse the prefix of a uri.
+
+        Args:
+            uri (str | Path): Uri to be parsed that contains the file prefix.
+
+        Examples:
+            >>> FileClient.parse_uri_prefix('s3://path/of/your/file')
+            's3'
+
+        Returns:
+            str | None: Return the prefix of uri if the uri contains '://' else
+            ``None``.
+        """
+        assert is_filepath(uri)
+        uri = str(uri)
+        if '://' not in uri:
+            return None
+        else:
+            prefix, _ = uri.split('://')
+            # In the case of PetrelBackend, the prefix may contains the cluster
+            # name like clusterName:s3
+            if ':' in prefix:
+                _, prefix = prefix.split(':')
+            return prefix
+
+    @classmethod
+    def infer_client(cls,
+                     file_client_args: Optional[dict] = None,
+                     uri: Optional[Union[str, Path]] = None) -> 'FileClient':
+        """Infer a suitable file client based on the URI and arguments.
+
+        Args:
+            file_client_args (dict, optional): Arguments to instantiate a
+                FileClient. Default: None.
+            uri (str | Path, optional): Uri to be parsed that contains the file
+                prefix. Default: None.
+
+        Examples:
+            >>> uri = 's3://path/of/your/file'
+            >>> file_client = FileClient.infer_client(uri=uri)
+            >>> file_client_args = {'backend': 'petrel'}
+            >>> file_client = FileClient.infer_client(file_client_args)
+
+        Returns:
+            FileClient: Instantiated FileClient object.
+        """
+        assert file_client_args is not None or uri is not None
+        if file_client_args is None:
+            file_prefix = cls.parse_uri_prefix(uri)  # type: ignore
+            return cls(prefix=file_prefix)
+        else:
+            return cls(**file_client_args)
+
+    @classmethod
+    def _register_backend(cls, name, backend, force=False, prefixes=None):
+        if not isinstance(name, str):
+            raise TypeError('the backend name should be a string, '
+                            f'but got {type(name)}')
+        if not inspect.isclass(backend):
+            raise TypeError(
+                f'backend should be a class but got {type(backend)}')
+        if not issubclass(backend, BaseStorageBackend):
+            raise TypeError(
+                f'backend {backend} is not a subclass of BaseStorageBackend')
+        if not force and name in cls._backends:
+            raise KeyError(
+                f'{name} is already registered as a storage backend, '
+                'add "force=True" if you want to override it')
+
+        if name in cls._backends and force:
+            cls._overridden_backends.add(name)
+        cls._backends[name] = backend
+
+        if prefixes is not None:
+            if isinstance(prefixes, str):
+                prefixes = [prefixes]
+            else:
+                assert isinstance(prefixes, (list, tuple))
+            for prefix in prefixes:
+                if prefix not in cls._prefix_to_backends:
+                    cls._prefix_to_backends[prefix] = backend
+                elif (prefix in cls._prefix_to_backends) and force:
+                    cls._overridden_prefixes.add(prefix)
+                    cls._prefix_to_backends[prefix] = backend
+                else:
+                    raise KeyError(
+                        f'{prefix} is already registered as a storage backend,'
+                        ' add "force=True" if you want to override it')
+
+    @classmethod
+    def register_backend(cls, name, backend=None, force=False, prefixes=None):
+        """Register a backend to FileClient.
+
+        This method can be used as a normal class method or a decorator.
+
+        .. code-block:: python
+
+            class NewBackend(BaseStorageBackend):
+
+                def get(self, filepath):
+                    return filepath
+
+                def get_text(self, filepath):
+                    return filepath
+
+            FileClient.register_backend('new', NewBackend)
+
+        or
+
+        .. code-block:: python
+
+            @FileClient.register_backend('new')
+            class NewBackend(BaseStorageBackend):
+
+                def get(self, filepath):
+                    return filepath
+
+                def get_text(self, filepath):
+                    return filepath
+
+        Args:
+            name (str): The name of the registered backend.
+            backend (class, optional): The backend class to be registered,
+                which must be a subclass of :class:`BaseStorageBackend`.
+                When this method is used as a decorator, backend is None.
+                Defaults to None.
+            force (bool, optional): Whether to override the backend if the name
+                has already been registered. Defaults to False.
+            prefixes (str or list[str] or tuple[str], optional): The prefixes
+                of the registered storage backend. Default: None.
+                `New in version 1.3.15.`
+        """
+        if backend is not None:
+            cls._register_backend(
+                name, backend, force=force, prefixes=prefixes)
+            return
+
+        def _register(backend_cls):
+            cls._register_backend(
+                name, backend_cls, force=force, prefixes=prefixes)
+            return backend_cls
+
+        return _register
+
+    def get(self, filepath: Union[str, Path]) -> Union[bytes, memoryview]:
+        """Read data from a given ``filepath`` with 'rb' mode.
+
+        Note:
+            There are two types of return values for ``get``, one is ``bytes``
+            and the other is ``memoryview``. The advantage of using memoryview
+            is that you can avoid copying, and if you want to convert it to
+            ``bytes``, you can use ``.tobytes()``.
+
+        Args:
+            filepath (str or Path): Path to read data.
+
+        Returns:
+            bytes | memoryview: Expected bytes object or a memory view of the
+            bytes object.
+        """
+        return self.client.get(filepath)
+
+    def get_text(self, filepath: Union[str, Path], encoding='utf-8') -> str:
+        """Read data from a given ``filepath`` with 'r' mode.
+
+        Args:
+            filepath (str or Path): Path to read data.
+            encoding (str): The encoding format used to open the ``filepath``.
+                Default: 'utf-8'.
+
+        Returns:
+            str: Expected text reading from ``filepath``.
+        """
+        return self.client.get_text(filepath, encoding)
+
+    def put(self, obj: bytes, filepath: Union[str, Path]) -> None:
+        """Write data to a given ``filepath`` with 'wb' mode.
+
+        Note:
+            ``put`` should create a directory if the directory of ``filepath``
+            does not exist.
+
+        Args:
+            obj (bytes): Data to be written.
+            filepath (str or Path): Path to write data.
+        """
+        self.client.put(obj, filepath)
+
+    def put_text(self, obj: str, filepath: Union[str, Path]) -> None:
+        """Write data to a given ``filepath`` with 'w' mode.
+
+        Note:
+            ``put_text`` should create a directory if the directory of
+            ``filepath`` does not exist.
+
+        Args:
+            obj (str): Data to be written.
+            filepath (str or Path): Path to write data.
+            encoding (str, optional): The encoding format used to open the
+                `filepath`. Default: 'utf-8'.
+        """
+        self.client.put_text(obj, filepath)
+
+    def remove(self, filepath: Union[str, Path]) -> None:
+        """Remove a file.
+
+        Args:
+            filepath (str, Path): Path to be removed.
+        """
+        self.client.remove(filepath)
+
+    def exists(self, filepath: Union[str, Path]) -> bool:
+        """Check whether a file path exists.
+
+        Args:
+            filepath (str or Path): Path to be checked whether exists.
+
+        Returns:
+            bool: Return ``True`` if ``filepath`` exists, ``False`` otherwise.
+        """
+        return self.client.exists(filepath)
+
+    def isdir(self, filepath: Union[str, Path]) -> bool:
+        """Check whether a file path is a directory.
+
+        Args:
+            filepath (str or Path): Path to be checked whether it is a
+                directory.
+
+        Returns:
+            bool: Return ``True`` if ``filepath`` points to a directory,
+            ``False`` otherwise.
+        """
+        return self.client.isdir(filepath)
+
+    def isfile(self, filepath: Union[str, Path]) -> bool:
+        """Check whether a file path is a file.
+
+        Args:
+            filepath (str or Path): Path to be checked whether it is a file.
+
+        Returns:
+            bool: Return ``True`` if ``filepath`` points to a file, ``False``
+            otherwise.
+        """
+        return self.client.isfile(filepath)
+
+    def join_path(self, filepath: Union[str, Path],
+                  *filepaths: Union[str, Path]) -> str:
+        """Concatenate all file paths.
+
+        Join one or more filepath components intelligently. The return value
+        is the concatenation of filepath and any members of *filepaths.
+
+        Args:
+            filepath (str or Path): Path to be concatenated.
+
+        Returns:
+            str: The result of concatenation.
+        """
+        return self.client.join_path(filepath, *filepaths)
+
+    @contextmanager
+    def get_local_path(self, filepath: Union[str, Path]) -> Iterable[str]:
+        """Download data from ``filepath`` and write the data to local path.
+
+        ``get_local_path`` is decorated by :meth:`contxtlib.contextmanager`. It
+        can be called with ``with`` statement, and when exists from the
+        ``with`` statement, the temporary path will be released.
+
+        Note:
+            If the ``filepath`` is a local path, just return itself.
+
+        .. warning::
+            ``get_local_path`` is an experimental interface that may change in
+            the future.
+
+        Args:
+            filepath (str or Path): Path to be read data.
+
+        Examples:
+            >>> file_client = FileClient(prefix='s3')
+            >>> with file_client.get_local_path('s3://bucket/abc.jpg') as path:
+            ...     # do something here
+
+        Yields:
+            Iterable[str]: Only yield one path.
+        """
+        with self.client.get_local_path(str(filepath)) as local_path:
+            yield local_path
+
+    def list_dir_or_file(self,
+                         dir_path: Union[str, Path],
+                         list_dir: bool = True,
+                         list_file: bool = True,
+                         suffix: Optional[Union[str, Tuple[str]]] = None,
+                         recursive: bool = False) -> Iterator[str]:
+        """Scan a directory to find the interested directories or files in
+        arbitrary order.
+
+        Note:
+            :meth:`list_dir_or_file` returns the path relative to ``dir_path``.
+
+        Args:
+            dir_path (str | Path): Path of the directory.
+            list_dir (bool): List the directories. Default: True.
+            list_file (bool): List the path of files. Default: True.
+            suffix (str or tuple[str], optional):  File suffix
+                that we are interested in. Default: None.
+            recursive (bool): If set to True, recursively scan the
+                directory. Default: False.
+
+        Yields:
+            Iterable[str]: A relative path to ``dir_path``.
+        """
+        yield from self.client.list_dir_or_file(dir_path, list_dir, list_file,
+                                                suffix, recursive)

groundingLMM/mmcv/mmcv/fileio/io.py

@@ -0,0 +1,151 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from io import BytesIO, StringIO
+from pathlib import Path
+
+from ..utils import is_list_of, is_str
+from .file_client import FileClient
+from .handlers import BaseFileHandler, JsonHandler, PickleHandler, YamlHandler
+
+file_handlers = {
+    'json': JsonHandler(),
+    'yaml': YamlHandler(),
+    'yml': YamlHandler(),
+    'pickle': PickleHandler(),
+    'pkl': PickleHandler()
+}
+
+
+def load(file, file_format=None, file_client_args=None, **kwargs):
+    """Load data from json/yaml/pickle files.
+
+    This method provides a unified api for loading data from serialized files.
+
+    Note:
+        In v1.3.16 and later, ``load`` supports loading data from serialized
+        files those can be storaged in different backends.
+
+    Args:
+        file (str or :obj:`Path` or file-like object): Filename or a file-like
+            object.
+        file_format (str, optional): If not specified, the file format will be
+            inferred from the file extension, otherwise use the specified one.
+            Currently supported formats include "json", "yaml/yml" and
+            "pickle/pkl".
+        file_client_args (dict, optional): Arguments to instantiate a
+            FileClient. See :class:`mmcv.fileio.FileClient` for details.
+            Default: None.
+
+    Examples:
+        >>> load('/path/of/your/file')  # file is storaged in disk
+        >>> load('https://path/of/your/file')  # file is storaged in Internet
+        >>> load('s3://path/of/your/file')  # file is storaged in petrel
+
+    Returns:
+        The content from the file.
+    """
+    if isinstance(file, Path):
+        file = str(file)
+    if file_format is None and is_str(file):
+        file_format = file.split('.')[-1]
+    if file_format not in file_handlers:
+        raise TypeError(f'Unsupported format: {file_format}')
+
+    handler = file_handlers[file_format]
+    if is_str(file):
+        file_client = FileClient.infer_client(file_client_args, file)
+        if handler.str_like:
+            with StringIO(file_client.get_text(file)) as f:
+                obj = handler.load_from_fileobj(f, **kwargs)
+        else:
+            with BytesIO(file_client.get(file)) as f:
+                obj = handler.load_from_fileobj(f, **kwargs)
+    elif hasattr(file, 'read'):
+        obj = handler.load_from_fileobj(file, **kwargs)
+    else:
+        raise TypeError('"file" must be a filepath str or a file-object')
+    return obj
+
+
+def dump(obj, file=None, file_format=None, file_client_args=None, **kwargs):
+    """Dump data to json/yaml/pickle strings or files.
+
+    This method provides a unified api for dumping data as strings or to files,
+    and also supports custom arguments for each file format.
+
+    Note:
+        In v1.3.16 and later, ``dump`` supports dumping data as strings or to
+        files which is saved to different backends.
+
+    Args:
+        obj (any): The python object to be dumped.
+        file (str or :obj:`Path` or file-like object, optional): If not
+            specified, then the object is dumped to a str, otherwise to a file
+            specified by the filename or file-like object.
+        file_format (str, optional): Same as :func:`load`.
+        file_client_args (dict, optional): Arguments to instantiate a
+            FileClient. See :class:`mmcv.fileio.FileClient` for details.
+            Default: None.
+
+    Examples:
+        >>> dump('hello world', '/path/of/your/file')  # disk
+        >>> dump('hello world', 's3://path/of/your/file')  # ceph or petrel
+
+    Returns:
+        bool: True for success, False otherwise.
+    """
+    if isinstance(file, Path):
+        file = str(file)
+    if file_format is None:
+        if is_str(file):
+            file_format = file.split('.')[-1]
+        elif file is None:
+            raise ValueError(
+                'file_format must be specified since file is None')
+    if file_format not in file_handlers:
+        raise TypeError(f'Unsupported format: {file_format}')
+
+    handler = file_handlers[file_format]
+    if file is None:
+        return handler.dump_to_str(obj, **kwargs)
+    elif is_str(file):
+        file_client = FileClient.infer_client(file_client_args, file)
+        if handler.str_like:
+            with StringIO() as f:
+                handler.dump_to_fileobj(obj, f, **kwargs)
+                file_client.put_text(f.getvalue(), file)
+        else:
+            with BytesIO() as f:
+                handler.dump_to_fileobj(obj, f, **kwargs)
+                file_client.put(f.getvalue(), file)
+    elif hasattr(file, 'write'):
+        handler.dump_to_fileobj(obj, file, **kwargs)
+    else:
+        raise TypeError('"file" must be a filename str or a file-object')
+
+
+def _register_handler(handler, file_formats):
+    """Register a handler for some file extensions.
+
+    Args:
+        handler (:obj:`BaseFileHandler`): Handler to be registered.
+        file_formats (str or list[str]): File formats to be handled by this
+            handler.
+    """
+    if not isinstance(handler, BaseFileHandler):
+        raise TypeError(
+            f'handler must be a child of BaseFileHandler, not {type(handler)}')
+    if isinstance(file_formats, str):
+        file_formats = [file_formats]
+    if not is_list_of(file_formats, str):
+        raise TypeError('file_formats must be a str or a list of str')
+    for ext in file_formats:
+        file_handlers[ext] = handler
+
+
+def register_handler(file_formats, **kwargs):
+
+    def wrap(cls):
+        _register_handler(cls(**kwargs), file_formats)
+        return cls
+
+    return wrap

groundingLMM/mmcv/mmcv/fileio/parse.py

@@ -0,0 +1,97 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+
+from io import StringIO
+
+from .file_client import FileClient
+
+
+def list_from_file(filename,
+                   prefix='',
+                   offset=0,
+                   max_num=0,
+                   encoding='utf-8',
+                   file_client_args=None):
+    """Load a text file and parse the content as a list of strings.
+
+    Note:
+        In v1.3.16 and later, ``list_from_file`` supports loading a text file
+        which can be storaged in different backends and parsing the content as
+        a list for strings.
+
+    Args:
+        filename (str): Filename.
+        prefix (str): The prefix to be inserted to the beginning of each item.
+        offset (int): The offset of lines.
+        max_num (int): The maximum number of lines to be read,
+            zeros and negatives mean no limitation.
+        encoding (str): Encoding used to open the file. Default utf-8.
+        file_client_args (dict, optional): Arguments to instantiate a
+            FileClient. See :class:`mmcv.fileio.FileClient` for details.
+            Default: None.
+
+    Examples:
+        >>> list_from_file('/path/of/your/file')  # disk
+        ['hello', 'world']
+        >>> list_from_file('s3://path/of/your/file')  # ceph or petrel
+        ['hello', 'world']
+
+    Returns:
+        list[str]: A list of strings.
+    """
+    cnt = 0
+    item_list = []
+    file_client = FileClient.infer_client(file_client_args, filename)
+    with StringIO(file_client.get_text(filename, encoding)) as f:
+        for _ in range(offset):
+            f.readline()
+        for line in f:
+            if 0 < max_num <= cnt:
+                break
+            item_list.append(prefix + line.rstrip('\n\r'))
+            cnt += 1
+    return item_list
+
+
+def dict_from_file(filename,
+                   key_type=str,
+                   encoding='utf-8',
+                   file_client_args=None):
+    """Load a text file and parse the content as a dict.
+
+    Each line of the text file will be two or more columns split by
+    whitespaces or tabs. The first column will be parsed as dict keys, and
+    the following columns will be parsed as dict values.
+
+    Note:
+        In v1.3.16 and later, ``dict_from_file`` supports loading a text file
+        which can be storaged in different backends and parsing the content as
+        a dict.
+
+    Args:
+        filename(str): Filename.
+        key_type(type): Type of the dict keys. str is user by default and
+            type conversion will be performed if specified.
+        encoding (str): Encoding used to open the file. Default utf-8.
+        file_client_args (dict, optional): Arguments to instantiate a
+            FileClient. See :class:`mmcv.fileio.FileClient` for details.
+            Default: None.
+
+    Examples:
+        >>> dict_from_file('/path/of/your/file')  # disk
+        {'key1': 'value1', 'key2': 'value2'}
+        >>> dict_from_file('s3://path/of/your/file')  # ceph or petrel
+        {'key1': 'value1', 'key2': 'value2'}
+
+    Returns:
+        dict: The parsed contents.
+    """
+    mapping = {}
+    file_client = FileClient.infer_client(file_client_args, filename)
+    with StringIO(file_client.get_text(filename, encoding)) as f:
+        for line in f:
+            items = line.rstrip('\n').split()
+            assert len(items) >= 2
+            key = key_type(items[0])
+            val = items[1:] if len(items) > 2 else items[1]
+            mapping[key] = val
+    return mapping

groundingLMM/mmcv/mmcv/image/__init__.py

@@ -0,0 +1,28 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from .colorspace import (bgr2gray, bgr2hls, bgr2hsv, bgr2rgb, bgr2ycbcr,
+                         gray2bgr, gray2rgb, hls2bgr, hsv2bgr, imconvert,
+                         rgb2bgr, rgb2gray, rgb2ycbcr, ycbcr2bgr, ycbcr2rgb)
+from .geometric import (cutout, imcrop, imflip, imflip_, impad,
+                        impad_to_multiple, imrescale, imresize, imresize_like,
+                        imresize_to_multiple, imrotate, imshear, imtranslate,
+                        rescale_size)
+from .io import imfrombytes, imread, imwrite, supported_backends, use_backend
+from .misc import tensor2imgs
+from .photometric import (adjust_brightness, adjust_color, adjust_contrast,
+                          adjust_lighting, adjust_sharpness, auto_contrast,
+                          clahe, imdenormalize, imequalize, iminvert,
+                          imnormalize, imnormalize_, lut_transform, posterize,
+                          solarize)
+
+__all__ = [
+    'bgr2gray', 'bgr2hls', 'bgr2hsv', 'bgr2rgb', 'gray2bgr', 'gray2rgb',
+    'hls2bgr', 'hsv2bgr', 'imconvert', 'rgb2bgr', 'rgb2gray', 'imrescale',
+    'imresize', 'imresize_like', 'imresize_to_multiple', 'rescale_size',
+    'imcrop', 'imflip', 'imflip_', 'impad', 'impad_to_multiple', 'imrotate',
+    'imfrombytes', 'imread', 'imwrite', 'supported_backends', 'use_backend',
+    'imdenormalize', 'imnormalize', 'imnormalize_', 'iminvert', 'posterize',
+    'solarize', 'rgb2ycbcr', 'bgr2ycbcr', 'ycbcr2rgb', 'ycbcr2bgr',
+    'tensor2imgs', 'imshear', 'imtranslate', 'adjust_color', 'imequalize',
+    'adjust_brightness', 'adjust_contrast', 'lut_transform', 'clahe',
+    'adjust_sharpness', 'auto_contrast', 'cutout', 'adjust_lighting'
+]

groundingLMM/mmcv/mmcv/image/colorspace.py

@@ -0,0 +1,306 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import cv2
+import numpy as np
+
+
+def imconvert(img, src, dst):
+    """Convert an image from the src colorspace to dst colorspace.
+
+    Args:
+        img (ndarray): The input image.
+        src (str): The source colorspace, e.g., 'rgb', 'hsv'.
+        dst (str): The destination colorspace, e.g., 'rgb', 'hsv'.
+
+    Returns:
+        ndarray: The converted image.
+    """
+    code = getattr(cv2, f'COLOR_{src.upper()}2{dst.upper()}')
+    out_img = cv2.cvtColor(img, code)
+    return out_img
+
+
+def bgr2gray(img, keepdim=False):
+    """Convert a BGR image to grayscale image.
+
+    Args:
+        img (ndarray): The input image.
+        keepdim (bool): If False (by default), then return the grayscale image
+            with 2 dims, otherwise 3 dims.
+
+    Returns:
+        ndarray: The converted grayscale image.
+    """
+    out_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+    if keepdim:
+        out_img = out_img[..., None]
+    return out_img
+
+
+def rgb2gray(img, keepdim=False):
+    """Convert a RGB image to grayscale image.
+
+    Args:
+        img (ndarray): The input image.
+        keepdim (bool): If False (by default), then return the grayscale image
+            with 2 dims, otherwise 3 dims.
+
+    Returns:
+        ndarray: The converted grayscale image.
+    """
+    out_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
+    if keepdim:
+        out_img = out_img[..., None]
+    return out_img
+
+
+def gray2bgr(img):
+    """Convert a grayscale image to BGR image.
+
+    Args:
+        img (ndarray): The input image.
+
+    Returns:
+        ndarray: The converted BGR image.
+    """
+    img = img[..., None] if img.ndim == 2 else img
+    out_img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
+    return out_img
+
+
+def gray2rgb(img):
+    """Convert a grayscale image to RGB image.
+
+    Args:
+        img (ndarray): The input image.
+
+    Returns:
+        ndarray: The converted RGB image.
+    """
+    img = img[..., None] if img.ndim == 2 else img
+    out_img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
+    return out_img
+
+
+def _convert_input_type_range(img):
+    """Convert the type and range of the input image.
+
+    It converts the input image to np.float32 type and range of [0, 1].
+    It is mainly used for pre-processing the input image in colorspace
+    conversion functions such as rgb2ycbcr and ycbcr2rgb.
+
+    Args:
+        img (ndarray): The input image. It accepts:
+            1. np.uint8 type with range [0, 255];
+            2. np.float32 type with range [0, 1].
+
+    Returns:
+        (ndarray): The converted image with type of np.float32 and range of
+            [0, 1].
+    """
+    img_type = img.dtype
+    img = img.astype(np.float32)
+    if img_type == np.float32:
+        pass
+    elif img_type == np.uint8:
+        img /= 255.
+    else:
+        raise TypeError('The img type should be np.float32 or np.uint8, '
+                        f'but got {img_type}')
+    return img
+
+
+def _convert_output_type_range(img, dst_type):
+    """Convert the type and range of the image according to dst_type.
+
+    It converts the image to desired type and range. If `dst_type` is np.uint8,
+    images will be converted to np.uint8 type with range [0, 255]. If
+    `dst_type` is np.float32, it converts the image to np.float32 type with
+    range [0, 1].
+    It is mainly used for post-processing images in colorspace conversion
+    functions such as rgb2ycbcr and ycbcr2rgb.
+
+    Args:
+        img (ndarray): The image to be converted with np.float32 type and
+            range [0, 255].
+        dst_type (np.uint8 | np.float32): If dst_type is np.uint8, it
+            converts the image to np.uint8 type with range [0, 255]. If
+            dst_type is np.float32, it converts the image to np.float32 type
+            with range [0, 1].
+
+    Returns:
+        (ndarray): The converted image with desired type and range.
+    """
+    if dst_type not in (np.uint8, np.float32):
+        raise TypeError('The dst_type should be np.float32 or np.uint8, '
+                        f'but got {dst_type}')
+    if dst_type == np.uint8:
+        img = img.round()
+    else:
+        img /= 255.
+    return img.astype(dst_type)
+
+
+def rgb2ycbcr(img, y_only=False):
+    """Convert a RGB image to YCbCr image.
+
+    This function produces the same results as Matlab's `rgb2ycbcr` function.
+    It implements the ITU-R BT.601 conversion for standard-definition
+    television. See more details in
+    https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion.
+
+    It differs from a similar function in cv2.cvtColor: `RGB <-> YCrCb`.
+    In OpenCV, it implements a JPEG conversion. See more details in
+    https://en.wikipedia.org/wiki/YCbCr#JPEG_conversion.
+
+    Args:
+        img (ndarray): The input image. It accepts:
+            1. np.uint8 type with range [0, 255];
+            2. np.float32 type with range [0, 1].
+        y_only (bool): Whether to only return Y channel. Default: False.
+
+    Returns:
+        ndarray: The converted YCbCr image. The output image has the same type
+        and range as input image.
+    """
+    img_type = img.dtype
+    img = _convert_input_type_range(img)
+    if y_only:
+        out_img = np.dot(img, [65.481, 128.553, 24.966]) + 16.0
+    else:
+        out_img = np.matmul(
+            img, [[65.481, -37.797, 112.0], [128.553, -74.203, -93.786],
+                  [24.966, 112.0, -18.214]]) + [16, 128, 128]
+    out_img = _convert_output_type_range(out_img, img_type)
+    return out_img
+
+
+def bgr2ycbcr(img, y_only=False):
+    """Convert a BGR image to YCbCr image.
+
+    The bgr version of rgb2ycbcr.
+    It implements the ITU-R BT.601 conversion for standard-definition
+    television. See more details in
+    https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion.
+
+    It differs from a similar function in cv2.cvtColor: `BGR <-> YCrCb`.
+    In OpenCV, it implements a JPEG conversion. See more details in
+    https://en.wikipedia.org/wiki/YCbCr#JPEG_conversion.
+
+    Args:
+        img (ndarray): The input image. It accepts:
+            1. np.uint8 type with range [0, 255];
+            2. np.float32 type with range [0, 1].
+        y_only (bool): Whether to only return Y channel. Default: False.
+
+    Returns:
+        ndarray: The converted YCbCr image. The output image has the same type
+        and range as input image.
+    """
+    img_type = img.dtype
+    img = _convert_input_type_range(img)
+    if y_only:
+        out_img = np.dot(img, [24.966, 128.553, 65.481]) + 16.0
+    else:
+        out_img = np.matmul(
+            img, [[24.966, 112.0, -18.214], [128.553, -74.203, -93.786],
+                  [65.481, -37.797, 112.0]]) + [16, 128, 128]
+    out_img = _convert_output_type_range(out_img, img_type)
+    return out_img
+
+
+def ycbcr2rgb(img):
+    """Convert a YCbCr image to RGB image.
+
+    This function produces the same results as Matlab's ycbcr2rgb function.
+    It implements the ITU-R BT.601 conversion for standard-definition
+    television. See more details in
+    https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion.
+
+    It differs from a similar function in cv2.cvtColor: `YCrCb <-> RGB`.
+    In OpenCV, it implements a JPEG conversion. See more details in
+    https://en.wikipedia.org/wiki/YCbCr#JPEG_conversion.
+
+    Args:
+        img (ndarray): The input image. It accepts:
+            1. np.uint8 type with range [0, 255];
+            2. np.float32 type with range [0, 1].
+
+    Returns:
+        ndarray: The converted RGB image. The output image has the same type
+        and range as input image.
+    """
+    img_type = img.dtype
+    img = _convert_input_type_range(img) * 255
+    out_img = np.matmul(img, [[0.00456621, 0.00456621, 0.00456621],
+                              [0, -0.00153632, 0.00791071],
+                              [0.00625893, -0.00318811, 0]]) * 255.0 + [
+                                  -222.921, 135.576, -276.836
+                              ]
+    out_img = _convert_output_type_range(out_img, img_type)
+    return out_img
+
+
+def ycbcr2bgr(img):
+    """Convert a YCbCr image to BGR image.
+
+    The bgr version of ycbcr2rgb.
+    It implements the ITU-R BT.601 conversion for standard-definition
+    television. See more details in
+    https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion.
+
+    It differs from a similar function in cv2.cvtColor: `YCrCb <-> BGR`.
+    In OpenCV, it implements a JPEG conversion. See more details in
+    https://en.wikipedia.org/wiki/YCbCr#JPEG_conversion.
+
+    Args:
+        img (ndarray): The input image. It accepts:
+            1. np.uint8 type with range [0, 255];
+            2. np.float32 type with range [0, 1].
+
+    Returns:
+        ndarray: The converted BGR image. The output image has the same type
+        and range as input image.
+    """
+    img_type = img.dtype
+    img = _convert_input_type_range(img) * 255
+    out_img = np.matmul(img, [[0.00456621, 0.00456621, 0.00456621],
+                              [0.00791071, -0.00153632, 0],
+                              [0, -0.00318811, 0.00625893]]) * 255.0 + [
+                                  -276.836, 135.576, -222.921
+                              ]
+    out_img = _convert_output_type_range(out_img, img_type)
+    return out_img
+
+
+def convert_color_factory(src, dst):
+
+    code = getattr(cv2, f'COLOR_{src.upper()}2{dst.upper()}')
+
+    def convert_color(img):
+        out_img = cv2.cvtColor(img, code)
+        return out_img
+
+    convert_color.__doc__ = f"""Convert a {src.upper()} image to {dst.upper()}
+        image.
+
+    Args:
+        img (ndarray or str): The input image.
+
+    Returns:
+        ndarray: The converted {dst.upper()} image.
+    """
+
+    return convert_color
+
+
+bgr2rgb = convert_color_factory('bgr', 'rgb')
+
+rgb2bgr = convert_color_factory('rgb', 'bgr')
+
+bgr2hsv = convert_color_factory('bgr', 'hsv')
+
+hsv2bgr = convert_color_factory('hsv', 'bgr')
+
+bgr2hls = convert_color_factory('bgr', 'hls')
+
+hls2bgr = convert_color_factory('hls', 'bgr')

groundingLMM/mmcv/mmcv/image/geometric.py

@@ -0,0 +1,728 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import numbers
+
+import cv2
+import numpy as np
+
+from ..utils import to_2tuple
+from .io import imread_backend
+
+try:
+    from PIL import Image
+except ImportError:
+    Image = None
+
+
+def _scale_size(size, scale):
+    """Rescale a size by a ratio.
+
+    Args:
+        size (tuple[int]): (w, h).
+        scale (float | tuple(float)): Scaling factor.
+
+    Returns:
+        tuple[int]: scaled size.
+    """
+    if isinstance(scale, (float, int)):
+        scale = (scale, scale)
+    w, h = size
+    return int(w * float(scale[0]) + 0.5), int(h * float(scale[1]) + 0.5)
+
+
+cv2_interp_codes = {
+    'nearest': cv2.INTER_NEAREST,
+    'bilinear': cv2.INTER_LINEAR,
+    'bicubic': cv2.INTER_CUBIC,
+    'area': cv2.INTER_AREA,
+    'lanczos': cv2.INTER_LANCZOS4
+}
+
+if Image is not None:
+    pillow_interp_codes = {
+        'nearest': Image.NEAREST,
+        'bilinear': Image.BILINEAR,
+        'bicubic': Image.BICUBIC,
+        'box': Image.BOX,
+        'lanczos': Image.LANCZOS,
+        'hamming': Image.HAMMING
+    }
+
+
+def imresize(img,
+             size,
+             return_scale=False,
+             interpolation='bilinear',
+             out=None,
+             backend=None):
+    """Resize image to a given size.
+
+    Args:
+        img (ndarray): The input image.
+        size (tuple[int]): Target size (w, h).
+        return_scale (bool): Whether to return `w_scale` and `h_scale`.
+        interpolation (str): Interpolation method, accepted values are
+            "nearest", "bilinear", "bicubic", "area", "lanczos" for 'cv2'
+            backend, "nearest", "bilinear" for 'pillow' backend.
+        out (ndarray): The output destination.
+        backend (str | None): The image resize backend type. Options are `cv2`,
+            `pillow`, `None`. If backend is None, the global imread_backend
+            specified by ``mmcv.use_backend()`` will be used. Default: None.
+
+    Returns:
+        tuple | ndarray: (`resized_img`, `w_scale`, `h_scale`) or
+        `resized_img`.
+    """
+    h, w = img.shape[:2]
+    if backend is None:
+        backend = imread_backend
+    if backend not in ['cv2', 'pillow']:
+        raise ValueError(f'backend: {backend} is not supported for resize.'
+                         f"Supported backends are 'cv2', 'pillow'")
+
+    if backend == 'pillow':
+        assert img.dtype == np.uint8, 'Pillow backend only support uint8 type'
+        pil_image = Image.fromarray(img)
+        pil_image = pil_image.resize(size, pillow_interp_codes[interpolation])
+        resized_img = np.array(pil_image)
+    else:
+        resized_img = cv2.resize(
+            img, size, dst=out, interpolation=cv2_interp_codes[interpolation])
+    if not return_scale:
+        return resized_img
+    else:
+        w_scale = size[0] / w
+        h_scale = size[1] / h
+        return resized_img, w_scale, h_scale
+
+
+def imresize_to_multiple(img,
+                         divisor,
+                         size=None,
+                         scale_factor=None,
+                         keep_ratio=False,
+                         return_scale=False,
+                         interpolation='bilinear',
+                         out=None,
+                         backend=None):
+    """Resize image according to a given size or scale factor and then rounds
+    up the the resized or rescaled image size to the nearest value that can be
+    divided by the divisor.
+
+    Args:
+        img (ndarray): The input image.
+        divisor (int | tuple): Resized image size will be a multiple of
+            divisor. If divisor is a tuple, divisor should be
+            (w_divisor, h_divisor).
+        size (None | int | tuple[int]): Target size (w, h). Default: None.
+        scale_factor (None | float | tuple[float]): Multiplier for spatial
+            size. Should match input size if it is a tuple and the 2D style is
+            (w_scale_factor, h_scale_factor). Default: None.
+        keep_ratio (bool): Whether to keep the aspect ratio when resizing the
+            image. Default: False.
+        return_scale (bool): Whether to return `w_scale` and `h_scale`.
+        interpolation (str): Interpolation method, accepted values are
+            "nearest", "bilinear", "bicubic", "area", "lanczos" for 'cv2'
+            backend, "nearest", "bilinear" for 'pillow' backend.
+        out (ndarray): The output destination.
+        backend (str | None): The image resize backend type. Options are `cv2`,
+            `pillow`, `None`. If backend is None, the global imread_backend
+            specified by ``mmcv.use_backend()`` will be used. Default: None.
+
+    Returns:
+        tuple | ndarray: (`resized_img`, `w_scale`, `h_scale`) or
+        `resized_img`.
+    """
+    h, w = img.shape[:2]
+    if size is not None and scale_factor is not None:
+        raise ValueError('only one of size or scale_factor should be defined')
+    elif size is None and scale_factor is None:
+        raise ValueError('one of size or scale_factor should be defined')
+    elif size is not None:
+        size = to_2tuple(size)
+        if keep_ratio:
+            size = rescale_size((w, h), size, return_scale=False)
+    else:
+        size = _scale_size((w, h), scale_factor)
+
+    divisor = to_2tuple(divisor)
+    size = tuple([int(np.ceil(s / d)) * d for s, d in zip(size, divisor)])
+    resized_img, w_scale, h_scale = imresize(
+        img,
+        size,
+        return_scale=True,
+        interpolation=interpolation,
+        out=out,
+        backend=backend)
+    if return_scale:
+        return resized_img, w_scale, h_scale
+    else:
+        return resized_img
+
+
+def imresize_like(img,
+                  dst_img,
+                  return_scale=False,
+                  interpolation='bilinear',
+                  backend=None):
+    """Resize image to the same size of a given image.
+
+    Args:
+        img (ndarray): The input image.
+        dst_img (ndarray): The target image.
+        return_scale (bool): Whether to return `w_scale` and `h_scale`.
+        interpolation (str): Same as :func:`resize`.
+        backend (str | None): Same as :func:`resize`.
+
+    Returns:
+        tuple or ndarray: (`resized_img`, `w_scale`, `h_scale`) or
+        `resized_img`.
+    """
+    h, w = dst_img.shape[:2]
+    return imresize(img, (w, h), return_scale, interpolation, backend=backend)
+
+
+def rescale_size(old_size, scale, return_scale=False):
+    """Calculate the new size to be rescaled to.
+
+    Args:
+        old_size (tuple[int]): The old size (w, h) of image.
+        scale (float | tuple[int]): The scaling factor or maximum size.
+            If it is a float number, then the image will be rescaled by this
+            factor, else if it is a tuple of 2 integers, then the image will
+            be rescaled as large as possible within the scale.
+        return_scale (bool): Whether to return the scaling factor besides the
+            rescaled image size.
+
+    Returns:
+        tuple[int]: The new rescaled image size.
+    """
+    w, h = old_size
+    if isinstance(scale, (float, int)):
+        if scale <= 0:
+            raise ValueError(f'Invalid scale {scale}, must be positive.')
+        scale_factor = scale
+    elif isinstance(scale, tuple):
+        max_long_edge = max(scale)
+        max_short_edge = min(scale)
+        scale_factor = min(max_long_edge / max(h, w),
+                           max_short_edge / min(h, w))
+    else:
+        raise TypeError(
+            f'Scale must be a number or tuple of int, but got {type(scale)}')
+
+    new_size = _scale_size((w, h), scale_factor)
+
+    if return_scale:
+        return new_size, scale_factor
+    else:
+        return new_size
+
+
+def imrescale(img,
+              scale,
+              return_scale=False,
+              interpolation='bilinear',
+              backend=None):
+    """Resize image while keeping the aspect ratio.
+
+    Args:
+        img (ndarray): The input image.
+        scale (float | tuple[int]): The scaling factor or maximum size.
+            If it is a float number, then the image will be rescaled by this
+            factor, else if it is a tuple of 2 integers, then the image will
+            be rescaled as large as possible within the scale.
+        return_scale (bool): Whether to return the scaling factor besides the
+            rescaled image.
+        interpolation (str): Same as :func:`resize`.
+        backend (str | None): Same as :func:`resize`.
+
+    Returns:
+        ndarray: The rescaled image.
+    """
+    h, w = img.shape[:2]
+    new_size, scale_factor = rescale_size((w, h), scale, return_scale=True)
+    rescaled_img = imresize(
+        img, new_size, interpolation=interpolation, backend=backend)
+    if return_scale:
+        return rescaled_img, scale_factor
+    else:
+        return rescaled_img
+
+
+def imflip(img, direction='horizontal'):
+    """Flip an image horizontally or vertically.
+
+    Args:
+        img (ndarray): Image to be flipped.
+        direction (str): The flip direction, either "horizontal" or
+            "vertical" or "diagonal".
+
+    Returns:
+        ndarray: The flipped image.
+    """
+    assert direction in ['horizontal', 'vertical', 'diagonal']
+    if direction == 'horizontal':
+        return np.flip(img, axis=1)
+    elif direction == 'vertical':
+        return np.flip(img, axis=0)
+    else:
+        return np.flip(img, axis=(0, 1))
+
+
+def imflip_(img, direction='horizontal'):
+    """Inplace flip an image horizontally or vertically.
+
+    Args:
+        img (ndarray): Image to be flipped.
+        direction (str): The flip direction, either "horizontal" or
+            "vertical" or "diagonal".
+
+    Returns:
+        ndarray: The flipped image (inplace).
+    """
+    assert direction in ['horizontal', 'vertical', 'diagonal']
+    if direction == 'horizontal':
+        return cv2.flip(img, 1, img)
+    elif direction == 'vertical':
+        return cv2.flip(img, 0, img)
+    else:
+        return cv2.flip(img, -1, img)
+
+
+def imrotate(img,
+             angle,
+             center=None,
+             scale=1.0,
+             border_value=0,
+             interpolation='bilinear',
+             auto_bound=False):
+    """Rotate an image.
+
+    Args:
+        img (ndarray): Image to be rotated.
+        angle (float): Rotation angle in degrees, positive values mean
+            clockwise rotation.
+        center (tuple[float], optional): Center point (w, h) of the rotation in
+            the source image. If not specified, the center of the image will be
+            used.
+        scale (float): Isotropic scale factor.
+        border_value (int): Border value.
+        interpolation (str): Same as :func:`resize`.
+        auto_bound (bool): Whether to adjust the image size to cover the whole
+            rotated image.
+
+    Returns:
+        ndarray: The rotated image.
+    """
+    if center is not None and auto_bound:
+        raise ValueError('`auto_bound` conflicts with `center`')
+    h, w = img.shape[:2]
+    if center is None:
+        center = ((w - 1) * 0.5, (h - 1) * 0.5)
+    assert isinstance(center, tuple)
+
+    matrix = cv2.getRotationMatrix2D(center, -angle, scale)
+    if auto_bound:
+        cos = np.abs(matrix[0, 0])
+        sin = np.abs(matrix[0, 1])
+        new_w = h * sin + w * cos
+        new_h = h * cos + w * sin
+        matrix[0, 2] += (new_w - w) * 0.5
+        matrix[1, 2] += (new_h - h) * 0.5
+        w = int(np.round(new_w))
+        h = int(np.round(new_h))
+    rotated = cv2.warpAffine(
+        img,
+        matrix, (w, h),
+        flags=cv2_interp_codes[interpolation],
+        borderValue=border_value)
+    return rotated
+
+
+def bbox_clip(bboxes, img_shape):
+    """Clip bboxes to fit the image shape.
+
+    Args:
+        bboxes (ndarray): Shape (..., 4*k)
+        img_shape (tuple[int]): (height, width) of the image.
+
+    Returns:
+        ndarray: Clipped bboxes.
+    """
+    assert bboxes.shape[-1] % 4 == 0
+    cmin = np.empty(bboxes.shape[-1], dtype=bboxes.dtype)
+    cmin[0::2] = img_shape[1] - 1
+    cmin[1::2] = img_shape[0] - 1
+    clipped_bboxes = np.maximum(np.minimum(bboxes, cmin), 0)
+    return clipped_bboxes
+
+
+def bbox_scaling(bboxes, scale, clip_shape=None):
+    """Scaling bboxes w.r.t the box center.
+
+    Args:
+        bboxes (ndarray): Shape(..., 4).
+        scale (float): Scaling factor.
+        clip_shape (tuple[int], optional): If specified, bboxes that exceed the
+            boundary will be clipped according to the given shape (h, w).
+
+    Returns:
+        ndarray: Scaled bboxes.
+    """
+    if float(scale) == 1.0:
+        scaled_bboxes = bboxes.copy()
+    else:
+        w = bboxes[..., 2] - bboxes[..., 0] + 1
+        h = bboxes[..., 3] - bboxes[..., 1] + 1
+        dw = (w * (scale - 1)) * 0.5
+        dh = (h * (scale - 1)) * 0.5
+        scaled_bboxes = bboxes + np.stack((-dw, -dh, dw, dh), axis=-1)
+    if clip_shape is not None:
+        return bbox_clip(scaled_bboxes, clip_shape)
+    else:
+        return scaled_bboxes
+
+
+def imcrop(img, bboxes, scale=1.0, pad_fill=None):
+    """Crop image patches.
+
+    3 steps: scale the bboxes -> clip bboxes -> crop and pad.
+
+    Args:
+        img (ndarray): Image to be cropped.
+        bboxes (ndarray): Shape (k, 4) or (4, ), location of cropped bboxes.
+        scale (float, optional): Scale ratio of bboxes, the default value
+            1.0 means no padding.
+        pad_fill (Number | list[Number]): Value to be filled for padding.
+            Default: None, which means no padding.
+
+    Returns:
+        list[ndarray] | ndarray: The cropped image patches.
+    """
+    chn = 1 if img.ndim == 2 else img.shape[2]
+    if pad_fill is not None:
+        if isinstance(pad_fill, (int, float)):
+            pad_fill = [pad_fill for _ in range(chn)]
+        assert len(pad_fill) == chn
+
+    _bboxes = bboxes[None, ...] if bboxes.ndim == 1 else bboxes
+    scaled_bboxes = bbox_scaling(_bboxes, scale).astype(np.int32)
+    clipped_bbox = bbox_clip(scaled_bboxes, img.shape)
+
+    patches = []
+    for i in range(clipped_bbox.shape[0]):
+        x1, y1, x2, y2 = tuple(clipped_bbox[i, :])
+        if pad_fill is None:
+            patch = img[y1:y2 + 1, x1:x2 + 1, ...]
+        else:
+            _x1, _y1, _x2, _y2 = tuple(scaled_bboxes[i, :])
+            if chn == 1:
+                patch_shape = (_y2 - _y1 + 1, _x2 - _x1 + 1)
+            else:
+                patch_shape = (_y2 - _y1 + 1, _x2 - _x1 + 1, chn)
+            patch = np.array(
+                pad_fill, dtype=img.dtype) * np.ones(
+                    patch_shape, dtype=img.dtype)
+            x_start = 0 if _x1 >= 0 else -_x1
+            y_start = 0 if _y1 >= 0 else -_y1
+            w = x2 - x1 + 1
+            h = y2 - y1 + 1
+            patch[y_start:y_start + h, x_start:x_start + w,
+                  ...] = img[y1:y1 + h, x1:x1 + w, ...]
+        patches.append(patch)
+
+    if bboxes.ndim == 1:
+        return patches[0]
+    else:
+        return patches
+
+
+def impad(img,
+          *,
+          shape=None,
+          padding=None,
+          pad_val=0,
+          padding_mode='constant'):
+    """Pad the given image to a certain shape or pad on all sides with
+    specified padding mode and padding value.
+
+    Args:
+        img (ndarray): Image to be padded.
+        shape (tuple[int]): Expected padding shape (h, w). Default: None.
+        padding (int or tuple[int]): Padding on each border. If a single int is
+            provided this is used to pad all borders. If tuple of length 2 is
+            provided this is the padding on left/right and top/bottom
+            respectively. If a tuple of length 4 is provided this is the
+            padding for the left, top, right and bottom borders respectively.
+            Default: None. Note that `shape` and `padding` can not be both
+            set.
+        pad_val (Number | Sequence[Number]): Values to be filled in padding
+            areas when padding_mode is 'constant'. Default: 0.
+        padding_mode (str): Type of padding. Should be: constant, edge,
+            reflect or symmetric. Default: constant.
+
+            - constant: pads with a constant value, this value is specified
+              with pad_val.
+            - edge: pads with the last value at the edge of the image.
+            - reflect: pads with reflection of image without repeating the last
+              value on the edge. For example, padding [1, 2, 3, 4] with 2
+              elements on both sides in reflect mode will result in
+              [3, 2, 1, 2, 3, 4, 3, 2].
+            - symmetric: pads with reflection of image repeating the last value
+              on the edge. For example, padding [1, 2, 3, 4] with 2 elements on
+              both sides in symmetric mode will result in
+              [2, 1, 1, 2, 3, 4, 4, 3]
+
+    Returns:
+        ndarray: The padded image.
+    """
+
+    assert (shape is not None) ^ (padding is not None)
+    if shape is not None:
+        padding = (0, 0, shape[1] - img.shape[1], shape[0] - img.shape[0])
+
+    # check pad_val
+    if isinstance(pad_val, tuple):
+        assert len(pad_val) == img.shape[-1]
+    elif not isinstance(pad_val, numbers.Number):
+        raise TypeError('pad_val must be a int or a tuple. '
+                        f'But received {type(pad_val)}')
+
+    # check padding
+    if isinstance(padding, tuple) and len(padding) in [2, 4]:
+        if len(padding) == 2:
+            padding = (padding[0], padding[1], padding[0], padding[1])
+    elif isinstance(padding, numbers.Number):
+        padding = (padding, padding, padding, padding)
+    else:
+        raise ValueError('Padding must be a int or a 2, or 4 element tuple.'
+                         f'But received {padding}')
+
+    # check padding mode
+    assert padding_mode in ['constant', 'edge', 'reflect', 'symmetric']
+
+    border_type = {
+        'constant': cv2.BORDER_CONSTANT,
+        'edge': cv2.BORDER_REPLICATE,
+        'reflect': cv2.BORDER_REFLECT_101,
+        'symmetric': cv2.BORDER_REFLECT
+    }
+    img = cv2.copyMakeBorder(
+        img,
+        padding[1],
+        padding[3],
+        padding[0],
+        padding[2],
+        border_type[padding_mode],
+        value=pad_val)
+
+    return img
+
+
+def impad_to_multiple(img, divisor, pad_val=0):
+    """Pad an image to ensure each edge to be multiple to some number.
+
+    Args:
+        img (ndarray): Image to be padded.
+        divisor (int): Padded image edges will be multiple to divisor.
+        pad_val (Number | Sequence[Number]): Same as :func:`impad`.
+
+    Returns:
+        ndarray: The padded image.
+    """
+    pad_h = int(np.ceil(img.shape[0] / divisor)) * divisor
+    pad_w = int(np.ceil(img.shape[1] / divisor)) * divisor
+    return impad(img, shape=(pad_h, pad_w), pad_val=pad_val)
+
+
+def cutout(img, shape, pad_val=0):
+    """Randomly cut out a rectangle from the original img.
+
+    Args:
+        img (ndarray): Image to be cutout.
+        shape (int | tuple[int]): Expected cutout shape (h, w). If given as a
+            int, the value will be used for both h and w.
+        pad_val (int | float | tuple[int | float]): Values to be filled in the
+            cut area. Defaults to 0.
+
+    Returns:
+        ndarray: The cutout image.
+    """
+
+    channels = 1 if img.ndim == 2 else img.shape[2]
+    if isinstance(shape, int):
+        cut_h, cut_w = shape, shape
+    else:
+        assert isinstance(shape, tuple) and len(shape) == 2, \
+            f'shape must be a int or a tuple with length 2, but got type ' \
+            f'{type(shape)} instead.'
+        cut_h, cut_w = shape
+    if isinstance(pad_val, (int, float)):
+        pad_val = tuple([pad_val] * channels)
+    elif isinstance(pad_val, tuple):
+        assert len(pad_val) == channels, \
+            'Expected the num of elements in tuple equals the channels' \
+            'of input image. Found {} vs {}'.format(
+                len(pad_val), channels)
+    else:
+        raise TypeError(f'Invalid type {type(pad_val)} for `pad_val`')
+
+    img_h, img_w = img.shape[:2]
+    y0 = np.random.uniform(img_h)
+    x0 = np.random.uniform(img_w)
+
+    y1 = int(max(0, y0 - cut_h / 2.))
+    x1 = int(max(0, x0 - cut_w / 2.))
+    y2 = min(img_h, y1 + cut_h)
+    x2 = min(img_w, x1 + cut_w)
+
+    if img.ndim == 2:
+        patch_shape = (y2 - y1, x2 - x1)
+    else:
+        patch_shape = (y2 - y1, x2 - x1, channels)
+
+    img_cutout = img.copy()
+    patch = np.array(
+        pad_val, dtype=img.dtype) * np.ones(
+            patch_shape, dtype=img.dtype)
+    img_cutout[y1:y2, x1:x2, ...] = patch
+
+    return img_cutout
+
+
+def _get_shear_matrix(magnitude, direction='horizontal'):
+    """Generate the shear matrix for transformation.
+
+    Args:
+        magnitude (int | float): The magnitude used for shear.
+        direction (str): The flip direction, either "horizontal"
+            or "vertical".
+
+    Returns:
+        ndarray: The shear matrix with dtype float32.
+    """
+    if direction == 'horizontal':
+        shear_matrix = np.float32([[1, magnitude, 0], [0, 1, 0]])
+    elif direction == 'vertical':
+        shear_matrix = np.float32([[1, 0, 0], [magnitude, 1, 0]])
+    return shear_matrix
+
+
+def imshear(img,
+            magnitude,
+            direction='horizontal',
+            border_value=0,
+            interpolation='bilinear'):
+    """Shear an image.
+
+    Args:
+        img (ndarray): Image to be sheared with format (h, w)
+            or (h, w, c).
+        magnitude (int | float): The magnitude used for shear.
+        direction (str): The flip direction, either "horizontal"
+            or "vertical".
+        border_value (int | tuple[int]): Value used in case of a
+            constant border.
+        interpolation (str): Same as :func:`resize`.
+
+    Returns:
+        ndarray: The sheared image.
+    """
+    assert direction in ['horizontal',
+                         'vertical'], f'Invalid direction: {direction}'
+    height, width = img.shape[:2]
+    if img.ndim == 2:
+        channels = 1
+    elif img.ndim == 3:
+        channels = img.shape[-1]
+    if isinstance(border_value, int):
+        border_value = tuple([border_value] * channels)
+    elif isinstance(border_value, tuple):
+        assert len(border_value) == channels, \
+            'Expected the num of elements in tuple equals the channels' \
+            'of input image. Found {} vs {}'.format(
+                len(border_value), channels)
+    else:
+        raise ValueError(
+            f'Invalid type {type(border_value)} for `border_value`')
+    shear_matrix = _get_shear_matrix(magnitude, direction)
+    sheared = cv2.warpAffine(
+        img,
+        shear_matrix,
+        (width, height),
+        # Note case when the number elements in `border_value`
+        # greater than 3 (e.g. shearing masks whose channels large
+        # than 3) will raise TypeError in `cv2.warpAffine`.
+        # Here simply slice the first 3 values in `border_value`.
+        borderValue=border_value[:3],
+        flags=cv2_interp_codes[interpolation])
+    return sheared
+
+
+def _get_translate_matrix(offset, direction='horizontal'):
+    """Generate the translate matrix.
+
+    Args:
+        offset (int | float): The offset used for translate.
+        direction (str): The translate direction, either
+            "horizontal" or "vertical".
+
+    Returns:
+        ndarray: The translate matrix with dtype float32.
+    """
+    if direction == 'horizontal':
+        translate_matrix = np.float32([[1, 0, offset], [0, 1, 0]])
+    elif direction == 'vertical':
+        translate_matrix = np.float32([[1, 0, 0], [0, 1, offset]])
+    return translate_matrix
+
+
+def imtranslate(img,
+                offset,
+                direction='horizontal',
+                border_value=0,
+                interpolation='bilinear'):
+    """Translate an image.
+
+    Args:
+        img (ndarray): Image to be translated with format
+            (h, w) or (h, w, c).
+        offset (int | float): The offset used for translate.
+        direction (str): The translate direction, either "horizontal"
+            or "vertical".
+        border_value (int | tuple[int]): Value used in case of a
+            constant border.
+        interpolation (str): Same as :func:`resize`.
+
+    Returns:
+        ndarray: The translated image.
+    """
+    assert direction in ['horizontal',
+                         'vertical'], f'Invalid direction: {direction}'
+    height, width = img.shape[:2]
+    if img.ndim == 2:
+        channels = 1
+    elif img.ndim == 3:
+        channels = img.shape[-1]
+    if isinstance(border_value, int):
+        border_value = tuple([border_value] * channels)
+    elif isinstance(border_value, tuple):
+        assert len(border_value) == channels, \
+            'Expected the num of elements in tuple equals the channels' \
+            'of input image. Found {} vs {}'.format(
+                len(border_value), channels)
+    else:
+        raise ValueError(
+            f'Invalid type {type(border_value)} for `border_value`.')
+    translate_matrix = _get_translate_matrix(offset, direction)
+    translated = cv2.warpAffine(
+        img,
+        translate_matrix,
+        (width, height),
+        # Note case when the number elements in `border_value`
+        # greater than 3 (e.g. translating masks whose channels
+        # large than 3) will raise TypeError in `cv2.warpAffine`.
+        # Here simply slice the first 3 values in `border_value`.
+        borderValue=border_value[:3],
+        flags=cv2_interp_codes[interpolation])
+    return translated

groundingLMM/mmcv/mmcv/image/io.py

@@ -0,0 +1,314 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import io
+import os.path as osp
+import warnings
+from pathlib import Path
+
+import cv2
+import numpy as np
+from cv2 import (IMREAD_COLOR, IMREAD_GRAYSCALE, IMREAD_IGNORE_ORIENTATION,
+                 IMREAD_UNCHANGED)
+
+from mmcv.fileio import FileClient
+from mmcv.utils import is_filepath, is_str
+
+try:
+    from turbojpeg import TJCS_RGB, TJPF_BGR, TJPF_GRAY, TurboJPEG
+except ImportError:
+    TJCS_RGB = TJPF_GRAY = TJPF_BGR = TurboJPEG = None
+
+try:
+    from PIL import Image, ImageOps
+except ImportError:
+    Image = None
+
+try:
+    import tifffile
+except ImportError:
+    tifffile = None
+
+jpeg = None
+supported_backends = ['cv2', 'turbojpeg', 'pillow', 'tifffile']
+
+imread_flags = {
+    'color': IMREAD_COLOR,
+    'grayscale': IMREAD_GRAYSCALE,
+    'unchanged': IMREAD_UNCHANGED,
+    'color_ignore_orientation': IMREAD_IGNORE_ORIENTATION | IMREAD_COLOR,
+    'grayscale_ignore_orientation':
+    IMREAD_IGNORE_ORIENTATION | IMREAD_GRAYSCALE
+}
+
+imread_backend = 'cv2'
+
+
+def use_backend(backend):
+    """Select a backend for image decoding.
+
+    Args:
+        backend (str): The image decoding backend type. Options are `cv2`,
+        `pillow`, `turbojpeg` (see https://github.com/lilohuang/PyTurboJPEG)
+        and `tifffile`. `turbojpeg` is faster but it only supports `.jpeg`
+        file format.
+    """
+    assert backend in supported_backends
+    global imread_backend
+    imread_backend = backend
+    if imread_backend == 'turbojpeg':
+        if TurboJPEG is None:
+            raise ImportError('`PyTurboJPEG` is not installed')
+        global jpeg
+        if jpeg is None:
+            jpeg = TurboJPEG()
+    elif imread_backend == 'pillow':
+        if Image is None:
+            raise ImportError('`Pillow` is not installed')
+    elif imread_backend == 'tifffile':
+        if tifffile is None:
+            raise ImportError('`tifffile` is not installed')
+
+
+def _jpegflag(flag='color', channel_order='bgr'):
+    channel_order = channel_order.lower()
+    if channel_order not in ['rgb', 'bgr']:
+        raise ValueError('channel order must be either "rgb" or "bgr"')
+
+    if flag == 'color':
+        if channel_order == 'bgr':
+            return TJPF_BGR
+        elif channel_order == 'rgb':
+            return TJCS_RGB
+    elif flag == 'grayscale':
+        return TJPF_GRAY
+    else:
+        raise ValueError('flag must be "color" or "grayscale"')
+
+
+def _pillow2array(img, flag='color', channel_order='bgr'):
+    """Convert a pillow image to numpy array.
+
+    Args:
+        img (:obj:`PIL.Image.Image`): The image loaded using PIL
+        flag (str): Flags specifying the color type of a loaded image,
+            candidates are 'color', 'grayscale' and 'unchanged'.
+            Default to 'color'.
+        channel_order (str): The channel order of the output image array,
+            candidates are 'bgr' and 'rgb'. Default to 'bgr'.
+
+    Returns:
+        np.ndarray: The converted numpy array
+    """
+    channel_order = channel_order.lower()
+    if channel_order not in ['rgb', 'bgr']:
+        raise ValueError('channel order must be either "rgb" or "bgr"')
+
+    if flag == 'unchanged':
+        array = np.array(img)
+        if array.ndim >= 3 and array.shape[2] >= 3:  # color image
+            array[:, :, :3] = array[:, :, (2, 1, 0)]  # RGB to BGR
+    else:
+        # Handle exif orientation tag
+        if flag in ['color', 'grayscale']:
+            img = ImageOps.exif_transpose(img)
+        # If the image mode is not 'RGB', convert it to 'RGB' first.
+        if img.mode != 'RGB':
+            if img.mode != 'LA':
+                # Most formats except 'LA' can be directly converted to RGB
+                img = img.convert('RGB')
+            else:
+                # When the mode is 'LA', the default conversion will fill in
+                #  the canvas with black, which sometimes shadows black objects
+                #  in the foreground.
+                #
+                # Therefore, a random color (124, 117, 104) is used for canvas
+                img_rgba = img.convert('RGBA')
+                img = Image.new('RGB', img_rgba.size, (124, 117, 104))
+                img.paste(img_rgba, mask=img_rgba.split()[3])  # 3 is alpha
+        if flag in ['color', 'color_ignore_orientation']:
+            array = np.array(img)
+            if channel_order != 'rgb':
+                array = array[:, :, ::-1]  # RGB to BGR
+        elif flag in ['grayscale', 'grayscale_ignore_orientation']:
+            img = img.convert('L')
+            array = np.array(img)
+        else:
+            raise ValueError(
+                'flag must be "color", "grayscale", "unchanged", '
+                f'"color_ignore_orientation" or "grayscale_ignore_orientation"'
+                f' but got {flag}')
+    return array
+
+
+def imread(img_or_path,
+           flag='color',
+           channel_order='bgr',
+           backend=None,
+           file_client_args=None):
+    """Read an image.
+
+    Note:
+        In v1.4.1 and later, add `file_client_args` parameters.
+
+    Args:
+        img_or_path (ndarray or str or Path): Either a numpy array or str or
+            pathlib.Path. If it is a numpy array (loaded image), then
+            it will be returned as is.
+        flag (str): Flags specifying the color type of a loaded image,
+            candidates are `color`, `grayscale`, `unchanged`,
+            `color_ignore_orientation` and `grayscale_ignore_orientation`.
+            By default, `cv2` and `pillow` backend would rotate the image
+            according to its EXIF info unless called with `unchanged` or
+            `*_ignore_orientation` flags. `turbojpeg` and `tifffile` backend
+            always ignore image's EXIF info regardless of the flag.
+            The `turbojpeg` backend only supports `color` and `grayscale`.
+        channel_order (str): Order of channel, candidates are `bgr` and `rgb`.
+        backend (str | None): The image decoding backend type. Options are
+            `cv2`, `pillow`, `turbojpeg`, `tifffile`, `None`.
+            If backend is None, the global imread_backend specified by
+            ``mmcv.use_backend()`` will be used. Default: None.
+        file_client_args (dict | None): Arguments to instantiate a
+            FileClient. See :class:`mmcv.fileio.FileClient` for details.
+            Default: None.
+
+    Returns:
+        ndarray: Loaded image array.
+
+    Examples:
+        >>> import mmcv
+        >>> img_path = '/path/to/img.jpg'
+        >>> img = mmcv.imread(img_path)
+        >>> img = mmcv.imread(img_path, flag='color', channel_order='rgb',
+        ...     backend='cv2')
+        >>> img = mmcv.imread(img_path, flag='color', channel_order='bgr',
+        ...     backend='pillow')
+        >>> s3_img_path = 's3://bucket/img.jpg'
+        >>> # infer the file backend by the prefix s3
+        >>> img = mmcv.imread(s3_img_path)
+        >>> # manually set the file backend petrel
+        >>> img = mmcv.imread(s3_img_path, file_client_args={
+        ...     'backend': 'petrel'})
+        >>> http_img_path = 'http://path/to/img.jpg'
+        >>> img = mmcv.imread(http_img_path)
+        >>> img = mmcv.imread(http_img_path, file_client_args={
+        ...     'backend': 'http'})
+    """
+
+    if isinstance(img_or_path, Path):
+        img_or_path = str(img_or_path)
+
+    if isinstance(img_or_path, np.ndarray):
+        return img_or_path
+    elif is_str(img_or_path):
+        file_client = FileClient.infer_client(file_client_args, img_or_path)
+        img_bytes = file_client.get(img_or_path)
+        return imfrombytes(img_bytes, flag, channel_order, backend)
+    else:
+        raise TypeError('"img" must be a numpy array or a str or '
+                        'a pathlib.Path object')
+
+
+def imfrombytes(content, flag='color', channel_order='bgr', backend=None):
+    """Read an image from bytes.
+
+    Args:
+        content (bytes): Image bytes got from files or other streams.
+        flag (str): Same as :func:`imread`.
+        channel_order (str): The channel order of the output, candidates
+            are 'bgr' and 'rgb'. Default to 'bgr'.
+        backend (str | None): The image decoding backend type. Options are
+            `cv2`, `pillow`, `turbojpeg`, `tifffile`, `None`. If backend is
+            None, the global imread_backend specified by ``mmcv.use_backend()``
+            will be used. Default: None.
+
+    Returns:
+        ndarray: Loaded image array.
+
+    Examples:
+        >>> img_path = '/path/to/img.jpg'
+        >>> with open(img_path, 'rb') as f:
+        >>>     img_buff = f.read()
+        >>> img = mmcv.imfrombytes(img_buff)
+        >>> img = mmcv.imfrombytes(img_buff, flag='color', channel_order='rgb')
+        >>> img = mmcv.imfrombytes(img_buff, backend='pillow')
+        >>> img = mmcv.imfrombytes(img_buff, backend='cv2')
+    """
+
+    if backend is None:
+        backend = imread_backend
+    if backend not in supported_backends:
+        raise ValueError(
+            f'backend: {backend} is not supported. Supported '
+            "backends are 'cv2', 'turbojpeg', 'pillow', 'tifffile'")
+    if backend == 'turbojpeg':
+        img = jpeg.decode(content, _jpegflag(flag, channel_order))
+        if img.shape[-1] == 1:
+            img = img[:, :, 0]
+        return img
+    elif backend == 'pillow':
+        with io.BytesIO(content) as buff:
+            img = Image.open(buff)
+            img = _pillow2array(img, flag, channel_order)
+        return img
+    elif backend == 'tifffile':
+        with io.BytesIO(content) as buff:
+            img = tifffile.imread(buff)
+        return img
+    else:
+        img_np = np.frombuffer(content, np.uint8)
+        flag = imread_flags[flag] if is_str(flag) else flag
+        img = cv2.imdecode(img_np, flag)
+        if flag == IMREAD_COLOR and channel_order == 'rgb':
+            cv2.cvtColor(img, cv2.COLOR_BGR2RGB, img)
+        return img
+
+
+def imwrite(img,
+            file_path,
+            params=None,
+            auto_mkdir=None,
+            file_client_args=None):
+    """Write image to file.
+
+    Note:
+        In v1.4.1 and later, add `file_client_args` parameters.
+
+    Warning:
+        The parameter `auto_mkdir` will be deprecated in the future and every
+        file clients will make directory automatically.
+
+    Args:
+        img (ndarray): Image array to be written.
+        file_path (str): Image file path.
+        params (None or list): Same as opencv :func:`imwrite` interface.
+        auto_mkdir (bool): If the parent folder of `file_path` does not exist,
+            whether to create it automatically. It will be deprecated.
+        file_client_args (dict | None): Arguments to instantiate a
+            FileClient. See :class:`mmcv.fileio.FileClient` for details.
+            Default: None.
+
+    Returns:
+        bool: Successful or not.
+
+    Examples:
+        >>> # write to hard disk client
+        >>> ret = mmcv.imwrite(img, '/path/to/img.jpg')
+        >>> # infer the file backend by the prefix s3
+        >>> ret = mmcv.imwrite(img, 's3://bucket/img.jpg')
+        >>> # manually set the file backend petrel
+        >>> ret = mmcv.imwrite(img, 's3://bucket/img.jpg', file_client_args={
+        ...     'backend': 'petrel'})
+    """
+    assert is_filepath(file_path)
+    file_path = str(file_path)
+    if auto_mkdir is not None:
+        warnings.warn(
+            'The parameter `auto_mkdir` will be deprecated in the future and '
+            'every file clients will make directory automatically.')
+    file_client = FileClient.infer_client(file_client_args, file_path)
+    img_ext = osp.splitext(file_path)[-1]
+    # Encode image according to image suffix.
+    # For example, if image path is '/path/your/img.jpg', the encode
+    # format is '.jpg'.
+    flag, img_buff = cv2.imencode(img_ext, img, params)
+    file_client.put(img_buff.tobytes(), file_path)
+    return flag

groundingLMM/mmcv/mmcv/image/misc.py

@@ -0,0 +1,53 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import numpy as np
+
+import mmcv
+
+try:
+    import torch
+except ImportError:
+    torch = None
+
+
+def tensor2imgs(tensor, mean=None, std=None, to_rgb=True):
+    """Convert tensor to 3-channel images or 1-channel gray images.
+
+    Args:
+        tensor (torch.Tensor): Tensor that contains multiple images, shape (
+            N, C, H, W). :math:`C` can be either 3 or 1.
+        mean (tuple[float], optional): Mean of images. If None,
+            (0, 0, 0) will be used for tensor with 3-channel,
+            while (0, ) for tensor with 1-channel. Defaults to None.
+        std (tuple[float], optional): Standard deviation of images. If None,
+            (1, 1, 1) will be used for tensor with 3-channel,
+            while (1, ) for tensor with 1-channel. Defaults to None.
+        to_rgb (bool, optional): Whether the tensor was converted to RGB
+            format in the first place. If so, convert it back to BGR.
+            For the tensor with 1 channel, it must be False. Defaults to True.
+
+    Returns:
+        list[np.ndarray]: A list that contains multiple images.
+    """
+
+    if torch is None:
+        raise RuntimeError('pytorch is not installed')
+    assert torch.is_tensor(tensor) and tensor.ndim == 4
+    channels = tensor.size(1)
+    assert channels in [1, 3]
+    if mean is None:
+        mean = (0, ) * channels
+    if std is None:
+        std = (1, ) * channels
+    assert (channels == len(mean) == len(std) == 3) or \
+        (channels == len(mean) == len(std) == 1 and not to_rgb)
+
+    num_imgs = tensor.size(0)
+    mean = np.array(mean, dtype=np.float32)
+    std = np.array(std, dtype=np.float32)
+    imgs = []
+    for img_id in range(num_imgs):
+        img = tensor[img_id, ...].cpu().numpy().transpose(1, 2, 0)
+        img = mmcv.imdenormalize(
+            img, mean, std, to_bgr=to_rgb).astype(np.uint8)
+        imgs.append(np.ascontiguousarray(img))
+    return imgs

groundingLMM/mmcv/mmcv/image/photometric.py

@@ -0,0 +1,428 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import cv2
+import numpy as np
+
+from ..utils import is_tuple_of
+from .colorspace import bgr2gray, gray2bgr
+
+
+def imnormalize(img, mean, std, to_rgb=True):
+    """Normalize an image with mean and std.
+
+    Args:
+        img (ndarray): Image to be normalized.
+        mean (ndarray): The mean to be used for normalize.
+        std (ndarray): The std to be used for normalize.
+        to_rgb (bool): Whether to convert to rgb.
+
+    Returns:
+        ndarray: The normalized image.
+    """
+    img = img.copy().astype(np.float32)
+    return imnormalize_(img, mean, std, to_rgb)
+
+
+def imnormalize_(img, mean, std, to_rgb=True):
+    """Inplace normalize an image with mean and std.
+
+    Args:
+        img (ndarray): Image to be normalized.
+        mean (ndarray): The mean to be used for normalize.
+        std (ndarray): The std to be used for normalize.
+        to_rgb (bool): Whether to convert to rgb.
+
+    Returns:
+        ndarray: The normalized image.
+    """
+    # cv2 inplace normalization does not accept uint8
+    assert img.dtype != np.uint8
+    mean = np.float64(mean.reshape(1, -1))
+    stdinv = 1 / np.float64(std.reshape(1, -1))
+    if to_rgb:
+        cv2.cvtColor(img, cv2.COLOR_BGR2RGB, img)  # inplace
+    cv2.subtract(img, mean, img)  # inplace
+    cv2.multiply(img, stdinv, img)  # inplace
+    return img
+
+
+def imdenormalize(img, mean, std, to_bgr=True):
+    assert img.dtype != np.uint8
+    mean = mean.reshape(1, -1).astype(np.float64)
+    std = std.reshape(1, -1).astype(np.float64)
+    img = cv2.multiply(img, std)  # make a copy
+    cv2.add(img, mean, img)  # inplace
+    if to_bgr:
+        cv2.cvtColor(img, cv2.COLOR_RGB2BGR, img)  # inplace
+    return img
+
+
+def iminvert(img):
+    """Invert (negate) an image.
+
+    Args:
+        img (ndarray): Image to be inverted.
+
+    Returns:
+        ndarray: The inverted image.
+    """
+    return np.full_like(img, 255) - img
+
+
+def solarize(img, thr=128):
+    """Solarize an image (invert all pixel values above a threshold)
+
+    Args:
+        img (ndarray): Image to be solarized.
+        thr (int): Threshold for solarizing (0 - 255).
+
+    Returns:
+        ndarray: The solarized image.
+    """
+    img = np.where(img < thr, img, 255 - img)
+    return img
+
+
+def posterize(img, bits):
+    """Posterize an image (reduce the number of bits for each color channel)
+
+    Args:
+        img (ndarray): Image to be posterized.
+        bits (int): Number of bits (1 to 8) to use for posterizing.
+
+    Returns:
+        ndarray: The posterized image.
+    """
+    shift = 8 - bits
+    img = np.left_shift(np.right_shift(img, shift), shift)
+    return img
+
+
+def adjust_color(img, alpha=1, beta=None, gamma=0):
+    r"""It blends the source image and its gray image:
+
+    .. math::
+        output = img * alpha + gray\_img * beta + gamma
+
+    Args:
+        img (ndarray): The input source image.
+        alpha (int | float): Weight for the source image. Default 1.
+        beta (int | float): Weight for the converted gray image.
+            If None, it's assigned the value (1 - `alpha`).
+        gamma (int | float): Scalar added to each sum.
+            Same as :func:`cv2.addWeighted`. Default 0.
+
+    Returns:
+        ndarray: Colored image which has the same size and dtype as input.
+    """
+    gray_img = bgr2gray(img)
+    gray_img = np.tile(gray_img[..., None], [1, 1, 3])
+    if beta is None:
+        beta = 1 - alpha
+    colored_img = cv2.addWeighted(img, alpha, gray_img, beta, gamma)
+    if not colored_img.dtype == np.uint8:
+        # Note when the dtype of `img` is not the default `np.uint8`
+        # (e.g. np.float32), the value in `colored_img` got from cv2
+        # is not guaranteed to be in range [0, 255], so here clip
+        # is needed.
+        colored_img = np.clip(colored_img, 0, 255)
+    return colored_img
+
+
+def imequalize(img):
+    """Equalize the image histogram.
+
+    This function applies a non-linear mapping to the input image,
+    in order to create a uniform distribution of grayscale values
+    in the output image.
+
+    Args:
+        img (ndarray): Image to be equalized.
+
+    Returns:
+        ndarray: The equalized image.
+    """
+
+    def _scale_channel(im, c):
+        """Scale the data in the corresponding channel."""
+        im = im[:, :, c]
+        # Compute the histogram of the image channel.
+        histo = np.histogram(im, 256, (0, 255))[0]
+        # For computing the step, filter out the nonzeros.
+        nonzero_histo = histo[histo > 0]
+        step = (np.sum(nonzero_histo) - nonzero_histo[-1]) // 255
+        if not step:
+            lut = np.array(range(256))
+        else:
+            # Compute the cumulative sum, shifted by step // 2
+            # and then normalized by step.
+            lut = (np.cumsum(histo) + (step // 2)) // step
+            # Shift lut, prepending with 0.
+            lut = np.concatenate([[0], lut[:-1]], 0)
+            # handle potential integer overflow
+            lut[lut > 255] = 255
+        # If step is zero, return the original image.
+        # Otherwise, index from lut.
+        return np.where(np.equal(step, 0), im, lut[im])
+
+    # Scales each channel independently and then stacks
+    # the result.
+    s1 = _scale_channel(img, 0)
+    s2 = _scale_channel(img, 1)
+    s3 = _scale_channel(img, 2)
+    equalized_img = np.stack([s1, s2, s3], axis=-1)
+    return equalized_img.astype(img.dtype)
+
+
+def adjust_brightness(img, factor=1.):
+    """Adjust image brightness.
+
+    This function controls the brightness of an image. An
+    enhancement factor of 0.0 gives a black image.
+    A factor of 1.0 gives the original image. This function
+    blends the source image and the degenerated black image:
+
+    .. math::
+        output = img * factor + degenerated * (1 - factor)
+
+    Args:
+        img (ndarray): Image to be brightened.
+        factor (float): A value controls the enhancement.
+            Factor 1.0 returns the original image, lower
+            factors mean less color (brightness, contrast,
+            etc), and higher values more. Default 1.
+
+    Returns:
+        ndarray: The brightened image.
+    """
+    degenerated = np.zeros_like(img)
+    # Note manually convert the dtype to np.float32, to
+    # achieve as close results as PIL.ImageEnhance.Brightness.
+    # Set beta=1-factor, and gamma=0
+    brightened_img = cv2.addWeighted(
+        img.astype(np.float32), factor, degenerated.astype(np.float32),
+        1 - factor, 0)
+    brightened_img = np.clip(brightened_img, 0, 255)
+    return brightened_img.astype(img.dtype)
+
+
+def adjust_contrast(img, factor=1.):
+    """Adjust image contrast.
+
+    This function controls the contrast of an image. An
+    enhancement factor of 0.0 gives a solid grey
+    image. A factor of 1.0 gives the original image. It
+    blends the source image and the degenerated mean image:
+
+    .. math::
+        output = img * factor + degenerated * (1 - factor)
+
+    Args:
+        img (ndarray): Image to be contrasted. BGR order.
+        factor (float): Same as :func:`mmcv.adjust_brightness`.
+
+    Returns:
+        ndarray: The contrasted image.
+    """
+    gray_img = bgr2gray(img)
+    hist = np.histogram(gray_img, 256, (0, 255))[0]
+    mean = round(np.sum(gray_img) / np.sum(hist))
+    degenerated = (np.ones_like(img[..., 0]) * mean).astype(img.dtype)
+    degenerated = gray2bgr(degenerated)
+    contrasted_img = cv2.addWeighted(
+        img.astype(np.float32), factor, degenerated.astype(np.float32),
+        1 - factor, 0)
+    contrasted_img = np.clip(contrasted_img, 0, 255)
+    return contrasted_img.astype(img.dtype)
+
+
+def auto_contrast(img, cutoff=0):
+    """Auto adjust image contrast.
+
+    This function maximize (normalize) image contrast by first removing cutoff
+    percent of the lightest and darkest pixels from the histogram and remapping
+    the image so that the darkest pixel becomes black (0), and the lightest
+    becomes white (255).
+
+    Args:
+        img (ndarray): Image to be contrasted. BGR order.
+        cutoff (int | float | tuple): The cutoff percent of the lightest and
+            darkest pixels to be removed. If given as tuple, it shall be
+            (low, high). Otherwise, the single value will be used for both.
+            Defaults to 0.
+
+    Returns:
+        ndarray: The contrasted image.
+    """
+
+    def _auto_contrast_channel(im, c, cutoff):
+        im = im[:, :, c]
+        # Compute the histogram of the image channel.
+        histo = np.histogram(im, 256, (0, 255))[0]
+        # Remove cut-off percent pixels from histo
+        histo_sum = np.cumsum(histo)
+        cut_low = histo_sum[-1] * cutoff[0] // 100
+        cut_high = histo_sum[-1] - histo_sum[-1] * cutoff[1] // 100
+        histo_sum = np.clip(histo_sum, cut_low, cut_high) - cut_low
+        histo = np.concatenate([[histo_sum[0]], np.diff(histo_sum)], 0)
+
+        # Compute mapping
+        low, high = np.nonzero(histo)[0][0], np.nonzero(histo)[0][-1]
+        # If all the values have been cut off, return the origin img
+        if low >= high:
+            return im
+        scale = 255.0 / (high - low)
+        offset = -low * scale
+        lut = np.array(range(256))
+        lut = lut * scale + offset
+        lut = np.clip(lut, 0, 255)
+        return lut[im]
+
+    if isinstance(cutoff, (int, float)):
+        cutoff = (cutoff, cutoff)
+    else:
+        assert isinstance(cutoff, tuple), 'cutoff must be of type int, ' \
+            f'float or tuple, but got {type(cutoff)} instead.'
+    # Auto adjusts contrast for each channel independently and then stacks
+    # the result.
+    s1 = _auto_contrast_channel(img, 0, cutoff)
+    s2 = _auto_contrast_channel(img, 1, cutoff)
+    s3 = _auto_contrast_channel(img, 2, cutoff)
+    contrasted_img = np.stack([s1, s2, s3], axis=-1)
+    return contrasted_img.astype(img.dtype)
+
+
+def adjust_sharpness(img, factor=1., kernel=None):
+    """Adjust image sharpness.
+
+    This function controls the sharpness of an image. An
+    enhancement factor of 0.0 gives a blurred image. A
+    factor of 1.0 gives the original image. And a factor
+    of 2.0 gives a sharpened image. It blends the source
+    image and the degenerated mean image:
+
+    .. math::
+        output = img * factor + degenerated * (1 - factor)
+
+    Args:
+        img (ndarray): Image to be sharpened. BGR order.
+        factor (float): Same as :func:`mmcv.adjust_brightness`.
+        kernel (np.ndarray, optional): Filter kernel to be applied on the img
+            to obtain the degenerated img. Defaults to None.
+
+    Note:
+        No value sanity check is enforced on the kernel set by users. So with
+        an inappropriate kernel, the ``adjust_sharpness`` may fail to perform
+        the function its name indicates but end up performing whatever
+        transform determined by the kernel.
+
+    Returns:
+        ndarray: The sharpened image.
+    """
+
+    if kernel is None:
+        # adopted from PIL.ImageFilter.SMOOTH
+        kernel = np.array([[1., 1., 1.], [1., 5., 1.], [1., 1., 1.]]) / 13
+    assert isinstance(kernel, np.ndarray), \
+        f'kernel must be of type np.ndarray, but got {type(kernel)} instead.'
+    assert kernel.ndim == 2, \
+        f'kernel must have a dimension of 2, but got {kernel.ndim} instead.'
+
+    degenerated = cv2.filter2D(img, -1, kernel)
+    sharpened_img = cv2.addWeighted(
+        img.astype(np.float32), factor, degenerated.astype(np.float32),
+        1 - factor, 0)
+    sharpened_img = np.clip(sharpened_img, 0, 255)
+    return sharpened_img.astype(img.dtype)
+
+
+def adjust_lighting(img, eigval, eigvec, alphastd=0.1, to_rgb=True):
+    """AlexNet-style PCA jitter.
+
+    This data augmentation is proposed in `ImageNet Classification with Deep
+    Convolutional Neural Networks
+    <https://dl.acm.org/doi/pdf/10.1145/3065386>`_.
+
+    Args:
+        img (ndarray): Image to be adjusted lighting. BGR order.
+        eigval (ndarray): the eigenvalue of the convariance matrix of pixel
+            values, respectively.
+        eigvec (ndarray): the eigenvector of the convariance matrix of pixel
+            values, respectively.
+        alphastd (float): The standard deviation for distribution of alpha.
+            Defaults to 0.1
+        to_rgb (bool): Whether to convert img to rgb.
+
+    Returns:
+        ndarray: The adjusted image.
+    """
+    assert isinstance(eigval, np.ndarray) and isinstance(eigvec, np.ndarray), \
+        f'eigval and eigvec should both be of type np.ndarray, got ' \
+        f'{type(eigval)} and {type(eigvec)} instead.'
+
+    assert eigval.ndim == 1 and eigvec.ndim == 2
+    assert eigvec.shape == (3, eigval.shape[0])
+    n_eigval = eigval.shape[0]
+    assert isinstance(alphastd, float), 'alphastd should be of type float, ' \
+        f'got {type(alphastd)} instead.'
+
+    img = img.copy().astype(np.float32)
+    if to_rgb:
+        cv2.cvtColor(img, cv2.COLOR_BGR2RGB, img)  # inplace
+
+    alpha = np.random.normal(0, alphastd, n_eigval)
+    alter = eigvec \
+        * np.broadcast_to(alpha.reshape(1, n_eigval), (3, n_eigval)) \
+        * np.broadcast_to(eigval.reshape(1, n_eigval), (3, n_eigval))
+    alter = np.broadcast_to(alter.sum(axis=1).reshape(1, 1, 3), img.shape)
+    img_adjusted = img + alter
+    return img_adjusted
+
+
+def lut_transform(img, lut_table):
+    """Transform array by look-up table.
+
+    The function lut_transform fills the output array with values from the
+    look-up table. Indices of the entries are taken from the input array.
+
+    Args:
+        img (ndarray): Image to be transformed.
+        lut_table (ndarray): look-up table of 256 elements; in case of
+            multi-channel input array, the table should either have a single
+            channel (in this case the same table is used for all channels) or
+            the same number of channels as in the input array.
+
+    Returns:
+        ndarray: The transformed image.
+    """
+    assert isinstance(img, np.ndarray)
+    assert 0 <= np.min(img) and np.max(img) <= 255
+    assert isinstance(lut_table, np.ndarray)
+    assert lut_table.shape == (256, )
+
+    return cv2.LUT(np.array(img, dtype=np.uint8), lut_table)
+
+
+def clahe(img, clip_limit=40.0, tile_grid_size=(8, 8)):
+    """Use CLAHE method to process the image.
+
+    See `ZUIDERVELD,K. Contrast Limited Adaptive Histogram Equalization[J].
+    Graphics Gems, 1994:474-485.` for more information.
+
+    Args:
+        img (ndarray): Image to be processed.
+        clip_limit (float): Threshold for contrast limiting. Default: 40.0.
+        tile_grid_size (tuple[int]): Size of grid for histogram equalization.
+            Input image will be divided into equally sized rectangular tiles.
+            It defines the number of tiles in row and column. Default: (8, 8).
+
+    Returns:
+        ndarray: The processed image.
+    """
+    assert isinstance(img, np.ndarray)
+    assert img.ndim == 2
+    assert isinstance(clip_limit, (float, int))
+    assert is_tuple_of(tile_grid_size, int)
+    assert len(tile_grid_size) == 2
+
+    clahe = cv2.createCLAHE(clip_limit, tile_grid_size)
+    return clahe.apply(np.array(img, dtype=np.uint8))

groundingLMM/mmcv/mmcv/model_zoo/deprecated.json

@@ -0,0 +1,6 @@
+{
+  "resnet50_caffe": "detectron/resnet50_caffe",
+  "resnet50_caffe_bgr": "detectron2/resnet50_caffe_bgr",
+  "resnet101_caffe": "detectron/resnet101_caffe",
+  "resnet101_caffe_bgr": "detectron2/resnet101_caffe_bgr"
+}

groundingLMM/mmcv/mmcv/model_zoo/mmcls.json

@@ -0,0 +1,59 @@
+{
+  "vgg11": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg11_batch256_imagenet_20210208-4271cd6c.pth",
+  "vgg13": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg13_batch256_imagenet_20210208-4d1d6080.pth",
+  "vgg16": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg16_batch256_imagenet_20210208-db26f1a5.pth",
+  "vgg19": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg19_batch256_imagenet_20210208-e6920e4a.pth",
+  "vgg11_bn": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg11_bn_batch256_imagenet_20210207-f244902c.pth",
+  "vgg13_bn": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg13_bn_batch256_imagenet_20210207-1a8b7864.pth",
+  "vgg16_bn": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg16_bn_batch256_imagenet_20210208-7e55cd29.pth",
+  "vgg19_bn": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg19_bn_batch256_imagenet_20210208-da620c4f.pth",
+  "resnet18": "https://download.openmmlab.com/mmclassification/v0/resnet/resnet18_8xb32_in1k_20210831-fbbb1da6.pth",
+  "resnet34": "https://download.openmmlab.com/mmclassification/v0/resnet/resnet34_8xb32_in1k_20210831-f257d4e6.pth",
+  "resnet50": "https://download.openmmlab.com/mmclassification/v0/resnet/resnet50_8xb32_in1k_20210831-ea4938fc.pth",
+  "resnet101": "https://download.openmmlab.com/mmclassification/v0/resnet/resnet101_8xb32_in1k_20210831-539c63f8.pth",
+  "resnet152": "https://download.openmmlab.com/mmclassification/v0/resnet/resnet152_8xb32_in1k_20210901-4d7582fa.pth",
+  "resnet50_v1d": "https://download.openmmlab.com/mmclassification/v0/resnet/resnetv1d50_b32x8_imagenet_20210531-db14775a.pth",
+  "resnet101_v1d": "https://download.openmmlab.com/mmclassification/v0/resnet/resnetv1d101_b32x8_imagenet_20210531-6e13bcd3.pth",
+  "resnet152_v1d": "https://download.openmmlab.com/mmclassification/v0/resnet/resnetv1d152_b32x8_imagenet_20210531-278cf22a.pth",
+  "resnext50_32x4d": "https://download.openmmlab.com/mmclassification/v0/resnext/resnext50_32x4d_b32x8_imagenet_20210429-56066e27.pth",
+  "resnext101_32x4d": "https://download.openmmlab.com/mmclassification/v0/resnext/resnext101_32x4d_b32x8_imagenet_20210506-e0fa3dd5.pth",
+  "resnext101_32x8d": "https://download.openmmlab.com/mmclassification/v0/resnext/resnext101_32x8d_b32x8_imagenet_20210506-23a247d5.pth",
+  "resnext152_32x4d": "https://download.openmmlab.com/mmclassification/v0/resnext/resnext152_32x4d_b32x8_imagenet_20210524-927787be.pth",
+  "se-resnet50": "https://download.openmmlab.com/mmclassification/v0/se-resnet/se-resnet50_batch256_imagenet_20200804-ae206104.pth",
+  "se-resnet101": "https://download.openmmlab.com/mmclassification/v0/se-resnet/se-resnet101_batch256_imagenet_20200804-ba5b51d4.pth",
+  "resnest50": "https://download.openmmlab.com/mmclassification/v0/resnest/resnest50_imagenet_converted-1ebf0afe.pth",
+  "resnest101": "https://download.openmmlab.com/mmclassification/v0/resnest/resnest101_imagenet_converted-032caa52.pth",
+  "resnest200": "https://download.openmmlab.com/mmclassification/v0/resnest/resnest200_imagenet_converted-581a60f2.pth",
+  "resnest269": "https://download.openmmlab.com/mmclassification/v0/resnest/resnest269_imagenet_converted-59930960.pth",
+  "shufflenet_v1": "https://download.openmmlab.com/mmclassification/v0/shufflenet_v1/shufflenet_v1_batch1024_imagenet_20200804-5d6cec73.pth",
+  "shufflenet_v2": "https://download.openmmlab.com/mmclassification/v0/shufflenet_v2/shufflenet_v2_batch1024_imagenet_20200812-5bf4721e.pth",
+  "mobilenet_v2": "https://download.openmmlab.com/mmclassification/v0/mobilenet_v2/mobilenet_v2_batch256_imagenet_20200708-3b2dc3af.pth",
+  "mobilenet_v3_small": "https://download.openmmlab.com/mmclassification/v0/mobilenet_v3/convert/mobilenet_v3_small-8427ecf0.pth",
+  "mobilenet_v3_large": "https://download.openmmlab.com/mmclassification/v0/mobilenet_v3/convert/mobilenet_v3_large-3ea3c186.pth",
+  "repvgg_A0": "https://download.openmmlab.com/mmclassification/v0/repvgg/repvgg-A0_3rdparty_4xb64-coslr-120e_in1k_20210909-883ab98c.pth",
+  "repvgg_A1": "https://download.openmmlab.com/mmclassification/v0/repvgg/repvgg-A1_3rdparty_4xb64-coslr-120e_in1k_20210909-24003a24.pth",
+  "repvgg_A2": "https://download.openmmlab.com/mmclassification/v0/repvgg/repvgg-A2_3rdparty_4xb64-coslr-120e_in1k_20210909-97d7695a.pth",
+  "repvgg_B0": "https://download.openmmlab.com/mmclassification/v0/repvgg/repvgg-B0_3rdparty_4xb64-coslr-120e_in1k_20210909-446375f4.pth",
+  "repvgg_B1": "https://download.openmmlab.com/mmclassification/v0/repvgg/repvgg-B1_3rdparty_4xb64-coslr-120e_in1k_20210909-750cdf67.pth",
+  "repvgg_B1g2": "https://download.openmmlab.com/mmclassification/v0/repvgg/repvgg-B1g2_3rdparty_4xb64-coslr-120e_in1k_20210909-344f6422.pth",
+  "repvgg_B1g4": "https://download.openmmlab.com/mmclassification/v0/repvgg/repvgg-B1g4_3rdparty_4xb64-coslr-120e_in1k_20210909-d4c1a642.pth",
+  "repvgg_B2": "https://download.openmmlab.com/mmclassification/v0/repvgg/repvgg-B2_3rdparty_4xb64-coslr-120e_in1k_20210909-bd6b937c.pth",
+  "repvgg_B2g4": "https://download.openmmlab.com/mmclassification/v0/repvgg/repvgg-B2g4_3rdparty_4xb64-autoaug-lbs-mixup-coslr-200e_in1k_20210909-7b7955f0.pth",
+  "repvgg_B3": "https://download.openmmlab.com/mmclassification/v0/repvgg/repvgg-B3_3rdparty_4xb64-autoaug-lbs-mixup-coslr-200e_in1k_20210909-dda968bf.pth",
+  "repvgg_B3g4": "https://download.openmmlab.com/mmclassification/v0/repvgg/repvgg-B3g4_3rdparty_4xb64-autoaug-lbs-mixup-coslr-200e_in1k_20210909-4e54846a.pth",
+  "repvgg_D2se": "https://download.openmmlab.com/mmclassification/v0/repvgg/repvgg-D2se_3rdparty_4xb64-autoaug-lbs-mixup-coslr-200e_in1k_20210909-cf3139b7.pth",
+  "res2net101_w26": "https://download.openmmlab.com/mmclassification/v0/res2net/res2net101-w26-s4_3rdparty_8xb32_in1k_20210927-870b6c36.pth",
+  "res2net50_w14": "https://download.openmmlab.com/mmclassification/v0/res2net/res2net50-w14-s8_3rdparty_8xb32_in1k_20210927-bc967bf1.pth",
+  "res2net50_w26": "https://download.openmmlab.com/mmclassification/v0/res2net/res2net50-w26-s8_3rdparty_8xb32_in1k_20210927-f547a94b.pth",
+  "swin_tiny": "https://download.openmmlab.com/mmclassification/v0/swin-transformer/swin_tiny_224_b16x64_300e_imagenet_20210616_090925-66df6be6.pth",
+  "swin_small": "https://download.openmmlab.com/mmclassification/v0/swin-transformer/swin_small_224_b16x64_300e_imagenet_20210615_110219-7f9d988b.pth",
+  "swin_base": "https://download.openmmlab.com/mmclassification/v0/swin-transformer/convert/swin_base_patch4_window7_224_22kto1k-f967f799.pth",
+  "swin_large": "https://download.openmmlab.com/mmclassification/v0/swin-transformer/convert/swin_large_patch4_window7_224_22kto1k-5f0996db.pth",
+  "t2t_vit_t_14": "https://download.openmmlab.com/mmclassification/v0/t2t-vit/t2t-vit-t-14_3rdparty_8xb64_in1k_20210928-b7c09b62.pth",
+  "t2t_vit_t_19": "https://download.openmmlab.com/mmclassification/v0/t2t-vit/t2t-vit-t-19_3rdparty_8xb64_in1k_20210928-7f1478d5.pth",
+  "t2t_vit_t_24": "https://download.openmmlab.com/mmclassification/v0/t2t-vit/t2t-vit-t-24_3rdparty_8xb64_in1k_20210928-fe95a61b.pth",
+  "tnt_small": "https://download.openmmlab.com/mmclassification/v0/tnt/tnt-small-p16_3rdparty_in1k_20210903-c56ee7df.pth",
+  "vit_base_p16": "https://download.openmmlab.com/mmclassification/v0/vit/finetune/vit-base-p16_in21k-pre-3rdparty_ft-64xb64_in1k-384_20210928-98e8652b.pth",
+  "vit_base_p32": "https://download.openmmlab.com/mmclassification/v0/vit/finetune/vit-base-p32_in21k-pre-3rdparty_ft-64xb64_in1k-384_20210928-9cea8599.pth",
+  "vit_large_p16": "https://download.openmmlab.com/mmclassification/v0/vit/finetune/vit-large-p16_in21k-pre-3rdparty_ft-64xb64_in1k-384_20210928-b20ba619.pth"
+}

groundingLMM/mmcv/mmcv/model_zoo/open_mmlab.json

@@ -0,0 +1,50 @@
+{
+  "vgg16_caffe": "https://download.openmmlab.com/pretrain/third_party/vgg16_caffe-292e1171.pth",
+  "detectron/resnet50_caffe": "https://download.openmmlab.com/pretrain/third_party/resnet50_caffe-788b5fa3.pth",
+  "detectron2/resnet50_caffe": "https://download.openmmlab.com/pretrain/third_party/resnet50_msra-5891d200.pth",
+  "detectron/resnet101_caffe": "https://download.openmmlab.com/pretrain/third_party/resnet101_caffe-3ad79236.pth",
+  "detectron2/resnet101_caffe": "https://download.openmmlab.com/pretrain/third_party/resnet101_msra-6cc46731.pth",
+  "detectron2/resnext101_32x8d": "https://download.openmmlab.com/pretrain/third_party/resnext101_32x8d-1516f1aa.pth",
+  "resnext50_32x4d": "https://download.openmmlab.com/pretrain/third_party/resnext50-32x4d-0ab1a123.pth",
+  "resnext101_32x4d": "https://download.openmmlab.com/pretrain/third_party/resnext101_32x4d-a5af3160.pth",
+  "resnext101_64x4d": "https://download.openmmlab.com/pretrain/third_party/resnext101_64x4d-ee2c6f71.pth",
+  "contrib/resnet50_gn": "https://download.openmmlab.com/pretrain/third_party/resnet50_gn_thangvubk-ad1730dd.pth",
+  "detectron/resnet50_gn": "https://download.openmmlab.com/pretrain/third_party/resnet50_gn-9186a21c.pth",
+  "detectron/resnet101_gn": "https://download.openmmlab.com/pretrain/third_party/resnet101_gn-cac0ab98.pth",
+  "jhu/resnet50_gn_ws": "https://download.openmmlab.com/pretrain/third_party/resnet50_gn_ws-15beedd8.pth",
+  "jhu/resnet101_gn_ws": "https://download.openmmlab.com/pretrain/third_party/resnet101_gn_ws-3e3c308c.pth",
+  "jhu/resnext50_32x4d_gn_ws": "https://download.openmmlab.com/pretrain/third_party/resnext50_32x4d_gn_ws-0d87ac85.pth",
+  "jhu/resnext101_32x4d_gn_ws": "https://download.openmmlab.com/pretrain/third_party/resnext101_32x4d_gn_ws-34ac1a9e.pth",
+  "jhu/resnext50_32x4d_gn": "https://download.openmmlab.com/pretrain/third_party/resnext50_32x4d_gn-c7e8b754.pth",
+  "jhu/resnext101_32x4d_gn": "https://download.openmmlab.com/pretrain/third_party/resnext101_32x4d_gn-ac3bb84e.pth",
+  "msra/hrnetv2_w18_small": "https://download.openmmlab.com/pretrain/third_party/hrnetv2_w18_small-b5a04e21.pth",
+  "msra/hrnetv2_w18": "https://download.openmmlab.com/pretrain/third_party/hrnetv2_w18-00eb2006.pth",
+  "msra/hrnetv2_w32": "https://download.openmmlab.com/pretrain/third_party/hrnetv2_w32-dc9eeb4f.pth",
+  "msra/hrnetv2_w40": "https://download.openmmlab.com/pretrain/third_party/hrnetv2_w40-ed0b031c.pth",
+  "msra/hrnetv2_w48": "https://download.openmmlab.com/pretrain/third_party/hrnetv2_w48-d2186c55.pth",
+  "bninception_caffe": "https://download.openmmlab.com/pretrain/third_party/bn_inception_caffe-ed2e8665.pth",
+  "kin400/i3d_r50_f32s2_k400": "https://download.openmmlab.com/pretrain/third_party/i3d_r50_f32s2_k400-2c57e077.pth",
+  "kin400/nl3d_r50_f32s2_k400": "https://download.openmmlab.com/pretrain/third_party/nl3d_r50_f32s2_k400-fa7e7caa.pth",
+  "res2net101_v1d_26w_4s": "https://download.openmmlab.com/pretrain/third_party/res2net101_v1d_26w_4s_mmdetv2-f0a600f9.pth",
+  "regnetx_400mf": "https://download.openmmlab.com/pretrain/third_party/regnetx_400mf-a5b10d96.pth",
+  "regnetx_800mf": "https://download.openmmlab.com/pretrain/third_party/regnetx_800mf-1f4be4c7.pth",
+  "regnetx_1.6gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_1.6gf-5791c176.pth",
+  "regnetx_3.2gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_3.2gf-c2599b0f.pth",
+  "regnetx_4.0gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_4.0gf-a88f671e.pth",
+  "regnetx_6.4gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_6.4gf-006af45d.pth",
+  "regnetx_8.0gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_8.0gf-3c68abe7.pth",
+  "regnetx_12gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_12gf-4c2a3350.pth",
+  "resnet18_v1c": "https://download.openmmlab.com/pretrain/third_party/resnet18_v1c-b5776b93.pth",
+  "resnet50_v1c": "https://download.openmmlab.com/pretrain/third_party/resnet50_v1c-2cccc1ad.pth",
+  "resnet101_v1c": "https://download.openmmlab.com/pretrain/third_party/resnet101_v1c-e67eebb6.pth",
+  "mmedit/vgg16": "https://download.openmmlab.com/mmediting/third_party/vgg_state_dict.pth",
+  "mmedit/res34_en_nomixup": "https://download.openmmlab.com/mmediting/third_party/model_best_resnet34_En_nomixup.pth",
+  "mmedit/mobilenet_v2": "https://download.openmmlab.com/mmediting/third_party/mobilenet_v2.pth",
+  "contrib/mobilenet_v3_large": "https://download.openmmlab.com/pretrain/third_party/mobilenet_v3_large-bc2c3fd3.pth",
+  "contrib/mobilenet_v3_small": "https://download.openmmlab.com/pretrain/third_party/mobilenet_v3_small-47085aa1.pth",
+  "resnest50": "https://download.openmmlab.com/pretrain/third_party/resnest50_d2-7497a55b.pth",
+  "resnest101": "https://download.openmmlab.com/pretrain/third_party/resnest101_d2-f3b931b2.pth",
+  "resnest200": "https://download.openmmlab.com/pretrain/third_party/resnest200_d2-ca88e41f.pth",
+  "darknet53": "https://download.openmmlab.com/pretrain/third_party/darknet53-a628ea1b.pth",
+  "mmdet/mobilenet_v2": "https://download.openmmlab.com/mmdetection/v2.0/third_party/mobilenet_v2_batch256_imagenet-ff34753d.pth"
+}

groundingLMM/mmcv/mmcv/onnx/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from .info import is_custom_op_loaded
+from .symbolic import register_extra_symbolics
+
+__all__ = ['register_extra_symbolics', 'is_custom_op_loaded']

groundingLMM/mmcv/mmcv/onnx/info.py

@@ -0,0 +1,35 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import os
+import warnings
+
+import torch
+
+
+def is_custom_op_loaded():
+
+    # Following strings of text style are from colorama package
+    bright_style, reset_style = '\x1b[1m', '\x1b[0m'
+    red_text, blue_text = '\x1b[31m', '\x1b[34m'
+    white_background = '\x1b[107m'
+
+    msg = white_background + bright_style + red_text
+    msg += 'DeprecationWarning: This function will be deprecated in future. '
+    msg += blue_text + 'Welcome to use the unified model deployment toolbox '
+    msg += 'MMDeploy: https://github.com/open-mmlab/mmdeploy'
+    msg += reset_style
+    warnings.warn(msg)
+
+    flag = False
+    try:
+        from ..tensorrt import is_tensorrt_plugin_loaded
+        flag = is_tensorrt_plugin_loaded()
+    except (ImportError, ModuleNotFoundError):
+        pass
+    if not flag:
+        try:
+            from ..ops import get_onnxruntime_op_path
+            ort_lib_path = get_onnxruntime_op_path()
+            flag = os.path.exists(ort_lib_path)
+        except (ImportError, ModuleNotFoundError):
+            pass
+    return flag or torch.__version__ == 'parrots'

groundingLMM/mmcv/mmcv/onnx/symbolic.py

@@ -0,0 +1,509 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+"""Modified from https://github.com/pytorch/pytorch."""
+import os
+import warnings
+
+import numpy as np
+import torch
+from torch.nn.modules.utils import _pair, _single, _triple
+from torch.onnx.symbolic_helper import parse_args
+from torch.onnx.symbolic_registry import register_op
+
+from .onnx_utils import symbolic_helper as sym_help
+
+
+def _interpolate(name, dim, interpolate_mode):
+
+    def symbolic_fn(g, input, output_size, *args):
+        scales, align_corners = sym_help._get_interpolate_attributes(
+            g, interpolate_mode, args)
+        align_corners = sym_help._maybe_get_scalar(align_corners)
+        transformation_mode = 'asymmetric' \
+            if interpolate_mode == 'nearest' \
+            else 'align_corners' if align_corners else 'pytorch_half_pixel'
+        empty_tensor = g.op(
+            'Constant', value_t=torch.tensor([], dtype=torch.float32))
+
+        if scales is None:
+            if 'ONNX_BACKEND' in os.environ and os.environ[
+                    'ONNX_BACKEND'] == 'TensorRT':
+                input_size = input.type().sizes()
+                # slice the first two dim
+                input_size = input_size[:2]
+                # convert output_size to int type
+                output_size = sym_help._maybe_get_const(output_size, 'is')
+                input_size.extend(output_size)
+                output_size = g.op(
+                    'Constant',
+                    value_t=torch.tensor(input_size, dtype=torch.int64))
+            else:
+                input_size = g.op('Shape', input)
+                input_size_beg = sym_help._slice_helper(
+                    g, input_size, axes=[0], ends=[2], starts=[0])
+                output_size = g.op(
+                    'Cast',
+                    output_size,
+                    to_i=sym_help.cast_pytorch_to_onnx['Long'])
+                output_size = g.op(
+                    'Concat', input_size_beg, output_size, axis_i=0)
+            scales = g.op(
+                'Constant', value_t=torch.tensor([], dtype=torch.float32))
+            return g.op(
+                'Resize',
+                input,
+                empty_tensor,
+                # roi only takes effect with
+                # coordinate_transformation_mode="tf_crop_and_resize"
+                scales,  # scales is not needed since we are sending out_size
+                output_size,
+                coordinate_transformation_mode_s=transformation_mode,
+                cubic_coeff_a_f=-0.75,  # only valid when mode="cubic"
+                mode_s=interpolate_mode,  # nearest, linear, or cubic
+                nearest_mode_s='floor')  # only valid when mode="nearest"
+        else:
+            return g.op(
+                'Resize',
+                input,
+                empty_tensor,
+                # roi only takes effect with
+                # coordinate_transformation_mode="tf_crop_and_resize"
+                scales,  # scales is not needed since we are sending out_size
+                coordinate_transformation_mode_s=transformation_mode,
+                cubic_coeff_a_f=-0.75,  # only valid when mode="cubic"
+                mode_s=interpolate_mode,  # nearest, linear, or cubic
+                nearest_mode_s='floor')  # only valid when mode="nearest"
+
+    return symbolic_fn
+
+
+upsample_nearest1d = _interpolate('upsample_nearest1d', 3, 'nearest')
+upsample_nearest2d = _interpolate('upsample_nearest2d', 4, 'nearest')
+upsample_nearest3d = _interpolate('upsample_nearest3d', 5, 'nearest')
+upsample_linear1d = _interpolate('upsample_linear1d', 3, 'linear')
+upsample_bilinear2d = _interpolate('upsample_bilinear2d', 4, 'linear')
+upsample_trilinear3d = _interpolate('upsample_trilinear3d', 5, 'linear')
+upsample_bicubic2d = _interpolate('upsample_bicubic2d', 4, 'cubic')
+
+
+@parse_args('v', 'v', 'i', 'i', 'i', 'none')
+def topk(g, self, k, dim, largest, sorted, out=None):
+    return sym_help._topk_helper(
+        g, self, k, dim, largest=largest, sorted=sorted, out=out)
+
+
+def masked_select(g, self, mask):
+    from torch.onnx.symbolic_opset9 import expand_as, nonzero
+    index = nonzero(g, expand_as(g, mask, self))
+    return g.op('GatherND', self, index)
+
+
+def _prepare_onnx_paddings(g, dim, pad):
+    pad_len = torch.onnx.symbolic_opset9.size(
+        g, pad, g.op('Constant', value_t=torch.tensor([0])))
+    # Set extension = [0] * (dim * 2 - len(pad))
+    extension = g.op(
+        'Sub',
+        g.op('Mul',
+             g.op('Constant', value_t=torch.tensor(dim, dtype=torch.int64)),
+             g.op('Constant', value_t=torch.tensor(2, dtype=torch.int64))),
+        pad_len)
+    pad = g.op('Cast', pad, to_i=sym_help.cast_pytorch_to_onnx['Long'])
+    paddings = g.op(
+        'Concat',
+        pad,
+        g.op(
+            'ConstantOfShape',
+            extension,
+            value_t=torch.tensor([0], dtype=torch.int64)),
+        axis_i=0)
+    paddings = g.op('Reshape', paddings,
+                    g.op('Constant', value_t=torch.tensor([-1, 2])))
+    paddings = g.op(
+        'Transpose',
+        torch.onnx.symbolic_opset10.flip(g, paddings, [0]),
+        perm_i=[1, 0])
+    paddings = g.op('Reshape', paddings,
+                    g.op('Constant', value_t=torch.tensor([-1])))
+    padding_c = g.op(
+        'Cast', paddings, to_i=sym_help.cast_pytorch_to_onnx['Long'])
+    return padding_c
+
+
+def constant_pad_nd(g, input, padding, value=None):
+    mode = 'constant'
+    value = sym_help._maybe_get_scalar(value)
+    value = sym_help._if_scalar_type_as(g, value, input)
+    pad = _prepare_onnx_paddings(g, input.type().dim(), padding)
+    return g.op('Pad', input, pad, value, mode_s=mode)
+
+
+def reflection_pad(g, input, padding):
+    mode = 'reflect'
+    paddings = _prepare_onnx_paddings(g, input.type().dim(), padding)
+    return g.op('Pad', input, paddings, mode_s=mode)
+
+
+reflection_pad1d = reflection_pad
+reflection_pad2d = reflection_pad
+reflection_pad3d = reflection_pad
+
+
+def _avg_pool(name, tuple_fn):
+
+    @parse_args('v', 'is', 'is', 'is', 'i', 'i', 'none')
+    def symbolic_fn(g,
+                    input,
+                    kernel_size,
+                    stride,
+                    padding,
+                    ceil_mode,
+                    count_include_pad,
+                    divisor_override=None):
+        padding = sym_help._avgpool_helper(tuple_fn, padding, kernel_size,
+                                           stride, divisor_override, name)
+        if not stride:
+            stride = kernel_size
+        if count_include_pad:
+            input = g.op(
+                'Pad',
+                input,
+                g.op(
+                    'Constant',
+                    value_t=torch.tensor(((0, ) * 2 + padding) * 2)),
+                mode_s='constant')
+            padding = (0, ) * len(padding)
+        output = g.op(
+            'AveragePool',
+            input,
+            kernel_shape_i=tuple_fn(kernel_size),
+            strides_i=tuple_fn(stride),
+            pads_i=padding * 2,
+            ceil_mode_i=ceil_mode)
+        return output
+
+    return symbolic_fn
+
+
+avg_pool1d = _avg_pool('avg_pool1d', _single)
+avg_pool2d = _avg_pool('avg_pool2d', _pair)
+avg_pool3d = _avg_pool('avg_pool3d', _triple)
+
+
+def _get_im2col_indices_along_dim(g, input_d, kernel_size_d, dilation_d,
+                                  padding_d, stride_d):
+    # Input is always 4-D (N, C, H, W)
+    # Calculate indices of sliding blocks along spatial dimension
+    # Slide kernel over input each dim d:
+    # each dimension d ranges from 0 to
+    # input[d]+2xpadding[d]-dilation[d]x(kernel_size[d]-1)
+    # with steps = stride
+
+    blocks_d = g.op('Add', input_d,
+                    g.op('Constant', value_t=torch.tensor(padding_d * 2)))
+    blocks_d = g.op(
+        'Sub', blocks_d,
+        g.op(
+            'Constant',
+            value_t=torch.tensor(dilation_d * (kernel_size_d - 1))))
+
+    # Stride kernel over input and find starting indices along dim d
+    blocks_d_indices = g.op('Range', g.op('Constant', value_t=torch.tensor(0)),
+                            blocks_d,
+                            g.op('Constant', value_t=torch.tensor(stride_d)))
+
+    # Apply dilation on kernel and find its indices along dim d
+    kernel_grid = np.arange(0, kernel_size_d * dilation_d, dilation_d)
+    kernel_grid = g.op('Constant', value_t=torch.tensor([kernel_grid]))
+
+    # Broadcast and add kernel staring positions (indices) with
+    # kernel_grid along dim d, to get block indices along dim d
+    blocks_d_indices = g.op(
+        'Unsqueeze', blocks_d_indices, axes_i=[0])  # Reshape to [1, -1]
+    kernel_mask = g.op('Reshape', kernel_grid,
+                       g.op('Constant', value_t=torch.tensor([-1, 1])))
+    block_mask = g.op('Add', blocks_d_indices, kernel_mask)
+
+    return block_mask
+
+
+def _get_im2col_padded_input(g, input, padding_h, padding_w):
+    # Input is always 4-D tensor (N, C, H, W)
+    # Padding tensor has the following format: (padding_h, padding_w)
+    # Reshape the padding to follow ONNX format:
+    # (dim1_begin, dim2_begin,...,dim1_end, dim2_end,...)
+    pad = g.op(
+        'Constant', value_t=torch.LongTensor([0, 0, padding_h, padding_w] * 2))
+    return g.op('Pad', input, pad)
+
+
+def _get_im2col_output_shape(g, input, kernel_h, kernel_w):
+    batch_dim = size(g, input, g.op('Constant', value_t=torch.tensor(0)))
+    channel_dim = size(g, input, g.op('Constant', value_t=torch.tensor(1)))
+    channel_unfolded = g.op(
+        'Mul', channel_dim,
+        g.op('Constant', value_t=torch.tensor(kernel_h * kernel_w)))
+
+    return g.op(
+        'Concat',
+        g.op('Unsqueeze', batch_dim, axes_i=[0]),
+        g.op('Unsqueeze', channel_unfolded, axes_i=[0]),
+        g.op('Constant', value_t=torch.tensor([-1])),
+        axis_i=0)
+
+
+def size(g, self, dim=None):
+    if dim is None:
+        return g.op('Shape', self)
+    return sym_help._size_helper(g, self, dim)
+
+
+@parse_args('v', 'is', 'is', 'is', 'is')
+def im2col(g, input, kernel_size, dilation, padding, stride):
+    # Input is always 4-D tensor (N, C, H, W)
+    # All other args are int[2]
+
+    input_h = size(g, input, g.op('Constant', value_t=torch.tensor(2)))
+    input_w = size(g, input, g.op('Constant', value_t=torch.tensor(3)))
+
+    stride_h, stride_w = stride[0], stride[1]
+    padding_h, padding_w = padding[0], padding[1]
+    dilation_h, dilation_w = dilation[0], dilation[1]
+    kernel_h, kernel_w = kernel_size[0], kernel_size[1]
+
+    blocks_row_indices = _get_im2col_indices_along_dim(g, input_h, kernel_h,
+                                                       dilation_h, padding_h,
+                                                       stride_h)
+    blocks_col_indices = _get_im2col_indices_along_dim(g, input_w, kernel_w,
+                                                       dilation_w, padding_w,
+                                                       stride_w)
+
+    output_shape = _get_im2col_output_shape(g, input, kernel_h, kernel_w)
+    padded_input = _get_im2col_padded_input(g, input, padding_h, padding_w)
+
+    output = g.op('Gather', padded_input, blocks_row_indices, axis_i=2)
+    output = g.op('Gather', output, blocks_col_indices, axis_i=4)
+    output = g.op('Transpose', output, perm_i=[0, 1, 2, 4, 3, 5])
+    return g.op('Reshape', output, output_shape)
+
+
+@parse_args('v', 'i')
+def one_hot(g, self, num_classes):
+    values = g.op('Constant', value_t=torch.LongTensor([0, 1]))
+    depth = g.op('Constant', value_t=torch.LongTensor([num_classes]))
+    return g.op('OneHot', self, depth, values, axis_i=-1)
+
+
+@parse_args('v', 'i', 'none')
+def softmax(g, input, dim, dtype=None):
+    input_dim = input.type().dim()
+    if input_dim:
+        # TODO: remove this as onnx opset 11 spec allows negative axes
+        if dim < 0:
+            dim = input_dim + dim
+        if input_dim == dim + 1:
+            softmax = g.op('Softmax', input, axis_i=dim)
+            if dtype and dtype.node().kind() != 'prim::Constant':
+                parsed_dtype = sym_help._get_const(dtype, 'i', 'dtype')
+                softmax = g.op(
+                    'Cast',
+                    softmax,
+                    to_i=sym_help.scalar_type_to_onnx[parsed_dtype])
+            return softmax
+
+    max_value = g.op('ReduceMax', input, axes_i=[dim], keepdims_i=1)
+    input = g.op('Sub', input, max_value)
+    exp = g.op('Exp', input)
+    sum = g.op('ReduceSum', exp, axes_i=[dim])
+    softmax = g.op('Div', exp, sum)
+    if dtype and dtype.node().kind() != 'prim::Constant':
+        parsed_dtype = sym_help._get_const(dtype, 'i', 'dtype')
+        softmax = g.op(
+            'Cast', softmax, to_i=sym_help.scalar_type_to_onnx[parsed_dtype])
+    return softmax
+
+
+def _adaptive_pool(name, type, tuple_fn, fn=None):
+
+    @parse_args('v', 'is')
+    def symbolic_fn(g, input, output_size):
+        if output_size == [1] * len(output_size) and type == 'AveragePool':
+            return g.op('GlobalAveragePool', input)
+        if not input.isCompleteTensor():
+            if output_size == [1] * len(output_size):
+                return g.op('GlobalMaxPool', input), None
+            raise NotImplementedError(
+                '[Adaptive pool]:input size not accessible')
+        dim = input.type().sizes()[2:]
+        if output_size == [1] * len(output_size) and type == 'MaxPool':
+            return g.op('GlobalMaxPool', input), None
+
+        # compute stride = floor(input_size / output_size)
+        s = [int(dim[i] / output_size[i]) for i in range(0, len(dim))]
+
+        # compute kernel_size = input_size - (output_size - 1) * stride
+        k = [dim[i] - (output_size[i] - 1) * s[i] for i in range(0, len(dim))]
+
+        # call max_poolxd_with_indices to get indices in the output
+        if type == 'MaxPool':
+            return fn(g, input, k, k, (0, ) * len(dim), (1, ) * len(dim),
+                      False)
+        output = g.op(
+            type,
+            input,
+            kernel_shape_i=tuple_fn(k),
+            strides_i=tuple_fn(s),
+            ceil_mode_i=False)
+        return output
+
+    return symbolic_fn
+
+
+adaptive_avg_pool1d = _adaptive_pool('adaptive_avg_pool1d', 'AveragePool',
+                                     _single)
+adaptive_avg_pool2d = _adaptive_pool('adaptive_avg_pool2d', 'AveragePool',
+                                     _pair)
+adaptive_avg_pool3d = _adaptive_pool('adaptive_avg_pool3d', 'AveragePool',
+                                     _triple)
+
+
+def new_full(g,
+             self,
+             size,
+             fill_value,
+             dtype,
+             layout,
+             device,
+             pin_memory=False):
+    from torch.onnx.symbolic_opset9 import full
+    if dtype is None and self.isCompleteTensor():
+        dtype = self.type().scalarType()
+        dtype = sym_help.scalar_type_to_onnx.index(
+            sym_help.cast_pytorch_to_onnx[dtype])
+    return full(g, size, fill_value, dtype, layout, device, pin_memory)
+
+
+@parse_args('v', 'v', 'i', 'i', 'i')
+def grid_sampler(g,
+                 input,
+                 grid,
+                 interpolation_mode,
+                 padding_mode,
+                 align_corners=False):
+    return g.op(
+        'mmcv::grid_sampler',
+        input,
+        grid,
+        interpolation_mode_i=interpolation_mode,
+        padding_mode_i=padding_mode,
+        align_corners_i=align_corners)
+
+
+@parse_args('v', 'i')
+def cummax(g, input, dim):
+    return g.op('mmcv::cummax', input, dim_i=dim, outputs=2)
+
+
+@parse_args('v', 'i')
+def cummin(g, input, dim):
+    return g.op('mmcv::cummin', input, dim_i=dim, outputs=2)
+
+
+@parse_args('v', 'v', 'is')
+def roll(g, input, shifts, dims):
+    from packaging import version
+    from torch.onnx.symbolic_opset9 import squeeze
+    input_shape = g.op('Shape', input)
+
+    need_flatten = len(dims) == 0
+    # If dims is not specified, the tensor will be flattened before
+    # rolling and then restored to the original shape.
+    if need_flatten:
+        resize_shape = input_shape
+        input = g.op('Reshape', input,
+                     g.op('Constant', value_t=torch.LongTensor([1, -1])))
+        input_shape = g.op('Shape', input)
+        dims = [1]
+
+    for index, dim in enumerate(dims):
+        end_size = sym_help._slice_helper(
+            g, input_shape, axes=[0], ends=[dim + 1], starts=[dim])
+        shift_size = sym_help._slice_helper(
+            g, shifts, axes=[0], ends=[index + 1], starts=[index])
+        slice_size = g.op('Sub', end_size, shift_size)
+
+        # Can not use Mod because tensorrt does not support
+        div_size = g.op('Div', slice_size, end_size)
+        slice_size = g.op('Sub', slice_size, g.op('Mul', end_size, div_size))
+
+        if version.parse(torch.__version__) >= version.parse('1.7.0'):
+            # add dim=0 for pytorch 1.9.0
+            end_size = squeeze(g, end_size, 0)
+            slice_size = squeeze(g, slice_size, 0)
+        else:
+            end_size = g.op('Squeeze', end_size)
+            slice_size = g.op('Squeeze', slice_size)
+            dim = torch.LongTensor([dim])
+
+        input_slice0 = sym_help._slice_helper(
+            g,
+            input,
+            axes=dim,
+            starts=torch.LongTensor([0]),
+            ends=slice_size,
+            dynamic_slice=True)
+        input_slice1 = sym_help._slice_helper(
+            g,
+            input,
+            axes=dim,
+            ends=end_size,
+            starts=slice_size,
+            dynamic_slice=True)
+
+        input = g.op('Concat', input_slice1, input_slice0, axis_i=dim)
+
+    if need_flatten:
+        input = g.op('Reshape', input, resize_shape)
+
+    return input
+
+
+def register_extra_symbolics(opset=11):
+    # Following strings of text style are from colorama package
+    bright_style, reset_style = '\x1b[1m', '\x1b[0m'
+    red_text, blue_text = '\x1b[31m', '\x1b[34m'
+    white_background = '\x1b[107m'
+
+    msg = white_background + bright_style + red_text
+    msg += 'DeprecationWarning: This function will be deprecated in future. '
+    msg += blue_text + 'Welcome to use the unified model deployment toolbox '
+    msg += 'MMDeploy: https://github.com/open-mmlab/mmdeploy'
+    msg += reset_style
+    warnings.warn(msg)
+
+    register_op('one_hot', one_hot, '', opset)
+    register_op('im2col', im2col, '', opset)
+    register_op('topk', topk, '', opset)
+    register_op('softmax', softmax, '', opset)
+    register_op('constant_pad_nd', constant_pad_nd, '', opset)
+    register_op('reflection_pad1d', reflection_pad1d, '', opset)
+    register_op('reflection_pad2d', reflection_pad2d, '', opset)
+    register_op('reflection_pad3d', reflection_pad3d, '', opset)
+    register_op('avg_pool1d', avg_pool1d, '', opset)
+    register_op('avg_pool2d', avg_pool2d, '', opset)
+    register_op('avg_pool3d', avg_pool3d, '', opset)
+    register_op('adaptive_avg_pool1d', adaptive_avg_pool1d, '', opset)
+    register_op('adaptive_avg_pool2d', adaptive_avg_pool2d, '', opset)
+    register_op('adaptive_avg_pool3d', adaptive_avg_pool3d, '', opset)
+    register_op('masked_select', masked_select, '', opset)
+    register_op('upsample_nearest1d', upsample_nearest1d, '', opset)
+    register_op('upsample_nearest2d', upsample_nearest2d, '', opset)
+    register_op('upsample_nearest3d', upsample_nearest3d, '', opset)
+    register_op('upsample_linear1d', upsample_linear1d, '', opset)
+    register_op('upsample_bilinear2d', upsample_bilinear2d, '', opset)
+    register_op('upsample_trilinear3d', upsample_trilinear3d, '', opset)
+    register_op('upsample_bicubic2d', upsample_bicubic2d, '', opset)
+    register_op('new_full', new_full, '', opset)
+    register_op('grid_sampler', grid_sampler, '', opset)
+    register_op('cummax', cummax, '', opset)
+    register_op('cummin', cummin, '', opset)
+    register_op('roll', roll, '', opset)

groundingLMM/mmcv/mmcv/parallel/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from .collate import collate
+from .data_container import DataContainer
+from .data_parallel import MMDataParallel
+from .distributed import MMDistributedDataParallel
+from .registry import MODULE_WRAPPERS
+from .scatter_gather import scatter, scatter_kwargs
+from .utils import is_module_wrapper
+
+__all__ = [
+    'collate', 'DataContainer', 'MMDataParallel', 'MMDistributedDataParallel',
+    'scatter', 'scatter_kwargs', 'is_module_wrapper', 'MODULE_WRAPPERS'
+]

groundingLMM/mmcv/mmcv/parallel/_functions.py

@@ -0,0 +1,76 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import torch
+from torch.nn.parallel._functions import _get_stream
+
+
+def scatter(input, devices, streams=None):
+    """Scatters tensor across multiple GPUs."""
+    if streams is None:
+        streams = [None] * len(devices)
+
+    if isinstance(input, list):
+        chunk_size = (len(input) - 1) // len(devices) + 1
+        outputs = [
+            scatter(input[i], [devices[i // chunk_size]],
+                    [streams[i // chunk_size]]) for i in range(len(input))
+        ]
+        return outputs
+    elif isinstance(input, torch.Tensor):
+        output = input.contiguous()
+        # TODO: copy to a pinned buffer first (if copying from CPU)
+        stream = streams[0] if output.numel() > 0 else None
+        if devices != [-1]:
+            with torch.cuda.device(devices[0]), torch.cuda.stream(stream):
+                output = output.cuda(devices[0], non_blocking=True)
+
+        return output
+    else:
+        raise Exception(f'Unknown type {type(input)}.')
+
+
+def synchronize_stream(output, devices, streams):
+    if isinstance(output, list):
+        chunk_size = len(output) // len(devices)
+        for i in range(len(devices)):
+            for j in range(chunk_size):
+                synchronize_stream(output[i * chunk_size + j], [devices[i]],
+                                   [streams[i]])
+    elif isinstance(output, torch.Tensor):
+        if output.numel() != 0:
+            with torch.cuda.device(devices[0]):
+                main_stream = torch.cuda.current_stream()
+                main_stream.wait_stream(streams[0])
+                output.record_stream(main_stream)
+    else:
+        raise Exception(f'Unknown type {type(output)}.')
+
+
+def get_input_device(input):
+    if isinstance(input, list):
+        for item in input:
+            input_device = get_input_device(item)
+            if input_device != -1:
+                return input_device
+        return -1
+    elif isinstance(input, torch.Tensor):
+        return input.get_device() if input.is_cuda else -1
+    else:
+        raise Exception(f'Unknown type {type(input)}.')
+
+
+class Scatter:
+
+    @staticmethod
+    def forward(target_gpus, input):
+        input_device = get_input_device(input)
+        streams = None
+        if input_device == -1 and target_gpus != [-1]:
+            # Perform CPU to GPU copies in a background stream
+            streams = [_get_stream(device) for device in target_gpus]
+
+        outputs = scatter(input, target_gpus, streams)
+        # Synchronize with the copy stream
+        if streams is not None:
+            synchronize_stream(outputs, target_gpus, streams)
+
+        return tuple(outputs) if isinstance(outputs, list) else (outputs, )

groundingLMM/mmcv/mmcv/parallel/collate.py

@@ -0,0 +1,84 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from collections.abc import Mapping, Sequence
+
+import torch
+import torch.nn.functional as F
+from torch.utils.data.dataloader import default_collate
+
+from .data_container import DataContainer
+
+
+def collate(batch, samples_per_gpu=1):
+    """Puts each data field into a tensor/DataContainer with outer dimension
+    batch size.
+
+    Extend default_collate to add support for
+    :type:`~mmcv.parallel.DataContainer`. There are 3 cases.
+
+    1. cpu_only = True, e.g., meta data
+    2. cpu_only = False, stack = True, e.g., images tensors
+    3. cpu_only = False, stack = False, e.g., gt bboxes
+    """
+
+    if not isinstance(batch, Sequence):
+        raise TypeError(f'{batch.dtype} is not supported.')
+
+    if isinstance(batch[0], DataContainer):
+        stacked = []
+        if batch[0].cpu_only:
+            for i in range(0, len(batch), samples_per_gpu):
+                stacked.append(
+                    [sample.data for sample in batch[i:i + samples_per_gpu]])
+            return DataContainer(
+                stacked, batch[0].stack, batch[0].padding_value, cpu_only=True)
+        elif batch[0].stack:
+            for i in range(0, len(batch), samples_per_gpu):
+                assert isinstance(batch[i].data, torch.Tensor)
+
+                if batch[i].pad_dims is not None:
+                    ndim = batch[i].dim()
+                    assert ndim > batch[i].pad_dims
+                    max_shape = [0 for _ in range(batch[i].pad_dims)]
+                    for dim in range(1, batch[i].pad_dims + 1):
+                        max_shape[dim - 1] = batch[i].size(-dim)
+                    for sample in batch[i:i + samples_per_gpu]:
+                        for dim in range(0, ndim - batch[i].pad_dims):
+                            assert batch[i].size(dim) == sample.size(dim)
+                        for dim in range(1, batch[i].pad_dims + 1):
+                            max_shape[dim - 1] = max(max_shape[dim - 1],
+                                                     sample.size(-dim))
+                    padded_samples = []
+                    for sample in batch[i:i + samples_per_gpu]:
+                        pad = [0 for _ in range(batch[i].pad_dims * 2)]
+                        for dim in range(1, batch[i].pad_dims + 1):
+                            pad[2 * dim -
+                                1] = max_shape[dim - 1] - sample.size(-dim)
+                        padded_samples.append(
+                            F.pad(
+                                sample.data, pad, value=sample.padding_value))
+                    stacked.append(default_collate(padded_samples))
+                elif batch[i].pad_dims is None:
+                    stacked.append(
+                        default_collate([
+                            sample.data
+                            for sample in batch[i:i + samples_per_gpu]
+                        ]))
+                else:
+                    raise ValueError(
+                        'pad_dims should be either None or integers (1-3)')
+
+        else:
+            for i in range(0, len(batch), samples_per_gpu):
+                stacked.append(
+                    [sample.data for sample in batch[i:i + samples_per_gpu]])
+        return DataContainer(stacked, batch[0].stack, batch[0].padding_value)
+    elif isinstance(batch[0], Sequence):
+        transposed = zip(*batch)
+        return [collate(samples, samples_per_gpu) for samples in transposed]
+    elif isinstance(batch[0], Mapping):
+        return {
+            key: collate([d[key] for d in batch], samples_per_gpu)
+            for key in batch[0]
+        }
+    else:
+        return default_collate(batch)

groundingLMM/mmcv/mmcv/parallel/data_container.py

@@ -0,0 +1,89 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import functools
+
+import torch
+
+
+def assert_tensor_type(func):
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        if not isinstance(args[0].data, torch.Tensor):
+            raise AttributeError(
+                f'{args[0].__class__.__name__} has no attribute '
+                f'{func.__name__} for type {args[0].datatype}')
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+class DataContainer:
+    """A container for any type of objects.
+
+    Typically tensors will be stacked in the collate function and sliced along
+    some dimension in the scatter function. This behavior has some limitations.
+    1. All tensors have to be the same size.
+    2. Types are limited (numpy array or Tensor).
+
+    We design `DataContainer` and `MMDataParallel` to overcome these
+    limitations. The behavior can be either of the following.
+
+    - copy to GPU, pad all tensors to the same size and stack them
+    - copy to GPU without stacking
+    - leave the objects as is and pass it to the model
+    - pad_dims specifies the number of last few dimensions to do padding
+    """
+
+    def __init__(self,
+                 data,
+                 stack=False,
+                 padding_value=0,
+                 cpu_only=False,
+                 pad_dims=2):
+        self._data = data
+        self._cpu_only = cpu_only
+        self._stack = stack
+        self._padding_value = padding_value
+        assert pad_dims in [None, 1, 2, 3]
+        self._pad_dims = pad_dims
+
+    def __repr__(self):
+        return f'{self.__class__.__name__}({repr(self.data)})'
+
+    def __len__(self):
+        return len(self._data)
+
+    @property
+    def data(self):
+        return self._data
+
+    @property
+    def datatype(self):
+        if isinstance(self.data, torch.Tensor):
+            return self.data.type()
+        else:
+            return type(self.data)
+
+    @property
+    def cpu_only(self):
+        return self._cpu_only
+
+    @property
+    def stack(self):
+        return self._stack
+
+    @property
+    def padding_value(self):
+        return self._padding_value
+
+    @property
+    def pad_dims(self):
+        return self._pad_dims
+
+    @assert_tensor_type
+    def size(self, *args, **kwargs):
+        return self.data.size(*args, **kwargs)
+
+    @assert_tensor_type
+    def dim(self):
+        return self.data.dim()

groundingLMM/mmcv/mmcv/parallel/data_parallel.py

@@ -0,0 +1,97 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from itertools import chain
+
+from torch.nn.parallel import DataParallel
+
+from .scatter_gather import scatter_kwargs
+
+
+class MMDataParallel(DataParallel):
+    """The DataParallel module that supports DataContainer.
+
+    MMDataParallel has two main differences with PyTorch DataParallel:
+
+    - It supports a custom type :class:`DataContainer` which allows more
+      flexible control of input data during both GPU and CPU inference.
+    - It implement two more APIs ``train_step()`` and ``val_step()``.
+
+    .. warning::
+        MMDataParallel only supports single GPU training, if you need to
+        train with multiple GPUs, please use MMDistributedDataParallel
+        instead. If you have multiple GPUs and you just want to use
+        MMDataParallel, you can set the environment variable
+        ``CUDA_VISIBLE_DEVICES=0`` or instantiate ``MMDataParallel`` with
+        ``device_ids=[0]``.
+
+    Args:
+        module (:class:`nn.Module`): Module to be encapsulated.
+        device_ids (list[int]): Device IDS of modules to be scattered to.
+            Defaults to None when GPU is not available.
+        output_device (str | int): Device ID for output. Defaults to None.
+        dim (int): Dimension used to scatter the data. Defaults to 0.
+    """
+
+    def __init__(self, *args, dim=0, **kwargs):
+        super(MMDataParallel, self).__init__(*args, dim=dim, **kwargs)
+        self.dim = dim
+
+    def forward(self, *inputs, **kwargs):
+        """Override the original forward function.
+
+        The main difference lies in the CPU inference where the data in
+        :class:`DataContainers` will still be gathered.
+        """
+        if not self.device_ids:
+            # We add the following line thus the module could gather and
+            # convert data containers as those in GPU inference
+            inputs, kwargs = self.scatter(inputs, kwargs, [-1])
+            return self.module(*inputs[0], **kwargs[0])
+        else:
+            return super().forward(*inputs, **kwargs)
+
+    def scatter(self, inputs, kwargs, device_ids):
+        return scatter_kwargs(inputs, kwargs, device_ids, dim=self.dim)
+
+    def train_step(self, *inputs, **kwargs):
+        if not self.device_ids:
+            # We add the following line thus the module could gather and
+            # convert data containers as those in GPU inference
+            inputs, kwargs = self.scatter(inputs, kwargs, [-1])
+            return self.module.train_step(*inputs[0], **kwargs[0])
+
+        assert len(self.device_ids) == 1, \
+            ('MMDataParallel only supports single GPU training, if you need to'
+             ' train with multiple GPUs, please use MMDistributedDataParallel'
+             ' instead.')
+
+        for t in chain(self.module.parameters(), self.module.buffers()):
+            if t.device != self.src_device_obj:
+                raise RuntimeError(
+                    'module must have its parameters and buffers '
+                    f'on device {self.src_device_obj} (device_ids[0]) but '
+                    f'found one of them on device: {t.device}')
+
+        inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)
+        return self.module.train_step(*inputs[0], **kwargs[0])
+
+    def val_step(self, *inputs, **kwargs):
+        if not self.device_ids:
+            # We add the following line thus the module could gather and
+            # convert data containers as those in GPU inference
+            inputs, kwargs = self.scatter(inputs, kwargs, [-1])
+            return self.module.val_step(*inputs[0], **kwargs[0])
+
+        assert len(self.device_ids) == 1, \
+            ('MMDataParallel only supports single GPU training, if you need to'
+             ' train with multiple GPUs, please use MMDistributedDataParallel'
+             ' instead.')
+
+        for t in chain(self.module.parameters(), self.module.buffers()):
+            if t.device != self.src_device_obj:
+                raise RuntimeError(
+                    'module must have its parameters and buffers '
+                    f'on device {self.src_device_obj} (device_ids[0]) but '
+                    f'found one of them on device: {t.device}')
+
+        inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)
+        return self.module.val_step(*inputs[0], **kwargs[0])

groundingLMM/mmcv/mmcv/parallel/distributed.py

@@ -0,0 +1,112 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import torch
+from torch.nn.parallel.distributed import (DistributedDataParallel,
+                                           _find_tensors)
+
+from mmcv import print_log
+from mmcv.utils import TORCH_VERSION, digit_version
+from .scatter_gather import scatter_kwargs
+
+
+class MMDistributedDataParallel(DistributedDataParallel):
+    """The DDP module that supports DataContainer.
+
+    MMDDP has two main differences with PyTorch DDP:
+
+    - It supports a custom type :class:`DataContainer` which allows more
+      flexible control of input data.
+    - It implement two APIs ``train_step()`` and ``val_step()``.
+    """
+
+    def to_kwargs(self, inputs, kwargs, device_id):
+        # Use `self.to_kwargs` instead of `self.scatter` in pytorch1.8
+        # to move all tensors to device_id
+        return scatter_kwargs(inputs, kwargs, [device_id], dim=self.dim)
+
+    def scatter(self, inputs, kwargs, device_ids):
+        return scatter_kwargs(inputs, kwargs, device_ids, dim=self.dim)
+
+    def train_step(self, *inputs, **kwargs):
+        """train_step() API for module wrapped by DistributedDataParallel.
+
+        This method is basically the same as
+        ``DistributedDataParallel.forward()``, while replacing
+        ``self.module.forward()`` with ``self.module.train_step()``.
+        It is compatible with PyTorch 1.1 - 1.5.
+        """
+
+        # In PyTorch >= 1.7, ``reducer._rebuild_buckets()`` is moved from the
+        # end of backward to the beginning of forward.
+        if ('parrots' not in TORCH_VERSION
+                and digit_version(TORCH_VERSION) >= digit_version('1.7')
+                and self.reducer._rebuild_buckets()):
+            print_log(
+                'Reducer buckets have been rebuilt in this iteration.',
+                logger='mmcv')
+
+        if getattr(self, 'require_forward_param_sync', True):
+            self._sync_params()
+        if self.device_ids:
+            inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)
+            if len(self.device_ids) == 1:
+                output = self.module.train_step(*inputs[0], **kwargs[0])
+            else:
+                outputs = self.parallel_apply(
+                    self._module_copies[:len(inputs)], inputs, kwargs)
+                output = self.gather(outputs, self.output_device)
+        else:
+            output = self.module.train_step(*inputs, **kwargs)
+
+        if torch.is_grad_enabled() and getattr(
+                self, 'require_backward_grad_sync', True):
+            if self.find_unused_parameters:
+                self.reducer.prepare_for_backward(list(_find_tensors(output)))
+            else:
+                self.reducer.prepare_for_backward([])
+        else:
+            if ('parrots' not in TORCH_VERSION
+                    and digit_version(TORCH_VERSION) > digit_version('1.2')):
+                self.require_forward_param_sync = False
+        return output
+
+    def val_step(self, *inputs, **kwargs):
+        """val_step() API for module wrapped by DistributedDataParallel.
+
+        This method is basically the same as
+        ``DistributedDataParallel.forward()``, while replacing
+        ``self.module.forward()`` with ``self.module.val_step()``.
+        It is compatible with PyTorch 1.1 - 1.5.
+        """
+        # In PyTorch >= 1.7, ``reducer._rebuild_buckets()`` is moved from the
+        # end of backward to the beginning of forward.
+        if ('parrots' not in TORCH_VERSION
+                and digit_version(TORCH_VERSION) >= digit_version('1.7')
+                and self.reducer._rebuild_buckets()):
+            print_log(
+                'Reducer buckets have been rebuilt in this iteration.',
+                logger='mmcv')
+
+        if getattr(self, 'require_forward_param_sync', True):
+            self._sync_params()
+        if self.device_ids:
+            inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)
+            if len(self.device_ids) == 1:
+                output = self.module.val_step(*inputs[0], **kwargs[0])
+            else:
+                outputs = self.parallel_apply(
+                    self._module_copies[:len(inputs)], inputs, kwargs)
+                output = self.gather(outputs, self.output_device)
+        else:
+            output = self.module.val_step(*inputs, **kwargs)
+
+        if torch.is_grad_enabled() and getattr(
+                self, 'require_backward_grad_sync', True):
+            if self.find_unused_parameters:
+                self.reducer.prepare_for_backward(list(_find_tensors(output)))
+            else:
+                self.reducer.prepare_for_backward([])
+        else:
+            if ('parrots' not in TORCH_VERSION
+                    and digit_version(TORCH_VERSION) > digit_version('1.2')):
+                self.require_forward_param_sync = False
+        return output

groundingLMM/mmcv/mmcv/parallel/distributed_deprecated.py

@@ -0,0 +1,70 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import torch
+import torch.distributed as dist
+import torch.nn as nn
+from torch._utils import (_flatten_dense_tensors, _take_tensors,
+                          _unflatten_dense_tensors)
+
+from mmcv.utils import TORCH_VERSION, digit_version
+from .registry import MODULE_WRAPPERS
+from .scatter_gather import scatter_kwargs
+
+
+@MODULE_WRAPPERS.register_module()
+class MMDistributedDataParallel(nn.Module):
+
+    def __init__(self,
+                 module,
+                 dim=0,
+                 broadcast_buffers=True,
+                 bucket_cap_mb=25):
+        super(MMDistributedDataParallel, self).__init__()
+        self.module = module
+        self.dim = dim
+        self.broadcast_buffers = broadcast_buffers
+
+        self.broadcast_bucket_size = bucket_cap_mb * 1024 * 1024
+        self._sync_params()
+
+    def _dist_broadcast_coalesced(self, tensors, buffer_size):
+        for tensors in _take_tensors(tensors, buffer_size):
+            flat_tensors = _flatten_dense_tensors(tensors)
+            dist.broadcast(flat_tensors, 0)
+            for tensor, synced in zip(
+                    tensors, _unflatten_dense_tensors(flat_tensors, tensors)):
+                tensor.copy_(synced)
+
+    def _sync_params(self):
+        module_states = list(self.module.state_dict().values())
+        if len(module_states) > 0:
+            self._dist_broadcast_coalesced(module_states,
+                                           self.broadcast_bucket_size)
+        if self.broadcast_buffers:
+            if (TORCH_VERSION != 'parrots'
+                    and digit_version(TORCH_VERSION) < digit_version('1.0')):
+                buffers = [b.data for b in self.module._all_buffers()]
+            else:
+                buffers = [b.data for b in self.module.buffers()]
+            if len(buffers) > 0:
+                self._dist_broadcast_coalesced(buffers,
+                                               self.broadcast_bucket_size)
+
+    def scatter(self, inputs, kwargs, device_ids):
+        return scatter_kwargs(inputs, kwargs, device_ids, dim=self.dim)
+
+    def forward(self, *inputs, **kwargs):
+        inputs, kwargs = self.scatter(inputs, kwargs,
+                                      [torch.cuda.current_device()])
+        return self.module(*inputs[0], **kwargs[0])
+
+    def train_step(self, *inputs, **kwargs):
+        inputs, kwargs = self.scatter(inputs, kwargs,
+                                      [torch.cuda.current_device()])
+        output = self.module.train_step(*inputs[0], **kwargs[0])
+        return output
+
+    def val_step(self, *inputs, **kwargs):
+        inputs, kwargs = self.scatter(inputs, kwargs,
+                                      [torch.cuda.current_device()])
+        output = self.module.val_step(*inputs[0], **kwargs[0])
+        return output