Release model: vov99_dd3d_1600x640_trainval_future (#2)

README.md

@@ -4,11 +4,12 @@ This is the official PyTorch implementation for paper [SparseBEV: High-Performan
 
 ## Model Zoo
 
-| Setting  | Pretrain | Training Cost | NDS | FPS | Weights |
-|----------|----------|---------------|-----|-----|---------|
-| [r50_nuimg_704x256](configs/r50_nuimg_704x256.py) | [nuImages](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/nuimages_semseg/cascade_mask_rcnn_r50_fpn_coco-20e_20e_nuim/cascade_mask_rcnn_r50_fpn_coco-20e_20e_nuim_20201009_124951-40963960.pth) | 21h (8x2080Ti) | 55.6 | 15.8 | [weights](https://drive.google.com/file/d/1ft34-pxLpHGo2Aw-jowEtCxyXcqszHNn/view) |
-| [r50_nuimg_704x256_400q_36ep](configs/r50_nuimg_704x256_400q_36ep.py) | [nuImages](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/nuimages_semseg/cascade_mask_rcnn_r50_fpn_coco-20e_20e_nuim/cascade_mask_rcnn_r50_fpn_coco-20e_20e_nuim_20201009_124951-40963960.pth) | 28h (8x2080Ti) | 55.8 | 23.5 | [weights](https://drive.google.com/file/d/1C_Vn3iiSnSW1Dw1r0DkjJMwvHC5Y3zTN/view) |
-| [r101_nuimg_1408x512](configs/r101_nuimg_1408x512.py) | [nuImages](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/nuimages_semseg/cascade_mask_rcnn_r101_fpn_1x_nuim/cascade_mask_rcnn_r101_fpn_1x_nuim_20201024_134804-45215b1e.pth) | 2d8h (8xV100) | 59.2 | 6.5 | [weights](https://drive.google.com/file/d/1dKu5cR1fuo-O0ynyBh-RCPtHrgut29mN/view) |
+| Setting  | Pretrain | Training Cost | NDS<sub>val</sub> | NDS<sub>test</sub> | FPS | Weights |
+|----------|:--------:|:-------------:|:-----------------:|:------------------:|:---:|:-------:|
+| [r50_nuimg_704x256](configs/r50_nuimg_704x256.py) | [nuImg](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/nuimages_semseg/cascade_mask_rcnn_r50_fpn_coco-20e_20e_nuim/cascade_mask_rcnn_r50_fpn_coco-20e_20e_nuim_20201009_124951-40963960.pth) | 21h (8x2080Ti) | 55.6 | - | 15.8 | [gdrive](https://drive.google.com/file/d/1ft34-pxLpHGo2Aw-jowEtCxyXcqszHNn/view) |
+| [r50_nuimg_704x256_400q_36ep](configs/r50_nuimg_704x256_400q_36ep.py) | [nuImg](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/nuimages_semseg/cascade_mask_rcnn_r50_fpn_coco-20e_20e_nuim/cascade_mask_rcnn_r50_fpn_coco-20e_20e_nuim_20201009_124951-40963960.pth) | 28h (8x2080Ti) | 55.8 | - | 23.5 | [gdrive](https://drive.google.com/file/d/1C_Vn3iiSnSW1Dw1r0DkjJMwvHC5Y3zTN/view) |
+| [r101_nuimg_1408x512](configs/r101_nuimg_1408x512.py) | [nuImg](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/nuimages_semseg/cascade_mask_rcnn_r101_fpn_1x_nuim/cascade_mask_rcnn_r101_fpn_1x_nuim_20201024_134804-45215b1e.pth) | 2d8h (8xV100) | 59.2 | - | 6.5 | [gdrive](https://drive.google.com/file/d/1dKu5cR1fuo-O0ynyBh-RCPtHrgut29mN/view) |
+| [vov99_dd3d_1600x640_trainval_future](configs/vov99_dd3d_1600x640_trainval_future.py) | [DD3D](https://drive.google.com/file/d/1gQkhWERCzAosBwG5bh2BKkt1k0TJZt-A/view) | 4d1h (8xA100) | 84.9 | 67.5 | - | [gdrive](https://drive.google.com/file/d/1TL0QoCiWD5uq8PCAWWE3A-g73ibK1R0S/view) |
 
 * We use `r50_nuimg_704x256` for ablation studies and `r50_nuimg_704x256_400q_36ep` for comparison with others.
 * We recommend using `r50_nuimg_704x256` to validate new ideas since it trains faster and the result is more stable.
@@ -72,13 +73,10 @@ python setup.py build_ext --inplace
 data/nuscenes
 ├── maps
 ├── nuscenes_infos_test_sweep.pkl
-├── nuscenes_infos_train_mini_sweep.pkl
 ├── nuscenes_infos_train_sweep.pkl
-├── nuscenes_infos_val_mini_sweep.pkl
 ├── nuscenes_infos_val_sweep.pkl
 ├── samples
 ├── sweeps
-├── v1.0-mini
 ├── v1.0-test
 └── v1.0-trainval
 ```
@@ -87,6 +85,14 @@ These `*.pkl` files can also be generated with our script: `gen_sweep_info.py`.
 
 ## Training
 
+Download pretrained weights and put it in directory `pretrain/`:
+
+```
+pretrain
+├── cascade_mask_rcnn_r101_fpn_1x_nuim_20201024_134804-45215b1e.pth
+├── cascade_mask_rcnn_r50_fpn_coco-20e_20e_nuim_20201009_124951-40963960.pth
+```
+
 Train SparseBEV with 8 GPUs:
 
 ```

configs/r101_nuimg_1408x512.py

@@ -70,7 +70,6 @@ test_pipeline = [
 ]
 
 data = dict(
-    workers_per_gpu=4,
     train=dict(pipeline=train_pipeline),
     val=dict(pipeline=test_pipeline),
     test=dict(pipeline=test_pipeline)

configs/r50_nuimg_704x256.py

@@ -175,7 +175,7 @@ data = dict(
     test=dict(
         type=dataset_type,
         data_root=dataset_root,
-        ann_file=dataset_root + 'nuscenes_custom_infos_test.pkl',
+        ann_file=dataset_root + 'nuscenes_infos_test_sweep.pkl',
         pipeline=test_pipeline,
         classes=class_names,
         modality=input_modality,

configs/r50_nuimg_704x256_400q_36ep.py

@@ -6,3 +6,5 @@ model = dict(
 
 total_epochs = 36
 eval_config = dict(interval=total_epochs)
+
+data = dict(workers_per_gpu=12)

configs/vov99_dd3d_1600x640_trainval_future.py

@@ -0,0 +1,107 @@
+_base_ = ['./r50_nuimg_704x256.py']
+
+# For nuScenes we usually do 10-class detection
+class_names = [
+    'car', 'truck', 'construction_vehicle', 'bus', 'trailer', 'barrier',
+    'motorcycle', 'bicycle', 'pedestrian', 'traffic_cone'
+]
+
+# If point cloud range is changed, the models should also change their point
+# cloud range accordingly
+point_cloud_range = [-51.2, -51.2, -5.0, 51.2, 51.2, 3.0]
+voxel_size = [0.2, 0.2, 8]
+
+img_backbone = dict(
+    _delete_=True,
+    type='VoVNet',
+    spec_name='V-99-eSE',
+    out_features=['stage2', 'stage3', 'stage4', 'stage5'],
+    norm_eval=True,
+    frozen_stages=1,
+    with_cp=True
+)
+img_neck=dict(
+    _delete_=True,
+    type='FPN',
+    in_channels=[256, 512, 768, 1024],
+    out_channels=256,
+    num_outs=5
+)
+img_norm_cfg = dict(
+    _delete_=True,
+    mean=[103.530, 116.280, 123.675],
+    std=[57.375, 57.120, 58.395],
+    to_rgb=False
+)
+
+model = dict(
+    data_aug=dict(
+        img_color_aug=True,
+        img_norm_cfg=img_norm_cfg,
+        img_pad_cfg=dict(size_divisor=32)
+    ),
+    img_backbone=img_backbone,
+    img_neck=img_neck,
+    pts_bbox_head=dict(
+        num_query=1600,
+        transformer=dict(
+            num_levels=5,
+            num_points=4,
+            num_frames=15
+        )
+    )
+)
+
+ida_aug_conf = {
+    'resize_lim': (0.94, 1.25),
+    'final_dim': (640, 1600),
+    'bot_pct_lim': (0.0, 0.0),
+    'rot_lim': (0.0, 0.0),
+    'H': 900, 'W': 1600,
+    'rand_flip': True,
+}
+
+train_pipeline = [
+    dict(type='LoadMultiViewImageFromFiles', to_float32=False, color_type='color'),
+    dict(type='LoadMultiViewImageFromMultiSweepsFuture', prev_sweeps_num=7, next_sweeps_num=7),
+    dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True, with_attr_label=False),
+    dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range),
+    dict(type='ObjectNameFilter', classes=class_names),
+    dict(type='RandomTransformImage', ida_aug_conf=ida_aug_conf, training=True),
+    dict(type='GlobalRotScaleTransImage', rot_range=[-0.3925, 0.3925], scale_ratio_range=[0.95, 1.05]),
+    dict(type='DefaultFormatBundle3D', class_names=class_names),
+    dict(type='Collect3D', keys=['gt_bboxes_3d', 'gt_labels_3d', 'img'], meta_keys=(
+        'filename', 'ori_shape', 'img_shape', 'pad_shape', 'lidar2img', 'img_timestamp'))
+]
+
+test_pipeline = [
+    dict(type='LoadMultiViewImageFromFiles', to_float32=False, color_type='color'),
+    dict(type='LoadMultiViewImageFromMultiSweepsFuture', prev_sweeps_num=7, next_sweeps_num=7, test_mode=True),
+    dict(type='RandomTransformImage', ida_aug_conf=ida_aug_conf, training=False),
+    dict(
+        type='MultiScaleFlipAug3D',
+        img_scale=(1600, 900),
+        pts_scale_ratio=1,
+        flip=False,
+        transforms=[
+            dict(type='DefaultFormatBundle3D', class_names=class_names, with_label=False),
+            dict(type='Collect3D', keys=['img'], meta_keys=(
+                'filename', 'box_type_3d', 'ori_shape', 'img_shape', 'pad_shape',
+                'lidar2img', 'img_timestamp'))
+        ])
+]
+
+data = dict(
+    train=dict(
+        ann_file=['data/nuscenes/nuscenes_infos_train_sweep.pkl',
+                  'data/nuscenes/nuscenes_infos_val_sweep.pkl'],
+        pipeline=train_pipeline),
+    val=dict(
+        ann_file='data/nuscenes/nuscenes_infos_val_sweep.pkl',  # use nuscenes_infos_test_sweep.pkl for submission
+        pipeline=test_pipeline),
+    test=dict(pipeline=test_pipeline)
+)
+
+# load pretrained weights
+load_from = 'pretrain/dd3d_det_final.pth'
+revise_keys = None

loaders/nuscenes_dataset.py

@@ -8,7 +8,7 @@ from pyquaternion import Quaternion
 @DATASETS.register_module()
 class CustomNuScenesDataset(NuScenesDataset):
 
-    def collect_sweeps(self, index, into_past=60, into_future=0):
+    def collect_sweeps(self, index, into_past=60, into_future=60):
         all_sweeps_prev = []
         curr_index = index
         while len(all_sweeps_prev) < into_past:

loaders/pipelines/loading.py

@@ -152,3 +152,106 @@ class LoadMultiViewImageFromMultiSweeps(object):
             return self.load_online(results)
         else:
             return self.load_offline(results)
+
+
+@PIPELINES.register_module()
+class LoadMultiViewImageFromMultiSweepsFuture(object):
+    def __init__(self,
+                 prev_sweeps_num=5,
+                 next_sweeps_num=5,
+                 color_type='color',
+                 test_mode=False):
+        self.prev_sweeps_num = prev_sweeps_num
+        self.next_sweeps_num = next_sweeps_num
+        self.color_type = color_type
+        self.test_mode = test_mode
+
+        assert prev_sweeps_num == next_sweeps_num
+
+        self.train_interval = [4, 8]
+        self.test_interval = 6
+
+        try:
+            mmcv.use_backend('turbojpeg')
+        except ImportError:
+            mmcv.use_backend('cv2')
+
+    def __call__(self, results):
+        if self.prev_sweeps_num == 0 and self.next_sweeps_num == 0:
+            return results
+
+        cam_types = [
+            'CAM_FRONT', 'CAM_FRONT_RIGHT', 'CAM_FRONT_LEFT',
+            'CAM_BACK', 'CAM_BACK_LEFT', 'CAM_BACK_RIGHT'
+        ]
+
+        if self.test_mode:
+            interval = self.test_interval
+        else:
+            interval = np.random.randint(self.train_interval[0], self.train_interval[1] + 1)
+
+        # previous sweeps
+        if len(results['sweeps']['prev']) == 0:
+            for _ in range(self.prev_sweeps_num):
+                for j in range(len(cam_types)):
+                    results['img'].append(results['img'][j])
+                    results['img_timestamp'].append(results['img_timestamp'][j])
+                    results['filename'].append(results['filename'][j])
+                    results['lidar2img'].append(np.copy(results['lidar2img'][j]))
+        else:
+            choices = [(k + 1) * interval - 1 for k in range(self.prev_sweeps_num)]
+
+            for idx in sorted(list(choices)):
+                sweep_idx = min(idx, len(results['sweeps']['prev']) - 1)
+                sweep = results['sweeps']['prev'][sweep_idx]
+
+                if len(sweep.keys()) < len(cam_types):
+                    sweep = results['sweeps']['prev'][sweep_idx - 1]
+
+                for sensor in cam_types:
+                    results['img'].append(mmcv.imread(sweep[sensor]['data_path'], self.color_type))
+                    results['img_timestamp'].append(sweep[sensor]['timestamp'] / 1e6)
+                    results['filename'].append(sweep[sensor]['data_path'])
+                    results['lidar2img'].append(compose_lidar2img(
+                        results['ego2global_translation'],
+                        results['ego2global_rotation'],
+                        results['lidar2ego_translation'],
+                        results['lidar2ego_rotation'],
+                        sweep[sensor]['sensor2global_translation'],
+                        sweep[sensor]['sensor2global_rotation'],
+                        sweep[sensor]['cam_intrinsic'],
+                    ))
+
+        # future sweeps
+        if len(results['sweeps']['next']) == 0:
+            for _ in range(self.next_sweeps_num):
+                for j in range(len(cam_types)):
+                    results['img'].append(results['img'][j])
+                    results['img_timestamp'].append(results['img_timestamp'][j])
+                    results['filename'].append(results['filename'][j])
+                    results['lidar2img'].append(np.copy(results['lidar2img'][j]))
+        else:
+            choices = [(k + 1) * interval - 1 for k in range(self.next_sweeps_num)]
+
+            for idx in sorted(list(choices)):
+                sweep_idx = min(idx, len(results['sweeps']['next']) - 1)
+                sweep = results['sweeps']['next'][sweep_idx]
+
+                if len(sweep.keys()) < len(cam_types):
+                    sweep = results['sweeps']['next'][sweep_idx - 1]
+
+                for sensor in cam_types:
+                    results['img'].append(mmcv.imread(sweep[sensor]['data_path'], self.color_type))
+                    results['img_timestamp'].append(sweep[sensor]['timestamp'] / 1e6)
+                    results['filename'].append(sweep[sensor]['data_path'])
+                    results['lidar2img'].append(compose_lidar2img(
+                        results['ego2global_translation'],
+                        results['ego2global_rotation'],
+                        results['lidar2ego_translation'],
+                        results['lidar2ego_rotation'],
+                        sweep[sensor]['sensor2global_translation'],
+                        sweep[sensor]['sensor2global_rotation'],
+                        sweep[sensor]['cam_intrinsic'],
+                    ))
+
+        return results

models/sparsebev.py

@@ -239,7 +239,7 @@ class SparseBEV(MVXTwoStageDetector):
         world_size = get_dist_info()[1]
         if world_size == 1:  # online
             return self.simple_test_online(img_metas, img, rescale)
-        elif world_size > 1:  # offline
+        else:  # offline
             return self.simple_test_offline(img_metas, img, rescale)
 
     def simple_test_offline(self, img_metas, img=None, rescale=False):
@@ -273,23 +273,21 @@ class SparseBEV(MVXTwoStageDetector):
         for i in range(num_frames):
             img_indices = list(np.arange(i * 6, (i + 1) * 6))
 
-            img_curr_large = img[:, 0]  # [B, 6, C, H, W]
-            img_metas_curr_large = [{}]
-
+            img_metas_curr = [{}]
             for k in img_metas[0].keys():
                 if isinstance(img_metas[0][k], list):
-                    img_metas_curr_large[0][k] = [img_metas[0][k][i] for i in img_indices]
+                    img_metas_curr[0][k] = [img_metas[0][k][i] for i in img_indices]
 
             if img_filenames[img_indices[0]] in self.memory:
-                img_feats_curr_large = self.memory[img_filenames[img_indices[0]]]
+                img_feats_curr = self.memory[img_filenames[img_indices[0]]]
             else:
-                assert i == 0
-                img_feats_curr_large = self.extract_feat(img_curr_large, img_metas_curr_large)
-                self.memory[img_filenames[img_indices[0]]] = img_feats_curr_large
+                img_curr_large = img[:, i]  # [B, 6, C, H, W]
+                img_feats_curr = self.extract_feat(img_curr_large, img_metas_curr)
+                self.memory[img_filenames[img_indices[0]]] = img_feats_curr
                 self.queue.put(img_filenames[img_indices[0]])
 
-            img_feats_large.append(img_feats_curr_large)
-            img_metas_large.append(img_metas_curr_large)
+            img_feats_large.append(img_feats_curr)
+            img_metas_large.append(img_metas_curr)
 
         # reorganize
         feat_levels = len(img_feats_large[0])
@@ -314,7 +312,7 @@ class SparseBEV(MVXTwoStageDetector):
         for result_dict, pts_bbox in zip(bbox_list, bbox_pts):
             result_dict['pts_bbox'] = pts_bbox
 
-        while self.queue.qsize() >= 8:
+        while self.queue.qsize() >= 16:
             pop_key = self.queue.get()
             self.memory.pop(pop_key)
 

val.py

@@ -17,7 +17,7 @@ from models.utils import VERSION
 
 
 def evaluate(dataset, results, epoch):
-    metrics = dataset.evaluate(results, jsonfile_prefix=None)
+    metrics = dataset.evaluate(results, jsonfile_prefix='submission')
 
     mAP = metrics['pts_bbox_NuScenes/mAP']
     mATE = metrics['pts_bbox_NuScenes/mATE']
@@ -126,7 +126,6 @@ def main():
 
     if 'version' in checkpoint:
         VERSION.name = checkpoint['version']
-        logging.info(VERSION.name)
 
     if world_size > 1:
         results = multi_gpu_test(model, val_loader, gpu_collect=True)