Upload 25 files

configs/huggingface/rsprompter_anchor_NWPU_config.py

@@ -0,0 +1,353 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'panoptic_head',
+    'data_preprocessor'
+]
+
+sub_model_optim = {
+    'panoptic_head': {'lr_mult': 1},
+}
+
+max_epochs = 1200
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 10
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+prompt_shape = (60, 5)
+
+model_cfg = dict(
+    type='SegSAMAnchorPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+    ),
+    need_train_names=sub_model_train,
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='vit_h',
+        # checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    ),
+    panoptic_head=dict(
+        type='SAMAnchorInstanceHead',
+        neck=dict(
+            type='SAMAggregatorNeck',
+            in_channels=[1280] * 32,
+            # in_channels=[768] * 12,
+            inner_channels=32,
+            selected_channels=range(4, 32, 2),
+            # selected_channels=range(4, 12, 2),
+            out_channels=256,
+            up_sample_scale=4,
+        ),
+        rpn_head=dict(
+            type='mmdet.RPNHead',
+            in_channels=256,
+            feat_channels=256,
+            anchor_generator=dict(
+                type='mmdet.AnchorGenerator',
+                scales=[2, 4, 8, 16, 32, 64],
+                ratios=[0.5, 1.0, 2.0],
+                strides=[8, 16, 32]),
+            bbox_coder=dict(
+                type='mmdet.DeltaXYWHBBoxCoder',
+                target_means=[.0, .0, .0, .0],
+                target_stds=[1.0, 1.0, 1.0, 1.0]),
+            loss_cls=dict(
+                type='mmdet.CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+            loss_bbox=dict(type='mmdet.SmoothL1Loss', loss_weight=1.0)),
+        roi_head=dict(
+            type='SAMAnchorPromptRoIHead',
+            bbox_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            bbox_head=dict(
+                type='mmdet.Shared2FCBBoxHead',
+                in_channels=256,
+                fc_out_channels=1024,
+                roi_feat_size=7,
+                num_classes=num_classes,
+                bbox_coder=dict(
+                    type='mmdet.DeltaXYWHBBoxCoder',
+                    target_means=[0., 0., 0., 0.],
+                    target_stds=[0.1, 0.1, 0.2, 0.2]),
+                reg_class_agnostic=False,
+                loss_cls=dict(
+                    type='mmdet.CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+                loss_bbox=dict(type='mmdet.SmoothL1Loss', loss_weight=1.0)),
+            mask_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            mask_head=dict(
+                type='SAMPromptMaskHead',
+                per_query_point=prompt_shape[1],
+                with_sincos=True,
+                class_agnostic=True,
+                loss_mask=dict(
+                    type='mmdet.CrossEntropyLoss', use_mask=True, loss_weight=1.0))),
+        # model training and testing settings
+        train_cfg=dict(
+            rpn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.7,
+                    neg_iou_thr=0.3,
+                    min_pos_iou=0.3,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=512,
+                    pos_fraction=0.5,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=False),
+                allowed_border=-1,
+                pos_weight=-1,
+                debug=False),
+            rpn_proposal=dict(
+                nms_pre=2000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.5,
+                    neg_iou_thr=0.5,
+                    min_pos_iou=0.5,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=256,
+                    pos_fraction=0.25,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=True),
+                mask_size=1024,
+                pos_weight=-1,
+                debug=False)),
+        test_cfg=dict(
+            rpn=dict(
+                nms_pre=1000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                score_thr=0.05,
+                nms=dict(type='nms', iou_threshold=0.5),
+                max_per_img=100,
+                mask_thr_binary=0.5)
+        )
+    )
+)
+
+
+task_name = 'nwpu_ins'
+exp_name = 'E20230629_1'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='sam-anchor',
+    name=exp_name
+)
+
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=3,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+vis_backends = [dict(type='mmdet.LocalVisBackend')]
+visualizer = dict(
+    type='mmdet.DetLocalVisualizer',
+    vis_backends=vis_backends,
+    name='visualizer',
+    fig_save_cfg=dict(
+        frameon=False,
+        figsize=(40, 20),
+        # dpi=300,
+    ),
+    line_width=2,
+    alpha=0.8
+)
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=5,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+predict_pipeline = [
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('ori_shape', 'img_shape', 'scale_factor'))
+]
+
+train_batch_size_per_gpu = 2
+train_num_workers = 2
+test_batch_size_per_gpu = 2
+test_num_workers = 2
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/NWPU10'
+train_data_prefix = ''
+val_data_prefix = ''
+dataset_type = 'NWPUInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_val.json',
+            data_prefix=dict(img_path='positive image set'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_train.json',
+            data_prefix=dict(img_path='positive image set'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/huggingface/rsprompter_anchor_SSDD_config.py

@@ -0,0 +1,369 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'panoptic_head',
+    'data_preprocessor'
+]
+
+sub_model_optim = {
+    'panoptic_head': {'lr_mult': 1},
+}
+
+max_epochs = 1000
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 1
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+prompt_shape = (30, 5)
+
+model_cfg = dict(
+    type='SegSAMAnchorPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='vit_h',
+        # checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    ),
+    panoptic_head=dict(
+        type='SAMAnchorInstanceHead',
+        neck=dict(
+            type='SAMAggregatorNeck',
+            in_channels=[1280] * 32,
+            # in_channels=[768] * 12,
+            inner_channels=32,
+            selected_channels=range(4, 32, 2),
+            # selected_channels=range(4, 12, 2),
+            out_channels=256,
+            up_sample_scale=4,
+        ),
+        rpn_head=dict(
+            type='mmdet.RPNHead',
+            in_channels=256,
+            feat_channels=256,
+            anchor_generator=dict(
+                type='mmdet.AnchorGenerator',
+                scales=[2, 4, 8, 16, 32, 64],
+                ratios=[0.5, 1.0, 2.0],
+                strides=[8, 16, 32]),
+            bbox_coder=dict(
+                type='mmdet.DeltaXYWHBBoxCoder',
+                target_means=[.0, .0, .0, .0],
+                target_stds=[1.0, 1.0, 1.0, 1.0]),
+            loss_cls=dict(
+                type='mmdet.CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+            loss_bbox=dict(type='mmdet.SmoothL1Loss', loss_weight=1.0)),
+        roi_head=dict(
+            type='SAMAnchorPromptRoIHead',
+            bbox_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            bbox_head=dict(
+                type='mmdet.Shared2FCBBoxHead',
+                in_channels=256,
+                fc_out_channels=1024,
+                roi_feat_size=7,
+                num_classes=num_classes,
+                bbox_coder=dict(
+                    type='mmdet.DeltaXYWHBBoxCoder',
+                    target_means=[0., 0., 0., 0.],
+                    target_stds=[0.1, 0.1, 0.2, 0.2]),
+                reg_class_agnostic=False,
+                loss_cls=dict(
+                    type='mmdet.CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+                loss_bbox=dict(type='mmdet.SmoothL1Loss', loss_weight=1.0)),
+            mask_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            mask_head=dict(
+                type='SAMPromptMaskHead',
+                per_query_point=prompt_shape[1],
+                with_sincos=True,
+                class_agnostic=True,
+                loss_mask=dict(
+                    type='mmdet.CrossEntropyLoss', use_mask=True, loss_weight=1.0))),
+        # model training and testing settings
+        train_cfg=dict(
+            rpn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.7,
+                    neg_iou_thr=0.3,
+                    min_pos_iou=0.3,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=512,
+                    pos_fraction=0.5,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=False),
+                allowed_border=-1,
+                pos_weight=-1,
+                debug=False),
+            rpn_proposal=dict(
+                nms_pre=2000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.5,
+                    neg_iou_thr=0.5,
+                    min_pos_iou=0.5,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=256,
+                    pos_fraction=0.25,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=True),
+                mask_size=1024,
+                pos_weight=-1,
+                debug=False)),
+        test_cfg=dict(
+            rpn=dict(
+                nms_pre=1000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                score_thr=0.05,
+                nms=dict(type='nms', iou_threshold=0.5),
+                max_per_img=100,
+                mask_thr_binary=0.5)
+        )
+    )
+)
+
+task_name = 'whu_ins'
+exp_name = 'E20230629_0'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='sam-anchor',
+    name=exp_name
+)
+
+
+vis_backends = [dict(type='mmdet.LocalVisBackend')]
+visualizer = dict(
+    type='mmdet.DetLocalVisualizer',
+    vis_backends=vis_backends,
+    name='visualizer',
+    fig_save_cfg=dict(
+        frameon=False,
+        figsize=(40, 20),
+        # dpi=300,
+    ),
+    line_width=2,
+    alpha=0.8
+)
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=3,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=5,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 2
+train_num_workers = 2
+test_batch_size_per_gpu = 2
+test_num_workers = 2
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/SSDD'
+dataset_type = 'SSDDInsSegDataset'
+
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            # ann_file='NWPU_instances_val.json',
+            # data_prefix=dict(img_path='positive image set'),
+            ann_file='annotations/SSDD_instances_val.json',
+            data_prefix=dict(img_path='imgs'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+predict_pipeline = [
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('ori_shape', 'img_shape', 'scale_factor'))
+]
+
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            # ann_file='NWPU_instances_train.json',
+            # data_prefix=dict(img_path='positive image set'),
+            ann_file='annotations/SSDD_instances_train.json',
+            data_prefix=dict(img_path='imgs'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/huggingface/rsprompter_anchor_WHU_config.py

@@ -0,0 +1,371 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'panoptic_head',
+    'data_preprocessor'
+]
+
+sub_model_optim = {
+    'panoptic_head': {'lr_mult': 1},
+}
+
+max_epochs = 2000
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 1
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+prompt_shape = (90, 4)
+
+model_cfg = dict(
+    type='SegSAMAnchorPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+    ),
+    need_train_names=sub_model_train,
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='vit_h'
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    ),
+    panoptic_head=dict(
+        type='SAMAnchorInstanceHead',
+        neck=dict(
+            type='SAMAggregatorNeck',
+            in_channels=[1280] * 32,
+            # in_channels=[768] * 12,
+            inner_channels=32,
+            selected_channels=range(4, 32, 2),
+            # selected_channels=range(4, 12, 2),
+            out_channels=256,
+            up_sample_scale=4,
+        ),
+        rpn_head=dict(
+            type='mmdet.RPNHead',
+            in_channels=256,
+            feat_channels=256,
+            anchor_generator=dict(
+                type='mmdet.AnchorGenerator',
+                scales=[2, 4, 8, 16, 32, 64],
+                ratios=[0.5, 1.0, 2.0],
+                strides=[8, 16, 32]),
+            bbox_coder=dict(
+                type='mmdet.DeltaXYWHBBoxCoder',
+                target_means=[.0, .0, .0, .0],
+                target_stds=[1.0, 1.0, 1.0, 1.0]),
+            loss_cls=dict(
+                type='mmdet.CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+            loss_bbox=dict(type='mmdet.SmoothL1Loss', loss_weight=1.0)),
+        roi_head=dict(
+            type='SAMAnchorPromptRoIHead',
+            bbox_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            bbox_head=dict(
+                type='mmdet.Shared2FCBBoxHead',
+                in_channels=256,
+                fc_out_channels=1024,
+                roi_feat_size=7,
+                num_classes=num_classes,
+                bbox_coder=dict(
+                    type='mmdet.DeltaXYWHBBoxCoder',
+                    target_means=[0., 0., 0., 0.],
+                    target_stds=[0.1, 0.1, 0.2, 0.2]),
+                reg_class_agnostic=False,
+                loss_cls=dict(
+                    type='mmdet.CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+                loss_bbox=dict(type='mmdet.SmoothL1Loss', loss_weight=1.0)),
+            mask_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            mask_head=dict(
+                type='SAMPromptMaskHead',
+                per_query_point=prompt_shape[1],
+                with_sincos=True,
+                class_agnostic=True,
+                loss_mask=dict(
+                    type='mmdet.CrossEntropyLoss', use_mask=True, loss_weight=1.0))),
+        # model training and testing settings
+        train_cfg=dict(
+            rpn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.7,
+                    neg_iou_thr=0.3,
+                    min_pos_iou=0.3,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=512,
+                    pos_fraction=0.5,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=False),
+                allowed_border=-1,
+                pos_weight=-1,
+                debug=False),
+            rpn_proposal=dict(
+                nms_pre=2000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.5,
+                    neg_iou_thr=0.5,
+                    min_pos_iou=0.5,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=256,
+                    pos_fraction=0.25,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=True),
+                mask_size=1024,
+                pos_weight=-1,
+                debug=False)),
+        test_cfg=dict(
+            rpn=dict(
+                nms_pre=1000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                score_thr=0.05,
+                nms=dict(type='nms', iou_threshold=0.5),
+                max_per_img=100,
+                mask_thr_binary=0.5)
+        )
+    )
+)
+
+task_name = 'whu_ins'
+exp_name = 'E20230629_0'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='sam-anchor',
+    name=exp_name
+)
+
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=3,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    ),
+    dict(
+        type='DetVisualizationHook',
+        draw=True,
+        interval=1,
+        score_thr=0.4,
+        show=False,
+        wait_time=1.,
+        test_out_dir='visualization',
+    )
+]
+
+vis_backends = [dict(type='mmdet.LocalVisBackend')]
+visualizer = dict(
+    type='mmdet.DetLocalVisualizer',
+    vis_backends=vis_backends,
+    name='visualizer',
+    fig_save_cfg=dict(
+        frameon=False,
+        figsize=(40, 20),
+        # dpi=300,
+    ),
+    line_width=2,
+    alpha=0.8
+)
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=10,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+predict_pipeline = [
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('ori_shape', 'img_shape', 'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 2
+train_num_workers = 2
+test_batch_size_per_gpu = 2
+test_num_workers = 2
+persistent_workers = True
+
+
+data_parent = '/mnt/search01/dataset/cky_data/WHU'
+train_data_prefix = 'train/'
+val_data_prefix = 'test/'
+dataset_type = 'WHUInsSegDataset'
+
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            # ann_file='NWPU_instances_val.json',
+            # data_prefix=dict(img_path='positive image set'),
+            # ann_file='annotations/SSDD_instances_val.json',
+            # data_prefix=dict(img_path='imgs'),
+            ann_file='annotations/WHU_building_test.json',
+            data_prefix=dict(img_path=val_data_prefix + '/image'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            # ann_file='NWPU_instances_train.json',
+            # data_prefix=dict(img_path='positive image set'),
+            # ann_file='annotations/SSDD_instances_train.json',
+            # data_prefix=dict(img_path='imgs'),
+            ann_file='annotations/WHU_building_train.json',
+            data_prefix=dict(img_path=train_data_prefix + '/image'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/rsprompter/mask2former_nwpu_config.py

@@ -0,0 +1,338 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models', 'mmdet.models'], allow_failed_imports=False)
+max_epochs = 2000
+
+optimizer = dict(
+    type='AdamW',
+    lr=0.0002,
+    weight_decay=1e-4
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    )
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    val_evaluator=evaluator_,
+    test_evaluator=evaluator_
+)
+
+
+image_size = (1024, 1024)
+data_preprocessor = dict(
+        type='mmdet.DetDataPreprocessor',
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        pad_mask=True,
+        mask_pad_value=0,
+        pad_size_divisor=32
+)
+
+num_things_classes = 10
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+num_queries = 60
+
+# model settings
+model = dict(
+    type='mmdet.Mask2Former',
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='mmdet.ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=-1,
+        norm_cfg=dict(type='BN', requires_grad=False),
+        norm_eval=True,
+        style='pytorch',
+        init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')),
+    panoptic_head=dict(
+        type='mmdet.Mask2FormerHead',
+        in_channels=[256, 512, 1024, 2048],  # pass to pixel_decoder inside
+        strides=[4, 8, 16, 32],
+        feat_channels=256,
+        out_channels=256,
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        num_queries=num_queries,
+        num_transformer_feat_level=3,
+        pixel_decoder=dict(
+            type='mmdet.MSDeformAttnPixelDecoder',
+            num_outs=3,
+            norm_cfg=dict(type='GN', num_groups=32),
+            act_cfg=dict(type='ReLU'),
+            encoder=dict(  # DeformableDetrTransformerEncoder
+                # num_layers=6,
+                num_layers=2,
+                layer_cfg=dict(  # DeformableDetrTransformerEncoderLayer
+                    self_attn_cfg=dict(  # MultiScaleDeformableAttention
+                        embed_dims=256,
+                        num_heads=8,
+                        num_levels=3,
+                        num_points=4,
+                        dropout=0.0,
+                        batch_first=True),
+                    ffn_cfg=dict(
+                        embed_dims=256,
+                        feedforward_channels=1024,
+                        num_fcs=2,
+                        ffn_drop=0.0,
+                        act_cfg=dict(type='ReLU', inplace=True)))),
+            positional_encoding=dict(num_feats=128, normalize=True)),
+        enforce_decoder_input_project=False,
+        positional_encoding=dict(num_feats=128, normalize=True),
+        transformer_decoder=dict(  # Mask2FormerTransformerDecoder
+            return_intermediate=True,
+            # num_layers=9,
+            num_layers=3,
+            layer_cfg=dict(  # Mask2FormerTransformerDecoderLayer
+                self_attn_cfg=dict(  # MultiheadAttention
+                    embed_dims=256,
+                    num_heads=8,
+                    dropout=0.0,
+                    batch_first=True),
+                cross_attn_cfg=dict(  # MultiheadAttention
+                    embed_dims=256,
+                    num_heads=8,
+                    dropout=0.0,
+                    batch_first=True),
+                ffn_cfg=dict(
+                    embed_dims=256,
+                    feedforward_channels=2048,
+                    num_fcs=2,
+                    ffn_drop=0.0,
+                    act_cfg=dict(type='ReLU', inplace=True))),
+            init_cfg=None),
+        loss_cls=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=False,
+            loss_weight=2.0,
+            reduction='mean',
+            class_weight=[1.0] * num_classes + [0.1]),
+        loss_mask=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=True,
+            reduction='mean',
+            loss_weight=5.0),
+        loss_dice=dict(
+            type='mmdet.DiceLoss',
+            use_sigmoid=True,
+            activate=True,
+            reduction='mean',
+            naive_dice=True,
+            eps=1.0,
+            loss_weight=5.0)),
+    panoptic_fusion_head=dict(
+        type='mmdet.MaskFormerFusionHead',
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        loss_panoptic=None,
+        init_cfg=None),
+    train_cfg=dict(
+        num_points=12544,
+        oversample_ratio=3.0,
+        importance_sample_ratio=0.75,
+        assigner=dict(
+            type='mmdet.HungarianAssigner',
+            match_costs=[
+                dict(type='mmdet.ClassificationCost', weight=2.0),
+                dict(
+                    type='mmdet.CrossEntropyLossCost', weight=5.0, use_sigmoid=True),
+                dict(type='mmdet.DiceCost', weight=5.0, pred_act=True, eps=1.0)
+            ]),
+        sampler=dict(type='mmdet.MaskPseudoSampler')),
+    test_cfg=dict(
+        panoptic_on=False,
+        # For now, the dataset does not support
+        # evaluating semantic segmentation metric.
+        semantic_on=False,
+        instance_on=True,
+        # max_per_image is for instance segmentation.
+        max_per_image=100,
+        iou_thr=0.8,
+        # In Mask2Former's panoptic postprocessing,
+        # it will filter mask area where score is less than 0.5 .
+        filter_low_score=True),
+    init_cfg=None)
+
+
+model_cfg = dict(
+    type='MMDetPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    whole_model=model,
+)
+
+task_name = 'nwpu_ins'
+exp_name = 'E20230604_4'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='mask2former',
+    name=exp_name
+)
+# logger = None
+
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=5,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 8
+train_num_workers = 4
+test_batch_size_per_gpu = 8
+test_num_workers = 4
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/NWPU10'
+train_data_prefix = ''
+val_data_prefix = ''
+
+dataset_type = 'NWPUInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_val.json',
+            data_prefix=dict(img_path='positive image set'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_train.json',
+            data_prefix=dict(img_path='positive image set'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    test_loader=val_loader,
+    predict_loader=val_loader
+)

configs/rsprompter/mask2former_ssdd_config.py

@@ -0,0 +1,335 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+max_epochs = 600
+
+optimizer = dict(
+    type='AdamW',
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    )
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+
+evaluator = dict(
+    # train_evaluator=evaluator_,
+    val_evaluator=evaluator_,
+    test_evaluator=evaluator_,
+)
+
+image_size = (512, 512)
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 1
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+num_queries = 30
+
+model = dict(
+    type='mmdet.Mask2Former',
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='mmdet.ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=-1,
+        norm_cfg=dict(type='BN', requires_grad=False),
+        norm_eval=True,
+        style='pytorch',
+        init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')),
+    panoptic_head=dict(
+        type='mmdet.Mask2FormerHead',
+        in_channels=[256, 512, 1024, 2048],  # pass to pixel_decoder inside
+        strides=[4, 8, 16, 32],
+        feat_channels=256,
+        out_channels=256,
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        num_queries=num_queries,
+        num_transformer_feat_level=3,
+        pixel_decoder=dict(
+            type='mmdet.MSDeformAttnPixelDecoder',
+            num_outs=3,
+            norm_cfg=dict(type='GN', num_groups=32),
+            act_cfg=dict(type='ReLU'),
+            encoder=dict(  # DeformableDetrTransformerEncoder
+                num_layers=3,
+                layer_cfg=dict(  # DeformableDetrTransformerEncoderLayer
+                    self_attn_cfg=dict(  # MultiScaleDeformableAttention
+                        embed_dims=256,
+                        num_heads=8,
+                        num_levels=3,
+                        num_points=4,
+                        dropout=0.0,
+                        batch_first=True),
+                    ffn_cfg=dict(
+                        embed_dims=256,
+                        feedforward_channels=1024,
+                        num_fcs=2,
+                        ffn_drop=0.0,
+                        act_cfg=dict(type='ReLU', inplace=True)))),
+            positional_encoding=dict(num_feats=128, normalize=True)),
+        enforce_decoder_input_project=False,
+        positional_encoding=dict(num_feats=128, normalize=True),
+        transformer_decoder=dict(  # Mask2FormerTransformerDecoder
+            return_intermediate=True,
+            num_layers=3,
+            layer_cfg=dict(  # Mask2FormerTransformerDecoderLayer
+                self_attn_cfg=dict(  # MultiheadAttention
+                    embed_dims=256,
+                    num_heads=8,
+                    dropout=0.0,
+                    batch_first=True),
+                cross_attn_cfg=dict(  # MultiheadAttention
+                    embed_dims=256,
+                    num_heads=8,
+                    dropout=0.0,
+                    batch_first=True),
+                ffn_cfg=dict(
+                    embed_dims=256,
+                    feedforward_channels=2048,
+                    num_fcs=2,
+                    ffn_drop=0.0,
+                    act_cfg=dict(type='ReLU', inplace=True))),
+            init_cfg=None),
+        loss_cls=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=False,
+            loss_weight=2.0,
+            reduction='mean',
+            class_weight=[1.0] * num_classes + [0.1]),
+        loss_mask=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=True,
+            reduction='mean',
+            loss_weight=5.0),
+        loss_dice=dict(
+            type='mmdet.DiceLoss',
+            use_sigmoid=True,
+            activate=True,
+            reduction='mean',
+            naive_dice=True,
+            eps=1.0,
+            loss_weight=5.0)),
+    panoptic_fusion_head=dict(
+        type='mmdet.MaskFormerFusionHead',
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        loss_panoptic=None,
+        init_cfg=None),
+    train_cfg=dict(
+        num_points=12544,
+        oversample_ratio=3.0,
+        importance_sample_ratio=0.75,
+        assigner=dict(
+            type='mmdet.HungarianAssigner',
+            match_costs=[
+                dict(type='mmdet.ClassificationCost', weight=2.0),
+                dict(
+                    type='mmdet.CrossEntropyLossCost', weight=5.0, use_sigmoid=True),
+                dict(type='mmdet.DiceCost', weight=5.0, pred_act=True, eps=1.0)
+            ]),
+        sampler=dict(type='mmdet.MaskPseudoSampler')),
+    test_cfg=dict(
+        panoptic_on=False,
+        # For now, the dataset does not support
+        # evaluating semantic segmentation metric.
+        semantic_on=False,
+        instance_on=True,
+        # max_per_image is for instance segmentation.
+        max_per_image=num_queries,
+        iou_thr=0.8,
+        # In Mask2Former's panoptic postprocessing,
+        # it will filter mask area where score is less than 0.5 .
+        filter_low_score=True),
+    init_cfg=None)
+
+
+model_cfg = dict(
+    type='MMDetPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    whole_model=model,
+)
+
+task_name = 'ssdd_ins'
+exp_name = 'E20230527_0'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='mask2former',
+    name=exp_name
+)
+# logger = None
+
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=4,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=10,
+    check_val_every_n_epoch=10,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 8
+train_num_workers = 4
+test_batch_size_per_gpu = 8
+test_num_workers = 4
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/SSDD'
+
+dataset_type = 'SSDDInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/SSDD_instances_val.json',
+            data_prefix=dict(img_path='imgs'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/SSDD_instances_train.json',
+            data_prefix=dict(img_path='imgs'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    test_loader=val_loader,
+    predict_loader=val_loader
+)

configs/rsprompter/mask2former_whu_config.py

@@ -0,0 +1,335 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+max_epochs = 400
+
+optimizer = dict(
+    type='AdamW',
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    )
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+    type='CocoPLMetric',
+    metric=['bbox', 'segm'],
+    proposal_nums=[1, 10, 100],
+)
+
+evaluator = dict(
+    val_evaluator=evaluator_,
+    test_evaluator=evaluator_,
+)
+
+
+image_size = (512, 512)
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 1
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+num_queries = 90
+
+model = dict(
+    type='mmdet.Mask2Former',
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='mmdet.ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=-1,
+        norm_cfg=dict(type='BN', requires_grad=False),
+        norm_eval=True,
+        style='pytorch',
+        init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')),
+    panoptic_head=dict(
+        type='mmdet.Mask2FormerHead',
+        in_channels=[256, 512, 1024, 2048],  # pass to pixel_decoder inside
+        strides=[4, 8, 16, 32],
+        feat_channels=256,
+        out_channels=256,
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        num_queries=num_queries,
+        num_transformer_feat_level=3,
+        pixel_decoder=dict(
+            type='mmdet.MSDeformAttnPixelDecoder',
+            num_outs=3,
+            norm_cfg=dict(type='GN', num_groups=32),
+            act_cfg=dict(type='ReLU'),
+            encoder=dict(  # DeformableDetrTransformerEncoder
+                num_layers=3,
+                layer_cfg=dict(  # DeformableDetrTransformerEncoderLayer
+                    self_attn_cfg=dict(  # MultiScaleDeformableAttention
+                        embed_dims=256,
+                        num_heads=8,
+                        num_levels=3,
+                        num_points=4,
+                        dropout=0.0,
+                        batch_first=True),
+                    ffn_cfg=dict(
+                        embed_dims=256,
+                        feedforward_channels=1024,
+                        num_fcs=2,
+                        ffn_drop=0.0,
+                        act_cfg=dict(type='ReLU', inplace=True)))),
+            positional_encoding=dict(num_feats=128, normalize=True)),
+        enforce_decoder_input_project=False,
+        positional_encoding=dict(num_feats=128, normalize=True),
+        transformer_decoder=dict(  # Mask2FormerTransformerDecoder
+            return_intermediate=True,
+            num_layers=3,
+            layer_cfg=dict(  # Mask2FormerTransformerDecoderLayer
+                self_attn_cfg=dict(  # MultiheadAttention
+                    embed_dims=256,
+                    num_heads=8,
+                    dropout=0.0,
+                    batch_first=True),
+                cross_attn_cfg=dict(  # MultiheadAttention
+                    embed_dims=256,
+                    num_heads=8,
+                    dropout=0.0,
+                    batch_first=True),
+                ffn_cfg=dict(
+                    embed_dims=256,
+                    feedforward_channels=2048,
+                    num_fcs=2,
+                    ffn_drop=0.0,
+                    act_cfg=dict(type='ReLU', inplace=True))),
+            init_cfg=None),
+        loss_cls=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=False,
+            loss_weight=2.0,
+            reduction='mean',
+            class_weight=[1.0] * num_classes + [0.1]),
+        loss_mask=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=True,
+            reduction='mean',
+            loss_weight=5.0),
+        loss_dice=dict(
+            type='mmdet.DiceLoss',
+            use_sigmoid=True,
+            activate=True,
+            reduction='mean',
+            naive_dice=True,
+            eps=1.0,
+            loss_weight=5.0)),
+    panoptic_fusion_head=dict(
+        type='mmdet.MaskFormerFusionHead',
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        loss_panoptic=None,
+        init_cfg=None),
+    train_cfg=dict(
+        num_points=12544,
+        oversample_ratio=3.0,
+        importance_sample_ratio=0.75,
+        assigner=dict(
+            type='mmdet.HungarianAssigner',
+            match_costs=[
+                dict(type='mmdet.ClassificationCost', weight=2.0),
+                dict(
+                    type='mmdet.CrossEntropyLossCost', weight=5.0, use_sigmoid=True),
+                dict(type='mmdet.DiceCost', weight=5.0, pred_act=True, eps=1.0)
+            ]),
+        sampler=dict(type='mmdet.MaskPseudoSampler')),
+    test_cfg=dict(
+        panoptic_on=False,
+        # For now, the dataset does not support
+        # evaluating semantic segmentation metric.
+        semantic_on=False,
+        instance_on=True,
+        # max_per_image is for instance segmentation.
+        max_per_image=100,
+        iou_thr=0.8,
+        # In Mask2Former's panoptic postprocessing,
+        # it will filter mask area where score is less than 0.5 .
+        filter_low_score=True),
+    init_cfg=None)
+
+
+model_cfg = dict(
+    type='MMDetPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    whole_model=model,
+)
+
+task_name = 'whu_ins'
+exp_name = 'E20230525_1'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='mask2former',
+    name=exp_name
+)
+# logger = None
+
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valmap_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valmap_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=4,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=20,
+    check_val_every_n_epoch=10,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 8
+train_num_workers = 4
+test_batch_size_per_gpu = 8
+test_num_workers = 4
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/WHU'
+train_data_prefix = 'train/'
+val_data_prefix = 'test/'
+
+dataset_type = 'WHUInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/WHU_building_test.json',
+            data_prefix=dict(img_path=val_data_prefix + '/image', seg_path=val_data_prefix + '/label'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/WHU_building_train.json',
+            data_prefix=dict(img_path=train_data_prefix + '/image', seg_path=train_data_prefix + '/label'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    test_loader=val_loader,
+    predict_loader=val_loader
+)

configs/rsprompter/maskrcnn_nwpu_config.py

@@ -0,0 +1,339 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models', 'mmdet.models'], allow_failed_imports=False)
+max_epochs = 500
+
+optimizer = dict(
+    type='AdamW',
+    lr=0.0005,
+    weight_decay=1e-4
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    )
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    val_evaluator=evaluator_,
+    test_evaluator=evaluator_
+)
+
+
+image_size = (1024, 1024)
+data_preprocessor = dict(
+        type='mmdet.DetDataPreprocessor',
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        pad_mask=True,
+        mask_pad_value=0,
+        pad_size_divisor=32
+)
+
+num_things_classes = 10
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+
+# model settings
+model = dict(
+    type='mmdet.MaskRCNN',
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='mmdet.ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=1,
+        norm_cfg=dict(type='BN', requires_grad=True),
+        norm_eval=True,
+        style='pytorch',
+        init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')
+    ),
+    neck=dict(
+        type='mmdet.FPN',
+        in_channels=[256, 512, 1024, 2048],
+        out_channels=256,
+        num_outs=5),
+    rpn_head=dict(
+        type='mmdet.RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='mmdet.AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='mmdet.DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='mmdet.CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0)),
+    roi_head=dict(
+        type='mmdet.StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='mmdet.SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='mmdet.Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=num_classes,
+            bbox_coder=dict(
+                type='mmdet.DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=False,
+            loss_cls=dict(
+                type='mmdet.CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0)),
+        mask_roi_extractor=dict(
+            type='mmdet.SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        mask_head=dict(
+            type='mmdet.FCNMaskHead',
+            num_convs=4,
+            in_channels=256,
+            conv_out_channels=256,
+            num_classes=num_classes,
+            loss_mask=dict(
+                type='mmdet.CrossEntropyLoss', use_mask=True, loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='mmdet.MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='mmdet.RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='mmdet.MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='mmdet.RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            mask_size=28,
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.05,
+            nms=dict(type='nms', iou_threshold=0.5),
+            max_per_img=100,
+            mask_thr_binary=0.5)
+    )
+)
+
+
+model_cfg = dict(
+    type='MMDetPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    whole_model=model,
+)
+
+task_name = 'nwpu_ins'
+exp_name = 'E20230520_0'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='maskrcnn',
+    name=exp_name
+)
+# logger = None
+
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valmap_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valmap_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="cpu",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=1,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=3,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 2
+train_num_workers = 4
+test_batch_size_per_gpu = 2
+test_num_workers = 4
+persistent_workers = True
+
+data_parent = '/Users/kyanchen/datasets/seg/VHR-10_dataset_coco/NWPUVHR-10_dataset/'
+train_data_prefix = ''
+val_data_prefix = ''
+
+dataset_type = 'NWPUInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_val.json',
+            data_prefix=dict(img_path='positive image set'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_train.json',
+            data_prefix=dict(img_path='positive image set'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    test_loader=val_loader,
+    predict_loader=val_loader
+)

configs/rsprompter/maskrcnn_ssdd_config.py

@@ -0,0 +1,345 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models', 'mmdet.models'], allow_failed_imports=False)
+
+max_epochs = 500
+
+optimizer = dict(
+    type='AdamW',
+    lr=0.0005,
+    weight_decay=1e-4
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    )
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (512, 512)
+data_preprocessor = dict(
+        type='mmdet.DetDataPreprocessor',
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        pad_mask=True,
+        mask_pad_value=0,
+        pad_size_divisor=32
+)
+
+num_things_classes = 1
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+num_queries = 100
+
+# model settings
+model = dict(
+    type='mmdet.MaskRCNN',
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='mmdet.ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=-1,
+        norm_cfg=dict(type='BN', requires_grad=True),
+        norm_eval=True,
+        style='pytorch',
+        init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')
+    ),
+    neck=dict(
+        type='mmdet.FPN',
+        in_channels=[256, 512, 1024, 2048],
+        out_channels=256,
+        num_outs=5),
+    rpn_head=dict(
+        type='mmdet.RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='mmdet.AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='mmdet.DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='mmdet.CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0)),
+    roi_head=dict(
+        type='mmdet.StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='mmdet.SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='mmdet.Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=num_classes,
+            bbox_coder=dict(
+                type='mmdet.DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=False,
+            loss_cls=dict(
+                type='mmdet.CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0)),
+        mask_roi_extractor=dict(
+            type='mmdet.SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        mask_head=dict(
+            type='mmdet.FCNMaskHead',
+            num_convs=4,
+            in_channels=256,
+            conv_out_channels=256,
+            num_classes=num_classes,
+            loss_mask=dict(
+                type='mmdet.CrossEntropyLoss', use_mask=True, loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='mmdet.MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='mmdet.RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='mmdet.MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='mmdet.RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            mask_size=28,
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.05,
+            nms=dict(type='nms', iou_threshold=0.5),
+            max_per_img=100,
+            mask_thr_binary=0.5)
+    )
+)
+
+
+model_cfg = dict(
+    type='MMDetPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    whole_model=model,
+)
+
+task_name = 'ssdd_ins'
+exp_name = 'E20230526_0'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='maskrcnn',
+    name=exp_name
+)
+# logger = None
+
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valmap_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valmap_0:.4f}'
+        # mode='min',
+        # monitor='train_loss',
+        # save_top_k=2,
+        # filename='epoch_{epoch}-trainloss_{train_loss:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=4,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=10,
+    check_val_every_n_epoch=10,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=1,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 8
+train_num_workers = 4
+test_batch_size_per_gpu = 8
+test_num_workers = 4
+persistent_workers = True
+
+data_parent = '/Users/kyanchen/datasets/seg/SSDD'
+# data_parent = '/mnt/search01/dataset/cky_data/SSDD'
+
+dataset_type = 'SSDDInsSegDataset'
+
+val_loader = dict(
+    batch_size=test_batch_size_per_gpu,
+    num_workers=test_num_workers,
+    persistent_workers=persistent_workers,
+    pin_memory=True,
+    dataset=dict(
+        type=dataset_type,
+        data_root=data_parent,
+        ann_file='annotations/SSDD_instances_val.json',
+        data_prefix=dict(img_path='imgs'),
+        test_mode=True,
+        filter_cfg=dict(filter_empty_gt=True, min_size=32),
+        pipeline=test_pipeline,
+        backend_args=backend_args
+    )
+)
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/SSDD_instances_train.json',
+            data_prefix=dict(img_path='imgs'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    test_loader=val_loader,
+    predict_loader=val_loader
+)

configs/rsprompter/maskrcnn_whu_config.py

@@ -0,0 +1,349 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models', 'mmdet.models'], allow_failed_imports=False)
+
+max_epochs = 150
+
+optimizer = dict(
+    type='AdamW',
+    lr=0.0005,
+    weight_decay=1e-4
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    )
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+        type='MeanAveragePrecision',
+        # iou_type='segm',
+        iou_type='bbox',
+        # dist_sync_on_step=True,
+        # compute_on_cpu=True,
+)
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (512, 512)
+data_preprocessor = dict(
+        type='mmdet.DetDataPreprocessor',
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True,
+        pad_mask=True,
+        mask_pad_value=0,
+        pad_size_divisor=32
+)
+
+num_things_classes = 1
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+num_queries = 90
+
+# model settings
+model = dict(
+    type='mmdet.MaskRCNN',
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='mmdet.ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=-1,
+        norm_cfg=dict(type='BN', requires_grad=True),
+        norm_eval=True,
+        style='pytorch',
+        init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')
+    ),
+    neck=dict(
+        type='mmdet.FPN',
+        in_channels=[256, 512, 1024, 2048],
+        out_channels=256,
+        num_outs=5),
+    rpn_head=dict(
+        type='mmdet.RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='mmdet.AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='mmdet.DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='mmdet.CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0)),
+    roi_head=dict(
+        type='mmdet.StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='mmdet.SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='mmdet.Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=num_classes,
+            bbox_coder=dict(
+                type='mmdet.DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=False,
+            loss_cls=dict(
+                type='mmdet.CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0)),
+        mask_roi_extractor=dict(
+            type='mmdet.SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        mask_head=dict(
+            type='mmdet.FCNMaskHead',
+            num_convs=4,
+            in_channels=256,
+            conv_out_channels=256,
+            num_classes=num_classes,
+            loss_mask=dict(
+                type='mmdet.CrossEntropyLoss', use_mask=True, loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='mmdet.MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='mmdet.RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='mmdet.MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='mmdet.RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            mask_size=28,
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.05,
+            nms=dict(type='nms', iou_threshold=0.5),
+            max_per_img=100,
+            mask_thr_binary=0.5)
+    )
+)
+
+
+model_cfg = dict(
+    type='MMDetPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    whole_model=model,
+)
+
+task_name = 'whu_ins'
+exp_name = 'E20230525_0'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='maskrcnn',
+    name=exp_name
+)
+# logger = None
+
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valmap_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valmap_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=4,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=20,
+    check_val_every_n_epoch=10,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=1,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 8
+train_num_workers = 4
+test_batch_size_per_gpu = 8
+test_num_workers = 4
+persistent_workers = True
+
+data_parent = '/Users/kyanchen/datasets/Building/WHU'
+train_data_prefix = 'train/'
+val_data_prefix = 'test/'
+
+dataset_type = 'WHUInsSegDataset'
+
+val_loader = dict(
+    batch_size=test_batch_size_per_gpu,
+    num_workers=test_num_workers,
+    persistent_workers=persistent_workers,
+    pin_memory=True,
+    dataset=dict(
+        type=dataset_type,
+        data_root=data_parent,
+        ann_file='annotations/WHU_building_test.json',
+        data_prefix=dict(img_path=val_data_prefix+'/image', seg_path=val_data_prefix+'/label'),
+        test_mode=True,
+        filter_cfg=dict(filter_empty_gt=True, min_size=32),
+        pipeline=test_pipeline,
+        backend_args=backend_args,
+    )
+)
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/WHU_building_train.json',
+            data_prefix=dict(img_path=train_data_prefix+'/image', seg_path=train_data_prefix+'/label'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    test_loader=val_loader,
+    predict_loader=val_loader
+)

configs/rsprompter/predict_rsprompter_anchor_nwpu.py

@@ -0,0 +1,277 @@
+custom_imports = dict(
+    imports=['mmseg.datasets', 'mmseg.models', 'mmdet.models'],
+    allow_failed_imports=False)
+
+sub_model_train = [
+    'panoptic_head',
+    'data_preprocessor'
+]
+
+sub_model_optim = {
+    'panoptic_head': {'lr_mult': 1},
+}
+
+
+max_epochs = 1200
+optimizer = dict(type='AdamW', lr=0.0005, weight_decay=0.0001)
+param_scheduler = [
+    dict(
+        type='LinearLR',
+        start_factor=0.0005,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        convert_to_iter_based=True),
+    dict(type='CosineAnnealingLR', T_max=120, by_epoch=True, begin=1, end=120)
+]
+
+param_scheduler_callback = dict(type='ParamSchedulerHook')
+evaluator_ = dict(type='MeanAveragePrecision', iou_type='segm')
+evaluator = dict(
+    val_evaluator=dict(type='MeanAveragePrecision', iou_type='segm'))
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 10
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+prompt_shape = (60, 4)
+
+
+model_cfg = dict(
+    type='SegSAMAnchorPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='vit_h',
+        checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    ),
+    panoptic_head=dict(
+        type='SAMAnchorInstanceHead',
+        neck=dict(
+            type='SAMAggregatorNeck',
+            in_channels=[1280] * 32,
+            # in_channels=[768] * 12,
+            inner_channels=32,
+            selected_channels=range(4, 32, 2),
+            # selected_channels=range(4, 12, 2),
+            out_channels=256,
+            up_sample_scale=4,
+        ),
+        rpn_head=dict(
+            type='mmdet.RPNHead',
+            in_channels=256,
+            feat_channels=256,
+            anchor_generator=dict(
+                type='mmdet.AnchorGenerator',
+                scales=[2, 4, 8, 16, 32, 64],
+                ratios=[0.5, 1.0, 2.0],
+                strides=[8, 16, 32]),
+            bbox_coder=dict(
+                type='mmdet.DeltaXYWHBBoxCoder',
+                target_means=[.0, .0, .0, .0],
+                target_stds=[1.0, 1.0, 1.0, 1.0]),
+            loss_cls=dict(
+                type='mmdet.CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+            loss_bbox=dict(type='mmdet.SmoothL1Loss', loss_weight=1.0)),
+        roi_head=dict(
+            type='SAMAnchorPromptRoIHead',
+            bbox_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            bbox_head=dict(
+                type='mmdet.Shared2FCBBoxHead',
+                in_channels=256,
+                fc_out_channels=1024,
+                roi_feat_size=7,
+                num_classes=num_classes,
+                bbox_coder=dict(
+                    type='mmdet.DeltaXYWHBBoxCoder',
+                    target_means=[0., 0., 0., 0.],
+                    target_stds=[0.1, 0.1, 0.2, 0.2]),
+                reg_class_agnostic=False,
+                loss_cls=dict(
+                    type='mmdet.CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+                loss_bbox=dict(type='mmdet.SmoothL1Loss', loss_weight=1.0)),
+            mask_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            mask_head=dict(
+                type='SAMPromptMaskHead',
+                per_query_point=prompt_shape[1],
+                with_sincos=True,
+                class_agnostic=True,
+                loss_mask=dict(
+                    type='mmdet.CrossEntropyLoss', use_mask=True, loss_weight=1.0))),
+        # model training and testing settings
+        train_cfg=dict(
+            rpn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.7,
+                    neg_iou_thr=0.3,
+                    min_pos_iou=0.3,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=512,
+                    pos_fraction=0.5,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=False),
+                allowed_border=-1,
+                pos_weight=-1,
+                debug=False),
+            rpn_proposal=dict(
+                nms_pre=2000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.5,
+                    neg_iou_thr=0.5,
+                    min_pos_iou=0.5,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=256,
+                    pos_fraction=0.25,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=True),
+                mask_size=1024,
+                pos_weight=-1,
+                debug=False)),
+        test_cfg=dict(
+            rpn=dict(
+                nms_pre=1000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                score_thr=0.05,
+                nms=dict(type='nms', iou_threshold=0.5),
+                max_per_img=100,
+                mask_thr_binary=0.5)
+        )
+    )
+)
+
+
+task_name = 'nwpu_ins'
+exp_name = 'rsprompter_anchor_E20230601_0'
+callbacks = [
+    dict(
+        type='DetVisualizationHook',
+        draw=True,
+        interval=1,
+        score_thr=0.1,
+        show=False,
+        wait_time=1.,
+        test_out_dir='visualization',
+    )
+]
+
+
+vis_backends = [dict(type='mmdet.LocalVisBackend')]
+visualizer = dict(
+    type='mmdet.DetLocalVisualizer',
+    vis_backends=vis_backends,
+    name='visualizer',
+    fig_save_cfg=dict(
+        frameon=False,
+        figsize=(40, 20),
+        # dpi=300,
+    ),
+    line_width=2,
+    alpha=0.8
+)
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator='auto',
+    strategy='auto',
+    devices=[0],
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    max_epochs=120,
+    logger=None,
+    callbacks=callbacks,
+    log_every_n_steps=20,
+    check_val_every_n_epoch=10,
+    benchmark=True,
+    use_distributed_sampler=True)
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+train_batch_size_per_gpu = 8
+train_num_workers = 4
+test_batch_size_per_gpu = 2
+test_num_workers = 0
+persistent_workers = False
+
+data_parent = '/mnt/search01/dataset/cky_data/NWPU10'
+train_data_prefix = ''
+val_data_prefix = ''
+
+dataset_type = 'NWPUInsSegDataset'
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_val.json',
+            data_prefix=dict(img_path='positive image set'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    predict_loader=val_loader,
+)

configs/rsprompter/rsprompter_anchor_nwpu_config.py

@@ -0,0 +1,345 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'panoptic_head',
+    'data_preprocessor'
+]
+
+sub_model_optim = {
+    'panoptic_head': {'lr_mult': 1},
+}
+
+max_epochs = 1200
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 10
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+prompt_shape = (60, 4)
+
+model_cfg = dict(
+    type='SegSAMAnchorPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='vit_h',
+        checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    ),
+    panoptic_head=dict(
+        type='SAMAnchorInstanceHead',
+        neck=dict(
+            type='SAMAggregatorNeck',
+            in_channels=[1280] * 32,
+            # in_channels=[768] * 12,
+            inner_channels=32,
+            selected_channels=range(8, 32, 2),
+            # selected_channels=range(4, 12, 2),
+            out_channels=256,
+            up_sample_scale=4,
+        ),
+        rpn_head=dict(
+            type='mmdet.RPNHead',
+            in_channels=256,
+            feat_channels=256,
+            anchor_generator=dict(
+                type='mmdet.AnchorGenerator',
+                scales=[2, 4, 8, 16, 32, 64],
+                ratios=[0.5, 1.0, 2.0],
+                strides=[8, 16, 32]),
+            bbox_coder=dict(
+                type='mmdet.DeltaXYWHBBoxCoder',
+                target_means=[.0, .0, .0, .0],
+                target_stds=[1.0, 1.0, 1.0, 1.0]),
+            loss_cls=dict(
+                type='mmdet.CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+            loss_bbox=dict(type='mmdet.SmoothL1Loss', loss_weight=1.0)),
+        roi_head=dict(
+            type='SAMAnchorPromptRoIHead',
+            bbox_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            bbox_head=dict(
+                type='mmdet.Shared2FCBBoxHead',
+                in_channels=256,
+                fc_out_channels=1024,
+                roi_feat_size=7,
+                num_classes=num_classes,
+                bbox_coder=dict(
+                    type='mmdet.DeltaXYWHBBoxCoder',
+                    target_means=[0., 0., 0., 0.],
+                    target_stds=[0.1, 0.1, 0.2, 0.2]),
+                reg_class_agnostic=False,
+                loss_cls=dict(
+                    type='mmdet.CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+                loss_bbox=dict(type='mmdet.SmoothL1Loss', loss_weight=1.0)),
+            mask_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            mask_head=dict(
+                type='SAMPromptMaskHead',
+                per_query_point=prompt_shape[1],
+                with_sincos=True,
+                class_agnostic=True,
+                loss_mask=dict(
+                    type='mmdet.CrossEntropyLoss', use_mask=True, loss_weight=1.0))),
+        # model training and testing settings
+        train_cfg=dict(
+            rpn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.7,
+                    neg_iou_thr=0.3,
+                    min_pos_iou=0.3,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=512,
+                    pos_fraction=0.5,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=False),
+                allowed_border=-1,
+                pos_weight=-1,
+                debug=False),
+            rpn_proposal=dict(
+                nms_pre=2000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.5,
+                    neg_iou_thr=0.5,
+                    min_pos_iou=0.5,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=256,
+                    pos_fraction=0.25,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=True),
+                mask_size=1024,
+                pos_weight=-1,
+                debug=False)),
+        test_cfg=dict(
+            rpn=dict(
+                nms_pre=1000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                score_thr=0.05,
+                nms=dict(type='nms', iou_threshold=0.5),
+                max_per_img=100,
+                mask_thr_binary=0.5)
+        )
+    )
+)
+
+
+task_name = 'nwpu_ins'
+exp_name = 'E20230629_1'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='sam-anchor',
+    name=exp_name
+)
+
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=3,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=5,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 2
+train_num_workers = 2
+test_batch_size_per_gpu = 2
+test_num_workers = 2
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/NWPU10'
+train_data_prefix = ''
+val_data_prefix = ''
+dataset_type = 'NWPUInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_val.json',
+            data_prefix=dict(img_path='positive image set'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_train.json',
+            data_prefix=dict(img_path='positive image set'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/rsprompter/rsprompter_anchor_ssdd_config.py

@@ -0,0 +1,347 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'panoptic_head',
+    'data_preprocessor'
+]
+
+sub_model_optim = {
+    'panoptic_head': {'lr_mult': 1},
+}
+
+max_epochs = 1000
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 1
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+prompt_shape = (30, 4)
+
+model_cfg = dict(
+    type='SegSAMAnchorPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='vit_h',
+        checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    ),
+    panoptic_head=dict(
+        type='SAMAnchorInstanceHead',
+        neck=dict(
+            type='SAMAggregatorNeck',
+            in_channels=[1280] * 32,
+            # in_channels=[768] * 12,
+            inner_channels=32,
+            selected_channels=range(8, 32, 2),
+            # selected_channels=range(4, 12, 2),
+            out_channels=256,
+            up_sample_scale=4,
+        ),
+        rpn_head=dict(
+            type='mmdet.RPNHead',
+            in_channels=256,
+            feat_channels=256,
+            anchor_generator=dict(
+                type='mmdet.AnchorGenerator',
+                scales=[2, 4, 8, 16, 32, 64],
+                ratios=[0.5, 1.0, 2.0],
+                strides=[8, 16, 32]),
+            bbox_coder=dict(
+                type='mmdet.DeltaXYWHBBoxCoder',
+                target_means=[.0, .0, .0, .0],
+                target_stds=[1.0, 1.0, 1.0, 1.0]),
+            loss_cls=dict(
+                type='mmdet.CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+            loss_bbox=dict(type='mmdet.SmoothL1Loss', loss_weight=1.0)),
+        roi_head=dict(
+            type='SAMAnchorPromptRoIHead',
+            bbox_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            bbox_head=dict(
+                type='mmdet.Shared2FCBBoxHead',
+                in_channels=256,
+                fc_out_channels=1024,
+                roi_feat_size=7,
+                num_classes=num_classes,
+                bbox_coder=dict(
+                    type='mmdet.DeltaXYWHBBoxCoder',
+                    target_means=[0., 0., 0., 0.],
+                    target_stds=[0.1, 0.1, 0.2, 0.2]),
+                reg_class_agnostic=False,
+                loss_cls=dict(
+                    type='mmdet.CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+                loss_bbox=dict(type='mmdet.SmoothL1Loss', loss_weight=1.0)),
+            mask_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            mask_head=dict(
+                type='SAMPromptMaskHead',
+                per_query_point=prompt_shape[1],
+                with_sincos=True,
+                class_agnostic=True,
+                loss_mask=dict(
+                    type='mmdet.CrossEntropyLoss', use_mask=True, loss_weight=1.0))),
+        # model training and testing settings
+        train_cfg=dict(
+            rpn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.7,
+                    neg_iou_thr=0.3,
+                    min_pos_iou=0.3,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=512,
+                    pos_fraction=0.5,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=False),
+                allowed_border=-1,
+                pos_weight=-1,
+                debug=False),
+            rpn_proposal=dict(
+                nms_pre=2000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.5,
+                    neg_iou_thr=0.5,
+                    min_pos_iou=0.5,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=256,
+                    pos_fraction=0.25,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=True),
+                mask_size=1024,
+                pos_weight=-1,
+                debug=False)),
+        test_cfg=dict(
+            rpn=dict(
+                nms_pre=1000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                score_thr=0.05,
+                nms=dict(type='nms', iou_threshold=0.5),
+                max_per_img=100,
+                mask_thr_binary=0.5)
+        )
+    )
+)
+
+task_name = 'ssdd_ins'
+exp_name = 'E20230629_2'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='sam-anchor',
+    name=exp_name
+)
+
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=3,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=5,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 2
+train_num_workers = 2
+test_batch_size_per_gpu = 2
+test_num_workers = 2
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/SSDD'
+dataset_type = 'SSDDInsSegDataset'
+
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            # ann_file='NWPU_instances_val.json',
+            # data_prefix=dict(img_path='positive image set'),
+            ann_file='annotations/SSDD_instances_val.json',
+            data_prefix=dict(img_path='imgs'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            # ann_file='NWPU_instances_train.json',
+            # data_prefix=dict(img_path='positive image set'),
+            ann_file='annotations/SSDD_instances_train.json',
+            data_prefix=dict(img_path='imgs'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/rsprompter/rsprompter_anchor_whu_config.py

@@ -0,0 +1,355 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'panoptic_head',
+    'data_preprocessor'
+]
+
+sub_model_optim = {
+    'panoptic_head': {'lr_mult': 1},
+}
+
+max_epochs = 2000
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 1
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+prompt_shape = (90, 4)
+
+
+model_cfg = dict(
+    type='SegSAMAnchorPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='vit_h',
+        checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    ),
+    panoptic_head=dict(
+        type='SAMAnchorInstanceHead',
+        neck=dict(
+            type='SAMAggregatorNeck',
+            in_channels=[1280] * 32,
+            # in_channels=[768] * 12,
+            inner_channels=32,
+            selected_channels=range(4, 32, 2),
+            # selected_channels=range(4, 12, 2),
+            out_channels=256,
+            up_sample_scale=4,
+        ),
+        rpn_head=dict(
+            type='mmdet.RPNHead',
+            in_channels=256,
+            feat_channels=256,
+            anchor_generator=dict(
+                type='mmdet.AnchorGenerator',
+                scales=[2, 4, 8, 16, 32, 64],
+                ratios=[0.5, 1.0, 2.0],
+                strides=[8, 16, 32]),
+            bbox_coder=dict(
+                type='mmdet.DeltaXYWHBBoxCoder',
+                target_means=[.0, .0, .0, .0],
+                target_stds=[1.0, 1.0, 1.0, 1.0]),
+            loss_cls=dict(
+                type='mmdet.CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+            loss_bbox=dict(type='mmdet.SmoothL1Loss', loss_weight=1.0)),
+        roi_head=dict(
+            type='SAMAnchorPromptRoIHead',
+            bbox_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            bbox_head=dict(
+                type='mmdet.Shared2FCBBoxHead',
+                in_channels=256,
+                fc_out_channels=1024,
+                roi_feat_size=7,
+                num_classes=num_classes,
+                bbox_coder=dict(
+                    type='mmdet.DeltaXYWHBBoxCoder',
+                    target_means=[0., 0., 0., 0.],
+                    target_stds=[0.1, 0.1, 0.2, 0.2]),
+                reg_class_agnostic=False,
+                loss_cls=dict(
+                    type='mmdet.CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+                loss_bbox=dict(type='mmdet.SmoothL1Loss', loss_weight=1.0)),
+            mask_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            mask_head=dict(
+                type='SAMPromptMaskHead',
+                per_query_point=prompt_shape[1],
+                with_sincos=True,
+                class_agnostic=True,
+                loss_mask=dict(
+                    type='mmdet.CrossEntropyLoss', use_mask=True, loss_weight=1.0))),
+        # model training and testing settings
+        train_cfg=dict(
+            rpn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.7,
+                    neg_iou_thr=0.3,
+                    min_pos_iou=0.3,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=512,
+                    pos_fraction=0.5,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=False),
+                allowed_border=-1,
+                pos_weight=-1,
+                debug=False),
+            rpn_proposal=dict(
+                nms_pre=2000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.5,
+                    neg_iou_thr=0.5,
+                    min_pos_iou=0.5,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=256,
+                    pos_fraction=0.25,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=True),
+                mask_size=1024,
+                pos_weight=-1,
+                debug=False)),
+        test_cfg=dict(
+            rpn=dict(
+                nms_pre=1000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                score_thr=0.05,
+                nms=dict(type='nms', iou_threshold=0.5),
+                max_per_img=100,
+                mask_thr_binary=0.5)
+        )
+    )
+)
+
+task_name = 'whu_ins'
+exp_name = 'E20230629_0'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='sam-anchor',
+    name=exp_name
+)
+
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=3,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=10,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 2
+train_num_workers = 2
+test_batch_size_per_gpu = 2
+test_num_workers = 2
+persistent_workers = True
+
+
+data_parent = '/mnt/search01/dataset/cky_data/WHU'
+train_data_prefix = 'train/'
+val_data_prefix = 'test/'
+dataset_type = 'WHUInsSegDataset'
+
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            # ann_file='NWPU_instances_val.json',
+            # data_prefix=dict(img_path='positive image set'),
+            # ann_file='annotations/SSDD_instances_val.json',
+            # data_prefix=dict(img_path='imgs'),
+            ann_file='annotations/WHU_building_test.json',
+            data_prefix=dict(img_path=val_data_prefix + '/image'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            # ann_file='NWPU_instances_train.json',
+            # data_prefix=dict(img_path='positive image set'),
+            # ann_file='annotations/SSDD_instances_train.json',
+            # data_prefix=dict(img_path='imgs'),
+            ann_file='annotations/WHU_building_train.json',
+            data_prefix=dict(img_path=train_data_prefix + '/image'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/rsprompter/rsprompter_query_nwpu_config.py

@@ -0,0 +1,300 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'panoptic_head',
+    'panoptic_fusion_head',
+    'data_preprocessor'
+]
+
+sub_model_optim = {
+    'panoptic_head': {'lr_mult': 1},
+    'panoptic_fusion_head': {'lr_mult': 1},
+}
+
+max_epochs = 5000
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 10
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+prompt_shape = (60, 4)
+
+
+model_cfg = dict(
+    type='SegSAMPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='vit_h',
+        checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    ),
+    panoptic_head=dict(
+        type='SAMInstanceHead',
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        with_multiscale=True,
+        with_sincos=True,
+        prompt_neck=dict(
+            type='SAMTransformerEDPromptGenNeck',
+            prompt_shape=prompt_shape,
+            in_channels=[1280] * 32,
+            inner_channels=32,
+            selected_channels=range(4, 32, 2),
+            # in_channels=[768] * 8,
+            num_encoders=1,
+            num_decoders=4,
+            out_channels=256
+        ),
+        loss_cls=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=False,
+            loss_weight=2.0,
+            reduction='mean',
+            class_weight=[1.0] * num_classes + [0.1]),
+        loss_mask=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=True,
+            reduction='mean',
+            loss_weight=5.0),
+        loss_dice=dict(
+            type='mmdet.DiceLoss',
+            use_sigmoid=True,
+            activate=True,
+            reduction='mean',
+            naive_dice=True,
+            eps=1.0,
+            loss_weight=5.0)),
+    panoptic_fusion_head=dict(
+        type='mmdet.MaskFormerFusionHead',
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        loss_panoptic=None,
+        init_cfg=None),
+    train_cfg=dict(
+        num_points=12544,
+        oversample_ratio=3.0,
+        importance_sample_ratio=0.75,
+        assigner=dict(
+            type='mmdet.HungarianAssigner',
+            match_costs=[
+                dict(type='mmdet.ClassificationCost', weight=2.0),
+                dict(
+                    type='mmdet.CrossEntropyLossCost', weight=5.0, use_sigmoid=True),
+                dict(type='mmdet.DiceCost', weight=5.0, pred_act=True, eps=1.0)
+            ]),
+        sampler=dict(type='mmdet.MaskPseudoSampler')),
+    test_cfg=dict(
+        panoptic_on=False,
+        # For now, the dataset does not support
+        # evaluating semantic segmentation metric.
+        semantic_on=False,
+        instance_on=True,
+        # max_per_image is for instance segmentation.
+        max_per_image=prompt_shape[0],
+        iou_thr=0.8,
+        # In Mask2Former's panoptic postprocessing,
+        # it will filter mask area where score is less than 0.5 .
+        filter_low_score=True),
+)
+
+task_name = 'nwpu_ins'
+exp_name = 'E20230623_1'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='sam-query',
+    name=exp_name
+)
+
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=3,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=5,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 3
+train_num_workers = 2
+test_batch_size_per_gpu = 3
+test_num_workers = 2
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/NWPU10'
+train_data_prefix = ''
+val_data_prefix = ''
+
+dataset_type = 'NWPUInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_val.json',
+            data_prefix=dict(img_path='positive image set'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_train.json',
+            data_prefix=dict(img_path='positive image set'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/rsprompter/rsprompter_query_ssdd_config.py

@@ -0,0 +1,298 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'panoptic_head',
+    'panoptic_fusion_head',
+    'data_preprocessor'
+]
+
+sub_model_optim = {
+    'panoptic_head': {'lr_mult': 1},
+    'panoptic_fusion_head': {'lr_mult': 1},
+}
+
+max_epochs = 5000
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 1
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+prompt_shape = (30, 4)
+
+
+model_cfg = dict(
+    type='SegSAMPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='vit_h',
+        checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    ),
+    panoptic_head=dict(
+        type='SAMInstanceHead',
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        with_multiscale=True,
+        with_sincos=True,
+        prompt_neck=dict(
+            type='SAMTransformerEDPromptGenNeck',
+            prompt_shape=prompt_shape,
+            in_channels=[1280] * 32,
+            inner_channels=32,
+            selected_channels=range(4, 32, 2),
+            # in_channels=[768] * 8,
+            num_encoders=1,
+            num_decoders=4,
+            out_channels=256
+        ),
+        loss_cls=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=False,
+            loss_weight=2.0,
+            reduction='mean',
+            class_weight=[1.0] * num_classes + [0.1]),
+        loss_mask=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=True,
+            reduction='mean',
+            loss_weight=5.0),
+        loss_dice=dict(
+            type='mmdet.DiceLoss',
+            use_sigmoid=True,
+            activate=True,
+            reduction='mean',
+            naive_dice=True,
+            eps=1.0,
+            loss_weight=5.0)),
+    panoptic_fusion_head=dict(
+        type='mmdet.MaskFormerFusionHead',
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        loss_panoptic=None,
+        init_cfg=None),
+    train_cfg=dict(
+        num_points=12544,
+        oversample_ratio=3.0,
+        importance_sample_ratio=0.75,
+        assigner=dict(
+            type='mmdet.HungarianAssigner',
+            match_costs=[
+                dict(type='mmdet.ClassificationCost', weight=2.0),
+                dict(
+                    type='mmdet.CrossEntropyLossCost', weight=5.0, use_sigmoid=True),
+                dict(type='mmdet.DiceCost', weight=5.0, pred_act=True, eps=1.0)
+            ]),
+        sampler=dict(type='mmdet.MaskPseudoSampler')),
+    test_cfg=dict(
+        panoptic_on=False,
+        # For now, the dataset does not support
+        # evaluating semantic segmentation metric.
+        semantic_on=False,
+        instance_on=True,
+        # max_per_image is for instance segmentation.
+        max_per_image=prompt_shape[0],
+        iou_thr=0.8,
+        # In Mask2Former's panoptic postprocessing,
+        # it will filter mask area where score is less than 0.5 .
+        filter_low_score=True),
+)
+
+task_name = 'ssdd_ins'
+exp_name = 'E20230527_1'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='sam',
+    name=exp_name
+)
+# logger = None
+
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=10,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 4
+train_num_workers = 2
+test_batch_size_per_gpu = 4
+test_num_workers = 2
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/SSDD'
+dataset_type = 'SSDDInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/SSDD_instances_val.json',
+            data_prefix=dict(img_path='imgs'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/SSDD_instances_train.json',
+            data_prefix=dict(img_path='imgs'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/rsprompter/rsprompter_query_whu_config.py

@@ -0,0 +1,303 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'panoptic_head',
+    'panoptic_fusion_head',
+    'data_preprocessor'
+]
+
+sub_model_optim = {
+    'panoptic_head': {'lr_mult': 1},
+    'panoptic_fusion_head': {'lr_mult': 1},
+}
+
+max_epochs = 5000
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    # train_evaluator=evaluator_,
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 1
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+prompt_shape = (90, 4)
+
+
+model_cfg = dict(
+    type='SegSAMPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='vit_h',
+        checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    ),
+    panoptic_head=dict(
+        type='SAMInstanceHead',
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        with_multiscale=True,
+        with_sincos=True,
+        prompt_neck=dict(
+            type='SAMTransformerEDPromptGenNeck',
+            prompt_shape=prompt_shape,
+            in_channels=[1280] * 32,
+            inner_channels=64,
+            selected_channels=range(4, 32, 2),
+            # in_channels=[768] * 8,
+            num_encoders=1,
+            num_decoders=4,
+            out_channels=256
+        ),
+        loss_cls=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=False,
+            loss_weight=2.0,
+            reduction='mean',
+            class_weight=[1.0] * num_classes + [0.1]),
+        loss_mask=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=True,
+            reduction='mean',
+            loss_weight=5.0),
+        loss_dice=dict(
+            type='mmdet.DiceLoss',
+            use_sigmoid=True,
+            activate=True,
+            reduction='mean',
+            naive_dice=True,
+            eps=1.0,
+            loss_weight=5.0)),
+    panoptic_fusion_head=dict(
+        type='mmdet.MaskFormerFusionHead',
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        loss_panoptic=None,
+        init_cfg=None),
+    train_cfg=dict(
+        num_points=12544,
+        oversample_ratio=3.0,
+        importance_sample_ratio=0.75,
+        assigner=dict(
+            type='mmdet.HungarianAssigner',
+            match_costs=[
+                dict(type='mmdet.ClassificationCost', weight=2.0),
+                dict(
+                    type='mmdet.CrossEntropyLossCost', weight=5.0, use_sigmoid=True),
+                dict(type='mmdet.DiceCost', weight=5.0, pred_act=True, eps=1.0)
+            ]),
+        sampler=dict(type='mmdet.MaskPseudoSampler')),
+    test_cfg=dict(
+        panoptic_on=False,
+        # For now, the dataset does not support
+        # evaluating semantic segmentation metric.
+        semantic_on=False,
+        instance_on=True,
+        # max_per_image is for instance segmentation.
+        max_per_image=80,
+        iou_thr=0.8,
+        # In Mask2Former's panoptic postprocessing,
+        # it will filter mask area where score is less than 0.5 .
+        filter_low_score=True),
+)
+
+task_name = 'whu_ins'
+exp_name = 'E20230603_0'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='sam',
+    name=exp_name
+)
+# logger = None
+
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=20,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 3
+train_num_workers = 2
+test_batch_size_per_gpu = 3
+test_num_workers = 2
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/WHU'
+train_data_prefix = 'train/'
+val_data_prefix = 'test/'
+
+dataset_type = 'WHUInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/WHU_building_test.json',
+            data_prefix=dict(img_path=val_data_prefix + '/image', seg_path=val_data_prefix + '/label'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/WHU_building_train.json',
+            data_prefix=dict(img_path=train_data_prefix + '/image', seg_path=train_data_prefix + '/label'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/rsprompter/samdet_fasterrcnn_nwpu_config.py

@@ -0,0 +1,338 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'whole_model'
+]
+
+sub_model_optim = {
+    'whole_model': {'lr_mult': 1},
+}
+
+max_epochs = 1000
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=5e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    # train_evaluator=evaluator_,
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 10
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+
+model = dict(
+    type='mmdet.FasterRCNN',
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='mmdet.ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=1,
+        norm_cfg=dict(type='BN', requires_grad=True),
+        norm_eval=True,
+        style='pytorch',
+        init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')),
+    neck=dict(
+        type='mmdet.FPN',
+        in_channels=[256, 512, 1024, 2048],
+        out_channels=256,
+        num_outs=5),
+    rpn_head=dict(
+        type='mmdet.RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='mmdet.AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='mmdet.DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='mmdet.CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0)),
+    roi_head=dict(
+        type='mmdet.StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='mmdet.SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='mmdet.Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=80,
+            bbox_coder=dict(
+                type='mmdet.DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=False,
+            loss_cls=dict(
+                type='mmdet.CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='mmdet.MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='mmdet.RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='mmdet.MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='mmdet.RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.05,
+            nms=dict(type='nms', iou_threshold=0.5),
+            max_per_img=100)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ))
+
+model_cfg = dict(
+    type='SegSAMDetPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    whole_model=model,
+    backbone=dict(
+        type='vit_h',
+        checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    )
+)
+
+task_name = 'nwpu_ins'
+exp_name = 'E20230531_9'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='samdet',
+    name=exp_name
+)
+# logger = None
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    # strategy="auto",
+    # strategy="ddp",
+    strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=5,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 4
+train_num_workers = 4
+test_batch_size_per_gpu = 4
+test_num_workers = 4
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/NWPU10'
+train_data_prefix = ''
+val_data_prefix = ''
+
+dataset_type = 'NWPUInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_val.json',
+            data_prefix=dict(img_path='positive image set'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_train.json',
+            data_prefix=dict(img_path='positive image set'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/rsprompter/samdet_fasterrcnn_ssdd_config.py

@@ -0,0 +1,344 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'whole_model'
+]
+
+sub_model_optim = {
+    'whole_model': {'lr_mult': 1},
+}
+
+max_epochs = 1000
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=5e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    # train_evaluator=evaluator_,
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 1
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+
+model = dict(
+    type='mmdet.FasterRCNN',
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='mmdet.ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=1,
+        norm_cfg=dict(type='BN', requires_grad=True),
+        norm_eval=True,
+        style='pytorch',
+        init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')),
+    neck=dict(
+        type='mmdet.FPN',
+        in_channels=[256, 512, 1024, 2048],
+        out_channels=256,
+        num_outs=5),
+    rpn_head=dict(
+        type='mmdet.RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='mmdet.AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='mmdet.DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='mmdet.CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0)),
+    roi_head=dict(
+        type='mmdet.StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='mmdet.SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='mmdet.Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=80,
+            bbox_coder=dict(
+                type='mmdet.DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=False,
+            loss_cls=dict(
+                type='mmdet.CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='mmdet.MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='mmdet.RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='mmdet.MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='mmdet.RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.05,
+            nms=dict(type='nms', iou_threshold=0.5),
+            max_per_img=100)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ))
+
+model_cfg = dict(
+    type='SegSAMDetPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    whole_model=model,
+    backbone=dict(
+        type='vit_h',
+        checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    )
+)
+
+task_name = 'ssdd_ins'
+exp_name = 'E20230531_8'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='samdet',
+    name=exp_name
+)
+# logger = None
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    # strategy="auto",
+    # strategy="ddp",
+    strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=5,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 4
+train_num_workers = 4
+test_batch_size_per_gpu = 4
+test_num_workers = 4
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/SSDD'
+dataset_type = 'SSDDInsSegDataset'
+
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            # ann_file='NWPU_instances_val.json',
+            # data_prefix=dict(img_path='positive image set'),
+            ann_file='annotations/SSDD_instances_val.json',
+            data_prefix=dict(img_path='imgs'),
+            # ann_file='annotations/WHU_building_test.json',
+            # data_prefix=dict(img_path=val_data_prefix + '/image'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            # ann_file='NWPU_instances_train.json',
+            # data_prefix=dict(img_path='positive image set'),
+            ann_file='annotations/SSDD_instances_train.json',
+            data_prefix=dict(img_path='imgs'),
+            # ann_file='NWPU_instances_train.json',
+            # data_prefix=dict(img_path='positive image set'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/rsprompter/samdet_fasterrcnn_whu_config.py

@@ -0,0 +1,345 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'whole_model'
+]
+
+sub_model_optim = {
+    'whole_model': {'lr_mult': 1},
+}
+
+max_epochs = 100
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0001,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    # train_evaluator=evaluator_,
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 1
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+
+model = dict(
+    type='mmdet.FasterRCNN',
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='mmdet.ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=1,
+        norm_cfg=dict(type='BN', requires_grad=True),
+        norm_eval=True,
+        style='pytorch',
+        init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')),
+    neck=dict(
+        type='mmdet.FPN',
+        in_channels=[256, 512, 1024, 2048],
+        out_channels=256,
+        num_outs=5),
+    rpn_head=dict(
+        type='mmdet.RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='mmdet.AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='mmdet.DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='mmdet.CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0)),
+    roi_head=dict(
+        type='mmdet.StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='mmdet.SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='mmdet.Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=80,
+            bbox_coder=dict(
+                type='mmdet.DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=False,
+            loss_cls=dict(
+                type='mmdet.CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='mmdet.MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='mmdet.RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='mmdet.MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='mmdet.RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.05,
+            nms=dict(type='nms', iou_threshold=0.5),
+            max_per_img=100)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ))
+
+model_cfg = dict(
+    type='SegSAMDetPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    whole_model=model,
+    backbone=dict(
+        type='vit_h',
+        checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    )
+)
+
+task_name = 'whu_ins'
+exp_name = 'E20230602_3'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='samdet',
+    name=exp_name
+)
+# logger = None
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    # strategy="auto",
+    # strategy="ddp",
+    strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=20,
+    check_val_every_n_epoch=3,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 4
+train_num_workers = 4
+test_batch_size_per_gpu = 4
+test_num_workers = 4
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/WHU'
+train_data_prefix = 'train/'
+val_data_prefix = 'test/'
+dataset_type = 'WHUInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            # ann_file='NWPU_instances_val.json',
+            # data_prefix=dict(img_path='positive image set'),
+            # ann_file='annotations/SSDD_instances_val.json',
+            # data_prefix=dict(img_path='imgs'),
+            ann_file='annotations/WHU_building_test.json',
+            data_prefix=dict(img_path=val_data_prefix + '/image'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            # ann_file='NWPU_instances_train.json',
+            # data_prefix=dict(img_path='positive image set'),
+            # ann_file='annotations/SSDD_instances_train.json',
+            # data_prefix=dict(img_path='imgs'),
+            ann_file='annotations/WHU_building_train.json',
+            data_prefix=dict(img_path=train_data_prefix + '/image'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/rsprompter/samseg_mask2former_nwpu_config.py

@@ -0,0 +1,350 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'panoptic_head',
+    'sam_neck',
+    'data_preprocessor'
+]
+
+sub_model_optim = {
+    'sam_neck': {'lr_mult': 1},
+    'panoptic_head': {'lr_mult': 1},
+}
+
+max_epochs = 500
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0001,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=1e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 10
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+num_queries = 90
+
+model_cfg = dict(
+    type='SegSAMPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='vit_h',
+        checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    ),
+    sam_neck=dict(
+            type='SAMAggregatorNeck',
+            in_channels=[1280] * 32,
+            # in_channels=[768] * 12,
+            inner_channels=32,
+            selected_channels=range(8, 32, 3),
+            # selected_channels=range(4, 12, 2),
+            out_channels=256,
+            up_sample_scale=4,
+    ),
+    panoptic_head=dict(
+        type='mmdet.Mask2FormerHead',
+        in_channels=[256, 256, 256],  # pass to pixel_decoder inside
+        strides=[8, 16, 32],
+        feat_channels=256,
+        out_channels=256,
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        num_queries=num_queries,
+        num_transformer_feat_level=3,
+        pixel_decoder=dict(
+            type='mmdet.MSDeformAttnPixelDecoder',
+            num_outs=3,
+            norm_cfg=dict(type='GN', num_groups=32),
+            act_cfg=dict(type='ReLU'),
+            encoder=dict(  # DeformableDetrTransformerEncoder
+                # num_layers=6,
+                num_layers=2,
+                layer_cfg=dict(  # DeformableDetrTransformerEncoderLayer
+                    self_attn_cfg=dict(  # MultiScaleDeformableAttention
+                        embed_dims=256,
+                        num_heads=8,
+                        num_levels=3,
+                        num_points=4,
+                        dropout=0.1,
+                        batch_first=True),
+                    ffn_cfg=dict(
+                        embed_dims=256,
+                        feedforward_channels=1024,
+                        num_fcs=2,
+                        ffn_drop=0.1,
+                        act_cfg=dict(type='ReLU', inplace=True)))),
+            positional_encoding=dict(num_feats=128, normalize=True)),
+        enforce_decoder_input_project=False,
+        positional_encoding=dict(num_feats=128, normalize=True),
+        transformer_decoder=dict(  # Mask2FormerTransformerDecoder
+            return_intermediate=True,
+            # num_layers=9,
+            num_layers=3,
+            layer_cfg=dict(  # Mask2FormerTransformerDecoderLayer
+                self_attn_cfg=dict(  # MultiheadAttention
+                    embed_dims=256,
+                    num_heads=8,
+                    dropout=0.1,
+                    batch_first=True),
+                cross_attn_cfg=dict(  # MultiheadAttention
+                    embed_dims=256,
+                    num_heads=8,
+                    dropout=0.1,
+                    batch_first=True),
+                ffn_cfg=dict(
+                    embed_dims=256,
+                    feedforward_channels=2048,
+                    num_fcs=2,
+                    ffn_drop=0.1,
+                    act_cfg=dict(type='ReLU', inplace=True))),
+            init_cfg=None),
+        loss_cls=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=False,
+            loss_weight=2.0,
+            reduction='mean',
+            class_weight=[1.0] * num_classes + [0.1]),
+        loss_mask=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=True,
+            reduction='mean',
+            loss_weight=5.0),
+        loss_dice=dict(
+            type='mmdet.DiceLoss',
+            use_sigmoid=True,
+            activate=True,
+            reduction='mean',
+            naive_dice=True,
+            eps=1.0,
+            loss_weight=5.0)),
+    panoptic_fusion_head=dict(
+        type='mmdet.MaskFormerFusionHead',
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        loss_panoptic=None,
+        init_cfg=None),
+    train_cfg=dict(
+        num_points=12544,
+        oversample_ratio=3.0,
+        importance_sample_ratio=0.75,
+        assigner=dict(
+            type='mmdet.HungarianAssigner',
+            match_costs=[
+                dict(type='mmdet.ClassificationCost', weight=2.0),
+                dict(
+                    type='mmdet.CrossEntropyLossCost', weight=5.0, use_sigmoid=True),
+                dict(type='mmdet.DiceCost', weight=5.0, pred_act=True, eps=1.0)
+            ]),
+        sampler=dict(type='mmdet.MaskPseudoSampler')),
+    test_cfg=dict(
+        panoptic_on=False,
+        # For now, the dataset does not support
+        # evaluating semantic segmentation metric.
+        semantic_on=False,
+        instance_on=True,
+        # max_per_image is for instance segmentation.
+        max_per_image=num_queries,
+        iou_thr=0.8,
+        # In Mask2Former's panoptic postprocessing,
+        # it will filter mask area where score is less than 0.5 .
+        filter_low_score=True),
+    init_cfg=None)
+
+
+task_name = 'nwpu_ins'
+exp_name = 'E20230604_5'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='samseg-mask2former',
+    name=exp_name
+)
+# logger = None
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=5,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 4
+train_num_workers = 4
+test_batch_size_per_gpu = 4
+test_num_workers = 4
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/NWPU10'
+train_data_prefix = ''
+val_data_prefix = ''
+
+dataset_type = 'NWPUInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_val.json',
+            data_prefix=dict(img_path='positive image set'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_train.json',
+            data_prefix=dict(img_path='positive image set'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/rsprompter/samseg_mask2former_ssdd_config.py

@@ -0,0 +1,346 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'panoptic_head',
+    'sam_neck',
+    'data_preprocessor'
+]
+
+sub_model_optim = {
+    'sam_neck': {'lr_mult': 1},
+    'panoptic_head': {'lr_mult': 1},
+}
+
+max_epochs = 600
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=5e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 1
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+num_queries = 30
+
+model_cfg = dict(
+    type='SegSAMPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='vit_h',
+        checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    ),
+    sam_neck=dict(
+            type='SAMAggregatorNeck',
+            in_channels=[1280] * 32,
+            # in_channels=[768] * 12,
+            inner_channels=32,
+            selected_channels=range(4, 32, 2),
+            # selected_channels=range(4, 12, 2),
+            out_channels=256,
+            up_sample_scale=4,
+    ),
+    panoptic_head=dict(
+        type='mmdet.Mask2FormerHead',
+        in_channels=[256, 256, 256],  # pass to pixel_decoder inside
+        strides=[8, 16, 32],
+        feat_channels=256,
+        out_channels=256,
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        num_queries=num_queries,
+        num_transformer_feat_level=3,
+        pixel_decoder=dict(
+            type='mmdet.MSDeformAttnPixelDecoder',
+            num_outs=3,
+            norm_cfg=dict(type='GN', num_groups=32),
+            act_cfg=dict(type='ReLU'),
+            encoder=dict(  # DeformableDetrTransformerEncoder
+                # num_layers=6,
+                num_layers=2,
+                layer_cfg=dict(  # DeformableDetrTransformerEncoderLayer
+                    self_attn_cfg=dict(  # MultiScaleDeformableAttention
+                        embed_dims=256,
+                        num_heads=8,
+                        num_levels=3,
+                        num_points=4,
+                        dropout=0.1,
+                        batch_first=True),
+                    ffn_cfg=dict(
+                        embed_dims=256,
+                        feedforward_channels=1024,
+                        num_fcs=2,
+                        ffn_drop=0.1,
+                        act_cfg=dict(type='ReLU', inplace=True)))),
+            positional_encoding=dict(num_feats=128, normalize=True)),
+        enforce_decoder_input_project=False,
+        positional_encoding=dict(num_feats=128, normalize=True),
+        transformer_decoder=dict(  # Mask2FormerTransformerDecoder
+            return_intermediate=True,
+            # num_layers=9,
+            num_layers=3,
+            layer_cfg=dict(  # Mask2FormerTransformerDecoderLayer
+                self_attn_cfg=dict(  # MultiheadAttention
+                    embed_dims=256,
+                    num_heads=8,
+                    dropout=0.1,
+                    batch_first=True),
+                cross_attn_cfg=dict(  # MultiheadAttention
+                    embed_dims=256,
+                    num_heads=8,
+                    dropout=0.1,
+                    batch_first=True),
+                ffn_cfg=dict(
+                    embed_dims=256,
+                    feedforward_channels=2048,
+                    num_fcs=2,
+                    ffn_drop=0.1,
+                    act_cfg=dict(type='ReLU', inplace=True))),
+            init_cfg=None),
+        loss_cls=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=False,
+            loss_weight=2.0,
+            reduction='mean',
+            class_weight=[1.0] * num_classes + [0.1]),
+        loss_mask=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=True,
+            reduction='mean',
+            loss_weight=5.0),
+        loss_dice=dict(
+            type='mmdet.DiceLoss',
+            use_sigmoid=True,
+            activate=True,
+            reduction='mean',
+            naive_dice=True,
+            eps=1.0,
+            loss_weight=5.0)),
+    panoptic_fusion_head=dict(
+        type='mmdet.MaskFormerFusionHead',
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        loss_panoptic=None,
+        init_cfg=None),
+    train_cfg=dict(
+        num_points=12544,
+        oversample_ratio=3.0,
+        importance_sample_ratio=0.75,
+        assigner=dict(
+            type='mmdet.HungarianAssigner',
+            match_costs=[
+                dict(type='mmdet.ClassificationCost', weight=2.0),
+                dict(
+                    type='mmdet.CrossEntropyLossCost', weight=5.0, use_sigmoid=True),
+                dict(type='mmdet.DiceCost', weight=5.0, pred_act=True, eps=1.0)
+            ]),
+        sampler=dict(type='mmdet.MaskPseudoSampler')),
+    test_cfg=dict(
+        panoptic_on=False,
+        # For now, the dataset does not support
+        # evaluating semantic segmentation metric.
+        semantic_on=False,
+        instance_on=True,
+        # max_per_image is for instance segmentation.
+        max_per_image=num_queries,
+        iou_thr=0.8,
+        # In Mask2Former's panoptic postprocessing,
+        # it will filter mask area where score is less than 0.5 .
+        filter_low_score=True),
+    init_cfg=None)
+
+task_name = 'ssdd_ins'
+exp_name = 'E20230531_1'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='samcls-mask2former',
+    name=exp_name
+)
+# logger = None
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=5,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 6
+train_num_workers = 4
+test_batch_size_per_gpu = 6
+test_num_workers = 4
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/SSDD'
+dataset_type = 'SSDDInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/SSDD_instances_val.json',
+            data_prefix=dict(img_path='imgs'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/SSDD_instances_train.json',
+            data_prefix=dict(img_path='imgs'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/rsprompter/samseg_mask2former_whu_config.py

@@ -0,0 +1,349 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'panoptic_head',
+    'sam_neck',
+    'data_preprocessor'
+]
+
+sub_model_optim = {
+    'sam_neck': {'lr_mult': 1},
+    'panoptic_head': {'lr_mult': 1},
+}
+
+max_epochs = 400
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=5e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    # train_evaluator=evaluator_,
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 1
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+
+num_queries = 100
+model_cfg = dict(
+    type='SegSAMPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='vit_h',
+        checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    ),
+    sam_neck=dict(
+            type='SAMAggregatorNeck',
+            in_channels=[1280] * 32,
+            # in_channels=[768] * 12,
+            inner_channels=32,
+            selected_channels=range(4, 32, 2),
+            # selected_channels=range(4, 12, 2),
+            out_channels=256,
+            up_sample_scale=4,
+    ),
+    panoptic_head=dict(
+        type='mmdet.Mask2FormerHead',
+        in_channels=[256, 256, 256],  # pass to pixel_decoder inside
+        strides=[8, 16, 32],
+        feat_channels=256,
+        out_channels=256,
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        num_queries=num_queries,
+        num_transformer_feat_level=3,
+        pixel_decoder=dict(
+            type='mmdet.MSDeformAttnPixelDecoder',
+            num_outs=3,
+            norm_cfg=dict(type='GN', num_groups=32),
+            act_cfg=dict(type='ReLU'),
+            encoder=dict(  # DeformableDetrTransformerEncoder
+                # num_layers=6,
+                num_layers=2,
+                layer_cfg=dict(  # DeformableDetrTransformerEncoderLayer
+                    self_attn_cfg=dict(  # MultiScaleDeformableAttention
+                        embed_dims=256,
+                        num_heads=8,
+                        num_levels=3,
+                        num_points=4,
+                        dropout=0.1,
+                        batch_first=True),
+                    ffn_cfg=dict(
+                        embed_dims=256,
+                        feedforward_channels=1024,
+                        num_fcs=2,
+                        ffn_drop=0.1,
+                        act_cfg=dict(type='ReLU', inplace=True)))),
+            positional_encoding=dict(num_feats=128, normalize=True)),
+        enforce_decoder_input_project=False,
+        positional_encoding=dict(num_feats=128, normalize=True),
+        transformer_decoder=dict(  # Mask2FormerTransformerDecoder
+            return_intermediate=True,
+            # num_layers=9,
+            num_layers=3,
+            layer_cfg=dict(  # Mask2FormerTransformerDecoderLayer
+                self_attn_cfg=dict(  # MultiheadAttention
+                    embed_dims=256,
+                    num_heads=8,
+                    dropout=0.1,
+                    batch_first=True),
+                cross_attn_cfg=dict(  # MultiheadAttention
+                    embed_dims=256,
+                    num_heads=8,
+                    dropout=0.1,
+                    batch_first=True),
+                ffn_cfg=dict(
+                    embed_dims=256,
+                    feedforward_channels=2048,
+                    num_fcs=2,
+                    ffn_drop=0.1,
+                    act_cfg=dict(type='ReLU', inplace=True))),
+            init_cfg=None),
+        loss_cls=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=False,
+            loss_weight=2.0,
+            reduction='mean',
+            class_weight=[1.0] * num_classes + [0.1]),
+        loss_mask=dict(
+            type='mmdet.CrossEntropyLoss',
+            use_sigmoid=True,
+            reduction='mean',
+            loss_weight=5.0),
+        loss_dice=dict(
+            type='mmdet.DiceLoss',
+            use_sigmoid=True,
+            activate=True,
+            reduction='mean',
+            naive_dice=True,
+            eps=1.0,
+            loss_weight=5.0)),
+    panoptic_fusion_head=dict(
+        type='mmdet.MaskFormerFusionHead',
+        num_things_classes=num_things_classes,
+        num_stuff_classes=num_stuff_classes,
+        loss_panoptic=None,
+        init_cfg=None),
+    train_cfg=dict(
+        num_points=12544,
+        oversample_ratio=3.0,
+        importance_sample_ratio=0.75,
+        assigner=dict(
+            type='mmdet.HungarianAssigner',
+            match_costs=[
+                dict(type='mmdet.ClassificationCost', weight=2.0),
+                dict(
+                    type='mmdet.CrossEntropyLossCost', weight=5.0, use_sigmoid=True),
+                dict(type='mmdet.DiceCost', weight=5.0, pred_act=True, eps=1.0)
+            ]),
+        sampler=dict(type='mmdet.MaskPseudoSampler')),
+    test_cfg=dict(
+        panoptic_on=False,
+        # For now, the dataset does not support
+        # evaluating semantic segmentation metric.
+        semantic_on=False,
+        instance_on=True,
+        # max_per_image is for instance segmentation.
+        max_per_image=num_queries,
+        iou_thr=0.8,
+        # In Mask2Former's panoptic postprocessing,
+        # it will filter mask area where score is less than 0.5 .
+        filter_low_score=True),
+    init_cfg=None)
+
+task_name = 'whu_ins'
+exp_name = 'E20230531_2'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='samcls-mask2former',
+    name=exp_name
+)
+# logger = None
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=20,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 6
+train_num_workers = 4
+test_batch_size_per_gpu = 6
+test_num_workers = 4
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/WHU'
+train_data_prefix = 'train/'
+val_data_prefix = 'test/'
+dataset_type = 'WHUInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/WHU_building_test.json',
+            data_prefix=dict(img_path=val_data_prefix + '/image', seg_path=val_data_prefix + '/label'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/WHU_building_train.json',
+            data_prefix=dict(img_path=train_data_prefix + '/image', seg_path=train_data_prefix + '/label'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/rsprompter/samseg_maskrcnn_nwpu_config.py

@@ -0,0 +1,348 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'panoptic_head',
+    'data_preprocessor'
+]
+
+sub_model_optim = {
+    'panoptic_head': {'lr_mult': 1},
+}
+
+max_epochs = 1000
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=5e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    # train_evaluator=evaluator_,
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 10
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+
+
+model_cfg = dict(
+    type='SegSAMAnchorPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='vit_h',
+        checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    ),
+    panoptic_head=dict(
+        type='SAMAnchorInstanceHead',
+        sam_head=False,
+        neck=dict(
+            type='SAMAggregatorNeck',
+            in_channels=[1280] * 32,
+            # in_channels=[768] * 12,
+            inner_channels=32,
+            selected_channels=range(4, 32, 2),
+            # selected_channels=range(4, 12, 2),
+            out_channels=256,
+            up_sample_scale=4,
+        ),
+        rpn_head=dict(
+            type='mmdet.RPNHead',
+            in_channels=256,
+            feat_channels=256,
+            anchor_generator=dict(
+                type='mmdet.AnchorGenerator',
+                scales=[2, 4, 8, 16, 32, 64],
+                ratios=[0.5, 1.0, 2.0],
+                strides=[8, 16, 32]),
+            bbox_coder=dict(
+                type='mmdet.DeltaXYWHBBoxCoder',
+                target_means=[.0, .0, .0, .0],
+                target_stds=[1.0, 1.0, 1.0, 1.0]),
+            loss_cls=dict(
+                type='mmdet.CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+            loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0)),
+        roi_head=dict(
+            type='mmdet.StandardRoIHead',
+            bbox_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            bbox_head=dict(
+                type='mmdet.Shared2FCBBoxHead',
+                in_channels=256,
+                fc_out_channels=1024,
+                roi_feat_size=7,
+                num_classes=num_classes,
+                bbox_coder=dict(
+                    type='mmdet.DeltaXYWHBBoxCoder',
+                    target_means=[0., 0., 0., 0.],
+                    target_stds=[0.1, 0.1, 0.2, 0.2]),
+                reg_class_agnostic=False,
+                loss_cls=dict(
+                    type='mmdet.CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+                loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0)),
+            mask_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            mask_head=dict(
+                type='mmdet.FCNMaskHead',
+                num_convs=4,
+                in_channels=256,
+                conv_out_channels=256,
+                num_classes=num_classes,
+                loss_mask=dict(
+                    type='mmdet.CrossEntropyLoss', use_mask=True, loss_weight=1.0))),
+        # model training and testing settings
+        train_cfg=dict(
+            rpn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.7,
+                    neg_iou_thr=0.3,
+                    min_pos_iou=0.3,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=256,
+                    pos_fraction=0.5,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=False),
+                allowed_border=-1,
+                pos_weight=-1,
+                debug=False),
+            rpn_proposal=dict(
+                nms_pre=2000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.5,
+                    neg_iou_thr=0.5,
+                    min_pos_iou=0.5,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=512,
+                    pos_fraction=0.25,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=True),
+                mask_size=28,
+                pos_weight=-1,
+                debug=False)),
+        test_cfg=dict(
+            rpn=dict(
+                nms_pre=1000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                score_thr=0.05,
+                nms=dict(type='nms', iou_threshold=0.5),
+                max_per_img=100,
+                mask_thr_binary=0.5)
+        )
+    )
+)
+
+task_name = 'nwpu_ins'
+exp_name = 'E20230530_0'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='samcls-rcnn',
+    name=exp_name
+)
+# logger = None
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=5,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 6
+train_num_workers = 4
+test_batch_size_per_gpu = 6
+test_num_workers = 4
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/NWPU10'
+train_data_prefix = ''
+val_data_prefix = ''
+
+dataset_type = 'NWPUInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_val.json',
+            data_prefix=dict(img_path='positive image set'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='NWPU_instances_train.json',
+            data_prefix=dict(img_path='positive image set'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/rsprompter/samseg_maskrcnn_ssdd_config.py

@@ -0,0 +1,345 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'panoptic_head',
+    'data_preprocessor'
+]
+
+sub_model_optim = {
+    'panoptic_head': {'lr_mult': 1},
+}
+
+max_epochs = 800
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=5e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    # train_evaluator=evaluator_,
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 1
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+
+
+model_cfg = dict(
+    type='SegSAMAnchorPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='vit_h',
+        checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    ),
+    panoptic_head=dict(
+        type='SAMAnchorInstanceHead',
+        sam_head=False,
+        neck=dict(
+            type='SAMAggregatorNeck',
+            in_channels=[1280] * 32,
+            # in_channels=[768] * 12,
+            inner_channels=32,
+            selected_channels=range(4, 32, 2),
+            # selected_channels=range(4, 12, 2),
+            out_channels=256,
+            up_sample_scale=4,
+        ),
+        rpn_head=dict(
+            type='mmdet.RPNHead',
+            in_channels=256,
+            feat_channels=256,
+            anchor_generator=dict(
+                type='mmdet.AnchorGenerator',
+                scales=[2, 4, 8, 16, 32, 64],
+                ratios=[0.5, 1.0, 2.0],
+                strides=[8, 16, 32]),
+            bbox_coder=dict(
+                type='mmdet.DeltaXYWHBBoxCoder',
+                target_means=[.0, .0, .0, .0],
+                target_stds=[1.0, 1.0, 1.0, 1.0]),
+            loss_cls=dict(
+                type='mmdet.CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+            loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0)),
+        roi_head=dict(
+            type='mmdet.StandardRoIHead',
+            bbox_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            bbox_head=dict(
+                type='mmdet.Shared2FCBBoxHead',
+                in_channels=256,
+                fc_out_channels=1024,
+                roi_feat_size=7,
+                num_classes=num_classes,
+                bbox_coder=dict(
+                    type='mmdet.DeltaXYWHBBoxCoder',
+                    target_means=[0., 0., 0., 0.],
+                    target_stds=[0.1, 0.1, 0.2, 0.2]),
+                reg_class_agnostic=False,
+                loss_cls=dict(
+                    type='mmdet.CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+                loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0)),
+            mask_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            mask_head=dict(
+                type='mmdet.FCNMaskHead',
+                num_convs=4,
+                in_channels=256,
+                conv_out_channels=256,
+                num_classes=num_classes,
+                loss_mask=dict(
+                    type='mmdet.CrossEntropyLoss', use_mask=True, loss_weight=1.0))),
+        # model training and testing settings
+        train_cfg=dict(
+            rpn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.7,
+                    neg_iou_thr=0.3,
+                    min_pos_iou=0.3,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=256,
+                    pos_fraction=0.5,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=False),
+                allowed_border=-1,
+                pos_weight=-1,
+                debug=False),
+            rpn_proposal=dict(
+                nms_pre=2000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.5,
+                    neg_iou_thr=0.5,
+                    min_pos_iou=0.5,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=512,
+                    pos_fraction=0.25,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=True),
+                mask_size=28,
+                pos_weight=-1,
+                debug=False)),
+        test_cfg=dict(
+            rpn=dict(
+                nms_pre=1000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                score_thr=0.05,
+                nms=dict(type='nms', iou_threshold=0.5),
+                max_per_img=100,
+                mask_thr_binary=0.5)
+        )
+    )
+)
+
+task_name = 'ssdd_ins'
+exp_name = 'E20230530_1'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='samcls-rcnn',
+    name=exp_name
+)
+# logger = None
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=5,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 6
+train_num_workers = 4
+test_batch_size_per_gpu = 6
+test_num_workers = 4
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/SSDD'
+dataset_type = 'SSDDInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/SSDD_instances_val.json',
+            data_prefix=dict(img_path='imgs'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/SSDD_instances_train.json',
+            data_prefix=dict(img_path='imgs'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)

configs/rsprompter/samseg_maskrcnn_whu_config.py

@@ -0,0 +1,346 @@
+custom_imports = dict(imports=['mmseg.datasets', 'mmseg.models'], allow_failed_imports=False)
+
+sub_model_train = [
+    'panoptic_head',
+    'data_preprocessor'
+]
+
+sub_model_optim = {
+    'panoptic_head': {'lr_mult': 1},
+}
+
+max_epochs = 400
+
+optimizer = dict(
+    type='AdamW',
+    sub_model=sub_model_optim,
+    lr=0.0005,
+    weight_decay=1e-3
+)
+
+param_scheduler = [
+    # warm up learning rate scheduler
+    dict(
+        type='LinearLR',
+        start_factor=5e-4,
+        by_epoch=True,
+        begin=0,
+        end=1,
+        # update by iter
+        convert_to_iter_based=True),
+    # main learning rate scheduler
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        by_epoch=True,
+        begin=1,
+        end=max_epochs,
+    ),
+]
+
+param_scheduler_callback = dict(
+    type='ParamSchedulerHook'
+)
+
+evaluator_ = dict(
+        type='CocoPLMetric',
+        metric=['bbox', 'segm'],
+        proposal_nums=[1, 10, 100]
+)
+
+evaluator = dict(
+    val_evaluator=evaluator_,
+)
+
+
+image_size = (1024, 1024)
+
+data_preprocessor = dict(
+    type='mmdet.DetDataPreprocessor',
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    bgr_to_rgb=True,
+    pad_size_divisor=32,
+    pad_mask=True,
+    mask_pad_value=0,
+)
+
+num_things_classes = 1
+num_stuff_classes = 0
+num_classes = num_things_classes + num_stuff_classes
+
+
+model_cfg = dict(
+    type='SegSAMAnchorPLer',
+    hyperparameters=dict(
+        optimizer=optimizer,
+        param_scheduler=param_scheduler,
+        evaluator=evaluator,
+    ),
+    need_train_names=sub_model_train,
+    data_preprocessor=data_preprocessor,
+    backbone=dict(
+        type='vit_h',
+        checkpoint='pretrain/sam/sam_vit_h_4b8939.pth',
+        # type='vit_b',
+        # checkpoint='pretrain/sam/sam_vit_b_01ec64.pth',
+    ),
+    panoptic_head=dict(
+        type='SAMAnchorInstanceHead',
+        sam_head=False,
+        neck=dict(
+            type='SAMAggregatorNeck',
+            in_channels=[1280] * 32,
+            # in_channels=[768] * 12,
+            inner_channels=32,
+            selected_channels=range(4, 32, 2),
+            # selected_channels=range(4, 12, 2),
+            out_channels=256,
+            up_sample_scale=4,
+        ),
+        rpn_head=dict(
+            type='mmdet.RPNHead',
+            in_channels=256,
+            feat_channels=256,
+            anchor_generator=dict(
+                type='mmdet.AnchorGenerator',
+                scales=[2, 4, 8, 16, 32, 64],
+                ratios=[0.5, 1.0, 2.0],
+                strides=[8, 16, 32]),
+            bbox_coder=dict(
+                type='mmdet.DeltaXYWHBBoxCoder',
+                target_means=[.0, .0, .0, .0],
+                target_stds=[1.0, 1.0, 1.0, 1.0]),
+            loss_cls=dict(
+                type='mmdet.CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+            loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0)),
+        roi_head=dict(
+            type='mmdet.StandardRoIHead',
+            bbox_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            bbox_head=dict(
+                type='mmdet.Shared2FCBBoxHead',
+                in_channels=256,
+                fc_out_channels=1024,
+                roi_feat_size=7,
+                num_classes=num_classes,
+                bbox_coder=dict(
+                    type='mmdet.DeltaXYWHBBoxCoder',
+                    target_means=[0., 0., 0., 0.],
+                    target_stds=[0.1, 0.1, 0.2, 0.2]),
+                reg_class_agnostic=False,
+                loss_cls=dict(
+                    type='mmdet.CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+                loss_bbox=dict(type='mmdet.L1Loss', loss_weight=1.0)),
+            mask_roi_extractor=dict(
+                type='mmdet.SingleRoIExtractor',
+                roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0),
+                out_channels=256,
+                featmap_strides=[8, 16, 32]),
+            mask_head=dict(
+                type='mmdet.FCNMaskHead',
+                num_convs=4,
+                in_channels=256,
+                conv_out_channels=256,
+                num_classes=num_classes,
+                loss_mask=dict(
+                    type='mmdet.CrossEntropyLoss', use_mask=True, loss_weight=1.0))),
+        # model training and testing settings
+        train_cfg=dict(
+            rpn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.7,
+                    neg_iou_thr=0.3,
+                    min_pos_iou=0.3,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=256,
+                    pos_fraction=0.5,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=False),
+                allowed_border=-1,
+                pos_weight=-1,
+                debug=False),
+            rpn_proposal=dict(
+                nms_pre=2000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                assigner=dict(
+                    type='mmdet.MaxIoUAssigner',
+                    pos_iou_thr=0.5,
+                    neg_iou_thr=0.5,
+                    min_pos_iou=0.5,
+                    match_low_quality=True,
+                    ignore_iof_thr=-1),
+                sampler=dict(
+                    type='mmdet.RandomSampler',
+                    num=512,
+                    pos_fraction=0.25,
+                    neg_pos_ub=-1,
+                    add_gt_as_proposals=True),
+                mask_size=28,
+                pos_weight=-1,
+                debug=False)),
+        test_cfg=dict(
+            rpn=dict(
+                nms_pre=1000,
+                max_per_img=1000,
+                nms=dict(type='nms', iou_threshold=0.7),
+                min_bbox_size=0),
+            rcnn=dict(
+                score_thr=0.05,
+                nms=dict(type='nms', iou_threshold=0.5),
+                max_per_img=100,
+                mask_thr_binary=0.5)
+        )
+    )
+)
+
+task_name = 'whu_ins'
+exp_name = 'E20230530_2'
+logger = dict(
+    type='WandbLogger',
+    project=task_name,
+    group='samcls-rcnn',
+    name=exp_name
+)
+# logger = None
+
+callbacks = [
+    param_scheduler_callback,
+    dict(
+        type='ModelCheckpoint',
+        dirpath=f'results/{task_name}/{exp_name}/checkpoints',
+        save_last=True,
+        mode='max',
+        monitor='valsegm_map_0',
+        save_top_k=2,
+        filename='epoch_{epoch}-map_{valsegm_map_0:.4f}'
+    ),
+    dict(
+        type='LearningRateMonitor',
+        logging_interval='step'
+    )
+]
+
+
+trainer_cfg = dict(
+    compiled_model=False,
+    accelerator="auto",
+    strategy="auto",
+    # strategy="ddp",
+    # strategy='ddp_find_unused_parameters_true',
+    # precision='32',
+    # precision='16-mixed',
+    devices=8,
+    default_root_dir=f'results/{task_name}/{exp_name}',
+    # default_root_dir='results/tmp',
+    max_epochs=max_epochs,
+    logger=logger,
+    callbacks=callbacks,
+    log_every_n_steps=20,
+    check_val_every_n_epoch=5,
+    benchmark=True,
+    # sync_batchnorm=True,
+    # fast_dev_run=True,
+
+    # limit_train_batches=1,
+    # limit_val_batches=0,
+    # limit_test_batches=None,
+    # limit_predict_batches=None,
+    # overfit_batches=0.0,
+
+    # val_check_interval=None,
+    # num_sanity_val_steps=0,
+    # enable_checkpointing=None,
+    # enable_progress_bar=None,
+    # enable_model_summary=None,
+    # accumulate_grad_batches=32,
+    # gradient_clip_val=15,
+    # gradient_clip_algorithm='norm',
+    # deterministic=None,
+    # inference_mode: bool=True,
+    use_distributed_sampler=True,
+    # profiler="simple",
+    # detect_anomaly=False,
+    # barebones=False,
+    # plugins=None,
+    # reload_dataloaders_every_n_epochs=0,
+)
+
+
+backend_args = None
+train_pipeline = [
+    dict(type='mmdet.LoadImageFromFile'),
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(type='mmdet.Resize', scale=image_size),
+    dict(type='mmdet.RandomFlip', prob=0.5),
+    dict(type='mmdet.PackDetInputs')
+]
+
+test_pipeline = [
+    dict(type='mmdet.LoadImageFromFile', backend_args=backend_args),
+    dict(type='mmdet.Resize', scale=image_size),
+    # If you don't have a gt annotation, delete the pipeline
+    dict(type='mmdet.LoadAnnotations', with_bbox=True, with_mask=True),
+    dict(
+        type='mmdet.PackDetInputs',
+        meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape',
+                   'scale_factor'))
+]
+
+
+train_batch_size_per_gpu = 6
+train_num_workers = 4
+test_batch_size_per_gpu = 6
+test_num_workers = 4
+persistent_workers = True
+
+data_parent = '/mnt/search01/dataset/cky_data/WHU'
+train_data_prefix = 'train/'
+val_data_prefix = 'test/'
+dataset_type = 'WHUInsSegDataset'
+
+val_loader = dict(
+        batch_size=test_batch_size_per_gpu,
+        num_workers=test_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/WHU_building_test.json',
+            data_prefix=dict(img_path=val_data_prefix + '/image', seg_path=val_data_prefix + '/label'),
+            test_mode=True,
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=test_pipeline,
+            backend_args=backend_args))
+
+datamodule_cfg = dict(
+    type='PLDataModule',
+    train_loader=dict(
+        batch_size=train_batch_size_per_gpu,
+        num_workers=train_num_workers,
+        persistent_workers=persistent_workers,
+        pin_memory=True,
+        dataset=dict(
+            type=dataset_type,
+            data_root=data_parent,
+            ann_file='annotations/WHU_building_train.json',
+            data_prefix=dict(img_path=train_data_prefix + '/image', seg_path=train_data_prefix + '/label'),
+            filter_cfg=dict(filter_empty_gt=True, min_size=32),
+            pipeline=train_pipeline,
+            backend_args=backend_args)
+    ),
+    val_loader=val_loader,
+    # test_loader=val_loader
+    predict_loader=val_loader
+)