Add files using upload-large-folder tool

approach/ovod/APE/configs/ADE20kFull_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py

@@ -0,0 +1,101 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detrex.modeling.neck import ChannelMapper
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from .ape_deta_vitl_eva02_lsj1024 import dataloader, lr_multiplier, model, optimizer, train
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = -1.0
+model.model_vision.transformer.proposal_ambiguous = 1
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 12800

approach/ovod/APE/configs/ADE20kFull_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py

@@ -0,0 +1,32 @@
+from detectron2.data import MetadataCatalog
+
+from ...COCO_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_12ep import (
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+from ...common.data.ade20kfull_semantic_lsj1024 import dataloader
+
+stuff_classes = MetadataCatalog.get("ade20k_full_sem_seg_train").stuff_classes
+del MetadataCatalog.get("ade20k_full_sem_seg_train").stuff_classes
+MetadataCatalog.get("ade20k_full_sem_seg_train").set(
+    stuff_classes=[x.split(",")[0] for x in stuff_classes]
+)
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.name_prompt_fusion_text = [False]
+model.model_vision.dataset_names = ["ade20k_full_sem_seg"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+model.model_vision.num_classes = 847
+model.model_vision.criterion[0].num_classes = 847
+model.model_vision.select_box_nums_for_evaluation = 300
+
+model.model_vision.instance_on = False
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+model.model_vision.stuff_prob_thing = -1.0
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/ADE20kFull_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py

@@ -0,0 +1,47 @@
+from detectron2.config import LazyCall as L
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+
+from .ape_deta_vitl_eva02_lsj1024 import dataloader, lr_multiplier, model, optimizer, train
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 12800

approach/ovod/APE/configs/ADE20k_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py

@@ -0,0 +1,101 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detrex.modeling.neck import ChannelMapper
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from .ape_deta_vitl_eva02_lsj1024 import dataloader, lr_multiplier, model, optimizer, train
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = -1.0
+model.model_vision.transformer.proposal_ambiguous = 1
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 12800

approach/ovod/APE/configs/ADE20k_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py

@@ -0,0 +1,22 @@
+from ...COCO_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_12ep import (
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+from ...common.data.ade20k_semantic_lsj1024 import dataloader
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.name_prompt_fusion_text = [False]
+model.model_vision.dataset_names = ["ade20k"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+model.model_vision.select_box_nums_for_evaluation = 300
+
+model.model_vision.instance_on = False
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+model.model_vision.stuff_prob_thing = -1.0
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/ADE20k_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py

@@ -0,0 +1,47 @@
+from detectron2.config import LazyCall as L
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+
+from .ape_deta_vitl_eva02_lsj1024 import dataloader, lr_multiplier, model, optimizer, train
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 12800

approach/ovod/APE/configs/ADE20k_SemanticSegmentation/deformable_deta/deformable_deta_segm_r50_160k.py

@@ -0,0 +1,45 @@
+from detectron2.config import LazyCall as L
+from detectron2.solver import WarmupParamScheduler
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ...COCO_InstanceSegmentation.deformable_deta.deformable_deta_segm_r50_12ep import (
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+from ...common.data.ade20k_semantic import dataloader
+
+num_classes = 150
+model.num_classes = num_classes
+model.criterion.num_classes = num_classes
+model.criterion.matcher_stage2.num_classes = num_classes
+model.criterion.eos_coef = 1.0
+
+model.instance_on = False
+model.semantic_on = True
+model.panoptic_on = False
+
+train.max_iter = 160000
+train.eval_period = 5000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1, 0.01],
+        milestones=[135000, 150000],
+        num_updates=160000,
+    ),
+    warmup_length=1000 / 160000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+train.init_checkpoint = "detectron2://ImageNetPretrained/torchvision/R-50.pkl"
+train.init_checkpoint = "models/torchvision/R-50.pkl"
+train.output_dir = "output/" + __file__[:-3]
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+train.ddp.find_unused_parameters = True
+
+model.dataset_metas = dataloader.train.dataset.names

approach/ovod/APE/configs/BDD10k_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py

@@ -0,0 +1,109 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detrex.modeling.neck import ChannelMapper
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from .ape_deta_vitl_eva02_lsj1024 import dataloader, lr_multiplier, model, optimizer, train
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = -1.0
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.panoptic_configs = {
+    "prob": 0.01,
+    "pano_temp": 0.06,
+    "transform_eval": True,
+    "object_mask_threshold": 0.0001,
+    "overlap_threshold": 0.4,
+}
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 12800

approach/ovod/APE/configs/BDD10k_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py

@@ -0,0 +1,23 @@
+from ...COCO_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_12ep import (
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+from ...common.data.bdd10k_panoptic_lsj1024 import dataloader
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.name_prompt_fusion_text = [False]
+model.model_vision.dataset_names = ["bdd10k"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+model.model_vision.select_box_nums_for_evaluation = 300
+
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = True
+
+model.model_vision.stuff_prob_thing = -1.0
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/BDD10k_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py

@@ -0,0 +1,47 @@
+from detectron2.config import LazyCall as L
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+
+from .ape_deta_vitl_eva02_lsj1024 import dataloader, lr_multiplier, model, optimizer, train
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 12800

approach/ovod/APE/configs/BDD10k_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py

@@ -0,0 +1,101 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detrex.modeling.neck import ChannelMapper
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from .ape_deta_vitl_eva02_lsj1024 import dataloader, lr_multiplier, model, optimizer, train
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = -1.0
+model.model_vision.transformer.proposal_ambiguous = 1
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 12800

approach/ovod/APE/configs/BDD10k_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py

@@ -0,0 +1,22 @@
+from ...COCO_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_12ep import (
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+from ...common.data.bdd10k_semantic_lsj1024 import dataloader
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.name_prompt_fusion_text = [False]
+model.model_vision.dataset_names = ["bdd10k"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+model.model_vision.select_box_nums_for_evaluation = 300
+
+model.model_vision.instance_on = False
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+model.model_vision.stuff_prob_thing = -1.0
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/BDD10k_SemanticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py

@@ -0,0 +1,47 @@
+from detectron2.config import LazyCall as L
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+
+from .ape_deta_vitl_eva02_lsj1024 import dataloader, lr_multiplier, model, optimizer, train
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 12800

approach/ovod/APE/configs/COCO_Detection/deformable_deta/deformable_deta_r50_12ep.py

@@ -0,0 +1,48 @@
+from detrex.config import get_config
+
+from .models.deformable_deta_r50 import model
+
+dataloader = get_config("common/data/coco_detr.py").dataloader
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [75000, 90000]
+optimizer = get_config("common/optim.py").AdamW
+train = get_config("common/train.py").train
+
+train.init_checkpoint = "detectron2://ImageNetPretrained/torchvision/R-50.pkl"
+train.init_checkpoint = "models/torchvision/R-50.pkl"
+train.output_dir = "output/" + __file__[:-3]
+
+train.max_iter = 90000
+
+train.eval_period = 5000
+
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "backbone" in module_name
+    or "reference_points" in module_name
+    or "sampling_offsets" in module_name
+    else 1
+)
+optimizer.params.weight_decay_norm = None
+
+dataloader.train.num_workers = 16
+
+dataloader.train.total_batch_size = 16
+
+
+train.amp.enabled = False
+train.ddp.fp16_compression = False

approach/ovod/APE/configs/COCO_Detection/deformable_deta/deformable_deta_r50_24ep.py

@@ -0,0 +1,47 @@
+from detrex.config import get_config
+
+from .models.deformable_deta_r50 import model
+
+dataloader = get_config("common/data/coco_detr.py").dataloader
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_24ep
+optimizer = get_config("common/optim.py").AdamW
+train = get_config("common/train.py").train
+
+train.init_checkpoint = "detectron2://ImageNetPretrained/torchvision/R-50.pkl"
+train.init_checkpoint = "models/torchvision/R-50.pkl"
+train.output_dir = "output/" + __file__[:-3]
+
+train.max_iter = 180000
+
+train.eval_period = 5000
+
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "backbone" in module_name
+    or "reference_points" in module_name
+    or "sampling_offsets" in module_name
+    else 1
+)
+optimizer.params.weight_decay_norm = None
+
+dataloader.train.num_workers = 16
+
+dataloader.train.total_batch_size = 16
+
+
+train.amp.enabled = False
+train.ddp.fp16_compression = False

approach/ovod/APE/configs/COCO_Detection/deformable_deta/deformable_deta_vitb_clip_openai_lsj1024_cp_12ep.py

@@ -0,0 +1,20 @@
+
+
+from ...common.data.coco_lsj1024_cp import dataloader
+from ...common.data.constants import constants
+from .deformable_deta_vitb_lsj1024_12ep import lr_multiplier, model, optimizer, train
+
+model.pixel_mean = constants.openai_imagenet_rgb256_mean
+model.pixel_std = constants.openai_imagenet_rgb256_std
+model.input_format = "RGB"
+dataloader.train.mapper.image_format = "RGB"
+
+
+
+
+train.init_checkpoint = "models/CLIP/ViT-B-16.pt"
+
+train.output_dir = "output/" + __file__[:-3]
+dataloader.evaluator.output_dir = train.output_dir
+dataloader.train.mapper.output_dir = train.output_dir
+dataloader.train.mapper.vis_period = 1

approach/ovod/APE/configs/COCO_Detection/deformable_deta/deformable_deta_vitb_lsj1024_12ep.py

@@ -0,0 +1,79 @@
+from functools import partial
+
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.modeling import SimpleFeaturePyramid, ViT
+from detectron2.modeling.backbone.fpn import LastLevelMaxPool
+from detectron2.modeling.backbone.vit import get_vit_lr_decay_rate
+
+from .....detectron2.configs.common.data.constants import constants
+from .....detectron2.projects.ViTDet.configs.common.coco_loader_lsj import dataloader
+from .deformable_deta_r50_12ep import lr_multiplier, model, optimizer, train
+
+model.pixel_mean = constants.imagenet_rgb256_mean
+model.pixel_std = constants.imagenet_rgb256_std
+model.input_format = "RGB"
+dataloader.train.mapper.image_format = "RGB"
+dataloader.train.total_batch_size = 16
+
+
+embed_dim, depth, num_heads, dp = 768, 12, 12, 0.1
+model.backbone = L(SimpleFeaturePyramid)(
+    net=L(ViT)(  # Single-scale ViT backbone
+        img_size=1024,
+        patch_size=16,
+        embed_dim=embed_dim,
+        depth=depth,
+        num_heads=num_heads,
+        drop_path_rate=dp,
+        window_size=14,
+        mlp_ratio=4,
+        qkv_bias=True,
+        norm_layer=partial(nn.LayerNorm, eps=1e-6),
+        window_block_indexes=[
+            0,
+            1,
+            3,
+            4,
+            6,
+            7,
+            9,
+            10,
+        ],
+        residual_block_indexes=[],
+        use_rel_pos=True,
+        out_feature="last_feat",
+    ),
+    in_feature="${.net.out_feature}",
+    out_channels=256,
+    scale_factors=(4.0, 2.0, 1.0, 0.5),
+    top_block=L(LastLevelMaxPool)(),
+    norm="LN",
+    square_pad=1024,
+)
+
+model.neck = None
+
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.7, num_layers=12)
+    if "backbone" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+
+
+lr_multiplier.warmup_length = 1000 / train.max_iter
+
+train.amp.enabled = False
+train.ddp.fp16_compression = False
+
+train.init_checkpoint = (
+    "detectron2://ImageNetPretrained/MAE/mae_pretrain_vit_base.pth?matching_heuristics=True"
+)
+train.init_checkpoint = "models/MAE/mae_pretrain_vit_base.pth?matching_heuristics=True"
+
+train.output_dir = "output/" + __file__[:-3]
+dataloader.evaluator.output_dir = train.output_dir

approach/ovod/APE/configs/COCO_Detection/deformable_deta/deformable_deta_vitg_eva_lsj1024_12ep.py

@@ -0,0 +1,62 @@
+from functools import partial
+
+from ape.modeling.backbone.vit_eva import SimpleFeaturePyramid, ViT, get_vit_lr_decay_rate
+
+from ..common.coco_loader_lsj1280 import dataloader
+from .deformable_deta_vitb_lsj1024_12ep import lr_multiplier, model, optimizer, train
+
+model.backbone.update(
+    _target_=SimpleFeaturePyramid,
+)
+model.backbone.net.update(
+    _target_=ViT,
+)
+
+dataloader.train.total_batch_size = 16
+
+model.backbone.net.beit_like_qkv_bias = True
+model.backbone.net.beit_like_gamma = False
+model.backbone.net.freeze_patch_embed = True
+model.backbone.square_pad = 1280
+model.backbone.net.img_size = 1280
+model.backbone.net.patch_size = 16
+model.backbone.net.window_size = 16
+model.backbone.net.embed_dim = 1408
+model.backbone.net.depth = 40
+model.backbone.net.num_heads = 16
+model.backbone.net.mlp_ratio = 6144 / 1408
+model.backbone.net.use_act_checkpoint = True
+model.backbone.net.drop_path_rate = 0.6  # 0.5 --> 0.6
+model.backbone.net.window_block_indexes = (
+    list(range(0, 3))
+    + list(range(4, 7))
+    + list(range(8, 11))
+    + list(range(12, 15))
+    + list(range(16, 19))
+    + list(range(20, 23))
+    + list(range(24, 27))
+    + list(range(28, 31))
+    + list(range(32, 35))
+    + list(range(36, 39))
+)
+
+optimizer.lr = 2e-4
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.9, num_layers=40)
+    if "backbone" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+optimizer.params.weight_decay_norm = None
+
+train.amp.enabled = False
+train.ddp.fp16_compression = False
+
+model.backbone.net.use_act_checkpoint = False
+model.backbone.net.frozen_stages = 41
+
+train.init_checkpoint = "models/BAAI/EVA/eva_o365.pth?matching_heuristics=True"
+train.output_dir = "output/" + __file__[:-3]
+dataloader.evaluator.output_dir = train.output_dir

approach/ovod/APE/configs/COCO_Detection/deformable_deta/deformable_deta_vitg_eva_lsj1024_cp_12ep.py

@@ -0,0 +1,12 @@
+from ....configs.common.data.coco_lsj1024_cp import dataloader
+from .deformable_deta_vitg_eva_lsj1024_12ep import lr_multiplier, model, optimizer, train
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+
+model.backbone.net.use_act_checkpoint = True
+model.backbone.net.frozen_stages = 20
+
+train.output_dir = "output/" + __file__[:-3]
+dataloader.evaluator.output_dir = train.output_dir
+dataloader.train.mapper.output_dir = train.output_dir

approach/ovod/APE/configs/COCO_Detection/deformable_deta/deformable_deta_vitl_eva02_lsj1024_cp_12ep.py

@@ -0,0 +1,101 @@
+from functools import partial
+
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.data.catalog import MetadataCatalog
+from detectron2.layers import ShapeSpec
+from detectron2.modeling.backbone.fpn import LastLevelMaxPool
+from detrex.config import get_config
+from ape.modeling.backbone.vit_eva02 import SimpleFeaturePyramid, ViT, get_vit_lr_decay_rate
+
+from .....detectron2.configs.common.data.constants import constants
+from ...common.data.coco_instance_lsj1024_cp import dataloader
+from .models.deformable_deta_r50 import model
+
+model.pixel_mean = constants.imagenet_rgb256_mean
+model.pixel_std = constants.imagenet_rgb256_std
+model.input_format = "RGB"
+
+model.backbone = L(SimpleFeaturePyramid)(
+    net=L(ViT)(  # Single-scale ViT backbone
+        img_size=1024,
+        patch_size=16,
+        embed_dim=1024,
+        depth=24,
+        num_heads=16,
+        drop_path_rate=0.4,
+        window_size=16,
+        mlp_ratio=4 * 2 / 3,
+        qkv_bias=True,
+        norm_layer=partial(nn.LayerNorm, eps=1e-6),
+        window_block_indexes=list(range(0, 5))
+        + list(range(6, 11))
+        + list(range(12, 17))
+        + list(range(18, 23)),
+        residual_block_indexes=[],
+        use_rel_pos=True,
+        out_feature="last_feat",
+        use_act_checkpoint=False,
+        xattn=True,
+    ),
+    in_feature="${.net.out_feature}",
+    out_channels=256,
+    scale_factors=(4.0, 2.0, 1.0, 0.5),
+    top_block=L(LastLevelMaxPool)(),
+    norm="LN",
+    square_pad=1024,
+)
+
+model.neck = None
+
+optimizer = get_config("common/optim.py").AdamW
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.8, num_layers=24)
+    if "backbone" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+optimizer.params.weight_decay_norm = None
+
+optimizer.lr = 2e-4
+optimizer.weight_decay = 1e-4
+
+train = get_config("common/train.py").train
+train.max_iter = 90000
+train.eval_period = 5000
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+train.init_checkpoint = (
+    "models/Yunxin-CV/EVA-02/eva02/pt/eva02_L_pt_in21k_p14to16.pt?matching_heuristics=True"
+)
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [75000, 90000]
+lr_multiplier.warmup_length = 1000 / train.max_iter
+
+dataloader.train.num_workers = 16
+dataloader.train.total_batch_size = 16
+dataloader.train.mapper.image_format = "RGB"
+
+if isinstance(dataloader.train.dataset.names, str):
+    model.metadata = MetadataCatalog.get(dataloader.train.dataset.names)
+else:
+    model.metadata = MetadataCatalog.get(dataloader.train.dataset.names[0])
+
+train.output_dir = "output/" + __file__[:-3]
+dataloader.train.mapper.output_dir = train.output_dir

approach/ovod/APE/configs/COCO_Detection/deformable_deta/deformable_deta_vitl_eva_lsj1024_cp_12ep.py

@@ -0,0 +1,25 @@
+from detectron2.modeling.backbone.vit import get_vit_lr_decay_rate
+
+from ...common.data.coco_lsj1024_cp import dataloader
+from .deformable_deta_vitl_lsj1024_12ep import lr_multiplier, model, optimizer, train
+
+train.init_checkpoint = "models/BAAI/EVA/eva_l_psz14to16.pt?matching_heuristics=True"
+
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.8, num_layers=24)
+    if "backbone" in module_name
+    else 1
+)
+
+optimizer.lr = 2e-4
+optimizer.weight_decay = 1e-4
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+model.backbone.net.use_act_checkpoint = False
+
+train.output_dir = "output/" + __file__[:-3]
+dataloader.evaluator.output_dir = train.output_dir
+dataloader.train.mapper.output_dir = train.output_dir

approach/ovod/APE/configs/COCO_Detection/deformable_deta/deformable_deta_vitl_lsj1024_12ep.py

@@ -0,0 +1,35 @@
+from detectron2.modeling.backbone.vit import get_vit_lr_decay_rate
+
+from .deformable_deta_vitb_lsj1024_12ep import dataloader, lr_multiplier, model, optimizer, train
+
+model.backbone.net.embed_dim = 1024
+model.backbone.net.depth = 24
+model.backbone.net.num_heads = 16
+model.backbone.net.drop_path_rate = 0.4
+model.backbone.net.window_block_indexes = (
+    list(range(0, 5)) + list(range(6, 11)) + list(range(12, 17)) + list(range(18, 23))
+)
+
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.8, num_layers=24)
+    if "backbone" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+
+optimizer.lr = 2e-4
+optimizer.weight_decay = 0.05
+
+train.init_checkpoint = (
+    "detectron2://ImageNetPretrained/MAE/mae_pretrain_vit_large.pth?matching_heuristics=True"
+)
+train.init_checkpoint = "models/MAE/mae_pretrain_vit_large.pth?matching_heuristics=True"
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+model.backbone.net.use_act_checkpoint = False
+
+train.output_dir = "output/" + __file__[:-3]
+dataloader.evaluator.output_dir = train.output_dir

approach/ovod/APE/configs/COCO_Detection/deformable_deta/models/deformable_deta_r50.py

@@ -0,0 +1,124 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.modeling.backbone import BasicStem, ResNet
+from detrex.layers import PositionEmbeddingSine
+from detrex.modeling.matcher import HungarianMatcher
+from detrex.modeling.neck import ChannelMapper
+from ape.modeling.deta import (
+    DeformableCriterion,
+    DeformableDETR,
+    DeformableDetrTransformer,
+    DeformableDetrTransformerDecoder,
+    DeformableDetrTransformerEncoder,
+    Stage1Assigner,
+    Stage2Assigner,
+)
+
+
+
+model = L(DeformableDETR)(
+    backbone=L(ResNet)(
+        stem=L(BasicStem)(in_channels=3, out_channels=64, norm="FrozenBN"),
+        stages=L(ResNet.make_default_stages)(
+            depth=50,
+            stride_in_1x1=False,
+            norm="FrozenBN",
+        ),
+        out_features=["res3", "res4", "res5"],
+        freeze_at=2,
+    ),
+    position_embedding=L(PositionEmbeddingSine)(
+        num_pos_feats=128,
+        temperature=10000,
+        normalize=True,
+        offset=-0.5,
+    ),
+    neck=L(ChannelMapper)(
+        input_shapes={
+            "res3": ShapeSpec(channels=512),
+            "res4": ShapeSpec(channels=1024),
+            "res5": ShapeSpec(channels=2048),
+        },
+        in_features=["res3", "res4", "res5"],
+        out_channels=256,
+        num_outs=5,
+        kernel_size=1,
+        norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+    ),
+    transformer=L(DeformableDetrTransformer)(
+        encoder=L(DeformableDetrTransformerEncoder)(
+            embed_dim=256,
+            num_heads=8,
+            feedforward_dim=2048,
+            attn_dropout=0.0,
+            ffn_dropout=0.0,
+            num_layers=6,
+            post_norm=False,
+            num_feature_levels="${..num_feature_levels}",
+        ),
+        decoder=L(DeformableDetrTransformerDecoder)(
+            embed_dim=256,
+            num_heads=8,
+            feedforward_dim=2048,
+            attn_dropout=0.0,
+            ffn_dropout=0.0,
+            num_layers=6,
+            return_intermediate=True,
+            num_feature_levels="${..num_feature_levels}",
+        ),
+        as_two_stage="${..as_two_stage}",
+        num_feature_levels=5,
+        two_stage_num_proposals="${..num_queries}",
+        assign_first_stage=True,
+    ),
+    embed_dim=256,
+    num_classes=80,
+    num_queries=900,
+    aux_loss=True,
+    with_box_refine=True,
+    as_two_stage=True,
+    criterion=L(DeformableCriterion)(
+        num_classes=80,
+        matcher=L(HungarianMatcher)(
+            cost_class=2.0,
+            cost_bbox=5.0,
+            cost_giou=2.0,
+            cost_class_type="focal_loss_cost",
+            alpha=0.25,
+            gamma=2.0,
+        ),
+        matcher_stage1=L(Stage1Assigner)(
+            t_low=0.3,
+            t_high=0.7,
+            max_k=4,
+        ),
+        matcher_stage2=L(Stage2Assigner)(
+            num_queries="${...num_queries}",
+            num_classes="${...num_classes}",
+            max_k=4,
+        ),
+        weight_dict={
+            "loss_class": 1.0,
+            "loss_bbox": 5.0,
+            "loss_giou": 2.0,
+        },
+        loss_class_type="focal_loss",
+        alpha=0.25,
+        gamma=2.0,
+    ),
+    pixel_mean=[123.675, 116.280, 103.530],
+    pixel_std=[58.395, 57.120, 57.375],
+    select_box_nums_for_evaluation=100,
+    input_format="RGB",
+)
+
+if model.aux_loss:
+    weight_dict = model.criterion.weight_dict
+    aux_weight_dict = {}
+    for i in range(model.transformer.decoder.num_layers - 1):
+        aux_weight_dict.update({k + f"_{i}": v for k, v in weight_dict.items()})
+    aux_weight_dict.update({k + "_enc": v for k, v in weight_dict.items()})
+    weight_dict.update(aux_weight_dict)
+    model.criterion.weight_dict = weight_dict

approach/ovod/APE/configs/COCO_Detection/deformable_detr/deformable_detr_r50_50ep.py

@@ -0,0 +1,38 @@
+from detrex.config import get_config
+
+from .models.deformable_detr_r50 import model
+
+dataloader = get_config("common/data/coco_detr.py").dataloader
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_50ep
+optimizer = get_config("common/optim.py").AdamW
+train = get_config("common/train.py").train
+
+train.init_checkpoint = "detectron2://ImageNetPretrained/torchvision/R-50.pkl"
+train.output_dir = "./output/deformable_detr_r50_50ep"
+
+train.max_iter = 375000
+
+train.eval_period = 5000
+
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+model.device = train.device
+
+optimizer.lr = 1e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+optimizer.params.lr_factor_func = lambda module_name: 0.1 if "backbone" in module_name else 1
+
+dataloader.train.num_workers = 16
+
+dataloader.train.total_batch_size = 16
+
+dataloader.evaluator.output_dir = train.output_dir

approach/ovod/APE/configs/COCO_Detection/deformable_detr/deformable_detr_r50_two_stage_50ep.py

@@ -0,0 +1,7 @@
+from .deformable_detr_r50_50ep import dataloader, lr_multiplier, model, optimizer, train
+
+model.with_box_refine = True
+model.as_two_stage = True
+
+train.init_checkpoint = "detectron2://ImageNetPretrained/torchvision/R-50.pkl"
+train.output_dir = "./output/deformable_detr_r50_two_stage_50ep"

approach/ovod/APE/configs/COCO_Detection/deformable_detr/deformable_detr_r50_with_box_refinement_50ep.py

@@ -0,0 +1,6 @@
+from .deformable_detr_r50_50ep import dataloader, lr_multiplier, model, optimizer, train
+
+model.with_box_refine = True
+
+train.init_checkpoint = "detectron2://ImageNetPretrained/torchvision/R-50.pkl"
+train.output_dir = "./output/deformable_detr_with_box_refinement_50ep"

approach/ovod/APE/configs/COCO_Detection/deformable_detr/improved_deformable_detr_r50_12ep.py

@@ -0,0 +1,46 @@
+from detrex.config import get_config
+
+from .models.improved_deformable_detr_r50 import model
+
+dataloader = get_config("common/data/coco_detr.py").dataloader
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [75000, 90000]
+optimizer = get_config("common/optim.py").AdamW
+train = get_config("common/train.py").train
+
+train.init_checkpoint = "detectron2://ImageNetPretrained/torchvision/R-50.pkl"
+train.output_dir = "./output/improved_deformable_detr_r50_12ep"
+
+train.max_iter = 90000
+
+train.eval_period = 5000
+
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+model.device = train.device
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "backbone" in module_name
+    or "reference_points" in module_name
+    or "sampling_offsets" in module_name
+    else 1
+)
+optimizer.params.weight_decay_norm = None
+
+dataloader.train.num_workers = 16
+
+dataloader.train.total_batch_size = 16
+
+dataloader.evaluator.output_dir = train.output_dir

approach/ovod/APE/configs/COCO_Detection/deformable_detr/improved_deformable_detr_r50_50ep.py

@@ -0,0 +1,45 @@
+from detrex.config import get_config
+
+from .models.improved_deformable_detr_r50 import model
+
+dataloader = get_config("common/data/coco_detr.py").dataloader
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_50ep
+optimizer = get_config("common/optim.py").AdamW
+train = get_config("common/train.py").train
+
+train.init_checkpoint = "detectron2://ImageNetPretrained/torchvision/R-50.pkl"
+train.output_dir = "./output/improved_deformable_detr_r50_50ep"
+
+train.max_iter = 375000
+
+train.eval_period = 5000
+
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+model.device = train.device
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "backbone" in module_name
+    or "reference_points" in module_name
+    or "sampling_offsets" in module_name
+    else 1
+)
+optimizer.params.weight_decay_norm = None
+
+dataloader.train.num_workers = 16
+
+dataloader.train.total_batch_size = 16
+
+dataloader.evaluator.output_dir = train.output_dir

approach/ovod/APE/configs/COCO_Detection/deformable_detr/improved_deformable_detr_r50_two_stage_12ep.py

@@ -0,0 +1,7 @@
+from .improved_deformable_detr_r50_12ep import dataloader, lr_multiplier, model, optimizer, train
+
+model.with_box_refine = True
+model.as_two_stage = True
+
+train.init_checkpoint = "detectron2://ImageNetPretrained/torchvision/R-50.pkl"
+train.output_dir = "./output/improved_deformable_detr_r50_two_stage_12ep"

approach/ovod/APE/configs/COCO_Detection/deformable_detr/improved_deformable_detr_r50_two_stage_50ep.py

@@ -0,0 +1,7 @@
+from .improved_deformable_detr_r50_50ep import dataloader, lr_multiplier, model, optimizer, train
+
+model.with_box_refine = True
+model.as_two_stage = True
+
+train.init_checkpoint = "detectron2://ImageNetPretrained/torchvision/R-50.pkl"
+train.output_dir = "./output/improved_deformable_detr_r50_two_stage_50ep"

approach/ovod/APE/configs/COCO_Detection/deformable_detr/models/deformable_detr_r50.py

@@ -0,0 +1,109 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.modeling.backbone import BasicStem, ResNet
+from detrex.layers import PositionEmbeddingSine
+from detrex.modeling.matcher import HungarianMatcher
+from detrex.modeling.neck import ChannelMapper
+from projects.deformable_detr.modeling import (
+    DeformableCriterion,
+    DeformableDETR,
+    DeformableDetrTransformer,
+    DeformableDetrTransformerDecoder,
+    DeformableDetrTransformerEncoder,
+)
+
+model = L(DeformableDETR)(
+    backbone=L(ResNet)(
+        stem=L(BasicStem)(in_channels=3, out_channels=64, norm="FrozenBN"),
+        stages=L(ResNet.make_default_stages)(
+            depth=50,
+            stride_in_1x1=False,
+            norm="FrozenBN",
+        ),
+        out_features=["res3", "res4", "res5"],
+        freeze_at=1,
+    ),
+    position_embedding=L(PositionEmbeddingSine)(
+        num_pos_feats=128,
+        temperature=10000,
+        normalize=True,
+        offset=-0.5,
+    ),
+    neck=L(ChannelMapper)(
+        input_shapes={
+            "res3": ShapeSpec(channels=512),
+            "res4": ShapeSpec(channels=1024),
+            "res5": ShapeSpec(channels=2048),
+        },
+        in_features=["res3", "res4", "res5"],
+        out_channels=256,
+        num_outs=4,
+        kernel_size=1,
+        norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+    ),
+    transformer=L(DeformableDetrTransformer)(
+        encoder=L(DeformableDetrTransformerEncoder)(
+            embed_dim=256,
+            num_heads=8,
+            feedforward_dim=1024,
+            attn_dropout=0.1,
+            ffn_dropout=0.1,
+            num_layers=6,
+            post_norm=False,
+            num_feature_levels="${..num_feature_levels}",
+        ),
+        decoder=L(DeformableDetrTransformerDecoder)(
+            embed_dim=256,
+            num_heads=8,
+            feedforward_dim=1024,
+            attn_dropout=0.1,
+            ffn_dropout=0.1,
+            num_layers=6,
+            return_intermediate=True,
+            num_feature_levels="${..num_feature_levels}",
+        ),
+        as_two_stage="${..as_two_stage}",
+        num_feature_levels=4,
+        two_stage_num_proposals="${..num_queries}",
+    ),
+    embed_dim=256,
+    num_classes=80,
+    num_queries=300,
+    aux_loss=True,
+    with_box_refine=False,
+    as_two_stage=False,
+    criterion=L(DeformableCriterion)(
+        num_classes=80,
+        matcher=L(HungarianMatcher)(
+            cost_class=2.0,
+            cost_bbox=5.0,
+            cost_giou=2.0,
+            cost_class_type="focal_loss_cost",
+            alpha=0.25,
+            gamma=2.0,
+        ),
+        weight_dict={
+            "loss_class": 1.0,
+            "loss_bbox": 5.0,
+            "loss_giou": 2.0,
+        },
+        loss_class_type="focal_loss",
+        alpha=0.25,
+        gamma=2.0,
+    ),
+    pixel_mean=[123.675, 116.280, 103.530],
+    pixel_std=[58.395, 57.120, 57.375],
+    select_box_nums_for_evaluation=300,
+    device="cuda",
+)
+
+if model.aux_loss:
+    weight_dict = model.criterion.weight_dict
+    aux_weight_dict = {}
+    for i in range(model.transformer.decoder.num_layers - 1):
+        aux_weight_dict.update({k + f"_{i}": v for k, v in weight_dict.items()})
+    aux_weight_dict.update({k + "_enc": v for k, v in weight_dict.items()})
+    weight_dict.update(aux_weight_dict)
+    model.criterion.weight_dict = weight_dict

approach/ovod/APE/configs/COCO_Detection/deformable_detr/models/improved_deformable_detr_r50.py

@@ -0,0 +1,109 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.modeling.backbone import BasicStem, ResNet
+from detrex.layers import PositionEmbeddingSine
+from detrex.modeling.matcher import HungarianMatcher
+from detrex.modeling.neck import ChannelMapper
+from projects.deformable_detr.modeling import (
+    DeformableCriterion,
+    DeformableDETR,
+    DeformableDetrTransformer,
+    DeformableDetrTransformerDecoder,
+    DeformableDetrTransformerEncoder,
+)
+
+model = L(DeformableDETR)(
+    backbone=L(ResNet)(
+        stem=L(BasicStem)(in_channels=3, out_channels=64, norm="FrozenBN"),
+        stages=L(ResNet.make_default_stages)(
+            depth=50,
+            stride_in_1x1=False,
+            norm="FrozenBN",
+        ),
+        out_features=["res3", "res4", "res5"],
+        freeze_at=2,
+    ),
+    position_embedding=L(PositionEmbeddingSine)(
+        num_pos_feats=128,
+        temperature=10000,
+        normalize=True,
+        offset=-0.5,
+    ),
+    neck=L(ChannelMapper)(
+        input_shapes={
+            "res3": ShapeSpec(channels=512),
+            "res4": ShapeSpec(channels=1024),
+            "res5": ShapeSpec(channels=2048),
+        },
+        in_features=["res3", "res4", "res5"],
+        out_channels=256,
+        num_outs=5,
+        kernel_size=1,
+        norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+    ),
+    transformer=L(DeformableDetrTransformer)(
+        encoder=L(DeformableDetrTransformerEncoder)(
+            embed_dim=256,
+            num_heads=8,
+            feedforward_dim=2048,
+            attn_dropout=0.0,
+            ffn_dropout=0.0,
+            num_layers=6,
+            post_norm=False,
+            num_feature_levels="${..num_feature_levels}",
+        ),
+        decoder=L(DeformableDetrTransformerDecoder)(
+            embed_dim=256,
+            num_heads=8,
+            feedforward_dim=2048,
+            attn_dropout=0.0,
+            ffn_dropout=0.0,
+            num_layers=6,
+            return_intermediate=True,
+            num_feature_levels="${..num_feature_levels}",
+        ),
+        as_two_stage="${..as_two_stage}",
+        num_feature_levels=5,
+        two_stage_num_proposals="${..num_queries}",
+    ),
+    embed_dim=256,
+    num_classes=80,
+    num_queries=900,
+    aux_loss=True,
+    with_box_refine=False,
+    as_two_stage=False,
+    criterion=L(DeformableCriterion)(
+        num_classes=80,
+        matcher=L(HungarianMatcher)(
+            cost_class=2.0,
+            cost_bbox=5.0,
+            cost_giou=2.0,
+            cost_class_type="focal_loss_cost",
+            alpha=0.25,
+            gamma=2.0,
+        ),
+        weight_dict={
+            "loss_class": 1.0,
+            "loss_bbox": 5.0,
+            "loss_giou": 2.0,
+        },
+        loss_class_type="focal_loss",
+        alpha=0.25,
+        gamma=2.0,
+    ),
+    pixel_mean=[123.675, 116.280, 103.530],
+    pixel_std=[58.395, 57.120, 57.375],
+    select_box_nums_for_evaluation=300,
+    device="cuda",
+)
+
+if model.aux_loss:
+    weight_dict = model.criterion.weight_dict
+    aux_weight_dict = {}
+    for i in range(model.transformer.decoder.num_layers - 1):
+        aux_weight_dict.update({k + f"_{i}": v for k, v in weight_dict.items()})
+    aux_weight_dict.update({k + "_enc": v for k, v in weight_dict.items()})
+    weight_dict.update(aux_weight_dict)
+    model.criterion.weight_dict = weight_dict

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/ape_deta_r50_12ep.py

@@ -0,0 +1,63 @@
+from detectron2.config import LazyCall as L
+from detrex.config import get_config
+from ape.modeling.text import EVA01CLIP
+
+from ...common.data.coco_instance import dataloader
+from .models.ape_deta_r50 import model
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [75000, 90000]
+optimizer = get_config("common/optim.py").AdamW
+train = get_config("common/train.py").train
+
+train.init_checkpoint = "detectron2://ImageNetPretrained/torchvision/R-50.pkl"
+train.init_checkpoint = "models/torchvision/R-50.pkl"
+
+train.max_iter = 90000
+
+train.eval_period = 5000
+
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "backbone" in module_name
+    or "reference_points" in module_name
+    or "sampling_offsets" in module_name
+    else 1
+)
+optimizer.params.weight_decay_norm = None
+
+dataloader.train.num_workers = 16
+
+dataloader.train.total_batch_size = 16
+
+dataloader.train.mapper.use_instance_mask = True
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.dataset_names = ["coco_2017"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]
+
+model.model_language = L(EVA01CLIP)(
+    clip_model="EVA_CLIP_g_14_X", cache_dir="models/BAAI/EVA/eva_clip_psz14.pt"
+)
+model.model_vision.embed_dim_language = 1024
+model.model_vision.text_feature_reduce_type = "last"
+model.model_vision.text_feature_reduce_before_fusion = True

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/ape_deta_r50_vlf_12ep.py

@@ -0,0 +1,46 @@
+from detectron2.config import LazyCall as L
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+
+from .ape_deta_r50_12ep import dataloader, lr_multiplier, model, optimizer, train
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 12800

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vite_eva02_clip_lsj1536_cp_64x90k.py

@@ -0,0 +1,85 @@
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detrex.config import get_config
+from ape.modeling.backbone.vit import get_vit_lr_decay_rate
+from ape.modeling.text import EVA02CLIP
+
+from .....detectron2.configs.common.data.constants import constants
+from ...common.backbone.vite_eva02_clip_1536 import backbone
+from ...common.data.coco_instance_lsj1536_cp import dataloader
+from .models.ape_deta_r50 import model
+
+model.model_vision.pixel_mean = constants.imagenet_rgb256_mean
+model.model_vision.pixel_std = constants.imagenet_rgb256_std
+model.model_vision.input_format = "RGB"
+
+model.model_vision.backbone = backbone
+
+model.model_vision.neck = None
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+optimizer = get_config("common/optim.py").AdamW
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.8, num_layers=64)
+    if "backbone.net" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+optimizer.params.weight_decay_norm = None
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+
+train = get_config("common/train.py").train
+train.max_iter = 90000
+train.eval_period = 5000
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14to16_plus_s9B.pt?matching_heuristics=True"
+)
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [75000, 90000]
+lr_multiplier.warmup_length = 1000 / train.max_iter
+
+dataloader.train.num_workers = 8
+dataloader.train.total_batch_size = 64
+dataloader.train.mapper.image_format = "RGB"
+dataloader.train.mapper.use_instance_mask = True
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.dataset_names = ["coco_2017"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitg_eva01_clip_lsj1536_cp_128x45k.py

@@ -0,0 +1,84 @@
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detrex.config import get_config
+from ape.modeling.backbone.vit import get_vit_lr_decay_rate
+from ape.modeling.text import EVA02CLIP
+
+from .....detectron2.configs.common.data.constants import constants
+from ...common.backbone.vitg_eva01_clip_1536 import backbone
+from ...common.data.coco_instance_lsj1536_cp import dataloader
+from .models.ape_deta_r50 import model
+
+model.model_vision.pixel_mean = constants.imagenet_rgb256_mean
+model.model_vision.pixel_std = constants.imagenet_rgb256_std
+model.model_vision.input_format = "RGB"
+
+model.model_vision.backbone = backbone
+
+model.model_vision.neck = None
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+optimizer = get_config("common/optim.py").AdamW
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.9, num_layers=40)
+    if "backbone.net" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+optimizer.params.weight_decay_norm = None
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+
+train = get_config("common/train.py").train
+train.max_iter = 45000
+train.eval_period = 5000
+train.log_period = 20
+
+train.checkpointer.period = 2500
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA01_CLIP_g_14_plus_psz14to16_s11B.pt?matching_heuristics=True"
+)
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [37500, 45000]
+lr_multiplier.warmup_length = 1000 / train.max_iter
+
+dataloader.train.num_workers = 16
+dataloader.train.total_batch_size = 128
+dataloader.train.mapper.image_format = "RGB"
+dataloader.train.mapper.use_instance_mask = True
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.dataset_names = ["coco_2017"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA01-CLIP-g-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA01_CLIP_g_14_plus_psz14_s11B.pt",
+)
+model.model_vision.embed_dim_language = 1024

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitg_eva01_clip_lsj1536_cp_64x90k.py

@@ -0,0 +1,84 @@
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detrex.config import get_config
+from ape.modeling.backbone.vit import get_vit_lr_decay_rate
+from ape.modeling.text import EVA02CLIP
+
+from .....detectron2.configs.common.data.constants import constants
+from ...common.backbone.vitg_eva01_clip_1536 import backbone
+from ...common.data.coco_instance_lsj1536_cp import dataloader
+from .models.ape_deta_r50 import model
+
+model.model_vision.pixel_mean = constants.imagenet_rgb256_mean
+model.model_vision.pixel_std = constants.imagenet_rgb256_std
+model.model_vision.input_format = "RGB"
+
+model.model_vision.backbone = backbone
+
+model.model_vision.neck = None
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+optimizer = get_config("common/optim.py").AdamW
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.9, num_layers=40)
+    if "backbone.net" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+optimizer.params.weight_decay_norm = None
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+
+train = get_config("common/train.py").train
+train.max_iter = 90000
+train.eval_period = 5000
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA01_CLIP_g_14_plus_psz14to16_s11B.pt?matching_heuristics=True"
+)
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [75000, 90000]
+lr_multiplier.warmup_length = 1000 / train.max_iter
+
+dataloader.train.num_workers = 16
+dataloader.train.total_batch_size = 64
+dataloader.train.mapper.image_format = "RGB"
+dataloader.train.mapper.use_instance_mask = True
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.dataset_names = ["coco_2017"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA01-CLIP-g-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA01_CLIP_g_14_plus_psz14_s11B.pt",
+)
+model.model_vision.embed_dim_language = 1024

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitg_eva01_lsj1536_cp_64x90k.py

@@ -0,0 +1,81 @@
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detrex.config import get_config
+from ape.modeling.backbone.vit import get_vit_lr_decay_rate
+from ape.modeling.text import EVA01CLIP
+
+from .....detectron2.configs.common.data.constants import constants
+from ...common.backbone.vitg_eva01_1536 import backbone
+from ...common.data.coco_instance_lsj1536_cp import dataloader
+from .models.ape_deta_r50 import model
+
+model.model_vision.pixel_mean = constants.imagenet_rgb256_mean
+model.model_vision.pixel_std = constants.imagenet_rgb256_std
+model.model_vision.input_format = "RGB"
+
+model.model_vision.backbone = backbone
+
+model.model_vision.neck = None
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+optimizer = get_config("common/optim.py").AdamW
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.9, num_layers=40)
+    if "backbone.net" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+optimizer.params.weight_decay_norm = None
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+
+train = get_config("common/train.py").train
+train.max_iter = 90000
+train.eval_period = 5000
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+train.init_checkpoint = "models/BAAI/EVA/eva_o365.pth?matching_heuristics=True"
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [75000, 90000]
+lr_multiplier.warmup_length = 1000 / train.max_iter
+
+dataloader.train.num_workers = 16
+dataloader.train.total_batch_size = 64
+dataloader.train.mapper.image_format = "RGB"
+dataloader.train.mapper.use_instance_mask = True
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.dataset_names = ["coco_2017"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]
+
+model.model_language = L(EVA01CLIP)(
+    clip_model="EVA_CLIP_g_14_X", cache_dir="models/BAAI/EVA/eva_clip_psz14.pt"
+)
+model.model_vision.embed_dim_language = 1024

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_clip_lsj1024_cp_12ep.py

@@ -0,0 +1,95 @@
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.model_zoo import get_config as get_config_d2
+from detrex.config import get_config as get_config_detrex
+from ape.modeling.backbone.vit import get_vit_lr_decay_rate
+
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from ...common.data.coco_instance_lsj1024_cp import dataloader
+from .models.ape_deta_r50 import model
+
+constants = get_config_d2("common/data/constants.py").constants
+
+model.model_vision.pixel_mean = constants.imagenet_rgb256_mean
+model.model_vision.pixel_std = constants.imagenet_rgb256_std
+model.model_vision.input_format = "RGB"
+
+model.model_vision.backbone = backbone
+
+model.model_vision.neck.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+model.model_vision.neck.in_features = ["p2", "p3", "p4", "p5", "p6"]
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+optimizer = get_config_detrex("common/optim.py").AdamW
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.8, num_layers=24)
+    if "backbone.net" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+optimizer.params.weight_decay_norm = None
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+
+train = get_config_detrex("common/train.py").train
+train.max_iter = 90000
+train.eval_period = 5000
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+
+lr_multiplier = get_config_detrex("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [75000, 90000]
+lr_multiplier.warmup_length = 1000 / train.max_iter
+
+dataloader.train.num_workers = 16
+dataloader.train.total_batch_size = 16
+dataloader.train.mapper.image_format = "RGB"
+dataloader.train.mapper.use_instance_mask = True
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.dataset_names = ["coco_2017"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_clip_lsj1536_cp_128x45k.py

@@ -0,0 +1,19 @@
+from .ape_deta_vitl_eva02_clip_lsj1536_cp_64x90k import (
+    dataloader,
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+
+train.max_iter = 45000
+
+train.eval_period = 2500
+
+train.checkpointer.period = 2500
+
+lr_multiplier.scheduler.milestones = [37500, 45000]
+
+dataloader.train.total_batch_size = 128
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_clip_lsj1536_cp_64x90k.py

@@ -0,0 +1,92 @@
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detrex.config import get_config
+from ape.modeling.backbone.vit import get_vit_lr_decay_rate
+
+from ape.modeling.text import EVA02CLIP
+
+from .....detectron2.configs.common.data.constants import constants
+from ...common.backbone.vitl_eva02_clip_1536 import backbone
+from ...common.data.coco_instance_lsj1536_cp import dataloader
+from .models.ape_deta_r50 import model
+
+model.model_vision.pixel_mean = constants.imagenet_rgb256_mean
+model.model_vision.pixel_std = constants.imagenet_rgb256_std
+model.model_vision.input_format = "RGB"
+
+model.model_vision.backbone = backbone
+
+model.model_vision.neck.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+model.model_vision.neck.in_features = ["p2", "p3", "p4", "p5", "p6"]
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+optimizer = get_config("common/optim.py").AdamW
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.8, num_layers=24)
+    if "backbone.net" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+optimizer.params.weight_decay_norm = None
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+
+train = get_config("common/train.py").train
+train.max_iter = 90000
+train.eval_period = 5000
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [75000, 90000]
+lr_multiplier.warmup_length = 1000 / train.max_iter
+
+dataloader.train.num_workers = 16
+dataloader.train.total_batch_size = 64
+dataloader.train.mapper.image_format = "RGB"
+dataloader.train.mapper.use_instance_mask = True
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.dataset_names = ["coco_2017"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+)
+model.model_vision.embed_dim_language = 1024

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_12ep.py

@@ -0,0 +1,52 @@
+from detectron2.config import LazyCall as L
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+
+from .ape_deta_vitl_eva02_clip_lsj1024_cp_12ep import (
+    dataloader,
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 12800

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024_cp_12ep.py

@@ -0,0 +1,86 @@
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+
+from detectron2.model_zoo import get_config as get_config_d2
+from detrex.config import get_config
+from ape.modeling.backbone.vit import get_vit_lr_decay_rate
+from ape.modeling.text import EVA01CLIP
+
+from ...common.backbone.vitl_eva02 import backbone
+from ...common.data.coco_instance_lsj1024_cp import dataloader
+from .models.ape_deta_r50 import model
+
+constants = get_config_d2("common/data/constants.py").constants
+
+model.model_vision.pixel_mean = constants.imagenet_rgb256_mean
+model.model_vision.pixel_std = constants.imagenet_rgb256_std
+model.model_vision.input_format = "RGB"
+
+model.model_vision.backbone = backbone
+
+model.model_vision.neck = None
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+optimizer = get_config("common/optim.py").AdamW
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.8, num_layers=24)
+    if "backbone.net" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+optimizer.params.weight_decay_norm = None
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+
+train = get_config("common/train.py").train
+train.max_iter = 90000
+train.eval_period = 5000
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+train.init_checkpoint = (
+    "models/Yuxin-CV/EVA-02/eva02/pt/eva02_L_pt_in21k_p14to16.pt?matching_heuristics=True"
+)
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [75000, 90000]
+lr_multiplier.warmup_length = 1000 / train.max_iter
+
+dataloader.train.num_workers = 16
+dataloader.train.total_batch_size = 16
+dataloader.train.mapper.image_format = "RGB"
+dataloader.train.mapper.use_instance_mask = True
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.dataset_names = ["coco_2017"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]
+
+model.model_language = L(EVA01CLIP)(
+    clip_model="EVA_CLIP_g_14_X", cache_dir="models/BAAI/EVA/eva_clip_psz14.pt"
+)
+model.model_vision.embed_dim_language = 1024

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1536_cp_128x90k.py

@@ -0,0 +1,11 @@
+from .ape_deta_vitl_eva02_lsj1536_cp_64x90k import (
+    dataloader,
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+
+dataloader.train.total_batch_size = 128
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1536_cp_12ep.py

@@ -0,0 +1,83 @@
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detrex.config import get_config
+from ape.modeling.backbone.vit import get_vit_lr_decay_rate
+from ape.modeling.text import EVA01CLIP
+
+from .....detectron2.configs.common.data.constants import constants
+from ...common.backbone.vitl_eva02_1536 import backbone
+from ...common.data.coco_instance_lsj1536_cp import dataloader
+from .models.ape_deta_r50 import model
+
+model.model_vision.pixel_mean = constants.imagenet_rgb256_mean
+model.model_vision.pixel_std = constants.imagenet_rgb256_std
+model.model_vision.input_format = "RGB"
+
+model.model_vision.backbone = backbone
+
+model.model_vision.neck = None
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+optimizer = get_config("common/optim.py").AdamW
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.8, num_layers=24)
+    if "backbone.net" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+optimizer.params.weight_decay_norm = None
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+
+train = get_config("common/train.py").train
+train.max_iter = 90000
+train.eval_period = 5000
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+train.init_checkpoint = (
+    "models/Yuxin-CV/EVA-02/eva02/pt/eva02_L_pt_in21k_p14to16.pt?matching_heuristics=True"
+)
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [75000, 90000]
+lr_multiplier.warmup_length = 1000 / train.max_iter
+
+dataloader.train.num_workers = 16
+dataloader.train.total_batch_size = 16
+dataloader.train.mapper.image_format = "RGB"
+dataloader.train.mapper.use_instance_mask = True
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.dataset_names = ["coco_2017"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]
+
+model.model_language = L(EVA01CLIP)(
+    clip_model="EVA_CLIP_g_14_X", cache_dir="models/BAAI/EVA/eva_clip_psz14.pt"
+)
+model.model_vision.embed_dim_language = 1024

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1536_cp_64x90k.py

@@ -0,0 +1,83 @@
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detrex.config import get_config
+from ape.modeling.backbone.vit import get_vit_lr_decay_rate
+from ape.modeling.text import EVA01CLIP
+
+from .....detectron2.configs.common.data.constants import constants
+from ...common.backbone.vitl_eva02_1536 import backbone
+from ...common.data.coco_instance_lsj1536_cp import dataloader
+from .models.ape_deta_r50 import model
+
+model.model_vision.pixel_mean = constants.imagenet_rgb256_mean
+model.model_vision.pixel_std = constants.imagenet_rgb256_std
+model.model_vision.input_format = "RGB"
+
+model.model_vision.backbone = backbone
+
+model.model_vision.neck = None
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+optimizer = get_config("common/optim.py").AdamW
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.8, num_layers=24)
+    if "backbone.net" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+optimizer.params.weight_decay_norm = None
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+
+train = get_config("common/train.py").train
+train.max_iter = 90000
+train.eval_period = 5000
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+train.init_checkpoint = (
+    "models/Yuxin-CV/EVA-02/eva02/pt/eva02_L_pt_in21k_p14to16.pt?matching_heuristics=True"
+)
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [75000, 90000]
+lr_multiplier.warmup_length = 1000 / train.max_iter
+
+dataloader.train.num_workers = 16
+dataloader.train.total_batch_size = 64
+dataloader.train.mapper.image_format = "RGB"
+dataloader.train.mapper.use_instance_mask = True
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.dataset_names = ["coco_2017"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]
+
+model.model_language = L(EVA01CLIP)(
+    clip_model="EVA_CLIP_g_14_X", cache_dir="models/BAAI/EVA/eva_clip_psz14.pt"
+)
+model.model_vision.embed_dim_language = 1024

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_12ep.py

@@ -0,0 +1,46 @@
+from detectron2.config import LazyCall as L
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+
+from .ape_deta_vitl_eva02_lsj1024_cp_12ep import dataloader, lr_multiplier, model, optimizer, train
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 12800

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/ape_deta_vitl_lsj1024_cp_12ep.py

@@ -0,0 +1,118 @@
+from functools import partial
+
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+
+from detectron2.model_zoo import get_config as get_config_d2
+from detectron2.modeling.backbone.fpn import LastLevelMaxPool
+from detectron2.modeling.backbone.vit import SimpleFeaturePyramid, ViT, get_vit_lr_decay_rate
+from detrex.config import get_config
+from ape.modeling.text import EVA01CLIP
+
+from ...common.data.coco_instance_lsj1024_cp import dataloader
+from .models.ape_deta_r50 import model
+
+constants = get_config_d2("common/data/constants.py").constants
+
+model.model_vision.pixel_mean = constants.imagenet_rgb256_mean
+model.model_vision.pixel_std = constants.imagenet_rgb256_std
+model.model_vision.input_format = "RGB"
+
+model.model_vision.backbone = L(SimpleFeaturePyramid)(
+    net=L(ViT)(  # Single-scale ViT backbone
+        img_size=1024,
+        patch_size=16,
+        embed_dim=1024,
+        depth=24,
+        num_heads=16,
+        drop_path_rate=0.4,
+        window_size=14,
+        mlp_ratio=4,
+        norm_layer=partial(nn.LayerNorm, eps=1e-6),
+        window_block_indexes=list(range(0, 5))
+        + list(range(6, 11))
+        + list(range(12, 17))
+        + list(range(18, 23)),
+        residual_block_indexes=[],
+        use_rel_pos=True,
+        out_feature="last_feat",
+        use_act_checkpoint=True,
+    ),
+    in_feature="${.net.out_feature}",
+    out_channels=256,
+    scale_factors=(4.0, 2.0, 1.0, 0.5),
+    top_block=L(LastLevelMaxPool)(),
+    norm="LN",
+    square_pad=1024,
+)
+
+model.model_vision.neck = None
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+optimizer = get_config("common/optim.py").AdamW
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.8, num_layers=24)
+    if "backbone" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+
+optimizer.lr = 2e-4
+optimizer.weight_decay = 0.05
+
+train = get_config("common/train.py").train
+train.max_iter = 90000
+train.eval_period = 5000
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+train.init_checkpoint = (
+    "detectron2://ImageNetPretrained/MAE/mae_pretrain_vit_large.pth?matching_heuristics=True"
+)
+train.init_checkpoint = "models/MAE/mae_pretrain_vit_large.pth?matching_heuristics=True"
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [75000, 90000]
+lr_multiplier.warmup_length = 1000 / train.max_iter
+
+dataloader.train.num_workers = 16
+dataloader.train.total_batch_size = 16
+dataloader.train.mapper.image_format = "RGB"
+dataloader.train.mapper.use_instance_mask = True
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.dataset_names = ["coco_2017"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.output_dir = train.output_dir
+dataloader.train.mapper.output_dir = train.output_dir
+dataloader.train.mapper.vis_period = 12800
+
+model.model_language = L(EVA01CLIP)(
+    clip_model="EVA_CLIP_g_14_X", cache_dir="models/BAAI/EVA/eva_clip_psz14.pt"
+)
+model.model_vision.embed_dim_language = 1024

approach/ovod/APE/configs/COCO_InstanceSegmentation/ape_deta/models/ape_deta_r50.py

@@ -0,0 +1,155 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.modeling.backbone import BasicStem, ResNet
+from detrex.layers import PositionEmbeddingSine
+from detrex.modeling.matcher import HungarianMatcher
+from detrex.modeling.neck import ChannelMapper
+from ape.modeling.ape_deta import (
+    DeformableCriterion,
+    DeformableDETR,
+    DeformableDETRSegm,
+    DeformableDetrTransformer,
+    DeformableDetrTransformerDecoder,
+    DeformableDetrTransformerEncoder,
+    SomeThing,
+    Stage1Assigner,
+    Stage2Assigner,
+)
+from ape.modeling.text import T5_warpper
+
+
+
+model_vision = L(DeformableDETRSegm)(
+    backbone=L(ResNet)(
+        stem=L(BasicStem)(in_channels=3, out_channels=64, norm="FrozenBN"),
+        stages=L(ResNet.make_default_stages)(
+            depth=50,
+            stride_in_1x1=False,
+            norm="FrozenBN",
+        ),
+        out_features=["res2", "res3", "res4", "res5"],
+        freeze_at=1,
+    ),
+    position_embedding=L(PositionEmbeddingSine)(
+        num_pos_feats=128,
+        temperature=10000,
+        normalize=True,
+        offset=-0.5,
+    ),
+    neck=L(ChannelMapper)(
+        input_shapes={
+            "res3": ShapeSpec(channels=512),
+            "res4": ShapeSpec(channels=1024),
+            "res5": ShapeSpec(channels=2048),
+        },
+        in_features=["res3", "res4", "res5"],
+        out_channels=256,
+        num_outs=5,
+        kernel_size=1,
+        norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+    ),
+    transformer=L(DeformableDetrTransformer)(
+        encoder=L(DeformableDetrTransformerEncoder)(
+            embed_dim=256,
+            num_heads=8,
+            feedforward_dim=2048,
+            attn_dropout=0.0,
+            ffn_dropout=0.0,
+            num_layers=6,
+            post_norm=False,
+            num_feature_levels="${..num_feature_levels}",
+        ),
+        decoder=L(DeformableDetrTransformerDecoder)(
+            embed_dim=256,
+            num_heads=8,
+            feedforward_dim=2048,
+            attn_dropout=0.0,
+            ffn_dropout=0.0,
+            num_layers=6,
+            return_intermediate=True,
+            num_feature_levels="${..num_feature_levels}",
+        ),
+        as_two_stage="${..as_two_stage}",
+        num_feature_levels=5,
+        two_stage_num_proposals="${..num_queries}",
+        assign_first_stage=True,
+    ),
+    embed_dim=256,
+    num_classes=80,
+    num_queries=900,
+    aux_loss=True,
+    with_box_refine=True,
+    as_two_stage=True,
+    criterion=[
+        L(DeformableCriterion)(
+            num_classes="${...num_classes}",
+            matcher=L(HungarianMatcher)(
+                cost_class=2.0,
+                cost_bbox=5.0,
+                cost_giou=2.0,
+                cost_class_type="focal_loss_cost",
+                alpha=0.25,
+                gamma=2.0,
+            ),
+            matcher_stage1=L(Stage1Assigner)(
+                t_low=0.3,
+                t_high=0.7,
+                max_k=4,
+            ),
+            matcher_stage2=L(Stage2Assigner)(
+                num_queries="${model.model_vision.num_queries}",
+                num_classes="${..num_classes}",
+                max_k=4,
+            ),
+            weight_dict={
+                "loss_class": 1.0,
+                "loss_bbox": 5.0,
+                "loss_giou": 2.0,
+                "loss_mask": 5,
+                "loss_dice": 5,
+            },
+            loss_class_type="focal_loss",
+            alpha=0.25,
+            gamma=2.0,
+            losses=["class", "boxes", "masks"],
+        ),
+    ],
+    pixel_mean=[123.675, 116.280, 103.530],
+    pixel_std=[58.395, 57.120, 57.375],
+    select_box_nums_for_evaluation=100,
+    input_format="RGB",
+    mask_encode_level=0,
+    mask_in_features=["res2"],
+    input_shapes={
+        "res2": ShapeSpec(channels=256),
+        "res3": ShapeSpec(channels=512),
+        "res4": ShapeSpec(channels=1024),
+        "res5": ShapeSpec(channels=2048),
+    },
+    output_dir=None,
+    vis_period=0,
+    embed_dim_language=1024,
+    instance_on=True,
+    semantic_on=False,
+    panoptic_on=False,
+)
+
+if model_vision.aux_loss:
+    for j in range(len(model_vision.criterion)):
+        weight_dict = model_vision.criterion[j].weight_dict
+        aux_weight_dict = {}
+        for i in range(model_vision.transformer.decoder.num_layers - 1):
+            aux_weight_dict.update({k + f"_{i}": v for k, v in weight_dict.items()})
+        aux_weight_dict.update({k + "_enc": v for k, v in weight_dict.items()})
+        weight_dict.update(aux_weight_dict)
+        model_vision.criterion[j].weight_dict = weight_dict
+
+model = L(SomeThing)(
+    model_vision=model_vision,
+    model_language=L(T5_warpper)(
+        pretrained_model_name_or_path="models/google/flan-t5-large/",
+        eval_only=True,
+    ),
+)

approach/ovod/APE/configs/COCO_InstanceSegmentation/deformable_deta/deformable_deta_segm_r50_12ep.py

@@ -0,0 +1,53 @@
+from detrex.config import get_config
+
+from ...common.data.coco_instance import dataloader
+from .models.deformable_deta_segm_r50 import model
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [75000, 90000]
+optimizer = get_config("common/optim.py").AdamW
+train = get_config("common/train.py").train
+
+train.init_checkpoint = "detectron2://ImageNetPretrained/torchvision/R-50.pkl"
+train.init_checkpoint = "models/torchvision/R-50.pkl"
+train.output_dir = "output/" + __file__[:-3]
+
+train.max_iter = 90000
+
+train.eval_period = 5000
+
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "backbone" in module_name
+    or "reference_points" in module_name
+    or "sampling_offsets" in module_name
+    else 1
+)
+optimizer.params.weight_decay_norm = None
+
+dataloader.train.num_workers = 16
+
+dataloader.train.total_batch_size = 16
+
+
+dataloader.train.mapper.use_instance_mask = True
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+train.ddp.find_unused_parameters = False
+
+model.dataset_metas = dataloader.train.dataset.names