Add files using upload-large-folder tool

approach/ovod/APE/.gitignore

@@ -0,0 +1,53 @@
+# output dir
+output
+instant_test_output
+inference_test_output
+
+
+*.png
+*.json
+*.diff
+*.jpg
+!/projects/DensePose/doc/images/*.jpg
+
+# compilation and distribution
+__pycache__
+_ext
+*.pyc
+*.pyd
+*.so
+*.dll
+*.egg-info/
+build/
+dist/
+wheels/
+
+# pytorch/python/numpy formats
+*.pth
+*.pkl
+*.npy
+*.ts
+model_ts*.txt
+
+# ipython/jupyter notebooks
+*.ipynb
+**/.ipynb_checkpoints/
+
+# Editor temporaries
+*.swn
+*.swo
+*.swp
+*~
+
+# editor settings
+.idea
+.vscode
+_darcs
+
+# project dirs
+/ape/model_zoo/configs
+/datasets/*
+!/datasets/*.*
+/projects/*/datasets
+/models
+/snippet

approach/ovod/APE/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

approach/ovod/APE/README.md

@@ -0,0 +1,315 @@
+# APE: Aligning and Prompting Everything All at Once for Universal Visual Perception
+
+
+<!-- 
+<a href='https://github.com/shenyunhang/APE'><img src='https://img.shields.io/badge/Project-Page-Green'></a>
+<a href='https://arxiv.org/abs/2312.02153'><img src='https://img.shields.io/badge/Paper-Arxiv-red'></a>
+<a href='https://huggingface.co/spaces/shenyunhang/APE'><img src='https://img.shields.io/badge/%F0%9F%A4%97-Demo-yellow'></a>
+<a href='https://huggingface.co/shenyunhang/APE'><img src='https://img.shields.io/badge/%F0%9F%A4%97-Model-yellow'></a>
+[![Code License](https://img.shields.io/badge/Code%20License-Apache_2.0-green.svg)](https://github.com/tatsu-lab/stanford_alpaca/blob/main/LICENSE)
+-->
+
+<p align="center">
+    <img src="./.asset/ape.png" width="96%" height="96%">
+</p>
+
+
+<font size=7><div align='center' > :grapes: \[[Read our arXiv Paper](https://arxiv.org/abs/2312.02153)\] &nbsp; :apple: \[[Try our Online Demo](https://huggingface.co/spaces/shenyunhang/APE)\] </div></font>
+
+
+---
+
+<p align="center">
+    <img src="./.asset/example_1.png" width="96%" height="96%">
+</p>
+
+
+## :bulb: Highlight
+
+- **High Performance.**  SotA (or competitive) performance on **160** datasets with only one model.
+- **Perception in the Wild.** Detect and segment **everything** with thousands of vocabularies or language descriptions all at once.
+- **Flexible.** Support both foreground objects and background stuff for instance segmentation and semantic segmentation.
+
+## :fire: News
+* **`2024.02.27`** APE has been accepted to CVPR 2024!
+* **`2023.12.05`** Release training codes!
+* **`2023.12.05`** Release checkpoints!
+* **`2023.12.05`** Release inference codes and demo!
+
+## :label: TODO 
+
+- [x] Release inference code and demo.
+- [x] Release checkpoints.
+- [x] Release training codes.
+- [ ] Add clean docs.
+
+
+## :hammer_and_wrench: Install 
+
+1. Clone the APE repository from GitHub:
+
+```bash
+git clone https://github.com/shenyunhang/APE
+cd APE
+```
+
+2. Install the required dependencies and APE:
+
+```bash
+pip3 install -r requirements.txt
+python3 -m pip install -e .
+```
+
+
+## :arrow_forward: Demo Localy
+
+**Web UI demo**
+```
+pip3 install gradio
+cd APE/demo
+python3 app.py
+```
+This demo will detect GPUs and use one GPU if you have GPUs.
+
+Please feel free to try our [Online Demo](https://huggingface.co/spaces/shenyunhang/APE)!
+
+<p align="center">
+<img src="./.asset/demo.png" width="96%" height="96%">
+</p>
+
+
+## :books: Data Prepare
+Following [here](https://github.com/shenyunhang/APE/blob/main/datasets/README.md) to prepare the following datasets:
+
+|       |   COCO  |   LVIS  | Objects365 | Openimages | VisualGenome |  SA-1B  | RefCOCO |   GQA   | PhraseCut | Flickr30k |  ODinW  |  SegInW | Roboflow100 |  ADE20k | ADE-full |  BDD10k | Cityscapes |  PC459  |   PC59  |   VOC   |    D3   |
+|:-----:|:-------:|:-------:|:----------:|:----------:|:------------:|:-------:|:-------:|:-------:|:---------:|:---------:|:-------:|:-------:|:-----------:|:-------:|:--------:|:-------:|:----------:|:-------:|:-------:|:-------:|:-------:|
+| Train | &check; | &check; |   &check;  |   &check;  |    &check;   | &check; | &check; | &check; |  &check;  |  &check;  | &cross; | &cross; |   &cross;   | &cross; |  &cross; | &cross; |   &cross;  | &cross; | &cross; | &cross; | &cross; |
+|  Test | &check; | &check; |   &check;  |   &check;  |    &cross;   | &cross; | &check; | &cross; |  &cross;  |  &cross;  | &check; | &check; |   &check;   | &check; |  &check; | &check; |   &check;  | &check; | &check; | &check; | &check; |
+
+Noted we do not use `coco_2017_train` for training.
+
+Instead, we augment `lvis_v1_train` with annotations from coco, and keep the image set unchanged.
+
+And we register it as `lvis_v1_train+coco` for instance segmentation and `lvis_v1_train+coco_panoptic_separated` for panoptic segmentation.
+
+
+## :test_tube: Inference
+
+### Infer on 160+ dataset
+We provide several scripts to evaluate all models.
+
+It is necessary to adjust the checkpoint location and GPU number in the scripts before running them.
+
+```bash
+scripts/eval_all_D.sh
+scripts/eval_all_C.sh
+scripts/eval_all_B.sh
+scripts/eval_all_A.sh
+```
+
+### Infer on images or videos
+
+APE-D
+```
+python3.9 demo/demo_lazy.py \
+--config-file configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k.py \
+--input image1.jpg image2.jpg image3.jpg \
+--output /path/to/output/dir \
+--confidence-threshold 0.1 \
+--text-prompt 'person,car,chess piece of horse head' \
+--with-box \
+--with-mask \
+--with-sseg \
+--opts \
+train.init_checkpoint=/path/to/APE-D/checkpoint \
+model.model_language.cache_dir="" \
+model.model_vision.select_box_nums_for_evaluation=500 \
+model.model_vision.text_feature_bank_reset=True \
+```
+
+To disable `xformers`, add the following option:
+```
+model.model_vision.backbone.net.xattn=False \
+```
+
+To use `pytorch` version of `MultiScaleDeformableAttention`, add the following option:
+```
+model.model_vision.transformer.encoder.pytorch_attn=True \
+model.model_vision.transformer.decoder.pytorch_attn=True \
+```
+
+
+## :train: Training
+
+### Prepare backbone and language models
+```bash
+git lfs install
+git clone https://huggingface.co/QuanSun/EVA-CLIP models/QuanSun/EVA-CLIP/
+git clone https://huggingface.co/BAAI/EVA models/BAAI/EVA/
+git clone https://huggingface.co/Yuxin-CV/EVA-02 models/Yuxin-CV/EVA-02/
+```
+
+Resize patch size:
+```bash
+python3.9 tools/eva_interpolate_patch_14to16.py --input models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt --output models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14to16_plus_s9B.pt --image_size 224
+python3.9 tools/eva_interpolate_patch_14to16.py --input models/QuanSun/EVA-CLIP/EVA01_CLIP_g_14_plus_psz14_s11B.pt --output models/QuanSun/EVA-CLIP/EVA01_CLIP_g_14_plus_psz14to16_s11B.pt --image_size 224
+python3.9 tools/eva_interpolate_patch_14to16.py --input models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14_s6B.pt --output models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt --image_size 336
+```
+
+### Train APE-D
+
+Single node:
+```bash
+python3.9 tools/train_net.py \
+--num-gpus 8 \
+--resume \
+--config-file configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k_mdl.py \
+train.output_dir=output/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k_mdl_`date +'%Y%m%d_%H%M%S'`
+```
+
+Multiple nodes:
+```bash
+python3.9 tools/train_net.py \
+--dist-url="tcp://${MASTER_IP}:${MASTER_PORT}" \
+--num-gpus ${HOST_GPU_NUM} \
+--num-machines ${HOST_NUM} \
+--machine-rank ${INDEX} \
+--resume \
+--config-file configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k_mdl.py \
+train.output_dir=output/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k_mdl_`date +'%Y%m%d_%H'`0000
+```
+
+### Train APE-C
+
+Single node:
+```bash
+python3.9 tools/train_net.py \
+--num-gpus 8 \
+--resume \
+--config-file configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_1080k.py \
+train.output_dir=output/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_1080k_`date +'%Y%m%d_%H%M%S'`
+```
+
+Multiple nodes:
+```bash
+python3.9 tools/train_net.py \
+--dist-url="tcp://${MASTER_IP}:${MASTER_PORT}" \
+--num-gpus ${HOST_GPU_NUM} \
+--num-machines ${HOST_NUM} \
+--machine-rank ${INDEX} \
+--resume \
+--config-file configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_1080k.py \
+train.output_dir=output/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_1080k_`date +'%Y%m%d_%H'`0000
+```
+
+### Train APE-B
+
+Single node:
+```bash
+python3.9 tools/train_net.py \
+--num-gpus 8 \
+--resume \
+--config-file configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_1080k.py \
+train.output_dir=output/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_1080k_`date +'%Y%m%d_%H%M%S'`
+```
+
+Multiple nodes:
+```bash
+python3.9 tools/train_net.py \
+--dist-url="tcp://${MASTER_IP}:${MASTER_PORT}" \
+--num-gpus ${HOST_GPU_NUM} \
+--num-machines ${HOST_NUM} \
+--machine-rank ${INDEX} \
+--resume \
+--config-file configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_1080k.py \
+train.output_dir=output/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_1080k_`date +'%Y%m%d_%H'`0000
+```
+
+### Train APE-A
+
+Single node:
+```bash
+python3.9 tools/train_net.py \
+--num-gpus 8 \
+--resume \
+--config-file configs/LVISCOCOCOCOSTUFF_O365_OID_VG/ape_deta/ape_deta_vitl_eva02_lsj1024_cp_720k.py \
+train.output_dir=output/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VG/ape_deta/ape_deta_vitl_eva02_lsj1024_cp_720k_`date +'%Y%m%d_%H%M%S'`
+```
+
+Multiple nodes:
+```bash
+python3.9 tools/train_net.py \
+--dist-url="tcp://${MASTER_IP}:${MASTER_PORT}" \
+--num-gpus ${HOST_GPU_NUM} \
+--num-machines ${HOST_NUM} \
+--machine-rank ${INDEX} \
+--resume \
+--config-file configs/LVISCOCOCOCOSTUFF_O365_OID_VG/ape_deta/ape_deta_vitl_eva02_lsj1024_cp_720k.py \
+train.output_dir=output/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VG/ape_deta/ape_deta_vitl_eva02_lsj1024_cp_720k_`date +'%Y%m%d_%H'`0000
+```
+
+
+
+## :luggage: Checkpoints
+
+```
+git lfs install
+git clone https://huggingface.co/shenyunhang/APE
+```
+
+<!-- insert a table -->
+<table>
+  <thead>
+    <tr style="text-align: right;">
+      <th></th>
+      <th>name</th>
+      <th>Checkpoint</th>
+      <th>Config</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <th>1</th>
+      <td>APE-A</td>
+      <td><a href="https://huggingface.co/shenyunhang/APE/blob/main/configs/LVISCOCOCOCOSTUFF_O365_OID_VG/ape_deta/ape_deta_vitl_eva02_lsj_cp_720k_20230504_002019/model_final.pth">HF link</a></td>
+      <td><a href="https://github.com/shenyunhang/APE/blob/main/configs/LVISCOCOCOCOSTUFF_O365_OID_VG/ape_deta/ape_deta_vitl_eva02_lsj1024_cp_720k.py">link</a></td>
+    </tr>
+    <tr>
+      <th>2</th>
+      <td>APE-B</td>
+      <td><a href="https://huggingface.co/shenyunhang/APE/blob/main/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj_cp_1080k_20230702_225418/model_final.pth">HF link</a> 
+      <td><a href="https://github.com/shenyunhang/APE/blob/main/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_1080k.py">link</a></td>
+    </tr>
+    <tr>
+      <th>3</th>
+      <td>APE-C</td>
+      <td><a href="https://huggingface.co/shenyunhang/APE/blob/main/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj_cp_1080k_20230702_210950/model_final.pth">HF link</a> 
+      <td><a href="https://github.com/shenyunhang/APE/blob/main/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_1080k.py">link</a></td>
+    </tr>
+    <tr>
+      <th>4</th>
+      <td>APE-D</td>
+      <td><a href="https://huggingface.co/shenyunhang/APE/blob/main/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k_mdl_20230829_162438/model_final.pth">HF link</a> 
+      <td><a href="https://github.com/shenyunhang/APE/blob/main/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k_mdl.py">link</a></td>
+    </tr>
+  </tbody>
+</table>
+
+
+## :medal_military: Results
+
+<img src=".asset/radar.png" alt="radar" width="100%">
+
+
+## :black_nib: Citation
+
+If you find our work helpful for your research, please consider citing the following BibTeX entry.   
+
+```bibtex
+@inproceedings{APE,
+  title={Aligning and Prompting Everything All at Once for Universal Visual Perception},
+  author={Shen, Yunhang and Fu, Chaoyou and Chen, Peixian and Zhang, Mengdan and Li, Ke and Sun, Xing and Wu, Yunsheng and Lin, Shaohui and Ji, Rongrong},
+  journal={CVPR},
+  year={2024}
+}
+```

approach/ovod/APE/configs/ADE20k_PanopticSegmentation/ape_deta/ape_deta_r50_160k.py

@@ -0,0 +1,46 @@
+from detectron2.config import LazyCall as L
+from detectron2.solver import WarmupParamScheduler
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ...COCO_InstanceSegmentation.ape_deta.ape_deta_r50_12ep import (
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+from ...common.data.ade20k_panoptic import dataloader
+
+num_classes = 150
+model.num_classes = num_classes
+model.criterion.num_classes = num_classes
+model.criterion.matcher_stage2.num_classes = num_classes
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.dataset_names = ["ade20k_panoptic"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = True
+
+model.model_vision.stuff_prob_thing = -1.0
+
+train.max_iter = 160000
+train.eval_period = 5000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1, 0.01],
+        milestones=[135000, 150000],
+        num_updates=160000,
+    ),
+    warmup_length=1000 / 160000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+model.model_vision.semantic_post_nms = False
+model.model_vision.panoptic_post_nms = True
+model.model_vision.aux_mask = True
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/ADE20k_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024.py

@@ -0,0 +1,101 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detrex.modeling.neck import ChannelMapper
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from .ape_deta_vitl_eva02_lsj1024 import dataloader, lr_multiplier, model, optimizer, train
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = -1.0
+model.model_vision.transformer.proposal_ambiguous = 1
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 12800

approach/ovod/APE/configs/ADE20k_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024.py

@@ -0,0 +1,22 @@
+from ...COCO_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_12ep import (
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+from ...common.data.ade20k_panoptic_lsj1024 import dataloader
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.name_prompt_fusion_text = [False]
+model.model_vision.dataset_names = ["ade20k"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+model.model_vision.select_box_nums_for_evaluation = 300
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = True
+
+model.model_vision.stuff_prob_thing = -1.0
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/ADE20k_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024.py

@@ -0,0 +1,47 @@
+from detectron2.config import LazyCall as L
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+
+from .ape_deta_vitl_eva02_lsj1024 import dataloader, lr_multiplier, model, optimizer, train
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 12800

approach/ovod/APE/configs/COCO_REFCOCO/ape_deta/ape_deta_r50_12ep.py

@@ -0,0 +1,29 @@
+from detectron2.config import LazyCall as L
+from detrex.config import get_config
+
+from ...COCO_InstanceSegmentation.ape_deta.ape_deta_r50_12ep import (
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+from ...common.data.coco_refcoco_instance import dataloader
+
+model.model_vision.num_classes = 80
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+model.model_vision.criterion = [criterion for _ in range(2)]
+for criterion, num_classes in zip(model.model_vision.criterion, [80, 1]):
+    criterion.num_classes = num_classes
+
+model.model_vision.criterion[1].weight_dict["loss_class_enc"] = 0.0
+
+dataloader.train.total_batch_size = 16
+dataloader.train.total_batch_size_list = [16, 16]
+
+model.model_vision.dataset_prompts = ["name", "expression"]
+model.model_vision.dataset_names = ["coco_2017", "refcoco"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/COCO_REFCOCO/ape_deta/ape_deta_r50_24ep.py

@@ -0,0 +1,12 @@
+from detrex.config import get_config
+
+from .ape_deta_r50_12ep import dataloader, model, optimizer, train
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_24ep
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.output_dir = train.output_dir
+
+train.max_iter = 180000
+
+train.eval_period = 10000

approach/ovod/APE/configs/COCO_REFCOCO/ape_deta/ape_deta_r50_36ep.py

@@ -0,0 +1,12 @@
+from detrex.config import get_config
+
+from .ape_deta_r50_12ep import dataloader, model, optimizer, train
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_36ep
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.output_dir = train.output_dir
+
+train.max_iter = 270000
+
+train.eval_period = 15000

approach/ovod/APE/configs/COCO_REFCOCO/ape_deta/ape_deta_r50_vlf_12ep.py

@@ -0,0 +1,52 @@
+from detectron2.config import LazyCall as L
+from omegaconf import OmegaConf
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+
+from .ape_deta_r50_12ep import dataloader, lr_multiplier, model, optimizer, train
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    cfg=OmegaConf.from_dotlist(
+        [
+            "MODEL.DYHEAD.FUSE_CONFIG.STABLE_SOFTMAX_2D=False",
+            "MODEL.DYHEAD.FUSE_CONFIG.CLAMP_MIN_FOR_UNDERFLOW=True",
+            "MODEL.DYHEAD.FUSE_CONFIG.CLAMP_MAX_FOR_OVERFLOW=True",
+            "MODEL.VL_FUSION_USE_CHECKPOINT=True",
+        ],
+    ),
+)
+
+
+model.model_vision.text_feature_bank = False
+model.model_vision.text_feature_reduce_before_fusion = False
+model.model_vision.text_feature_batch_repeat = False
+model.model_vision.expression_cumulative_gt_class = False
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 5120

approach/ovod/APE/configs/COCO_REFCOCO/ape_deta/ape_deta_r50_vlf_36ep.py

@@ -0,0 +1,12 @@
+from detrex.config import get_config
+
+from .ape_deta_r50_vlf_12ep import dataloader, model, optimizer, train
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_36ep
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.output_dir = train.output_dir
+
+train.max_iter = 270000
+
+train.eval_period = 15000

approach/ovod/APE/configs/COCO_REFCOCO/ape_deta/ape_deta_r50_vlf_bert_36ep.py

@@ -0,0 +1,21 @@
+from detectron2.config import LazyCall as L
+from ape.modeling.text import Bert
+
+from .ape_deta_r50_vlf_36ep import dataloader, lr_multiplier, model, optimizer, train
+
+model.model_vision.criterion[1].num_classes = 1
+
+model.model_language = L(Bert)(
+    pretrained_model_name_or_path="models/huggingface/bert-base-uncased/"
+)
+model.model_vision.embed_dim_language = 768
+model.model_vision.text_feature_reduce_type = "average"
+
+model.model_vision.text_feature_bank = False
+model.model_vision.text_feature_reduce_before_fusion = False
+model.model_vision.text_feature_batch_repeat = False
+model.model_vision.expression_cumulative_gt_class = False
+
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 5120

approach/ovod/APE/configs/COCO_REFCOCO/ape_deta/ape_deta_vitl_eva02_lsj1024_12ep.py

@@ -0,0 +1,118 @@
+from functools import partial
+
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.modeling.backbone.fpn import LastLevelMaxPool
+from detrex.config import get_config
+from ape.modeling.backbone.vit import get_vit_lr_decay_rate
+from ape.modeling.backbone.vit_eva02 import SimpleFeaturePyramid, ViT
+from ape.modeling.text import EVA01CLIP
+
+from .....detectron2.configs.common.data.constants import constants
+from ...common.data.coco_refcoco_instance_lsj1024 import dataloader
+from .models.ape_deta_r50 import model
+
+model.model_vision.pixel_mean = constants.imagenet_rgb256_mean
+model.model_vision.pixel_std = constants.imagenet_rgb256_std
+model.model_vision.input_format = "RGB"
+
+model.model_vision.backbone = L(SimpleFeaturePyramid)(
+    net=L(ViT)(  # Single-scale ViT backbone
+        img_size=1024,
+        patch_size=16,
+        embed_dim=1024,
+        depth=24,
+        num_heads=16,
+        drop_path_rate=0.4,
+        window_size=16,
+        mlp_ratio=4 * 2 / 3,
+        qkv_bias=True,
+        norm_layer=partial(nn.LayerNorm, eps=1e-6),
+        window_block_indexes=list(range(0, 5))
+        + list(range(6, 11))
+        + list(range(12, 17))
+        + list(range(18, 23)),
+        residual_block_indexes=[],
+        use_rel_pos=True,
+        out_feature="last_feat",
+        use_act_checkpoint=True,
+        xattn=True,
+    ),
+    in_feature="${.net.out_feature}",
+    out_channels=256,
+    scale_factors=(4.0, 2.0, 1.0, 0.5),
+    top_block=L(LastLevelMaxPool)(),
+    norm="LN",
+    square_pad=1024,
+)
+
+model.model_vision.neck = None
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+optimizer = get_config("common/optim.py").AdamW
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.8, num_layers=24)
+    if "backbone.net" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+optimizer.params.weight_decay_norm = None
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+
+train = get_config("common/train.py").train
+train.max_iter = 90000
+train.eval_period = 5000
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+train.init_checkpoint = (
+    "models/Yunxin-CV/EVA-02/eva02/pt/eva02_L_pt_in21k_p14to16.pt?matching_heuristics=True"
+)
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [75000, 90000]
+lr_multiplier.warmup_length = 1000 / train.max_iter
+
+dataloader.train.num_workers = 16
+dataloader.train.total_batch_size = 16
+dataloader.train.total_batch_size_list = [16, 16]
+dataloader.train.mapper.image_format = "RGB"
+dataloader.train.mapper.use_instance_mask = True
+
+model.model_vision.dataset_prompts = ["name", "expression"]
+model.model_vision.dataset_names = ["coco_2017", "refcoco"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.output_dir = train.output_dir
+
+model.model_language = L(EVA01CLIP)(
+    clip_model="EVA_CLIP_g_14_X", cache_dir="models/BAAI/EVA/eva_clip_psz14.pt"
+)
+model.model_vision.embed_dim_language = 1024

approach/ovod/APE/configs/COCO_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_36ep.py

@@ -0,0 +1,118 @@
+from functools import partial
+
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.modeling.backbone.fpn import LastLevelMaxPool
+from detrex.config import get_config
+from ape.modeling.backbone.vit import get_vit_lr_decay_rate
+from ape.modeling.backbone.vit_eva02 import SimpleFeaturePyramid, ViT
+from ape.modeling.text import EVA01CLIP
+
+from .....detectron2.configs.common.data.constants import constants
+from ...common.data.coco_refcoco_instance_lsj1024 import dataloader
+from .models.ape_deta_r50_vlf import model
+
+model.model_vision.pixel_mean = constants.imagenet_rgb256_mean
+model.model_vision.pixel_std = constants.imagenet_rgb256_std
+model.model_vision.input_format = "RGB"
+
+model.model_vision.backbone = L(SimpleFeaturePyramid)(
+    net=L(ViT)(  # Single-scale ViT backbone
+        img_size=1024,
+        patch_size=16,
+        embed_dim=1024,
+        depth=24,
+        num_heads=16,
+        drop_path_rate=0.4,
+        window_size=16,
+        mlp_ratio=4 * 2 / 3,
+        qkv_bias=True,
+        norm_layer=partial(nn.LayerNorm, eps=1e-6),
+        window_block_indexes=list(range(0, 5))
+        + list(range(6, 11))
+        + list(range(12, 17))
+        + list(range(18, 23)),
+        residual_block_indexes=[],
+        use_rel_pos=True,
+        out_feature="last_feat",
+        use_act_checkpoint=False,
+        xattn=True,
+    ),
+    in_feature="${.net.out_feature}",
+    out_channels=256,
+    scale_factors=(4.0, 2.0, 1.0, 0.5),
+    top_block=L(LastLevelMaxPool)(),
+    norm="LN",
+    square_pad=1024,
+)
+
+model.model_vision.neck = None
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+optimizer = get_config("common/optim.py").AdamW
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.8, num_layers=24)
+    if "backbone.net" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+optimizer.params.weight_decay_norm = None
+
+optimizer.lr = 2e-4
+optimizer.betas = (0.9, 0.999)
+optimizer.weight_decay = 1e-4
+
+train = get_config("common/train.py").train
+train.max_iter = 270000
+train.eval_period = 27000
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+train.init_checkpoint = (
+    "models/Yunxin-CV/EVA-02/eva02/pt/eva02_L_pt_in21k_p14to16.pt?matching_heuristics=True"
+)
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_36ep
+lr_multiplier.warmup_length = 1000 / train.max_iter
+
+dataloader.train.num_workers = 16
+dataloader.train.total_batch_size = 16
+dataloader.train.total_batch_size_list = [16, 16]
+dataloader.train.mapper.image_format = "RGB"
+dataloader.train.mapper.use_instance_mask = True
+
+model.model_vision.dataset_prompts = ["name", "expression"]
+model.model_vision.dataset_names = ["coco_2017", "refcoco"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.output_dir = train.output_dir
+
+model.model_language = L(EVA01CLIP)(
+    clip_model="EVA_CLIP_g_14_X", cache_dir="models/BAAI/EVA/eva_clip_psz14.pt"
+)
+model.model_vision.embed_dim_language = 1024
+

approach/ovod/APE/configs/COCO_REFCOCO/ape_deta/ape_deta_vitl_lsj1024_12ep.py

@@ -0,0 +1,114 @@
+from functools import partial
+
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.modeling.backbone.fpn import LastLevelMaxPool
+from detectron2.modeling.backbone.vit import SimpleFeaturePyramid, ViT, get_vit_lr_decay_rate
+from detrex.config import get_config
+from ape.modeling.text import EVA01CLIP
+
+from .....detectron2.configs.common.data.constants import constants
+from ...common.data.coco_refcoco_instance_lsj1024 import dataloader
+from .models.ape_deta_r50 import model
+
+model.model_vision.pixel_mean = constants.imagenet_rgb256_mean
+model.model_vision.pixel_std = constants.imagenet_rgb256_std
+model.model_vision.input_format = "RGB"
+
+model.model_vision.backbone = L(SimpleFeaturePyramid)(
+    net=L(ViT)(  # Single-scale ViT backbone
+        img_size=1024,
+        patch_size=16,
+        embed_dim=1024,
+        depth=24,
+        num_heads=16,
+        drop_path_rate=0.4,
+        window_size=14,
+        mlp_ratio=4,
+        norm_layer=partial(nn.LayerNorm, eps=1e-6),
+        window_block_indexes=list(range(0, 5))
+        + list(range(6, 11))
+        + list(range(12, 17))
+        + list(range(18, 23)),
+        residual_block_indexes=[],
+        use_rel_pos=True,
+        out_feature="last_feat",
+        use_act_checkpoint=True,
+    ),
+    in_feature="${.net.out_feature}",
+    out_channels=256,
+    scale_factors=(4.0, 2.0, 1.0, 0.5),
+    top_block=L(LastLevelMaxPool)(),
+    norm="LN",
+    square_pad=1024,
+)
+
+model.model_vision.neck = None
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+optimizer = get_config("common/optim.py").AdamW
+optimizer.params.lr_factor_func = (
+    lambda module_name: 0.1
+    if "reference_points" in module_name or "sampling_offsets" in module_name
+    else get_vit_lr_decay_rate(module_name, lr_decay_rate=0.8, num_layers=24)
+    if "backbone" in module_name
+    else 1
+)
+optimizer.params.overrides = {"pos_embed": {"weight_decay": 0.0}}
+
+optimizer.lr = 2e-4
+optimizer.weight_decay = 0.05
+
+train = get_config("common/train.py").train
+train.max_iter = 90000
+train.eval_period = 5000
+train.log_period = 20
+
+train.checkpointer.period = 5000
+train.checkpointer.max_to_keep = 2
+
+train.clip_grad.enabled = True
+train.clip_grad.params.max_norm = 0.1
+train.clip_grad.params.norm_type = 2
+
+train.device = "cuda"
+
+train.init_checkpoint = (
+    "detectron2://ImageNetPretrained/MAE/mae_pretrain_vit_large.pth?matching_heuristics=True"
+)
+train.init_checkpoint = "models/MAE/mae_pretrain_vit_large.pth?matching_heuristics=True"
+
+train.amp.enabled = True
+train.ddp.fp16_compression = True
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_12ep
+lr_multiplier.scheduler.milestones = [75000, 90000]
+lr_multiplier.warmup_length = 1000 / train.max_iter
+
+dataloader.train.num_workers = 16
+dataloader.train.total_batch_size = 16
+dataloader.train.total_batch_size_list = [16, 16]
+dataloader.train.mapper.image_format = "RGB"
+dataloader.train.mapper.use_instance_mask = True
+
+model.model_vision.dataset_tasks = ["name", "expression"]
+model.model_vision.dataset_names = ["coco_2017", "refcoco"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.output_dir = train.output_dir
+
+model.model_language = L(EVA01CLIP)(
+    clip_model="EVA_CLIP_g_14_X", cache_dir="models/BAAI/EVA/eva_clip_psz14.pt"
+)
+model.model_vision.embed_dim_language = 1024

approach/ovod/APE/configs/GQA_VisualGrounding/ape_deta/ape_deta_r50_12ep.py

@@ -0,0 +1,20 @@
+from ...COCO_InstanceSegmentation.ape_deta.ape_deta_r50_12ep import (
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+from ...common.data.gqa_region_instance import dataloader
+
+model.model_vision.num_classes = 200
+
+model.model_vision.criterion[0].num_classes = 200
+
+dataloader.train.mapper.max_num_phrase = 100
+
+model.model_vision.dataset_prompts = ["phrase", "expression"]
+model.model_vision.dataset_names = ["gqa", "refcoco"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names + ["refcoco-mixed"]
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 1280

approach/ovod/APE/configs/GQA_VisualGrounding/ape_deta/ape_deta_r50_12ep_eval_odinw13.py

@@ -0,0 +1,10 @@
+from ...common.data.odinw13_instance import dataloader
+from .ape_deta_r50_12ep import lr_multiplier, model, optimizer, train
+
+model.model_vision.dataset_prompts = ["name" for _ in dataloader.tests]
+model.model_vision.dataset_names = [
+    test.dataset.names.replace("_val", "") for test in dataloader.tests
+]
+model.model_vision.dataset_metas = [test.dataset.names for test in dataloader.tests]
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/GQA_VisualGrounding/ape_deta/ape_deta_r50_12ep_eval_odinw35.py

@@ -0,0 +1,10 @@
+from ...common.data.odinw35_instance import dataloader
+from .ape_deta_r50_12ep import lr_multiplier, model, optimizer, train
+
+model.model_vision.dataset_prompts = ["name" for _ in dataloader.tests]
+model.model_vision.dataset_names = [
+    test.dataset.names.replace("_val", "") for test in dataloader.tests
+]
+model.model_vision.dataset_metas = [test.dataset.names for test in dataloader.tests]
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/GQA_VisualGrounding/ape_deta/ape_deta_r50_vlf_12ep.py

@@ -0,0 +1,46 @@
+from detectron2.config import LazyCall as L
+from omegaconf import OmegaConf
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+
+from .ape_deta_r50_12ep import dataloader, lr_multiplier, model, optimizer, train
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    cfg=OmegaConf.from_dotlist(
+        [
+            "MODEL.DYHEAD.FUSE_CONFIG.STABLE_SOFTMAX_2D=False",
+            "MODEL.DYHEAD.FUSE_CONFIG.CLAMP_MIN_FOR_UNDERFLOW=True",
+            "MODEL.DYHEAD.FUSE_CONFIG.CLAMP_MAX_FOR_OVERFLOW=True",
+            "MODEL.VL_FUSION_USE_CHECKPOINT=True",
+        ],
+    ),
+)
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 1280

approach/ovod/APE/configs/GQA_VisualGrounding/ape_deta/ape_deta_r50_vlf_12ep_eval_odinw13.py

@@ -0,0 +1,10 @@
+from ...common.data.odinw13_instance import dataloader
+from .ape_deta_r50_vlf_12ep import lr_multiplier, model, optimizer, train
+
+model.model_vision.dataset_prompts = ["name" for _ in dataloader.tests]
+model.model_vision.dataset_names = [
+    test.dataset.names.replace("_val", "") for test in dataloader.tests
+]
+model.model_vision.dataset_metas = [test.dataset.names for test in dataloader.tests]
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/GQA_VisualGrounding/ape_deta/ape_deta_r50_vlf_12ep_eval_odinw35.py

@@ -0,0 +1,10 @@
+from ...common.data.odinw35_instance import dataloader
+from .ape_deta_r50_vlf_12ep import lr_multiplier, model, optimizer, train
+
+model.model_vision.dataset_prompts = ["name" for _ in dataloader.tests]
+model.model_vision.dataset_names = [
+    test.dataset.names.replace("_val", "") for test in dataloader.tests
+]
+model.model_vision.dataset_metas = [test.dataset.names for test in dataloader.tests]
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_REFCOCO/ape_deta/ape_deta_vitl_eva02_lsj1024_cp_180k.py

@@ -0,0 +1,21 @@
+from detectron2.config import LazyCall as L
+from detectron2.solver import WarmupParamScheduler
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from .ape_deta_vitl_eva02_lsj1024_cp_720k import dataloader, lr_multiplier, model, optimizer, train
+
+train.max_iter = 180000
+train.eval_period = 180000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[150000],
+        num_updates=180000,
+    ),
+    warmup_length=1000 / 180000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_REFCOCO/ape_deta/ape_deta_vitl_eva02_lsj1024_cp_720k.py

@@ -0,0 +1,83 @@
+from detectron2.config import LazyCall as L
+from detectron2.solver import WarmupParamScheduler
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+
+from ...common.data.lviscocococostuff_o365_oid_vgr_refcoco_group_by_image_panoptic_lsj1024_cp import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(6)]
+for criterion, num_classes in zip(model.model_vision.criterion, [1256, 365, 601, 200, 200, 200]):
+    criterion.num_classes = num_classes
+
+dataloader.train.mapper.max_num_phrase = 100
+dataloader.train.mapper.nms_thresh_phrase = 0.6
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[0], 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[4].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+model.model_vision.neck = None
+
+train.max_iter = 720000
+train.eval_period = 720000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[640000],
+        num_updates=720000,
+    ),
+    warmup_length=1000 / 720000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+dataloader.train.total_batch_size = 16
+dataloader.train.total_batch_size_list = [16, 16, 16, 16, 16]
+
+model.model_vision.dataset_prompts = ["name", "name", "name", "phrase", "phrase", "expression"]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "refcoco-mixed_group-by-image",
+    "refcoco",
+]
+model.model_vision.dataset_metas = dataloader.train.dataset.names + ["refcoco-mixed"]
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_1080k.py

@@ -0,0 +1,27 @@
+from detectron2.config import LazyCall as L
+from detectron2.solver import WarmupParamScheduler
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from .ape_deta_vitl_eva02_vlf_lsj1024_cp_720k import (
+    dataloader,
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+
+train.max_iter = 1080000
+train.eval_period = 1080000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[900000],
+        num_updates=1080000,
+    ),
+    warmup_length=2000 / 1080000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_180k.py

@@ -0,0 +1,27 @@
+from detectron2.config import LazyCall as L
+from detectron2.solver import WarmupParamScheduler
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from .ape_deta_vitl_eva02_vlf_lsj1024_cp_720k import (
+    dataloader,
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+
+train.max_iter = 180000
+train.eval_period = 180000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[150000],
+        num_updates=180000,
+    ),
+    warmup_length=1000 / 180000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_720k.py

@@ -0,0 +1,47 @@
+from detectron2.config import LazyCall as L
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+
+from .ape_deta_vitl_eva02_lsj1024_cp_720k import dataloader, lr_multiplier, model, optimizer, train
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 12800

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_08x8x270k.py

@@ -0,0 +1,228 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.solver import WarmupParamScheduler
+from detrex.modeling.neck import ChannelMapper
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from ...common.data.lviscocococostuff_o365_oid_vgr_sa1b_refcoco_group_by_image_gqa_phrasecut_flickr30k_panoptic_lsj1024_cp import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+model.model_vision.transformer.encoder.use_act_checkpoint = True
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(10)]
+for criterion, num_classes in zip(
+    model.model_vision.criterion, [1256, 365, 601, 256, 1, 256, 256, 256, 256, 256]
+):
+    criterion.num_classes = num_classes
+
+dataloader.train.mapper.max_num_phrase = 128
+dataloader.train.mapper.nms_thresh_phrase = 0.6
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[0], 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[2].use_fed_loss = True
+model.model_vision.criterion[2].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[2], 0.5
+)
+model.model_vision.criterion[2].fed_loss_num_classes = 50
+model.model_vision.criterion[2].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+for k, v in model.model_vision.criterion[4].weight_dict.items():
+    if "_class" in k and "_enc" not in k:
+        model.model_vision.criterion[4].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.criterion[6].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[6].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[7].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[7].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[8].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[8].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 270000
+train.eval_period = 270000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[225000],
+        num_updates=270000,
+    ),
+    warmup_length=2000 / 270000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+dataloader.train.total_batch_size = 8
+dataloader.train.total_batch_size_list = [8, 8, 8, 8, 8, 8, 8, 8, 8]
+dataloader.train.num_workers = 2
+train.iter_size = 8
+
+dataloader.wait_group = 2
+dataloader.wait_time = 30 * 60
+
+model.model_vision.dataset_prompts = [
+    "name",
+    "name",
+    "name",
+    "phrase",
+    "name",
+    "phrase",
+    "phrase",
+    "phrase",
+    "phrase",
+    "expression",
+]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "sa1b",
+    "refcoco-mixed_group-by-image",
+    "gqa",
+    "phrasecut",
+    "flickr30k",
+    "refcoco",
+]
+model.model_vision.dataset_metas = dataloader.train.dataset.names + ["refcoco-mixed"]
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_1080k.py

@@ -0,0 +1,225 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.solver import WarmupParamScheduler
+from detrex.modeling.neck import ChannelMapper
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from ...common.data.lviscocococostuff_o365_oid_vgr_sa1b_refcoco_group_by_image_gqa_phrasecut_flickr30k_panoptic_lsj1024_cp import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+model.model_vision.transformer.encoder.use_act_checkpoint = True
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(10)]
+for criterion, num_classes in zip(
+    model.model_vision.criterion, [1256, 365, 601, 200, 1, 200, 200, 200, 200, 200]
+):
+    criterion.num_classes = num_classes
+
+dataloader.train.mapper.max_num_phrase = 100
+dataloader.train.mapper.nms_thresh_phrase = 0.6
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[0], 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[2].use_fed_loss = True
+model.model_vision.criterion[2].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[2], 0.5
+)
+model.model_vision.criterion[2].fed_loss_num_classes = 50
+model.model_vision.criterion[2].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+for k, v in model.model_vision.criterion[4].weight_dict.items():
+    if "_class" in k and "_enc" not in k:
+        model.model_vision.criterion[4].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.criterion[6].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[6].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[7].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[7].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[8].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[8].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 1080000
+train.eval_period = 1080000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[900000],
+        num_updates=1080000,
+    ),
+    warmup_length=2000 / 270000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+dataloader.train.total_batch_size = 16
+dataloader.train.total_batch_size_list = [16, 16, 16, 16, 16, 16, 16, 16, 16]
+dataloader.train.num_workers = 4
+
+
+model.model_vision.dataset_prompts = [
+    "name",
+    "name",
+    "name",
+    "phrase",
+    "name",
+    "phrase",
+    "phrase",
+    "phrase",
+    "phrase",
+    "expression",
+]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "sa1b",
+    "refcoco-mixed_group-by-image",
+    "gqa",
+    "phrasecut",
+    "flickr30k",
+    "refcoco",
+]
+model.model_vision.dataset_metas = dataloader.train.dataset.names + ["refcoco-mixed"]
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 5120

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k.py

@@ -0,0 +1,227 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.solver import WarmupParamScheduler
+from detrex.modeling.neck import ChannelMapper
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from ...common.data.lviscocococostuff_o365_oid_vgr_sa1b_refcoco_group_by_image_gqa_phrasecut_flickr30k_panoptic_lsj1024_cp import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+model.model_vision.transformer.encoder.use_act_checkpoint = True
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(10)]
+for criterion, num_classes in zip(
+    model.model_vision.criterion, [1256, 365, 601, 256, 1, 256, 256, 256, 256, 256]
+):
+    criterion.num_classes = num_classes
+
+dataloader.train.mapper.max_num_phrase = 128
+dataloader.train.mapper.nms_thresh_phrase = 0.6
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[0], 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[2].use_fed_loss = True
+model.model_vision.criterion[2].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[2], 0.5
+)
+model.model_vision.criterion[2].fed_loss_num_classes = 50
+model.model_vision.criterion[2].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+for k, v in model.model_vision.criterion[4].weight_dict.items():
+    if "_class" in k and "_enc" not in k:
+        model.model_vision.criterion[4].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.criterion[6].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[6].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[7].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[7].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[8].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[8].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 1080000
+train.eval_period = 1080000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[900000],
+        num_updates=1080000,
+    ),
+    warmup_length=2000 / 270000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+dataloader.train.total_batch_size = 16
+dataloader.train.total_batch_size_list = [16, 16, 16, 16, 16, 16, 16, 16, 16]
+dataloader.train.num_workers = 0
+train.iter_size = 4
+train.iter_loop = False
+
+
+model.model_vision.dataset_prompts = [
+    "name",
+    "name",
+    "name",
+    "phrase",
+    "name",
+    "phrase",
+    "phrase",
+    "phrase",
+    "phrase",
+    "expression",
+]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "sa1b",
+    "refcoco-mixed_group-by-image",
+    "gqa",
+    "phrasecut",
+    "flickr30k",
+    "refcoco",
+]
+model.model_vision.dataset_metas = dataloader.train.dataset.names + ["refcoco-mixed"]
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 5120

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k_mdl.py

@@ -0,0 +1,230 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.solver import WarmupParamScheduler
+from detrex.modeling.neck import ChannelMapper
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from ...common.data.lviscocococostuff_o365_oid_vgr_sa1b_refcoco_group_by_image_gqa_phrasecut_flickr30k_panoptic_lsj1024_cp_mdl import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+model.model_vision.transformer.encoder.use_act_checkpoint = True
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(10)]
+for criterion, num_classes in zip(
+    model.model_vision.criterion, [1256, 365, 601, 256, 1, 256, 256, 256, 256, 256]
+):
+    criterion.num_classes = num_classes
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train[0].dataset.names, 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[2].use_fed_loss = True
+model.model_vision.criterion[2].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train[2].dataset.names, 0.5
+)
+model.model_vision.criterion[2].fed_loss_num_classes = 50
+model.model_vision.criterion[2].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+for k, v in model.model_vision.criterion[4].weight_dict.items():
+    if "_class" in k and "_enc" not in k:
+        model.model_vision.criterion[4].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.criterion[6].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[6].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[7].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[7].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[8].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[8].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 1080000
+train.eval_period = 1080000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[900000],
+        num_updates=1080000,
+    ),
+    warmup_length=2000 / 270000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+for i in range(len(dataloader.train)):
+    dataloader.train[i].mapper.max_num_phrase = 128
+    dataloader.train[i].mapper.nms_thresh_phrase = 0.6
+    dataloader.train[i].total_batch_size = 16
+    dataloader.train[i].total_batch_size_list = [16]
+    dataloader.train[i].num_workers = 2
+
+train.iter_size = 4
+train.iter_loop = False
+train.dataset_ratio = [1, 1, 1, 1, 1, 0.1, 0.1, 0.1, 0.1]
+
+model.model_vision.dataset_prompts = [
+    "name",
+    "name",
+    "name",
+    "phrase",
+    "name",
+    "phrase",
+    "phrase",
+    "phrase",
+    "phrase",
+    "expression",
+]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "sa1b",
+    "refcoco-mixed_group-by-image",
+    "gqa",
+    "phrasecut",
+    "flickr30k",
+    "refcoco",
+]
+model.model_vision.dataset_metas = [xx for x in dataloader.train for xx in x.dataset.names] + [
+    "refcoco-mixed"
+]
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 5120

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4_1080k_mdl_llama2.py

@@ -0,0 +1,235 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.solver import WarmupParamScheduler
+from detrex.modeling.neck import ChannelMapper
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import Llama2
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from ...common.data.lviscocococostuff_o365_oid_vgr_sa1b_refcoco_group_by_image_gqa_phrasecut_flickr30k_panoptic_lsj1024_cp_mdl import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(Llama2)(
+    pretrained_model_name_or_path="models/meta-llama/Llama-2-7b-hf/",
+    dtype="float32",
+    vision_port="decoder",
+    eval_only=True,
+    load_in_4bit=True,
+    load_in_8bit=False,
+)
+model.model_vision.embed_dim_language = 4096
+model.model_vision.text_feature_reduce_type = "average"
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+model.model_vision.transformer.encoder.use_act_checkpoint = True
+model.model_vision.transformer.decoder.use_act_checkpoint = True
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(10)]
+for criterion, num_classes in zip(
+    model.model_vision.criterion, [1256, 365, 601, 256, 1, 256, 256, 256, 256, 256]
+):
+    criterion.num_classes = num_classes
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train[0].dataset.names, 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[2].use_fed_loss = True
+model.model_vision.criterion[2].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train[2].dataset.names, 0.5
+)
+model.model_vision.criterion[2].fed_loss_num_classes = 50
+model.model_vision.criterion[2].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+for k, v in model.model_vision.criterion[4].weight_dict.items():
+    if "_class" in k and "_enc" not in k:
+        model.model_vision.criterion[4].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.criterion[6].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[6].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[7].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[7].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[8].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[8].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 1080000
+train.eval_period = 1080000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[900000],
+        num_updates=1080000,
+    ),
+    warmup_length=2000 / 270000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+for i in range(len(dataloader.train)):
+    dataloader.train[i].mapper.max_num_phrase = 128
+    dataloader.train[i].mapper.nms_thresh_phrase = 0.6
+    dataloader.train[i].total_batch_size = 16
+    dataloader.train[i].total_batch_size_list = [16]
+    dataloader.train[i].num_workers = 2
+
+train.iter_size = 4
+train.iter_loop = False
+train.dataset_ratio = [1, 1, 1, 1, 1, 0.1, 0.1, 0.1, 0.1]
+
+model.model_vision.dataset_prompts = [
+    "name",
+    "name",
+    "name",
+    "phrase",
+    "name",
+    "phrase",
+    "phrase",
+    "phrase",
+    "phrase",
+    "expression",
+]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "sa1b",
+    "refcoco-mixed_group-by-image",
+    "gqa",
+    "phrasecut",
+    "flickr30k",
+    "refcoco",
+]
+model.model_vision.dataset_metas = [xx for x in dataloader.train for xx in x.dataset.names] + [
+    "refcoco-mixed"
+]
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 5120

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4x270k.py

@@ -0,0 +1,227 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.solver import WarmupParamScheduler
+from detrex.modeling.neck import ChannelMapper
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from ...common.data.lviscocococostuff_o365_oid_vgr_sa1b_refcoco_group_by_image_gqa_phrasecut_flickr30k_panoptic_lsj1024_cp import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+    use_attention_mask_v=True,
+)
+model.model_vision.transformer.encoder.use_act_checkpoint = True
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(10)]
+for criterion, num_classes in zip(
+    model.model_vision.criterion, [1256, 365, 601, 256, 1, 256, 256, 256, 256, 256]
+):
+    criterion.num_classes = num_classes
+
+dataloader.train.mapper.max_num_phrase = 128
+dataloader.train.mapper.nms_thresh_phrase = 0.6
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[0], 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[2].use_fed_loss = True
+model.model_vision.criterion[2].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[2], 0.5
+)
+model.model_vision.criterion[2].fed_loss_num_classes = 50
+model.model_vision.criterion[2].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+for k, v in model.model_vision.criterion[4].weight_dict.items():
+    if "_class" in k and "_enc" not in k:
+        model.model_vision.criterion[4].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.criterion[6].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[6].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[7].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[7].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[8].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[8].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 270000
+train.eval_period = 270000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[225000],
+        num_updates=270000,
+    ),
+    warmup_length=2000 / 270000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+dataloader.train.total_batch_size = 16
+dataloader.train.total_batch_size_list = [16, 16, 16, 16, 16, 16, 16, 16, 16]
+dataloader.train.num_workers = 0
+train.iter_size = 4
+
+
+model.model_vision.dataset_prompts = [
+    "name",
+    "name",
+    "name",
+    "phrase",
+    "name",
+    "phrase",
+    "phrase",
+    "phrase",
+    "phrase",
+    "expression",
+]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "sa1b",
+    "refcoco-mixed_group-by-image",
+    "gqa",
+    "phrasecut",
+    "flickr30k",
+    "refcoco",
+]
+model.model_vision.dataset_metas = dataloader.train.dataset.names + ["refcoco-mixed"]
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 5120

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4x270k_mdl.py

@@ -0,0 +1,230 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.solver import WarmupParamScheduler
+from detrex.modeling.neck import ChannelMapper
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from ...common.data.lviscocococostuff_o365_oid_vgr_sa1b_refcoco_group_by_image_gqa_phrasecut_flickr30k_panoptic_lsj1024_cp_mdl import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+    use_attention_mask_v=True,
+)
+model.model_vision.transformer.encoder.use_act_checkpoint = True
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(10)]
+for criterion, num_classes in zip(
+    model.model_vision.criterion, [1256, 365, 601, 256, 1, 256, 256, 256, 256, 256]
+):
+    criterion.num_classes = num_classes
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train[0].dataset.names, 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[2].use_fed_loss = True
+model.model_vision.criterion[2].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train[2].dataset.names, 0.5
+)
+model.model_vision.criterion[2].fed_loss_num_classes = 50
+model.model_vision.criterion[2].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+for k, v in model.model_vision.criterion[4].weight_dict.items():
+    if "_class" in k and "_enc" not in k:
+        model.model_vision.criterion[4].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.criterion[6].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[6].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[7].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[7].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[8].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[8].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 270000
+train.eval_period = 270000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[225000],
+        num_updates=270000,
+    ),
+    warmup_length=2000 / 270000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+for i in range(len(dataloader.train)):
+    dataloader.train[i].mapper.max_num_phrase = 128
+    dataloader.train[i].mapper.nms_thresh_phrase = 0.6
+    dataloader.train[i].total_batch_size = 16
+    dataloader.train[i].total_batch_size_list = [16]
+    dataloader.train[i].num_workers = 2
+
+train.iter_size = 4
+train.dataset_ratio = [1, 1, 1, 1, 1, 0.1, 0.1, 0.1, 0.1]
+
+model.model_vision.dataset_prompts = [
+    "name",
+    "name",
+    "name",
+    "phrase",
+    "name",
+    "phrase",
+    "phrase",
+    "phrase",
+    "phrase",
+    "expression",
+]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "sa1b",
+    "refcoco-mixed_group-by-image",
+    "gqa",
+    "phrasecut",
+    "flickr30k",
+    "refcoco",
+]
+model.model_vision.dataset_metas = [xx for x in dataloader.train for xx in x.dataset.names] + [
+    "refcoco-mixed"
+]
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 5120

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4x270k_mdl_llama2.py

@@ -0,0 +1,235 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.solver import WarmupParamScheduler
+from detrex.modeling.neck import ChannelMapper
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import Llama2
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from ...common.data.lviscocococostuff_o365_oid_vgr_sa1b_refcoco_group_by_image_gqa_phrasecut_flickr30k_panoptic_lsj1024_cp_mdl import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(Llama2)(
+    pretrained_model_name_or_path="models/meta-llama/Llama-2-7b-hf/",
+    dtype="float32",
+    vision_port="decoder",
+    eval_only=True,
+    load_in_4bit=True,
+    load_in_8bit=False,
+)
+model.model_vision.embed_dim_language = 4096
+model.model_vision.text_feature_reduce_type = "average"
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+    use_attention_mask_v=True,
+)
+model.model_vision.transformer.encoder.use_act_checkpoint = True
+model.model_vision.transformer.decoder.use_act_checkpoint = True
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(10)]
+for criterion, num_classes in zip(
+    model.model_vision.criterion, [1256, 365, 601, 256, 1, 256, 256, 256, 256, 256]
+):
+    criterion.num_classes = num_classes
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train[0].dataset.names, 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[2].use_fed_loss = True
+model.model_vision.criterion[2].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train[2].dataset.names, 0.5
+)
+model.model_vision.criterion[2].fed_loss_num_classes = 50
+model.model_vision.criterion[2].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+for k, v in model.model_vision.criterion[4].weight_dict.items():
+    if "_class" in k and "_enc" not in k:
+        model.model_vision.criterion[4].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.criterion[6].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[6].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[7].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[7].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[8].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[8].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 270000
+train.eval_period = 270000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[225000],
+        num_updates=270000,
+    ),
+    warmup_length=2000 / 270000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+for i in range(len(dataloader.train)):
+    dataloader.train[i].mapper.max_num_phrase = 128
+    dataloader.train[i].mapper.nms_thresh_phrase = 0.6
+    dataloader.train[i].total_batch_size = 16
+    dataloader.train[i].total_batch_size_list = [16]
+    dataloader.train[i].num_workers = 2
+
+train.iter_size = 4
+train.dataset_ratio = [1, 1, 1, 1, 1, 0.1, 0.1, 0.1, 0.1]
+
+model.model_vision.dataset_prompts = [
+    "name",
+    "name",
+    "name",
+    "phrase",
+    "name",
+    "phrase",
+    "phrase",
+    "phrase",
+    "phrase",
+    "expression",
+]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "sa1b",
+    "refcoco-mixed_group-by-image",
+    "gqa",
+    "phrasecut",
+    "flickr30k",
+    "refcoco",
+]
+model.model_vision.dataset_metas = [xx for x in dataloader.train for xx in x.dataset.names] + [
+    "refcoco-mixed"
+]
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 5120

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_16x4x337k_mdl.py

@@ -0,0 +1,230 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.solver import WarmupParamScheduler
+from detrex.modeling.neck import ChannelMapper
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from ...common.data.lviscocococostuff_o365_oid_vgr_sa1b_refcoco_group_by_image_gqa_phrasecut_flickr30k_panoptic_lsj1024_cp_mdl import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+    use_attention_mask_v=True,
+)
+model.model_vision.transformer.encoder.use_act_checkpoint = True
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(10)]
+for criterion, num_classes in zip(
+    model.model_vision.criterion, [1256, 365, 601, 256, 1, 256, 256, 256, 256, 256]
+):
+    criterion.num_classes = num_classes
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train[0].dataset.names, 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[2].use_fed_loss = True
+model.model_vision.criterion[2].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train[2].dataset.names, 0.5
+)
+model.model_vision.criterion[2].fed_loss_num_classes = 50
+model.model_vision.criterion[2].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+for k, v in model.model_vision.criterion[4].weight_dict.items():
+    if "_class" in k and "_enc" not in k:
+        model.model_vision.criterion[4].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.criterion[6].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[6].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[7].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[7].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[8].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[8].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 337500
+train.eval_period = 337500
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1, 0.01],
+        milestones=[225000, 300000],
+        num_updates=337500,
+    ),
+    warmup_length=2000 / 270000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+for i in range(len(dataloader.train)):
+    dataloader.train[i].mapper.max_num_phrase = 128
+    dataloader.train[i].mapper.nms_thresh_phrase = 0.6
+    dataloader.train[i].total_batch_size = 16
+    dataloader.train[i].total_batch_size_list = [16]
+    dataloader.train[i].num_workers = 2
+
+train.iter_size = 4
+train.dataset_ratio = [1, 1, 1, 1, 1, 0.1, 0.1, 0.1, 0.1]
+
+model.model_vision.dataset_prompts = [
+    "name",
+    "name",
+    "name",
+    "phrase",
+    "name",
+    "phrase",
+    "phrase",
+    "phrase",
+    "phrase",
+    "expression",
+]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "sa1b",
+    "refcoco-mixed_group-by-image",
+    "gqa",
+    "phrasecut",
+    "flickr30k",
+    "refcoco",
+]
+model.model_vision.dataset_metas = [xx for x in dataloader.train for xx in x.dataset.names] + [
+    "refcoco-mixed"
+]
+
+train.output_dir = "output/" + __file__[:-3]
+model.model_vision.vis_period = 5120

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_32x2x270k.py

@@ -0,0 +1,228 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.solver import WarmupParamScheduler
+from detrex.modeling.neck import ChannelMapper
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from ...common.data.lviscocococostuff_o365_oid_vgr_sa1b_refcoco_group_by_image_gqa_phrasecut_flickr30k_panoptic_lsj1024_cp import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+model.model_vision.transformer.encoder.use_act_checkpoint = True
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(10)]
+for criterion, num_classes in zip(
+    model.model_vision.criterion, [1256, 365, 601, 256, 1, 256, 256, 256, 256, 256]
+):
+    criterion.num_classes = num_classes
+
+dataloader.train.mapper.max_num_phrase = 128
+dataloader.train.mapper.nms_thresh_phrase = 0.6
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[0], 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[2].use_fed_loss = True
+model.model_vision.criterion[2].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[2], 0.5
+)
+model.model_vision.criterion[2].fed_loss_num_classes = 50
+model.model_vision.criterion[2].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+for k, v in model.model_vision.criterion[4].weight_dict.items():
+    if "_class" in k and "_enc" not in k:
+        model.model_vision.criterion[4].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.criterion[6].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[6].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[7].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[7].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[8].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[8].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 270000
+train.eval_period = 270000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[225000],
+        num_updates=270000,
+    ),
+    warmup_length=2000 / 270000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+dataloader.train.total_batch_size = 32
+dataloader.train.total_batch_size_list = [32, 32, 32, 32, 32, 32, 32, 32, 32]
+dataloader.train.num_workers = 2
+train.iter_size = 2
+
+dataloader.wait_group = 2
+dataloader.wait_time = 30 * 60
+
+model.model_vision.dataset_prompts = [
+    "name",
+    "name",
+    "name",
+    "phrase",
+    "name",
+    "phrase",
+    "phrase",
+    "phrase",
+    "phrase",
+    "expression",
+]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "sa1b",
+    "refcoco-mixed_group-by-image",
+    "gqa",
+    "phrasecut",
+    "flickr30k",
+    "refcoco",
+]
+model.model_vision.dataset_metas = dataloader.train.dataset.names + ["refcoco-mixed"]
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_48x2x270k.py

@@ -0,0 +1,225 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.solver import WarmupParamScheduler
+from detrex.modeling.neck import ChannelMapper
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from ...common.data.lviscocococostuff_o365_oid_vgr_sa1b_refcoco_group_by_image_gqa_phrasecut_flickr30k_panoptic_lsj1024_cp import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+model.model_vision.transformer.encoder.use_act_checkpoint = True
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(10)]
+for criterion, num_classes in zip(
+    model.model_vision.criterion, [1256, 365, 601, 200, 1, 200, 200, 200, 200, 200]
+):
+    criterion.num_classes = num_classes
+
+dataloader.train.mapper.max_num_phrase = 100
+dataloader.train.mapper.nms_thresh_phrase = 0.6
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[0], 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[2].use_fed_loss = True
+model.model_vision.criterion[2].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[2], 0.5
+)
+model.model_vision.criterion[2].fed_loss_num_classes = 50
+model.model_vision.criterion[2].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+for k, v in model.model_vision.criterion[4].weight_dict.items():
+    if "_class" in k and "_enc" not in k:
+        model.model_vision.criterion[4].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.criterion[6].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[6].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[7].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[7].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[8].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[8].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 270000 * 2
+train.eval_period = 270000 * 2
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[225000 * 2],
+        num_updates=270000 * 2,
+    ),
+    warmup_length=2000 / 270000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+dataloader.train.total_batch_size = 48
+dataloader.train.total_batch_size_list = [48, 48, 48, 48, 48, 48, 48, 48, 48]
+dataloader.train.num_workers = 2
+train.iter_size = 2
+
+
+model.model_vision.dataset_prompts = [
+    "name",
+    "name",
+    "name",
+    "phrase",
+    "name",
+    "phrase",
+    "phrase",
+    "phrase",
+    "phrase",
+    "expression",
+]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "sa1b",
+    "refcoco-mixed_group-by-image",
+    "gqa",
+    "phrasecut",
+    "flickr30k",
+    "refcoco",
+]
+model.model_vision.dataset_metas = dataloader.train.dataset.names + ["refcoco-mixed"]
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1024_cp_64x1x270k.py

@@ -0,0 +1,228 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.solver import WarmupParamScheduler
+from detrex.modeling.neck import ChannelMapper
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip import backbone
+from ...common.data.lviscocococostuff_o365_oid_vgr_sa1b_refcoco_group_by_image_gqa_phrasecut_flickr30k_panoptic_lsj1024_cp import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+model.model_vision.transformer.encoder.use_act_checkpoint = True
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(10)]
+for criterion, num_classes in zip(
+    model.model_vision.criterion, [1256, 365, 601, 256, 1, 256, 256, 256, 256, 256]
+):
+    criterion.num_classes = num_classes
+
+dataloader.train.mapper.max_num_phrase = 128
+dataloader.train.mapper.nms_thresh_phrase = 0.6
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[0], 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[2].use_fed_loss = True
+model.model_vision.criterion[2].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[2], 0.5
+)
+model.model_vision.criterion[2].fed_loss_num_classes = 50
+model.model_vision.criterion[2].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+for k, v in model.model_vision.criterion[4].weight_dict.items():
+    if "_class" in k and "_enc" not in k:
+        model.model_vision.criterion[4].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.criterion[6].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[6].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[7].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[7].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[8].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[8].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 270000
+train.eval_period = 270000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[225000],
+        num_updates=270000,
+    ),
+    warmup_length=2000 / 270000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+dataloader.train.total_batch_size = 64
+dataloader.train.total_batch_size_list = [64, 64, 64, 64, 64, 64, 64, 64, 64]
+dataloader.train.num_workers = 2
+train.iter_size = 1
+
+dataloader.wait_group = 2
+dataloader.wait_time = 30 * 60
+
+model.model_vision.dataset_prompts = [
+    "name",
+    "name",
+    "name",
+    "phrase",
+    "name",
+    "phrase",
+    "phrase",
+    "phrase",
+    "phrase",
+    "expression",
+]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "sa1b",
+    "refcoco-mixed_group-by-image",
+    "gqa",
+    "phrasecut",
+    "flickr30k",
+    "refcoco",
+]
+model.model_vision.dataset_metas = dataloader.train.dataset.names + ["refcoco-mixed"]
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1536_cp_08x8x270k.py

@@ -0,0 +1,225 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.solver import WarmupParamScheduler
+from detrex.modeling.neck import ChannelMapper
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip_1536 import backbone
+from ...common.data.lviscocococostuff_o365_oid_vgr_sa1b_refcoco_group_by_image_gqa_phrasecut_flickr30k_panoptic_lsj1536_cp import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+    dtype="float16",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 6
+model.model_vision.transformer.decoder.num_layers = 6
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+model.model_vision.transformer.encoder.use_act_checkpoint = True
+model.model_vision.transformer.decoder.use_act_checkpoint = True
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(10)]
+for criterion, num_classes in zip(
+    model.model_vision.criterion, [1256, 365, 601, 200, 1, 200, 200, 200, 200, 200]
+):
+    criterion.num_classes = num_classes
+
+dataloader.train.mapper.max_num_phrase = 100
+dataloader.train.mapper.nms_thresh_phrase = 0.6
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[0], 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[2].use_fed_loss = True
+model.model_vision.criterion[2].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[2], 0.5
+)
+model.model_vision.criterion[2].fed_loss_num_classes = 50
+model.model_vision.criterion[2].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+for k, v in model.model_vision.criterion[4].weight_dict.items():
+    if "_class" in k and "_enc" not in k:
+        model.model_vision.criterion[4].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.criterion[6].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[6].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[7].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[7].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[8].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[8].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 270000 * 8
+train.eval_period = 270000 * 8
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[225000 * 8],
+        num_updates=270000 * 8,
+    ),
+    warmup_length=2000 / 270000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+dataloader.train.total_batch_size = 8
+dataloader.train.total_batch_size_list = [8, 8, 8, 8, 8, 8, 8, 8, 8]
+train.iter_size = 8
+
+model.model_vision.dataset_prompts = [
+    "name",
+    "name",
+    "name",
+    "phrase",
+    "name",
+    "phrase",
+    "phrase",
+    "phrase",
+    "phrase",
+    "expression",
+]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "sa1b",
+    "refcoco-mixed_group-by-image",
+    "gqa",
+    "phrasecut",
+    "flickr30k",
+    "refcoco",
+]
+model.model_vision.dataset_metas = dataloader.train.dataset.names + ["refcoco-mixed"]
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1536_cp_32x2x270k.py

@@ -0,0 +1,222 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.solver import WarmupParamScheduler
+from detrex.modeling.neck import ChannelMapper
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip_1536 import backbone
+from ...common.data.lviscocococostuff_o365_oid_vgr_sa1b_refcoco_group_by_image_gqa_phrasecut_flickr30k_panoptic_lsj1536_cp import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 9
+model.model_vision.transformer.decoder.num_layers = 9
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(10)]
+for criterion, num_classes in zip(
+    model.model_vision.criterion, [1256, 365, 601, 200, 1, 200, 200, 200, 200, 200]
+):
+    criterion.num_classes = num_classes
+
+dataloader.train.mapper.max_num_phrase = 100
+dataloader.train.mapper.nms_thresh_phrase = 0.6
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[0], 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[2].use_fed_loss = True
+model.model_vision.criterion[2].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[2], 0.5
+)
+model.model_vision.criterion[2].fed_loss_num_classes = 50
+model.model_vision.criterion[2].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+for k, v in model.model_vision.criterion[4].weight_dict.items():
+    if "_class" in k and "_enc" not in k:
+        model.model_vision.criterion[4].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.criterion[6].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[6].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[7].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[7].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[8].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[8].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 270000 * 2
+train.eval_period = 270000 * 2
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[225000 * 2],
+        num_updates=270000 * 2,
+    ),
+    warmup_length=2000 / 270000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+dataloader.train.total_batch_size = 32
+dataloader.train.total_batch_size_list = [32, 32, 32, 32, 32, 32, 32, 32, 32]
+train.iter_size = 2
+
+model.model_vision.dataset_prompts = [
+    "name",
+    "name",
+    "name",
+    "phrase",
+    "name",
+    "phrase",
+    "phrase",
+    "phrase",
+    "phrase",
+    "expression",
+]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "sa1b",
+    "refcoco-mixed_group-by-image",
+    "gqa",
+    "phrasecut",
+    "flickr30k",
+    "refcoco",
+]
+model.model_vision.dataset_metas = dataloader.train.dataset.names + ["refcoco-mixed"]
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_clip_vlf_lsj1536_cp_64x270k.py

@@ -0,0 +1,222 @@
+import torch.nn as nn
+
+from detectron2.config import LazyCall as L
+from detectron2.layers import ShapeSpec
+from detectron2.solver import WarmupParamScheduler
+from detrex.modeling.neck import ChannelMapper
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+from ape.modeling.text import EVA02CLIP
+
+from ...common.backbone.vitl_eva02_clip_1536 import backbone
+from ...common.data.lviscocococostuff_o365_oid_vgr_sa1b_refcoco_group_by_image_gqa_phrasecut_flickr30k_panoptic_lsj1536_cp import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.backbone = backbone
+
+train.init_checkpoint = (
+    "models/QuanSun/EVA-CLIP/EVA02_CLIP_L_336_psz14to16_s6B.pt?matching_heuristics=True"
+)
+
+model.model_language = L(EVA02CLIP)(
+    clip_model="EVA02-CLIP-bigE-14-plus",
+    cache_dir="models/QuanSun/EVA-CLIP/EVA02_CLIP_E_psz14_plus_s9B.pt",
+)
+model.model_vision.embed_dim_language = 1024
+
+model.model_vision.neck = L(ChannelMapper)(
+    input_shapes={
+        "p2": ShapeSpec(channels=256),
+        "p3": ShapeSpec(channels=256),
+        "p4": ShapeSpec(channels=256),
+        "p5": ShapeSpec(channels=256),
+        "p6": ShapeSpec(channels=256),
+    },
+    in_features=["p2", "p3", "p4", "p5", "p6"],
+    out_channels=256,
+    num_outs=5,
+    kernel_size=1,
+    norm_layer=L(nn.GroupNorm)(num_groups=32, num_channels=256),
+)
+
+model.model_vision.mask_in_features = ["p2"]
+model.model_vision.input_shapes = {
+    "p2": ShapeSpec(channels=256),
+    "p3": ShapeSpec(channels=256),
+    "p4": ShapeSpec(channels=256),
+    "p5": ShapeSpec(channels=256),
+    "p6": ShapeSpec(channels=256),
+}
+
+model.model_vision.transformer.encoder.num_layers = 9
+model.model_vision.transformer.decoder.num_layers = 9
+model.model_vision.transformer.encoder.embed_dim = 256
+model.model_vision.transformer.decoder.embed_dim = 256
+model.model_vision.embed_dim = 256
+model.model_vision.backbone.out_channels = 256
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(10)]
+for criterion, num_classes in zip(
+    model.model_vision.criterion, [1256, 365, 601, 200, 1, 200, 200, 200, 200, 200]
+):
+    criterion.num_classes = num_classes
+
+dataloader.train.mapper.max_num_phrase = 100
+dataloader.train.mapper.nms_thresh_phrase = 0.6
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[0], 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[2].use_fed_loss = True
+model.model_vision.criterion[2].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[2], 0.5
+)
+model.model_vision.criterion[2].fed_loss_num_classes = 50
+model.model_vision.criterion[2].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+for k, v in model.model_vision.criterion[4].weight_dict.items():
+    if "_class" in k and "_enc" not in k:
+        model.model_vision.criterion[4].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.criterion[6].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[6].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[7].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[7].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[8].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[8].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 270000
+train.eval_period = 270000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[225000],
+        num_updates=270000,
+    ),
+    warmup_length=2000 / 270000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+dataloader.train.total_batch_size = 64
+dataloader.train.total_batch_size_list = [64, 64, 64, 64, 64, 64, 64, 64, 64]
+
+
+model.model_vision.dataset_prompts = [
+    "name",
+    "name",
+    "name",
+    "phrase",
+    "name",
+    "phrase",
+    "phrase",
+    "phrase",
+    "phrase",
+    "expression",
+]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "sa1b",
+    "refcoco-mixed_group-by-image",
+    "gqa",
+    "phrasecut",
+    "flickr30k",
+    "refcoco",
+]
+model.model_vision.dataset_metas = dataloader.train.dataset.names + ["refcoco-mixed"]
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_O365_OID_VGR_SA1B_REFCOCO_GQA_PhraseCut_Flickr30k/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_1080k.py

@@ -0,0 +1,173 @@
+from detectron2.config import LazyCall as L
+from detectron2.solver import WarmupParamScheduler
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+
+from ...common.data.lviscocococostuff_o365_oid_vgr_sa1b_refcoco_group_by_image_gqa_phrasecut_flickr30k_panoptic_lsj1024_cp import (
+    dataloader,
+)
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=True,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+model.model_vision.text_feature_reduce_before_fusion = True
+model.model_vision.text_feature_batch_repeat = True
+model.model_vision.expression_cumulative_gt_class = True
+model.model_vision.name_prompt_fusion_type = "zero"
+
+model.model_vision.num_classes = 1256
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+del criterion.fed_loss_num_classes
+model.model_vision.criterion = [criterion for _ in range(10)]
+for criterion, num_classes in zip(
+    model.model_vision.criterion, [1256, 365, 601, 200, 1, 200, 200, 200, 200, 200]
+):
+    criterion.num_classes = num_classes
+
+dataloader.train.mapper.max_num_phrase = 100
+dataloader.train.mapper.nms_thresh_phrase = 0.6
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[0], 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[2].use_fed_loss = True
+model.model_vision.criterion[2].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[2], 0.5
+)
+model.model_vision.criterion[2].fed_loss_num_classes = 50
+model.model_vision.criterion[2].fed_loss_pad_type = "cat"
+
+model.model_vision.criterion[3].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[3].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[3].weight_dict.update({k: 0.0})
+
+for k, v in model.model_vision.criterion[4].weight_dict.items():
+    if "_class" in k and "_enc" not in k:
+        model.model_vision.criterion[4].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+model.model_vision.criterion[6].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[6].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[6].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[7].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[7].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[7].weight_dict.update({k: 0.0})
+
+model.model_vision.criterion[8].weight_dict["loss_class_enc"] = 0.0
+for k, v in model.model_vision.criterion[8].weight_dict.items():
+    if "_enc" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+    if "_bbox" in k or "_giou" in k or "_dice" in k or "_mask" in k:
+        model.model_vision.criterion[8].weight_dict.update({k: 0.0})
+
+model.model_vision.stuff_dataset_learn_thing = False
+model.model_vision.stuff_prob_thing = 0.9
+model.model_vision.transformer.proposal_ambiguous = 1
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 1080000
+train.eval_period = 1080000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[900000],
+        num_updates=1080000,
+    ),
+    warmup_length=2000 / 1080000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+dataloader.train.total_batch_size = 16
+dataloader.train.total_batch_size_list = [16, 16, 16, 16, 16, 16, 16, 16, 16]
+dataloader.train.num_workers = 4
+
+model.model_vision.dataset_prompts = [
+    "name",
+    "name",
+    "name",
+    "phrase",
+    "name",
+    "phrase",
+    "phrase",
+    "phrase",
+    "phrase",
+    "expression",
+]
+model.model_vision.dataset_names = [
+    "lvis+stuffonly",
+    "objects365",
+    "openimages",
+    "vgregion",
+    "sa1b",
+    "refcoco-mixed_group-by-image",
+    "gqa",
+    "phrasecut",
+    "flickr30k",
+    "refcoco",
+]
+model.model_vision.dataset_metas = dataloader.train.dataset.names + ["refcoco-mixed"]
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_PanopticSegmentation/ape_deta/ape_deta_r50_lsj1024_cp_50ep.py

@@ -0,0 +1,36 @@
+from detectron2.data.detection_utils import get_fed_loss_cls_weights
+from detrex.config import get_config
+
+from ...common.data.lviscocococostuff_panoptic_lsj1024_cp import dataloader
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_r50_24ep import (
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.num_classes = 1256
+model.model_vision.criterion[0].num_classes = 1256
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names, 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+model.model_vision.criterion[0].fed_loss_pad_type = "cat"
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+train.max_iter = 375000
+train.eval_period = 20000
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_50ep
+
+dataloader.train.total_batch_size = 16
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.dataset_names = ["lvis+stuffonly"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVISCOCOCOCOSTUFF_PanopticSegmentation/ape_deta/ape_deta_vitl_eva02_lsj1024_cp_24ep.py

@@ -0,0 +1,20 @@
+from ...common.data.lviscocococostuff_panoptic_lsj1024_cp import dataloader
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.num_classes = 1256
+model.model_vision.criterion[0].num_classes = 1256
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+model.model_vision.dataset_prompts = ["name"]
+model.model_vision.dataset_names = ["lvis+stuffonly"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVISCOCO_COCOSTUFF_O365_OID_VG_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_180k.py

@@ -0,0 +1,27 @@
+from detectron2.config import LazyCall as L
+from detectron2.solver import WarmupParamScheduler
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from .ape_deta_vitl_eva02_vlf_lsj1024_cp_720k import (
+    dataloader,
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+
+train.max_iter = 180000
+train.eval_period = 180000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[150000],
+        num_updates=180000,
+    ),
+    warmup_length=1000 / 180000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVISCOCO_COCOSTUFF_O365_OID_VG_REFCOCO/ape_deta/ape_deta_vitl_eva02_vlf_lsj1024_cp_720k.py

@@ -0,0 +1,120 @@
+import random
+
+from detectron2.config import LazyCall as L
+from detectron2.solver import WarmupParamScheduler
+from fvcore.common.param_scheduler import MultiStepParamScheduler
+
+from ape.data.detection_utils import get_fed_loss_cls_weights
+from ape.data.samplers import MultiDatasetTrainingSampler
+from ape.layers import VisionLanguageFusion
+from ape.modeling.ape_deta import (
+    DeformableDETRSegmVL,
+    DeformableDetrTransformerDecoderVL,
+    DeformableDetrTransformerEncoderVL,
+    DeformableDetrTransformerVL,
+)
+
+from ...common.data.lviscoco_cocostuff_o365_oid_vg_refcoco_panoptic_lsj1024_cp import dataloader
+from ...LVIS_InstanceSegmentation.ape_deta.ape_deta_vitl_eva02_lsj1024_cp_24ep import (
+    lr_multiplier,
+    model,
+    optimizer,
+    train,
+)
+
+model.model_vision.update(
+    _target_=DeformableDETRSegmVL,
+)
+model.model_vision.transformer.update(
+    _target_=DeformableDetrTransformerVL,
+)
+model.model_vision.transformer.encoder.update(
+    _target_=DeformableDetrTransformerEncoderVL,
+)
+model.model_vision.transformer.decoder.update(
+    _target_=DeformableDetrTransformerDecoderVL,
+)
+
+model.model_vision.transformer.encoder.vl_layer = L(VisionLanguageFusion)(
+    v_dim="${....embed_dim}",
+    l_dim="${....embed_dim_language}",
+    embed_dim=2048,
+    num_heads=8,
+    dropout=0.1,
+    drop_path=0.0,
+    init_values=1.0 / 6,
+    stable_softmax_2d=False,
+    clamp_min_for_underflow=True,
+    clamp_max_for_overflow=True,
+    use_checkpoint=True,
+)
+
+model.model_vision.text_feature_bank = True
+
+model.model_vision.num_classes = 1203
+model.model_vision.select_box_nums_for_evaluation = 300
+
+criterion = model.model_vision.criterion[0]
+del criterion.use_fed_loss
+del criterion.get_fed_loss_cls_weights
+model.model_vision.criterion = [criterion for _ in range(6)]
+for criterion, num_classes in zip(model.model_vision.criterion, [1203, 54, 365, 601, 150, 200]):
+    criterion.num_classes = num_classes
+
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[0], 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+
+model.model_vision.criterion[5].weight_dict["loss_class_enc"] = 0.0
+
+
+model.model_vision.instance_on = True
+model.model_vision.semantic_on = True
+model.model_vision.panoptic_on = False
+
+model.model_vision.neck = None
+
+train.max_iter = 720000
+train.eval_period = 720000
+
+lr_multiplier = L(WarmupParamScheduler)(
+    scheduler=L(MultiStepParamScheduler)(
+        values=[1.0, 0.1],
+        milestones=[640000],
+        num_updates=720000,
+    ),
+    warmup_length=1000 / 720000,
+    warmup_method="linear",
+    warmup_factor=0.001,
+)
+
+dataloader.train.total_batch_size = 16
+dataloader.train.total_batch_size_list = [16, 16, 16, 16, 16]
+
+model.model_vision.dataset_prompts = ["name", "name", "name", "name", "name", "expression"]
+model.model_vision.dataset_names = [
+    "lvis",
+    "stuffonly",
+    "objects365",
+    "openimages",
+    "visualgenome",
+    "refcoco",
+]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]
+
+dataloader.train.sampler = lambda dataset_dicts: MultiDatasetTrainingSampler(
+    repeat_factors=MultiDatasetTrainingSampler.get_repeat_factors(
+        dataset_dicts=dataset_dicts,
+        num_datasets=6,
+        dataset_ratio=[1, 1, 1, 1, 1, 0],
+        use_rfs=[True, False, True, True, True, True],
+        use_cas=[False, False, False, False, False, False],
+        repeat_thresh=0.001,
+        cas_lambda=1.0,
+    ),
+    seed=random.randint(0, 2**31),
+)

approach/ovod/APE/configs/LVIS_SA1B_InstanceSegmentation/ape_deta/ape_deta_r50_50ep.py

@@ -0,0 +1,29 @@
+from detectron2.data.detection_utils import get_fed_loss_cls_weights
+from detrex.config import get_config
+
+from ...COCO_SA1B_InstanceSegmentation.ape_deta.ape_deta_r50_24ep import model, optimizer, train
+
+from ...common.data.lvis_sa1b_instance import dataloader
+
+model.model_vision.num_classes = 1203
+model.model_vision.criterion[0].num_classes = 1203
+model.model_vision.select_box_nums_for_evaluation = 300
+model.model_vision.criterion[0].use_fed_loss = True
+model.model_vision.criterion[0].get_fed_loss_cls_weights = lambda: get_fed_loss_cls_weights(
+    dataloader.train.dataset.names[0], 0.5
+)
+model.model_vision.criterion[0].fed_loss_num_classes = 50
+
+model.model_vision.semantic_on = False
+model.model_vision.panoptic_on = False
+
+train.max_iter = 375000
+train.eval_period = 20000
+
+lr_multiplier = get_config("common/coco_schedule.py").lr_multiplier_50ep
+
+model.model_vision.dataset_prompts = ["name", "name"]
+model.model_vision.dataset_names = ["lvis", "sa1b"]
+model.model_vision.dataset_metas = dataloader.train.dataset.names
+
+train.output_dir = "output/" + __file__[:-3]

approach/ovod/APE/configs/LVIS_SA1B_InstanceSegmentation/ape_deta/ape_deta_r50_50ep_eval_odinw13.py

@@ -0,0 +1,10 @@
+from ...common.data.odinw13_instance import dataloader
+from .ape_deta_r50_50ep import lr_multiplier, model, optimizer, train
+
+model.model_vision.dataset_prompts = ["name" for _ in dataloader.tests]
+model.model_vision.dataset_names = [
+    test.dataset.names.replace("_val", "") for test in dataloader.tests
+]
+model.model_vision.dataset_metas = [test.dataset.names for test in dataloader.tests]
+
+train.output_dir = "output/" + __file__[:-3]