Upload 16 files

.clang-format

@@ -0,0 +1,85 @@
+AccessModifierOffset: -1
+AlignAfterOpenBracket: AlwaysBreak
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlinesLeft: true
+AlignOperands:   false
+AlignTrailingComments: false
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Empty
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: false
+BinPackParameters: false
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: false
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ForEachMacros:   [ FOR_EACH, FOR_EACH_R, FOR_EACH_RANGE, ]
+IncludeCategories:
+  - Regex:           '^<.*\.h(pp)?>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IndentCaseLabels: true
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Left
+ReflowComments:  true
+SortIncludes:    true
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Cpp11
+TabWidth:        8
+UseTab:          Never

.flake8

@@ -0,0 +1,15 @@
+# This is an example .flake8 config, used when developing *Black* itself.
+# Keep in sync with setup.cfg which is used for source packages.
+
+[flake8]
+ignore = W503, E203, E221, C901, C408, E741, C407, E741, B017
+max-line-length = 100
+max-complexity = 18
+select = B,C,E,F,W,T4,B9
+exclude = build
+per-file-ignores =
+  **/__init__.py:F401,F403,E402
+  **/configs/**.py:F401,E402
+  configs/**.py:F401,E402
+  **/tests/config/**.py:F401,E402
+  tests/config/**.py:F401,E402

.gitignore

@@ -0,0 +1,65 @@
+# output dir
+output
+instant_test_output
+inference_test_output
+oai_clip_weights
+
+pt_output
+pretrained_ckpt
+datasets/coco
+datasets/lvis
+datasets/cifar-10-batches-py
+
+*.png
+*.json
+!vscode_launch.json
+!/detectron2/data/classnames/*.json
+*.diff
+*.jpg
+!/projects/DensePose/doc/images/*.jpg
+
+# compilation and distribution
+__pycache__
+_ext
+*.pyc
+*.pyd
+*.so
+*.dll
+*.egg-info/
+build/
+dist/
+wheels/
+
+# pytorch/python/numpy formats
+*.pth
+*.pkl
+*.npy
+*.ts
+model_ts*.txt
+
+# ipython/jupyter notebooks
+*.ipynb
+**/.ipynb_checkpoints/
+
+# Editor temporaries
+*.swn
+*.swo
+*.swp
+*~
+
+# editor settings
+.idea
+.vscode
+_darcs
+
+# project dirs
+/detectron2/model_zoo/configs
+/datasets/*
+!/datasets/*.*
+!/datasets/custom_images/
+!/datasets/custom_images/*.jpg
+!/docs
+!/docs/*.*
+/projects/*/datasets
+/models
+/snippet

LICENSE

@@ -0,0 +1,202 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction,
+and distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by
+the copyright owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all
+other entities that control, are controlled by, or are under common
+control with that entity. For the purposes of this definition,
+"control" means (i) the power, direct or indirect, to cause the
+direction or management of such entity, whether by contract or
+otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity
+exercising permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications,
+including but not limited to software source code, documentation
+source, and configuration files.
+
+"Object" form shall mean any form resulting from mechanical
+transformation or translation of a Source form, including but
+not limited to compiled object code, generated documentation,
+and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or
+Object form, made available under the License, as indicated by a
+copyright notice that is included in or attached to the work
+(an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object
+form, that is based on (or derived from) the Work and for which the
+editorial revisions, annotations, elaborations, or other modifications
+represent, as a whole, an original work of authorship. For the purposes
+of this License, Derivative Works shall not include works that remain
+separable from, or merely link (or bind by name) to the interfaces of,
+the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including
+the original version of the Work and any modifications or additions
+to that Work or Derivative Works thereof, that is intentionally
+submitted to Licensor for inclusion in the Work by the copyright owner
+or by an individual or Legal Entity authorized to submit on behalf of
+the copyright owner. For the purposes of this definition, "submitted"
+means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems,
+and issue tracking systems that are managed by, or on behalf of, the
+Licensor for the purpose of discussing and improving the Work, but
+excluding communication that is conspicuously marked or otherwise
+designated in writing by the copyright owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity
+on behalf of whom a Contribution has been received by Licensor and
+subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+this License, each Contributor hereby grants to You a perpetual,
+worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the
+Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+this License, each Contributor hereby grants to You a perpetual,
+worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+(except as stated in this section) patent license to make, have made,
+use, offer to sell, sell, import, and otherwise transfer the Work,
+where such license applies only to those patent claims licensable
+by such Contributor that are necessarily infringed by their
+Contribution(s) alone or by combination of their Contribution(s)
+with the Work to which such Contribution(s) was submitted. If You
+institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work
+or a Contribution incorporated within the Work constitutes direct
+or contributory patent infringement, then any patent licenses
+granted to You under this License for that Work shall terminate
+as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+Work or Derivative Works thereof in any medium, with or without
+modifications, and in Source or Object form, provided that You
+meet the following conditions:
+
+(a) You must give any other recipients of the Work or
+Derivative Works a copy of this License; and
+
+(b) You must cause any modified files to carry prominent notices
+stating that You changed the files; and
+
+(c) You must retain, in the Source form of any Derivative Works
+that You distribute, all copyright, patent, trademark, and
+attribution notices from the Source form of the Work,
+excluding those notices that do not pertain to any part of
+the Derivative Works; and
+
+(d) If the Work includes a "NOTICE" text file as part of its
+distribution, then any Derivative Works that You distribute must
+include a readable copy of the attribution notices contained
+within such NOTICE file, excluding those notices that do not
+pertain to any part of the Derivative Works, in at least one
+of the following places: within a NOTICE text file distributed
+as part of the Derivative Works; within the Source form or
+documentation, if provided along with the Derivative Works; or,
+within a display generated by the Derivative Works, if and
+wherever such third-party notices normally appear. The contents
+of the NOTICE file are for informational purposes only and
+do not modify the License. You may add Your own attribution
+notices within Derivative Works that You distribute, alongside
+or as an addendum to the NOTICE text from the Work, provided
+that such additional attribution notices cannot be construed
+as modifying the License.
+
+You may add Your own copyright statement to Your modifications and
+may provide additional or different license terms and conditions
+for use, reproduction, or distribution of Your modifications, or
+for any such Derivative Works as a whole, provided Your use,
+reproduction, and distribution of the Work otherwise complies with
+the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+any Contribution intentionally submitted for inclusion in the Work
+by You to the Licensor shall be under the terms and conditions of
+this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify
+the terms of any separate license agreement you may have executed
+with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+names, trademarks, service marks, or product names of the Licensor,
+except as required for reasonable and customary use in describing the
+origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+agreed to in writing, Licensor provides the Work (and each
+Contributor provides its Contributions) on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied, including, without limitation, any warranties or conditions
+of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+PARTICULAR PURPOSE. You are solely responsible for determining the
+appropriateness of using or redistributing the Work and assume any
+risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+whether in tort (including negligence), contract, or otherwise,
+unless required by applicable law (such as deliberate and grossly
+negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special,
+incidental, or consequential damages of any character arising as a
+result of this License or out of the use or inability to use the
+Work (including but not limited to damages for loss of goodwill,
+work stoppage, computer failure or malfunction, or any and all
+other commercial damages or losses), even if such Contributor
+has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+the Work or Derivative Works thereof, You may choose to offer,
+and charge a fee for, acceptance of support, warranty, indemnity,
+or other liability obligations and/or rights consistent with this
+License. However, in accepting such obligations, You may act only
+on Your own behalf and on Your sole responsibility, not on behalf
+of any other Contributor, and only if You agree to indemnify,
+defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason
+of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+To apply the Apache License to your work, attach the following
+boilerplate notice, with the fields enclosed by brackets "[]"
+replaced with your own identifying information. (Don't include
+the brackets!)  The text should be enclosed in the appropriate
+comment syntax for the file format. We also recommend that a
+file or class name and description of purpose be included on the
+same "printed page" as the copyright notice for easier
+identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

README.md

@@ -0,0 +1,481 @@
+# RegionCLIP: Region-based Language-Image Pretraining
+
+This is the official PyTorch implementation of RegionCLIP (CVPR 2022).
+
+[**Paper**](https://arxiv.org/abs/2112.09106) | [**Demo on Hugging Face**](https://huggingface.co/spaces/CVPR/regionclip-demo) | [**Slides**](https://drive.google.com/file/d/1EepNVJGo_d73Glr4vNjR4Av0dNkBCGcj/view?usp=sharing)
+
+> **RegionCLIP: Region-based Language-Image Pretraining (CVPR 2022)** <br>
+> [Yiwu Zhong](https://pages.cs.wisc.edu/~yiwuzhong/), [Jianwei Yang](https://jwyang.github.io/), [Pengchuan Zhang](https://pzzhang.github.io/pzzhang/), [Chunyuan Li](https://chunyuan.li/), [Noel Codella](https://noelcodella.github.io/publicwebsite/), [Liunian Li](https://liunian-harold-li.github.io/), [Luowei Zhou](https://luoweizhou.github.io/), [Xiyang Dai](https://sites.google.com/site/xiyangdai/), [Lu Yuan](https://scholar.google.com/citations?user=k9TsUVsAAAAJ&hl=en), [Yin Li](https://www.biostat.wisc.edu/~yli/), and [Jianfeng Gao](https://www.microsoft.com/en-us/research/people/jfgao/?from=http%3A%2F%2Fresearch.microsoft.com%2Fen-us%2Fum%2Fpeople%2Fjfgao%2F) <br>
+
+<p align="center">
+<img src="docs/regionclip.png" width=80% height=80%
+class="center">
+</p>
+
+## Overview
+
+We propose RegionCLIP that significantly extends CLIP to learn region-level visual representations. RegionCLIP enables fine-grained alignment between image regions and textual concepts, and thus supports region-based reasoning tasks including zero-shot object detection and open-vocabulary object detection.
+
+- **Pretraining**: We leverage a CLIP model to match image regions with template captions, and then pretrain our model to align these region-text pairs.
+- **Zero-shot inference**: Once pretrained, the learned region representations support zero-shot inference for object detection.
+- **Transfer learning**: The learned RegionCLIP model can be further fine-tuned with additional object detection annotations, allowing our model to be used for fully supervised or open-vocabulary object detection.
+- **Results**: Our method demonstrates **state-of-the-art** results for zero-shot object detection and open-vocabulary object detection.
+
+## Updates
+* :collision: [10/05/2022] RegionCLIP now supports not only resnet but also many vision transformers (e.g., vit, swin, davit, focalnet) for zero-shot object detection! Please checkout the [zero-shot branch](https://github.com/microsoft/RegionCLIP/tree/zero-shot)!
+* [09/23/2022] As requested by researchers, we release the [configs](configs/pretrain) and [scripts](pretrain.sh) of pre-training. A full tutorial and pre-training data will be released later. Stay tuned!
+* [09/18/2022] Organizing ECCV Workshop [Computer Vision in the Wild (CVinW)](https://computer-vision-in-the-wild.github.io/eccv-2022/), where two challenges are hosted to evaluate the zero-shot, few-shot and full-shot performance of pre-trained vision models in downstream tasks:
+  - [Image Classification in the Wild (ICinW)](https://eval.ai/web/challenges/challenge-page/1832/overview) Challenge evaluates on 20 image classification tasks.
+  - [Object Detection in the Wild (ODinW)](https://eval.ai/web/challenges/challenge-page/1839/overview) Challenge evaluates on 35 object detection tasks.
+* [07/11/2022] We included the scripts for concept feature extraction. It can be used for your own costomized concept pool!
+* [07/07/2022] We included the scripts for region feature extraction. The extracted visual features can be used for various downstream tasks!
+* [06/24/2022] We released [**a Web demo using Gradio on Hugging Face**](https://huggingface.co/spaces/CVPR/regionclip-demo). It uses our pretrained RegionCLIP for zero-shot inference. Check it out!
+* [06/20/2022] We released models and inference code for our RegionCLIP!
+
+## Outline
+
+1. [Installation](#Installation)
+2. [Datasets](#Datasets)
+3. [Model Zoo](#Model-Zoo)
+4. [Zero-shot Inference](#Zero-shot-Inference)
+5. [Transfer Learning](#Transfer-Learning)
+6. [Extract Region Features](#Extract-Region-Features)
+7. [Extract Concept Features](#Extract-Concept-Features)
+8. [Citation and Acknowledgement](#Citation-and-Acknowledgement)
+9. [Contributing](#Contributing)
+
+## Installation
+
+Check [`INSTALL.md`](docs/INSTALL.md) for installation instructions.
+
+## Datasets
+
+Check [`datasets/README.md`](datasets/README.md) for dataset preparation.
+
+## Model Zoo
+
+Check [`MODEL_ZOO.md`](docs/MODEL_ZOO.md) for our pretrained models.
+
+
+## Zero-shot Inference
+
+After pretraining, RegionCLIP can directly support the challenging zero-shot object detection task **without finetuning on detection annotation**. Given an input image, our pretrained RegionCLIP can match image region features to object concept embeddings, and thus recognize image regions into many object categories. The image regions are produced by a region localizer (e.g., RPN), where the object class names come from a dictionary **specifiied by users**.
+
+
+### Visualization on custom images
+
+We provide an example below for zero-shot object detection with pretrained RegionCLIP on custom images and for visualizing the results.
+
+<details>
+
+<summary>
+Before detecting objects, please prepare pretrained models, label files, and the custom images. See details below.
+</summary>
+  
+- Check [`MODEL_ZOO.md`](docs/MODEL_ZOO.md) to 
+  - download the pretrained model checkpoint `regionclip_pretrained-cc_rn50x4.pth` (RegionCLIP with ResNet50x4) to the folder `./pretrained_ckpt/regionclip`.
+  - download the class embeddings `lvis_1203_cls_emb_rn50x4.pth` to the folder `./pretrained_ckpt/concept_emb`.
+- Check [`datasets/README.md`](datasets/README.md) to download LVIS label file `lvis_v1_val.json` and put it in the folder `./datasets/lvis/lvis_v1_val.json`. The file is used to specify object class names.
+- Put all custom images in the folder `./datasets/custom_images/`.
+
+</details>
+  
+<details>
+
+<summary>
+After preparation, run the following script to detect objects.
+</summary>
+  
+```
+python3 ./tools/train_net.py \
+--eval-only \
+--num-gpus 1 \
+--config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_custom_img.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50x4.pth \
+MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb_rn50x4.pth \
+MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+MODEL.CLIP.TEXT_EMB_DIM 640 \
+MODEL.RESNETS.DEPTH 200 \
+MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION 18 \
+```
+
+</details>
+
+<details>
+
+<summary>
+The detection results will be stored as the file "./output/inference/lvis_instances_results.json". To visualize it, run the script below.
+</summary>
+ 
+```
+ python ./tools/visualize_json_results.py \
+--input ./output/inference/lvis_instances_results.json \
+--output ./output/regions \
+--dataset lvis_v1_val_custom_img \
+--conf-threshold 0.05 \
+--show-unique-boxes \
+--max-boxes 25 \
+--small-region-px 8100\ 
+```
+</details> 
+
+The visualized images will be placed at `./output/regions/`. The visualized images would look like:
+
+<p align="center">
+<img src="docs/sample_img1_vis.jpg" width=80% height=80%
+class="center">
+</p>
+
+In this example, the detection results come from our pretrained RegionCLIP with ResNet50x4 architecture. The regions are proposed by an RPN trained by 866 object categories from LVIS dataset. For now, we use 1203 object class names from LVIS dataset for this visualization example. We also include an example in `visualize_zeroshot_inference.sh` with our pretrained RegionCLIP (ResNet50 architecture).
+
+
+### Evaluation for zero-shot inference
+
+We provide an example below for evaluating our pretrained RegionCLIP (ResNet50) using ground-truth boxes on COCO dataset. This will reproduce our results in Table 4 of the paper.
+
+<details>
+
+<summary>
+Before evaluation, please prepare pretrained models and set up the dataset.
+</summary>
+  
+- Check [`MODEL_ZOO.md`](docs/MODEL_ZOO.md) to 
+  - download the pretrained RegionCLIP checkpoint `regionclip_pretrained-cc_rn50.pth` to the folder `./pretrained_ckpt/regionclip`.
+  - download the class embeddings `coco_65_cls_emb.pth` to the folder `./pretrained_ckpt/concept_emb`.
+- Check [`datasets/README.md`](datasets/README.md) to set up COCO dataset.
+
+</details>
+  
+<details>
+
+<summary>
+After preparation, run the following script to evaluate the pretrained model in zero-shot inference setting.
+</summary>
+  
+```
+python3 ./tools/train_net.py \
+--eval-only  \
+--num-gpus 1 \
+--config-file ./configs/COCO-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_ovd_zsinf.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50.pth \
+MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_65_cls_emb.pth \
+MODEL.CLIP.CROP_REGION_TYPE GT \
+MODEL.CLIP.MULTIPLY_RPN_SCORE False \
+```
+
+</details>
+
+For more examples, please refer to `test_zeroshot_inference.sh`. This script covers a wide combination of pretrained models (ResNet50, ResNet50x4), datasets (COCO, LVIS) and region proposal types (ground-truth regions, RPN proposals). Also, please refer to [MODEL_ZOO.md](docs/MODEL_ZOO.md) for available trained models and [`datasets/README.md`](datasets/README.md) for setting up COCO and LVIS datasets.
+
+## Transfer Learning
+
+Our pretrained RegionCLIP can be further **fine-tuned** when human annotations of objects are available. In this transfer learning setting, we demonstrate results on **open-vocabulary object detection**, where the object detector is trained on base categories and evaluated on both base and **novel** categories.
+
+We show an example for running a trained detector on custom images. Further, we provide scripts of training and evaluation for the benchmark of **open-vocabulary object detection**, including COCO and LVIS datasets (Table 1 & 2 in paper).
+
+
+### Visualization on custom images
+
+We provide an example below for running a trained open-vocabulary object detector on custom images and for visualizing the results. In this example, the detector is initialized using RegionCLIP (RN50x4), trained on 866 LVIS base categories, and is tasked to detect all 1203 categories on LVIS.
+
+<details>
+
+<summary>
+Before detecting objects, please prepare the trained detectors, label files, and the custom images.
+</summary>
+  
+- Check [`MODEL_ZOO.md`](docs/MODEL_ZOO.md) to 
+  - download the trained detector checkpoint `regionclip_finetuned-lvis_rn50x4.pth` to the folder `./pretrained_ckpt/regionclip`.
+  - download the trained RPN checkpoint `rpn_lvis_866_lsj.pth` to the folder `./pretrained_ckpt/rpn`.
+  - download the class embeddings `lvis_1203_cls_emb_rn50x4.pth` to the folder `./pretrained_ckpt/concept_emb`.
+- Check [`datasets/README.md`](datasets/README.md) to download label file `lvis_v1_val.json` and put it in the folder `./datasets/lvis/lvis_v1_val.json`.
+- Put all custom images in the folder `./datasets/custom_images/`.
+
+</details>
+  
+<details>
+
+<summary>
+After preparation, run the following script to detect objects and visualize the results.
+</summary>
+
+```
+# for simplicity, we integrate the script in visualize_transfer_learning.sh
+bash visualize_transfer_learning.sh
+```
+
+</details>
+
+  
+The visualized images will be placed at `./output/regions/`.
+
+
+### Evaluate the trained detectors
+
+We provide an example below for evaluating our open-vocabulary object detector, initialized by RegionCLIP (ResNet50) and trained on COCO dataset.
+
+<details>
+
+<summary>
+Before evaluation, please prepare the trained detector and set up the dataset.
+</summary>
+  
+- Check [`MODEL_ZOO.md`](docs/MODEL_ZOO.md) to 
+  - download the trained detector checkpoint `regionclip_finetuned-coco_rn50.pth` to the folder `./pretrained_ckpt/regionclip`, 
+  - download the trained RPN checkpoint `rpn_coco_48.pth` to the folder `./pretrained_ckpt/rpn`,
+  - download the class embeddings `coco_48_base_cls_emb.pth` and `coco_65_cls_emb.pth` to the folder `./pretrained_ckpt/concept_emb`.
+- Check [`datasets/README.md`](datasets/README.md) to set up COCO dataset.
+
+</details>
+  
+<details>
+
+<summary>
+After preparation, run the following script to evaluate the trained open-vocabulary detector.
+</summary>
+  
+```
+python3 ./tools/train_net.py \
+--eval-only  \
+--num-gpus 1 \
+--config-file ./configs/COCO-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_ovd.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_finetuned-coco_rn50.pth \
+MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/COCO-InstanceSegmentation/mask_rcnn_R_50_C4_1x_ovd_FSD.yaml \
+MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_coco_48.pth \
+MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_48_base_cls_emb.pth \
+MODEL.CLIP.OPENSET_TEST_TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_65_cls_emb.pth \
+MODEL.ROI_HEADS.SOFT_NMS_ENABLED True \
+```
+
+</details>
+
+
+For more examples, please refer to `test_transfer_learning.sh`. This script includes benchmark evaluation for various combination of trained detectors (ResNet50, ResNet50x4) and datasets (COCO, LVIS). Also, please refer to [MODEL_ZOO.md](docs/MODEL_ZOO.md) for available trained models and [`datasets/README.md`](datasets/README.md) for setting up COCO and LVIS datasets.
+
+
+### Train detectors on your own
+
+We provide an example below for training an open-vocabulary object detector on COCO dataset, with pretrained RegionCLIP (ResNet50) as the initialization.
+
+<details>
+
+<summary>
+Before training, please prepare our pretrained RegionCLIP model and set up the dataset.
+</summary>
+  
+- Check [`MODEL_ZOO.md`](docs/MODEL_ZOO.md) to 
+  - download the pretrained RegionCLIP checkpoint `regionclip_pretrained-cc_rn50.pth` to the folder `./pretrained_ckpt/regionclip`, 
+  - download the trained RPN checkpoint `rpn_coco_48.pth` to the folder `./pretrained_ckpt/rpn`,
+  - download the class embeddings `coco_48_base_cls_emb.pth` and `coco_65_cls_emb.pth` to the folder `./pretrained_ckpt/concept_emb`.
+- Check [`datasets/README.md`](datasets/README.md) to set up COCO dataset.
+
+</details>
+  
+<details>
+
+<summary>
+After preparation, run the following script to train an open-vocabulary detector.
+</summary>
+  
+```
+python3 ./tools/train_net.py \
+--num-gpus 1 \
+--config-file ./configs/COCO-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_ovd.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50.pth \
+MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/COCO-InstanceSegmentation/mask_rcnn_R_50_C4_1x_ovd_FSD.yaml \
+MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_coco_48.pth \
+MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_48_base_cls_emb.pth \
+MODEL.CLIP.OPENSET_TEST_TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_65_cls_emb.pth \
+```
+
+</details>
+
+For more examples, please refer to `train_transfer_learning.sh`. This script provides training scripts for various combination of detector backbones (ResNet50, ResNet50x4) and datasets (COCO, LVIS). Also, please refer to [MODEL_ZOO.md](docs/MODEL_ZOO.md) for available trained models and [`datasets/README.md`](datasets/README.md) for setting up COCO and LVIS datasets.
+
+
+## Extract Region Features
+
+We provide scripts for extracting region features from our pre-trained RegionCLIP. Given a folder of images, our scripts extract region features (along with other detection results such as box coordinates) and save them as local files.
+
+The following is an example using pretrained RegionCLIP with ResNet50. We extend the scripts from zero-shot inference (section above) with minor changes, such as the input and output folders.
+
+<details>
+
+<summary>
+The following is a brief introduction for the settings.
+</summary>
+
+We enable feature extraction for two types of regions:
+
+- RPN regions: This setting is class-agnostic. The regions are the top-scored RPN proposals.
+
+- Detection regions: This setting requires additional input as a concept embedding file (the concepts of interests). The regions are the final detection output boxes (after 2nd-stage NMS). As a reference, the [Bottom-Up features](https://openaccess.thecvf.com/content_cvpr_2018/papers/Anderson_Bottom-Up_and_Top-Down_CVPR_2018_paper.pdf) (widely-used in vision-language tasks) also come from the final detection boxes.
+
+</details>
+
+
+
+<details>
+
+<summary>
+Before running scripts, please prepare pretrained models and your custom images.
+</summary>
+  
+- Check [`MODEL_ZOO.md`](docs/MODEL_ZOO.md) to 
+  - download the pretrained RegionCLIP checkpoint `regionclip_pretrained-cc_rn50.pth` to the folder `./pretrained_ckpt/regionclip`.
+  - download the trained RPN checkpoint `rpn_lvis_866.pth` to the folder `./pretrained_ckpt/rpn`.
+  - (optional) if you want to extract features of the boxes detected for 1203 LVIS concepts, download the class embeddings `lvis_1203_cls_emb.pth` to the folder `./pretrained_ckpt/concept_emb`.
+- Put all custom images in a folder. It can be specified in the script (check `INPUT_DIR` below).
+
+
+</details>
+
+
+<details>
+
+<summary>
+After preparation, run the following script to extract region features.
+</summary>
+
+The following script extracts features from **RPN regions**.
+```
+# RN50, features of RPN regions
+python3 ./tools/extract_region_features.py \
+--config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_zsinf.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50.pth \
+MODEL.CLIP.CROP_REGION_TYPE RPN \
+MODEL.CLIP.MULTIPLY_RPN_SCORE True \
+MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866.pth \
+INPUT_DIR ./datasets/custom_images \
+OUTPUT_DIR ./output/region_feats \
+MODEL.CLIP.OFFLINE_RPN_POST_NMS_TOPK_TEST 100 \
+```
+
+The following script extracts features from **detection regions** (after 2nd-stage NMS).
+
+```
+# You can simply run "bash extract_region_features.sh"
+python3 ./tools/extract_region_features.py \
+--config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_zsinf.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50.pth \
+MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb.pth \
+MODEL.CLIP.CROP_REGION_TYPE RPN \
+MODEL.CLIP.MULTIPLY_RPN_SCORE True \
+MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866.pth \
+INPUT_DIR ./datasets/custom_images \
+OUTPUT_DIR ./output/region_feats \
+TEST.DETECTIONS_PER_IMAGE 100 \
+```
+
+The region features of each image will be saved into a `.pth` file in the folder `OUTPUT_DIR`. For simplicity, the current script only supports single GPU inference. As a reference, it takes roughly 0.76 seconds on single Titan-Xp GPU with RegionCLIP-ResNet50 and 1203 LVIS object concepts.
+
+The following is a list of key arguments for feature extraction. You can specify them in the script as needed.
+
+- `INPUT_DIR` and `OUTPUT_DIR`: specify a folder of input images and an output folder where region features will be saved, respectively.
+
+- `MODEL.CLIP.BB_RPN_WEIGHTS`: specifies which trained RPN to use. You can replace it as needed. For more details, please check [`MODEL_ZOO.md`](docs/MODEL_ZOO.md).
+
+- `MODEL.CLIP.TEXT_EMB_PATH` (optional): specifies which object concept embedding file to use. The selection of concepts will affect the per-class NMS (2nd stage) and thus final output boxes.
+
+- `TEST.DETECTIONS_PER_IMAGE`: defines the number of final output regions (e.g., default value is 100 in COCO configs and 300 in LVIS configs) 
+
+- `MODEL.CLIP.OFFLINE_RPN_POST_NMS_TOPK_TEST`: defines the number of region proposals from RPN (e.g., default is 1000). Lowering this value can significantly reduce inference time and memory cost, but might affect the final detection quality.
+
+- `MODEL.CLIP.OFFLINE_RPN_NMS_THRESH` and `MODEL.ROI_HEADS.NMS_THRESH_TEST`: control the NMS IoU thresholds in RPN (1st stage, default is 0.9) and prediction head (2nd stage, default is 0.5), respectively. If you extract features using RPN regions, you might want to change `MODEL.CLIP.OFFLINE_RPN_NMS_THRESH` as needed.
+
+</details>
+
+## Extract Concept Features
+
+Along with the region feature extraction, we also provide scripts for extracting concept features from our pre-trained RegionCLIP. Given a list of concepts, our scripts extract textual embeddings and save them as local files. The following is an example using pretrained RegionCLIP. We extend the scripts from region feature extraction (section above) with minor changes.
+
+
+<details>
+
+<summary>
+Before running scripts, please prepare pretrained models and your custom concepts.
+</summary>
+  
+- Check [`MODEL_ZOO.md`](docs/MODEL_ZOO.md) to 
+  - download the pretrained RegionCLIP checkpoint `regionclip_pretrained-cc_rn50.pth` to the folder `./pretrained_ckpt/regionclip`.
+- Put all concepts in the file `concepts.txt` with each line as a concept name. Place this file in a folder which can be specified in the script (check `INPUT_DIR` below).
+
+
+</details>
+
+
+<details>
+
+<summary>
+After preparation, run the following script to extract region features.
+</summary>
+
+The following script extracts features from ResNet50.
+```
+# RN50 concept embeddings
+python3 ./tools/extract_concept_features.py \
+--config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_zsinf.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50.pth \
+MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+INPUT_DIR ./datasets/custom_concepts \
+OUTPUT_DIR ./output/concept_feats \
+MODEL.CLIP.GET_CONCEPT_EMB True \
+```
+
+And for ResNet50x4, use the following command:
+```
+# RN50x4 concept embeddings
+python3 ./tools/extract_concept_features.py \
+--config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_zsinf.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50x4.pth \
+MODEL.CLIP.TEXT_EMB_DIM 640 \
+MODEL.RESNETS.DEPTH 200 \
+MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+INPUT_DIR ./datasets/custom_concepts \
+OUTPUT_DIR ./output/concept_feats \
+MODEL.CLIP.GET_CONCEPT_EMB True \
+```
+
+The language embeddings of all concepts will be saved into a `.pth` file in the folder `OUTPUT_DIR`. These language embeddings have not been normalized yet, for the consistency with concept embeddings provided in [`MODEL_ZOO.md`](docs/MODEL_ZOO.md).
+
+The following is a list of key arguments for feature extraction. You can specify them in the script as needed.
+
+- `INPUT_DIR` and `OUTPUT_DIR`: specify a folder of input concepts and an output folder where region features will be saved, respectively.
+
+</details>
+
+## Citation and Acknowledgement
+
+### Citation
+
+If you find this repo useful, please consider citing our paper:
+
+```
+@inproceedings{zhong2022regionclip,
+  title={Regionclip: Region-based language-image pretraining},
+  author={Zhong, Yiwu and Yang, Jianwei and Zhang, Pengchuan and Li, Chunyuan and Codella, Noel and Li, Liunian Harold and Zhou, Luowei and Dai, Xiyang and Yuan, Lu and Li, Yin and others},
+  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
+  pages={16793--16803},
+  year={2022}
+}
+```
+
+### Acknowledgement
+
+This repository was built on top of [Detectron2](https://github.com/facebookresearch/detectron2), [CLIP](https://github.com/openai/CLIP), [OVR-CNN](https://github.com/alirezazareian/ovr-cnn), and [maskrcnn-benchmark](https://github.com/facebookresearch/maskrcnn-benchmark). We thank the effort from our community.
+
+## Contributing
+
+This project welcomes contributions and suggestions.  Most contributions require you to agree to a
+Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
+the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
+
+When you submit a pull request, a CLA bot will automatically determine whether you need to provide
+a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions
+provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
+contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.

SECURITY.md

@@ -0,0 +1,41 @@
+<!-- BEGIN MICROSOFT SECURITY.MD V0.0.7 BLOCK -->
+
+## Security
+
+Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
+
+If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below.
+
+## Reporting Security Issues
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report).
+
+If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com).  If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey).
+
+You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+  * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+  * Full paths of source file(s) related to the manifestation of the issue
+  * The location of the affected source code (tag/branch/commit or direct URL)
+  * Any special configuration required to reproduce the issue
+  * Step-by-step instructions to reproduce the issue
+  * Proof-of-concept or exploit code (if possible)
+  * Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs.
+
+## Preferred Languages
+
+We prefer all communications to be in English.
+
+## Policy
+
+Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd).
+
+<!-- END MICROSOFT SECURITY.MD BLOCK -->

extract_concept_features.sh

@@ -0,0 +1,21 @@
+# Extract concept features for a list of concepts
+
+# RN50 concept embeddings
+python3 ./tools/extract_concept_features.py \
+--config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_zsinf.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50.pth \
+MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+INPUT_DIR ./datasets/custom_concepts \
+OUTPUT_DIR ./output/concept_feats \
+MODEL.CLIP.GET_CONCEPT_EMB True \
+
+# RN50x4 concept embeddings
+# python3 ./tools/extract_concept_features.py \
+# --config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_zsinf.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50x4.pth \
+# MODEL.CLIP.TEXT_EMB_DIM 640 \
+# MODEL.RESNETS.DEPTH 200 \
+# MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+# INPUT_DIR ./datasets/custom_concepts \
+# OUTPUT_DIR ./output/concept_feats \
+# MODEL.CLIP.GET_CONCEPT_EMB True \

extract_region_features.sh

@@ -0,0 +1,42 @@
+# Extract region features for a folder of images
+
+# RN50, LVIS 1203 concepts
+python3 ./tools/extract_region_features.py \
+--config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_zsinf.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50.pth \
+MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb.pth \
+MODEL.CLIP.CROP_REGION_TYPE RPN \
+MODEL.CLIP.MULTIPLY_RPN_SCORE True \
+MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866.pth \
+INPUT_DIR ./datasets/custom_images \
+OUTPUT_DIR ./output/region_feats \
+TEST.DETECTIONS_PER_IMAGE 100 \
+
+# # RN50, features of RPN proposals
+# python3 ./tools/extract_region_features.py \
+# --config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_zsinf.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50.pth \
+# MODEL.CLIP.CROP_REGION_TYPE RPN \
+# MODEL.CLIP.MULTIPLY_RPN_SCORE True \
+# MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+# MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866.pth \
+# INPUT_DIR ./datasets/custom_images \
+# OUTPUT_DIR ./output/region_feats \
+# MODEL.CLIP.OFFLINE_RPN_POST_NMS_TOPK_TEST 100 \
+
+# # RN50x4, LVIS 1203 concepts
+# python3 ./tools/extract_region_features.py \
+# --config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_zsinf.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50x4.pth \
+# MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb_rn50x4.pth \
+# MODEL.CLIP.CROP_REGION_TYPE RPN \
+# MODEL.CLIP.MULTIPLY_RPN_SCORE True \
+# MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+# MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866.pth \
+# MODEL.CLIP.TEXT_EMB_DIM 640 \
+# MODEL.RESNETS.DEPTH 200 \
+# MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION 18 \
+# INPUT_DIR ./datasets/custom_images \
+# OUTPUT_DIR ./output/region_feats \
+# TEST.DETECTIONS_PER_IMAGE 100 \

pretrain.sh

@@ -0,0 +1,27 @@
+# NOTE: The pre-training section is still under construction.
+#       The pre-training code was already released (PretrainFastRCNN class).
+#       Now we release the config files and scripts (un-tested yet), as requested by researchers.
+#       We will release the pre-training data (image-text pairs) in near future.
+
+
+# Distributed training across multiple nodes
+# ResNet50 (default: batch 96, lr 0.002, 32 GPUs)
+python3 -m launch --nnodes=2 --nproc_per_node=16 --master_port 12345 ./tools/train_net.py \
+--num-gpus 16 \
+--config-file ./configs/pretrain/RegionCLIP_RN50.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/clip/teacher_RN50_student_RN50_OAI_CLIP.pth \
+MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866.pth \
+MODEL.CLIP.CONCEPT_POOL_EMB ./pretrained_ckpt/concept_emb/coco_nouns_4764_emb.pth \
+OUTPUT_DIR ./output/pretrain \
+
+
+# ResNet50x4 (default: batch 96, lr 0.002, 32 GPUs)
+python3 -m launch --nnodes=2 --nproc_per_node=16 --master_port 12345 ./tools/train_net.py \
+--num-gpus 16 \
+--config-file ./configs/pretrain/RegionCLIP_RN50x4.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/clip/teacher_RN50x4_student_RN50x4_OAI_CLIP.pth \
+MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866.pth \
+MODEL.CLIP.CONCEPT_POOL_EMB ./pretrained_ckpt/concept_emb/coco_nouns_4764_emb_rn50x4.pth \
+OUTPUT_DIR ./output/pretrain \

setup.cfg

@@ -0,0 +1,26 @@
+[isort]
+line_length=100
+multi_line_output=3
+include_trailing_comma=True
+known_standard_library=numpy,setuptools,mock
+skip=./datasets,docs
+skip_glob=*/__init__.py,**/configs/**,tests/config/**
+known_myself=detectron2
+known_third_party=fvcore,matplotlib,cv2,torch,torchvision,PIL,pycocotools,yacs,termcolor,cityscapesscripts,tabulate,tqdm,scipy,lvis,psutil,pkg_resources,caffe2,onnx,panopticapi,black,isort,av,iopath,omegaconf,hydra,yaml,pydoc,submitit,cloudpickle
+no_lines_before=STDLIB,THIRDPARTY
+sections=FUTURE,STDLIB,THIRDPARTY,myself,FIRSTPARTY,LOCALFOLDER
+default_section=FIRSTPARTY
+
+[mypy]
+python_version=3.6
+ignore_missing_imports = True
+warn_unused_configs = True
+disallow_untyped_defs = True
+check_untyped_defs = True
+warn_unused_ignores = True
+warn_redundant_casts = True
+show_column_numbers = True
+follow_imports = silent
+allow_redefinition = True
+; Require all functions to be annotated
+disallow_incomplete_defs = True

setup.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python
+# Copyright (c) Facebook, Inc. and its affiliates.
+
+import glob
+import os
+import shutil
+from os import path
+from setuptools import find_packages, setup
+from typing import List
+import torch
+from torch.utils.cpp_extension import CUDA_HOME, CppExtension, CUDAExtension
+from torch.utils.hipify import hipify_python
+
+torch_ver = [int(x) for x in torch.__version__.split(".")[:2]]
+assert torch_ver >= [1, 6], "Requires PyTorch >= 1.6"
+
+
+def get_version():
+    init_py_path = path.join(path.abspath(path.dirname(__file__)), "detectron2", "__init__.py")
+    init_py = open(init_py_path, "r").readlines()
+    version_line = [l.strip() for l in init_py if l.startswith("__version__")][0]
+    version = version_line.split("=")[-1].strip().strip("'\"")
+
+    # The following is used to build release packages.
+    # Users should never use it.
+    suffix = os.getenv("D2_VERSION_SUFFIX", "")
+    version = version + suffix
+    if os.getenv("BUILD_NIGHTLY", "0") == "1":
+        from datetime import datetime
+
+        date_str = datetime.today().strftime("%y%m%d")
+        version = version + ".dev" + date_str
+
+        new_init_py = [l for l in init_py if not l.startswith("__version__")]
+        new_init_py.append('__version__ = "{}"\n'.format(version))
+        with open(init_py_path, "w") as f:
+            f.write("".join(new_init_py))
+    return version
+
+
+def get_extensions():
+    this_dir = path.dirname(path.abspath(__file__))
+    extensions_dir = path.join(this_dir, "detectron2", "layers", "csrc")
+
+    main_source = path.join(extensions_dir, "vision.cpp")
+    sources = glob.glob(path.join(extensions_dir, "**", "*.cpp"))
+
+    from torch.utils.cpp_extension import ROCM_HOME
+
+    is_rocm_pytorch = (
+        True if ((torch.version.hip is not None) and (ROCM_HOME is not None)) else False
+    )
+
+    hipify_ver = (
+        [int(x) for x in torch.utils.hipify.__version__.split(".")]
+        if hasattr(torch.utils.hipify, "__version__")
+        else [0, 0, 0]
+    )
+
+    if is_rocm_pytorch and hipify_ver < [1, 0, 0]:  # TODO not needed since pt1.8
+
+        # Earlier versions of hipification and extension modules were not
+        # transparent, i.e. would require an explicit call to hipify, and the
+        # hipification would introduce "hip" subdirectories, possibly changing
+        # the relationship between source and header files.
+        # This path is maintained for backwards compatibility.
+
+        hipify_python.hipify(
+            project_directory=this_dir,
+            output_directory=this_dir,
+            includes="/detectron2/layers/csrc/*",
+            show_detailed=True,
+            is_pytorch_extension=True,
+        )
+
+        source_cuda = glob.glob(path.join(extensions_dir, "**", "hip", "*.hip")) + glob.glob(
+            path.join(extensions_dir, "hip", "*.hip")
+        )
+
+        shutil.copy(
+            "detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_utils.h",
+            "detectron2/layers/csrc/box_iou_rotated/hip/box_iou_rotated_utils.h",
+        )
+        shutil.copy(
+            "detectron2/layers/csrc/deformable/deform_conv.h",
+            "detectron2/layers/csrc/deformable/hip/deform_conv.h",
+        )
+
+        sources = [main_source] + sources
+        sources = [
+            s
+            for s in sources
+            if not is_rocm_pytorch or torch_ver < [1, 7] or not s.endswith("hip/vision.cpp")
+        ]
+
+    else:
+
+        # common code between cuda and rocm platforms,
+        # for hipify version [1,0,0] and later.
+
+        source_cuda = glob.glob(path.join(extensions_dir, "**", "*.cu")) + glob.glob(
+            path.join(extensions_dir, "*.cu")
+        )
+
+        sources = [main_source] + sources
+
+    extension = CppExtension
+
+    extra_compile_args = {"cxx": []}
+    define_macros = []
+
+    if (torch.cuda.is_available() and ((CUDA_HOME is not None) or is_rocm_pytorch)) or os.getenv(
+        "FORCE_CUDA", "0"
+    ) == "1":
+        extension = CUDAExtension
+        sources += source_cuda
+
+        if not is_rocm_pytorch:
+            define_macros += [("WITH_CUDA", None)]
+            extra_compile_args["nvcc"] = [
+                "-O3",
+                "-DCUDA_HAS_FP16=1",
+                "-D__CUDA_NO_HALF_OPERATORS__",
+                "-D__CUDA_NO_HALF_CONVERSIONS__",
+                "-D__CUDA_NO_HALF2_OPERATORS__",
+            ]
+        else:
+            define_macros += [("WITH_HIP", None)]
+            extra_compile_args["nvcc"] = []
+
+        if torch_ver < [1, 7]:
+            # supported by https://github.com/pytorch/pytorch/pull/43931
+            CC = os.environ.get("CC", None)
+            if CC is not None:
+                extra_compile_args["nvcc"].append("-ccbin={}".format(CC))
+
+    include_dirs = [extensions_dir]
+
+    ext_modules = [
+        extension(
+            "detectron2._C",
+            sources,
+            include_dirs=include_dirs,
+            define_macros=define_macros,
+            extra_compile_args=extra_compile_args,
+        )
+    ]
+
+    return ext_modules
+
+
+def get_model_zoo_configs() -> List[str]:
+    """
+    Return a list of configs to include in package for model zoo. Copy over these configs inside
+    detectron2/model_zoo.
+    """
+
+    # Use absolute paths while symlinking.
+    source_configs_dir = path.join(path.dirname(path.realpath(__file__)), "configs")
+    destination = path.join(
+        path.dirname(path.realpath(__file__)), "detectron2", "model_zoo", "configs"
+    )
+    # Symlink the config directory inside package to have a cleaner pip install.
+
+    # Remove stale symlink/directory from a previous build.
+    if path.exists(source_configs_dir):
+        if path.islink(destination):
+            os.unlink(destination)
+        elif path.isdir(destination):
+            shutil.rmtree(destination)
+
+    if not path.exists(destination):
+        try:
+            os.symlink(source_configs_dir, destination)
+        except OSError:
+            # Fall back to copying if symlink fails: ex. on Windows.
+            shutil.copytree(source_configs_dir, destination)
+
+    config_paths = glob.glob("configs/**/*.yaml", recursive=True) + glob.glob(
+        "configs/**/*.py", recursive=True
+    )
+    return config_paths
+
+
+# For projects that are relative small and provide features that are very close
+# to detectron2's core functionalities, we install them under detectron2.projects
+PROJECTS = {
+#     "detectron2.projects.point_rend": "projects/PointRend/point_rend",
+#     "detectron2.projects.deeplab": "projects/DeepLab/deeplab",
+#     "detectron2.projects.panoptic_deeplab": "projects/Panoptic-DeepLab/panoptic_deeplab",
+}
+
+setup(
+    name="detectron2",
+    version=get_version(),
+    author="FAIR",
+    url="https://github.com/facebookresearch/detectron2",
+    description="Detectron2 is FAIR's next-generation research "
+    "platform for object detection and segmentation.",
+    packages=find_packages(exclude=("configs", "tests*")) + list(PROJECTS.keys()),
+    package_dir=PROJECTS,
+    package_data={"detectron2.model_zoo": get_model_zoo_configs()},
+    python_requires=">=3.6",
+    install_requires=[
+        # Do not add opencv here. Just like pytorch, user should install
+        # opencv themselves, preferrably by OS's package manager, or by
+        # choosing the proper pypi package name at https://github.com/skvark/opencv-python
+        "termcolor>=1.1",
+        "Pillow>=7.1",  # or use pillow-simd for better performance
+        "yacs>=0.1.6",
+        "tabulate",
+        "cloudpickle",
+        "matplotlib",
+        "tqdm>4.29.0",
+        "tensorboard",
+        # Lock version of fvcore/iopath because they may have breaking changes
+        # NOTE: when updating fvcore/iopath version, make sure fvcore depends
+        # on compatible version of iopath.
+        "fvcore>=0.1.5,<0.1.6",  # required like this to make it pip installable
+        "iopath>=0.1.7,<0.1.9",
+        "pycocotools>=2.0.2",  # corresponds to https://github.com/ppwwyyxx/cocoapi
+        "future",  # used by caffe2
+        "pydot",  # used to save caffe2 SVGs
+        "dataclasses; python_version<'3.7'",
+        "omegaconf>=2.1.0rc1",
+        "hydra-core>=1.1.0rc1",
+        "black==21.4b2",
+        # When adding to the list, may need to update docs/requirements.txt
+        # or add mock in docs/conf.py
+    ],
+    extras_require={
+        "all": [
+            "shapely",
+            "pygments>=2.2",
+            "psutil",
+            "panopticapi @ https://github.com/cocodataset/panopticapi/archive/master.zip",
+        ],
+        "dev": [
+            "flake8==3.8.1",
+            "isort==4.3.21",
+            "flake8-bugbear",
+            "flake8-comprehensions",
+        ],
+    },
+    ext_modules=get_extensions(),
+    cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension},
+)

test_transfer_learning.sh

@@ -0,0 +1,79 @@
+# evaluate our trained open-vocabulary object detectors, {RN50, RN50x4} x {COCO, LVIS}
+
+# RN50, COCO (Generalized: Novel + Base)
+python3 ./tools/train_net.py \
+--eval-only  \
+--num-gpus 1 \
+--config-file ./configs/COCO-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_ovd.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_finetuned-coco_rn50.pth \
+MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/COCO-InstanceSegmentation/mask_rcnn_R_50_C4_1x_ovd_FSD.yaml \
+MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_coco_48.pth \
+MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_48_base_cls_emb.pth \
+MODEL.CLIP.OPENSET_TEST_TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_65_cls_emb.pth \
+MODEL.ROI_HEADS.SOFT_NMS_ENABLED True \
+
+# # RN50, COCO (only Novel)
+# # --config-file ./configs/COCO-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_ovd_testt.yaml \
+# # MODEL.CLIP.OPENSET_TEST_TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_17_target_cls_emb.pth \
+
+# # RN50, COCO (only Base)
+# # --config-file ./configs/COCO-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_ovd_testb.yaml \
+# # MODEL.CLIP.OPENSET_TEST_TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_48_base_cls_emb.pth \
+
+
+# # RN50, LVIS
+# python3 ./tools/train_net.py \
+# --eval-only  \
+# --num-gpus 1 \
+# --config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_finetuned-lvis_rn50.pth \
+# MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+# MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866_lsj.pth \
+# MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_866_base_cls_emb.pth \
+# MODEL.CLIP.OPENSET_TEST_TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb.pth \
+# MODEL.CLIP.OFFLINE_RPN_LSJ_PRETRAINED True \
+# MODEL.ROI_HEADS.SOFT_NMS_ENABLED True \
+
+
+
+# # RN50x4, COCO (Generalized: Novel + Base)
+# python3 ./tools/train_net.py \
+# --eval-only  \
+# --num-gpus 1 \
+# --config-file ./configs/COCO-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_ovd.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_finetuned-coco_rn50x4.pth \
+# MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/COCO-InstanceSegmentation/mask_rcnn_R_50_C4_1x_ovd_FSD.yaml \
+# MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_coco_48.pth \
+# MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_48_base_cls_emb_rn50x4.pth \
+# MODEL.CLIP.OPENSET_TEST_TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_65_cls_emb_rn50x4.pth \
+# MODEL.CLIP.TEXT_EMB_DIM 640 \
+# MODEL.RESNETS.DEPTH 200 \
+# MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION 18 \
+# MODEL.ROI_HEADS.SOFT_NMS_ENABLED True \
+
+# # RN50x4, COCO (only Novel)
+# # --config-file ./configs/COCO-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_ovd_testt.yaml \
+# # MODEL.CLIP.OPENSET_TEST_TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_17_target_cls_emb_rn50x4.pth \
+
+# # RN50x4, COCO (only Base)
+# # --config-file ./configs/COCO-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_ovd_testb.yaml \
+# # MODEL.CLIP.OPENSET_TEST_TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_48_base_cls_emb_rn50x4.pth \
+
+
+# # RN50x4, LVIS
+# python3 ./tools/train_net.py \
+# --eval-only  \
+# --num-gpus 1 \
+# --config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_finetuned-lvis_rn50x4 \
+# MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+# MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866_lsj.pth \
+# MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_866_base_cls_emb_rn50x4.pth \
+# MODEL.CLIP.OPENSET_TEST_TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb_rn50x4.pth \
+# MODEL.CLIP.OFFLINE_RPN_LSJ_PRETRAINED True \
+# MODEL.CLIP.TEXT_EMB_DIM 640 \
+# MODEL.RESNETS.DEPTH 200 \
+# MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION 18 \
+# MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION 18 \
+# MODEL.RESNETS.RES2_OUT_CHANNELS 320 \
+# MODEL.ROI_HEADS.SOFT_NMS_ENABLED True \

test_zeroshot_inference.sh

@@ -0,0 +1,107 @@
+# evaluate zero-shot inference {RN50, RN50x4} x {COCO, LVIS} x {GT, RPN}
+
+# RN50, GT, COCO
+python3 ./tools/train_net.py \
+--eval-only  \
+--num-gpus 1 \
+--config-file ./configs/COCO-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_ovd_zsinf.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50.pth \
+MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_65_cls_emb.pth \
+MODEL.CLIP.CROP_REGION_TYPE GT \
+MODEL.CLIP.MULTIPLY_RPN_SCORE False \
+
+# # RN50, RPN, COCO
+# python3 ./tools/train_net.py \
+# --eval-only  \
+# --num-gpus 1 \
+# --config-file ./configs/COCO-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_ovd_zsinf.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50.pth \
+# MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_65_cls_emb.pth \
+# MODEL.CLIP.CROP_REGION_TYPE RPN \
+# MODEL.CLIP.MULTIPLY_RPN_SCORE True \
+# MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+# MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866.pth \
+
+
+# # RN50, GT, LVIS
+# python3 ./tools/train_net.py \
+# --eval-only  \
+# --num-gpus 1 \
+# --config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_zsinf.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50.pth \
+# MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb.pth \
+# MODEL.CLIP.CROP_REGION_TYPE GT \
+# MODEL.CLIP.MULTIPLY_RPN_SCORE False \
+# MODEL.ROI_HEADS.SCORE_THRESH_TEST 0.0001 \
+
+# # RN50, RPN, LVIS
+# python3 ./tools/train_net.py \
+# --eval-only  \
+# --num-gpus 1 \
+# --config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_zsinf.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50.pth \
+# MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb.pth \
+# MODEL.CLIP.CROP_REGION_TYPE RPN \
+# MODEL.CLIP.MULTIPLY_RPN_SCORE True \
+# MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+# MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866.pth \
+
+
+
+# # RN50x4, GT, COCO
+# python3 ./tools/train_net.py \
+# --eval-only  \
+# --num-gpus 1 \
+# --config-file ./configs/COCO-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_ovd_zsinf.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50x4.pth \
+# MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_65_cls_emb_rn50x4.pth \
+# MODEL.CLIP.CROP_REGION_TYPE GT \
+# MODEL.CLIP.MULTIPLY_RPN_SCORE False \
+# MODEL.CLIP.TEXT_EMB_DIM 640 \
+# MODEL.RESNETS.DEPTH 200 \
+# MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION 18 \
+
+# # RN50x4, RPN, COCO
+# python3 ./tools/train_net.py \
+# --eval-only  \
+# --num-gpus 1 \
+# --config-file ./configs/COCO-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_ovd_zsinf.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50x4.pth \
+# MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_65_cls_emb_rn50x4.pth \
+# MODEL.CLIP.CROP_REGION_TYPE RPN \
+# MODEL.CLIP.MULTIPLY_RPN_SCORE True \
+# MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+# MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866.pth \
+# MODEL.CLIP.TEXT_EMB_DIM 640 \
+# MODEL.RESNETS.DEPTH 200 \
+# MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION 18 \
+
+
+# # RN50x4, GT, LVIS
+# python3 ./tools/train_net.py \
+# --eval-only  \
+# --num-gpus 1 \
+# --config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_zsinf.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50x4.pth \
+# MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb_rn50x4.pth \
+# MODEL.CLIP.CROP_REGION_TYPE GT \
+# MODEL.CLIP.MULTIPLY_RPN_SCORE False \
+# MODEL.ROI_HEADS.SCORE_THRESH_TEST 0.0001 \
+# MODEL.CLIP.TEXT_EMB_DIM 640 \
+# MODEL.RESNETS.DEPTH 200 \
+# MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION 18 \
+
+# # RN50x4, RPN, LVIS
+# python3 ./tools/train_net.py \
+# --eval-only  \
+# --num-gpus 1 \
+# --config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_zsinf.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50x4.pth \
+# MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb_rn50x4.pth \
+# MODEL.CLIP.CROP_REGION_TYPE RPN \
+# MODEL.CLIP.MULTIPLY_RPN_SCORE True \
+# MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+# MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866.pth \
+# MODEL.CLIP.TEXT_EMB_DIM 640 \
+# MODEL.RESNETS.DEPTH 200 \
+# MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION 18 \

train_transfer_learning.sh

@@ -0,0 +1,53 @@
+# train open-vocabulary object detectors (initialized by our pretrained RegionCLIP), {RN50, RN50x4} x {COCO, LVIS}
+
+# RN50, COCO
+python3 ./tools/train_net.py \
+--num-gpus 1 \
+--config-file ./configs/COCO-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_ovd.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50.pth \
+MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/COCO-InstanceSegmentation/mask_rcnn_R_50_C4_1x_ovd_FSD.yaml \
+MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_coco_48.pth \
+MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_48_base_cls_emb.pth \
+MODEL.CLIP.OPENSET_TEST_TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_65_cls_emb.pth \
+
+# # RN50, LVIS
+# python3 ./tools/train_net.py \
+# --num-gpus 1 \
+# --config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50.pth \
+# MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+# MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866_lsj.pth \
+# MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_866_base_cls_emb.pth \
+# MODEL.CLIP.OPENSET_TEST_TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb.pth \
+# MODEL.CLIP.OFFLINE_RPN_LSJ_PRETRAINED True \
+
+
+
+# # RN50x4, COCO
+# python3 ./tools/train_net.py \
+# --num-gpus 1 \
+# --config-file ./configs/COCO-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_ovd.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50x4.pth \
+# MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/COCO-InstanceSegmentation/mask_rcnn_R_50_C4_1x_ovd_FSD.yaml \
+# MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_coco_48.pth \
+# MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_48_base_cls_emb_rn50x4.pth \
+# MODEL.CLIP.OPENSET_TEST_TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/coco_65_cls_emb_rn50x4.pth \
+# MODEL.CLIP.TEXT_EMB_DIM 640 \
+# MODEL.RESNETS.DEPTH 200 \
+# MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION 18 \
+
+# # RN50x4, LVIS
+# python3 ./tools/train_net.py \
+# --num-gpus 1 \
+# --config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50x4.pth \
+# MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+# MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866_lsj.pth \
+# MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_866_base_cls_emb_rn50x4.pth \
+# MODEL.CLIP.OPENSET_TEST_TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb_rn50x4.pth \
+# MODEL.CLIP.OFFLINE_RPN_LSJ_PRETRAINED True \
+# MODEL.CLIP.TEXT_EMB_DIM 640 \
+# MODEL.RESNETS.DEPTH 200 \
+# MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION 18 \
+# MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION 18 \
+# MODEL.RESNETS.RES2_OUT_CHANNELS 320 \

visualize_transfer_learning.sh

@@ -0,0 +1,60 @@
+# visualize detection results from finetuned detectors on custom images
+
+########################################################
+
+# Open-vocabulary detector trained by 866 LVIS base categories, with RegionCLIP (RN50x4) as initialization
+python3 ./tools/train_net.py \
+--eval-only \
+--num-gpus 1 \
+--config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_custom_img.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_finetuned-lvis_rn50x4.pth \
+MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb_rn50x4.pth \
+MODEL.CLIP.OPENSET_TEST_TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb_rn50x4.pth \
+MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866_lsj.pth \
+MODEL.CLIP.OFFLINE_RPN_LSJ_PRETRAINED True \
+MODEL.CLIP.TEXT_EMB_DIM 640 \
+MODEL.RESNETS.DEPTH 200 \
+MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION 18 \
+MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION 18 \
+MODEL.RESNETS.RES2_OUT_CHANNELS 320 \
+
+# visualize the prediction json file
+python ./tools/visualize_json_results.py \
+--input ./output/inference/lvis_instances_results.json \
+--output ./output/regions \
+--dataset lvis_v1_val_custom_img \
+--conf-threshold 0.05 \
+--show-unique-boxes \
+--max-boxes 25 \
+--small-region-px 8100\
+
+
+########################################################
+
+# Open-vocabulary detector trained by 866 LVIS base categories, with RegionCLIP (RN50) as initialization
+# python3 ./tools/train_net.py \
+# --eval-only \
+# --num-gpus 1 \
+# --config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_custom_img.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_finetuned-lvis_rn50.pth \
+# MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb.pth \
+# MODEL.CLIP.OPENSET_TEST_TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb.pth \
+# MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+# MODEL.CLIP.BB_RPN_WEIGHTS ./pretrained_ckpt/rpn/rpn_lvis_866_lsj.pth \
+# MODEL.CLIP.OFFLINE_RPN_LSJ_PRETRAINED True \
+
+# # visualize the prediction json file
+# python ./tools/visualize_json_results.py \
+# --input ./output/inference/lvis_instances_results.json \
+# --output ./output/regions \
+# --dataset lvis_v1_val_custom_img \
+# --conf-threshold 0.05 \
+# --show-unique-boxes \
+# --max-boxes 25 \
+# --small-region-px 8100\
+
+
+
+
+

visualize_zeroshot_inference.sh

@@ -0,0 +1,52 @@
+# visualize zero-shot inference results on custom images
+
+########################################################
+
+# RegionCLIP (RN50x4)
+python3 ./tools/train_net.py \
+--eval-only \
+--num-gpus 1 \
+--config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_custom_img.yaml \
+MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50x4.pth \
+MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb_rn50x4.pth \
+MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+MODEL.CLIP.TEXT_EMB_DIM 640 \
+MODEL.RESNETS.DEPTH 200 \
+MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION 18 \
+
+# visualize the prediction json file
+python ./tools/visualize_json_results.py \
+--input ./output/inference/lvis_instances_results.json \
+--output ./output/regions \
+--dataset lvis_v1_val_custom_img \
+--conf-threshold 0.05 \
+--show-unique-boxes \
+--max-boxes 25 \
+--small-region-px 8100\
+
+
+########################################################
+
+# RegionCLIP (RN50)
+# python3 ./tools/train_net.py \
+# --eval-only \
+# --num-gpus 1 \
+# --config-file ./configs/LVISv1-InstanceSegmentation/CLIP_fast_rcnn_R_50_C4_custom_img.yaml \
+# MODEL.WEIGHTS ./pretrained_ckpt/regionclip/regionclip_pretrained-cc_rn50.pth \
+# MODEL.CLIP.TEXT_EMB_PATH ./pretrained_ckpt/concept_emb/lvis_1203_cls_emb.pth \
+# MODEL.CLIP.OFFLINE_RPN_CONFIG ./configs/LVISv1-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml \
+
+# # visualize the prediction json file
+# python ./tools/visualize_json_results.py \
+# --input ./output/inference/lvis_instances_results.json \
+# --output ./output/regions \
+# --dataset lvis_v1_val_custom_img \
+# --conf-threshold 0.05 \
+# --show-unique-boxes \
+# --max-boxes 25 \
+# --small-region-px 8100\
+
+
+
+
+