Spaces:
Running on Zero
Running on Zero
Add CorridorKey v2 (GPU) with v1/v2 switch, v2 default; v1 unchanged behind --version v1
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitignore +5 -0
- LICENSE_CorridorKey2.0 +597 -0
- README.md +76 -12
- THIRD_PARTY_NOTICES.md +39 -0
- app.py +199 -50
- ck2.py +337 -0
- corridorkey/__init__.py +8 -0
- corridorkey/blocks.py +0 -0
- corridorkey/color_utils.py +208 -0
- corridorkey/foundations.py +327 -0
- corridorkey/model.py +307 -0
- corridorkey/rope.py +320 -0
- corridorkey/runtime.py +937 -0
- corridorkey/rvm.py +562 -0
- corridorkey/stage1.py +920 -0
- corridorkey/stage2.py +532 -0
- corridorkey/video_io.py +820 -0
- requirements.txt +7 -0
- third_party/MoGe/LICENSE +224 -0
- third_party/MoGe/README.md +295 -0
- third_party/MoGe/moge/__init__.py +0 -0
- third_party/MoGe/moge/model/__init__.py +18 -0
- third_party/MoGe/moge/model/dinov2/__init__.py +6 -0
- third_party/MoGe/moge/model/dinov2/hub/__init__.py +4 -0
- third_party/MoGe/moge/model/dinov2/hub/backbones.py +156 -0
- third_party/MoGe/moge/model/dinov2/hub/utils.py +39 -0
- third_party/MoGe/moge/model/dinov2/layers/__init__.py +11 -0
- third_party/MoGe/moge/model/dinov2/layers/attention.py +100 -0
- third_party/MoGe/moge/model/dinov2/layers/block.py +259 -0
- third_party/MoGe/moge/model/dinov2/layers/dino_head.py +58 -0
- third_party/MoGe/moge/model/dinov2/layers/drop_path.py +34 -0
- third_party/MoGe/moge/model/dinov2/layers/layer_scale.py +27 -0
- third_party/MoGe/moge/model/dinov2/layers/mlp.py +40 -0
- third_party/MoGe/moge/model/dinov2/layers/patch_embed.py +88 -0
- third_party/MoGe/moge/model/dinov2/layers/swiglu_ffn.py +72 -0
- third_party/MoGe/moge/model/dinov2/models/__init__.py +43 -0
- third_party/MoGe/moge/model/dinov2/models/vision_transformer.py +407 -0
- third_party/MoGe/moge/model/dinov2/utils/__init__.py +4 -0
- third_party/MoGe/moge/model/dinov2/utils/cluster.py +95 -0
- third_party/MoGe/moge/model/dinov2/utils/config.py +72 -0
- third_party/MoGe/moge/model/dinov2/utils/dtype.py +37 -0
- third_party/MoGe/moge/model/dinov2/utils/param_groups.py +103 -0
- third_party/MoGe/moge/model/dinov2/utils/utils.py +95 -0
- third_party/MoGe/moge/model/modules.py +254 -0
- third_party/MoGe/moge/model/utils.py +49 -0
- third_party/MoGe/moge/model/v1.py +392 -0
- third_party/MoGe/moge/model/v2.py +303 -0
- third_party/MoGe/moge/utils/__init__.py +0 -0
- third_party/MoGe/moge/utils/alignment.py +416 -0
- third_party/MoGe/moge/utils/data_augmentation.py +250 -0
.gitignore
CHANGED
|
@@ -1 +1,6 @@
|
|
| 1 |
*.onnx
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
*.onnx
|
| 2 |
+
|
| 3 |
+
weights/
|
| 4 |
+
vfms/
|
| 5 |
+
__pycache__/
|
| 6 |
+
.gradio/
|
LICENSE_CorridorKey2.0
ADDED
|
@@ -0,0 +1,597 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CORRIDOR KEY LICENCE
|
| 2 |
+
=======================================================================
|
| 3 |
+
|
| 4 |
+
Version 2.0
|
| 5 |
+
|
| 6 |
+
Copyright (c) Corridor Digital LLC. All rights reserved.
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
ADDITIONAL TERMS AND CONDITIONS
|
| 10 |
+
=======================================================================
|
| 11 |
+
|
| 12 |
+
This work is licensed under the Creative Commons
|
| 13 |
+
Attribution-NonCommercial-ShareAlike 4.0 International Public License
|
| 14 |
+
(CC BY-NC-SA 4.0), the full text of which is included below, subject
|
| 15 |
+
to the following additional terms and conditions. These additional
|
| 16 |
+
terms supplement the CC BY-NC-SA 4.0 licence and take precedence
|
| 17 |
+
where they conflict.
|
| 18 |
+
|
| 19 |
+
"This tool" means the CorridorKey software in this repository and
|
| 20 |
+
the CorridorKey model weights distributed for use with it.
|
| 21 |
+
|
| 22 |
+
By exercising any rights to the Licensed Material, You accept and
|
| 23 |
+
agree to be bound by both these Additional Terms and the
|
| 24 |
+
CC BY-NC-SA 4.0 Public License.
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
1. PERMITTED USE
|
| 28 |
+
|
| 29 |
+
a. You may use this tool for any lawful purpose, including for
|
| 30 |
+
processing images as part of a commercial project, provided
|
| 31 |
+
that such use complies with the restrictions set out below
|
| 32 |
+
and the terms of the CC BY-NC-SA 4.0 licence.
|
| 33 |
+
|
| 34 |
+
b. The images and video You produce with this tool are Yours.
|
| 35 |
+
The Licensor claims no rights in them, and You may use them
|
| 36 |
+
commercially.
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
2. RESTRICTIONS
|
| 40 |
+
|
| 41 |
+
In addition to the restrictions set out in the CC BY-NC-SA 4.0
|
| 42 |
+
licence, the following restrictions apply:
|
| 43 |
+
|
| 44 |
+
a. NO REPACKAGING OR RESALE
|
| 45 |
+
You may not repackage, redistribute, sublicense, or sell
|
| 46 |
+
this tool, in whole or in part, as a standalone product or
|
| 47 |
+
as part of a competing product, and You may not charge for
|
| 48 |
+
access to this tool's code or model weights.
|
| 49 |
+
|
| 50 |
+
b. NO PAID API OR INFERENCE SERVICES
|
| 51 |
+
You may not use this tool to provide inference as a paid
|
| 52 |
+
API service, whether directly or indirectly. This includes,
|
| 53 |
+
but is not limited to, offering access to this tool behind
|
| 54 |
+
a paywall, subscription model, or usage-based billing
|
| 55 |
+
system.
|
| 56 |
+
|
| 57 |
+
c. DERIVATIVES MUST BE FREELY ACCESSIBLE
|
| 58 |
+
You may build on this tool, but any project, fork,
|
| 59 |
+
derivative work, or modified version that You release and
|
| 60 |
+
that incorporates this tool must be made available to the
|
| 61 |
+
public free of charge. Voluntary donations and
|
| 62 |
+
pay-what-you-want pricing are permitted, provided that
|
| 63 |
+
paying nothing is always an available option.
|
| 64 |
+
|
| 65 |
+
d. COMMERCIAL SOFTWARE INTEGRATION
|
| 66 |
+
If you operate a commercial software package or inference
|
| 67 |
+
service and wish to incorporate this tool into your
|
| 68 |
+
software, you must obtain a separate written agreement
|
| 69 |
+
from the Licensor. Please contact:
|
| 70 |
+
|
| 71 |
+
contact@corridordigital.com
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
3. LARGE ORGANISATIONS
|
| 75 |
+
|
| 76 |
+
a. If You are, or are using this tool on behalf of, a Legal
|
| 77 |
+
Entity whose total gross revenue exceeded US $20,000,000
|
| 78 |
+
(twenty million US dollars) in its most recently completed
|
| 79 |
+
fiscal year, the permissions in Section 1 do not apply
|
| 80 |
+
unless that Legal Entity has purchased a seat of the
|
| 81 |
+
CorridorKey plugin from the Licensor for each individual
|
| 82 |
+
who uses this tool.
|
| 83 |
+
|
| 84 |
+
b. "Legal Entity" means the entity using this tool together
|
| 85 |
+
with all other entities that control, are controlled by,
|
| 86 |
+
or are under common control with it. Revenue is measured
|
| 87 |
+
across the whole Legal Entity.
|
| 88 |
+
|
| 89 |
+
c. Each seat of the CorridorKey plugin includes a licence
|
| 90 |
+
for one individual to download and use this tool and its
|
| 91 |
+
model weights. Seats are available at:
|
| 92 |
+
|
| 93 |
+
https://corridorkey.com
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
4. ATTRIBUTION
|
| 97 |
+
|
| 98 |
+
a. In addition to the attribution requirements set out in
|
| 99 |
+
Section 3(a) of the CC BY-NC-SA 4.0 licence, You must
|
| 100 |
+
include the name "CorridorKey" in any attribution notice.
|
| 101 |
+
|
| 102 |
+
b. Any fork, derivative work, variation, improvement, or
|
| 103 |
+
subsequent release of this tool must retain the
|
| 104 |
+
"CorridorKey" name in a reasonably prominent position.
|
| 105 |
+
|
| 106 |
+
c. You may interchange "Corridor Key" for "CorridorKey" as
|
| 107 |
+
needed.
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
5. SHARE-ALIKE
|
| 111 |
+
|
| 112 |
+
Any variations, improvements, derivative works, or modified
|
| 113 |
+
versions of this tool that are publicly released must be
|
| 114 |
+
distributed under this same licence, including these Additional
|
| 115 |
+
Terms, or a licence with substantially equivalent terms.
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
6. THIRD-PARTY COMPONENTS
|
| 119 |
+
|
| 120 |
+
This tool depends on third-party code and model weights that
|
| 121 |
+
are not owned by the Licensor and are not covered by this
|
| 122 |
+
licence. Each remains under its own licence, as listed in
|
| 123 |
+
THIRD_PARTY_NOTICES.md.
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
7. DISCLAIMER
|
| 127 |
+
|
| 128 |
+
This licence is provided on an "as-is" basis. The Licensor makes
|
| 129 |
+
no representations or warranties regarding the enforceability or
|
| 130 |
+
legal sufficiency of these terms. You are encouraged to seek
|
| 131 |
+
independent legal advice if you have questions about the scope
|
| 132 |
+
or applicability of this licence.
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
8. CONTRIBUTIONS
|
| 136 |
+
|
| 137 |
+
By submitting any contribution (including source code,
|
| 138 |
+
documentation, or images) to this repository, You agree that:
|
| 139 |
+
|
| 140 |
+
a. Your contribution is provided under the terms of this
|
| 141 |
+
Corridor Key Licence.
|
| 142 |
+
|
| 143 |
+
b. You grant the Licensor (Corridor Digital LLC) a perpetual,
|
| 144 |
+
worldwide, non-exclusive, no-charge, royalty-free,
|
| 145 |
+
irrevocable license to use, reproduce, prepare derivative
|
| 146 |
+
works of, publicly display, and sublicense your
|
| 147 |
+
contribution, including for commercial purposes.
|
| 148 |
+
|
| 149 |
+
c. You represent that you are the legal owner of the
|
| 150 |
+
contribution or have the authority to submit it under
|
| 151 |
+
these terms.
|
| 152 |
+
|
| 153 |
+
=======================================================================
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
https://creativecommons.org/licenses/by-nc-sa/4.0/
|
| 157 |
+
https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode.txt
|
| 158 |
+
|
| 159 |
+
=======================================================================
|
| 160 |
+
|
| 161 |
+
Attribution-NonCommercial-ShareAlike 4.0 International
|
| 162 |
+
|
| 163 |
+
=======================================================================
|
| 164 |
+
|
| 165 |
+
Creative Commons Corporation ("Creative Commons") is not a law firm and
|
| 166 |
+
does not provide legal services or legal advice. Distribution of
|
| 167 |
+
Creative Commons public licenses does not create a lawyer-client or
|
| 168 |
+
other relationship. Creative Commons makes its licenses and related
|
| 169 |
+
information available on an "as-is" basis. Creative Commons gives no
|
| 170 |
+
warranties regarding its licenses, any material licensed under their
|
| 171 |
+
terms and conditions, or any related information. Creative Commons
|
| 172 |
+
disclaims all liability for damages resulting from their use to the
|
| 173 |
+
fullest extent possible.
|
| 174 |
+
|
| 175 |
+
Using Creative Commons Public Licenses
|
| 176 |
+
|
| 177 |
+
Creative Commons public licenses provide a standard set of terms and
|
| 178 |
+
conditions that creators and other rights holders may use to share
|
| 179 |
+
original works of authorship and other material subject to copyright
|
| 180 |
+
and certain other rights specified in the public license below. The
|
| 181 |
+
following considerations are for informational purposes only, are not
|
| 182 |
+
exhaustive, and do not form part of our licenses.
|
| 183 |
+
|
| 184 |
+
Considerations for licensors: Our public licenses are
|
| 185 |
+
intended for use by those authorized to give the public
|
| 186 |
+
permission to use material in ways otherwise restricted by
|
| 187 |
+
copyright and certain other rights. Our licenses are
|
| 188 |
+
irrevocable. Licensors should read and understand the terms
|
| 189 |
+
and conditions of the license they choose before applying it.
|
| 190 |
+
Licensors should also secure all rights necessary before
|
| 191 |
+
applying our licenses so that the public can reuse the
|
| 192 |
+
material as expected. Licensors should clearly mark any
|
| 193 |
+
material not subject to the license. This includes other CC-
|
| 194 |
+
licensed material, or material used under an exception or
|
| 195 |
+
limitation to copyright. More considerations for licensors:
|
| 196 |
+
wiki.creativecommons.org/Considerations_for_licensors
|
| 197 |
+
|
| 198 |
+
Considerations for the public: By using one of our public
|
| 199 |
+
licenses, a licensor grants the public permission to use the
|
| 200 |
+
licensed material under specified terms and conditions. If
|
| 201 |
+
the licensor's permission is not necessary for any reason--for
|
| 202 |
+
example, because of any applicable exception or limitation to
|
| 203 |
+
copyright--then that use is not regulated by the license. Our
|
| 204 |
+
licenses grant only permissions under copyright and certain
|
| 205 |
+
other rights that a licensor has authority to grant. Use of
|
| 206 |
+
the licensed material may still be restricted for other
|
| 207 |
+
reasons, including because others have copyright or other
|
| 208 |
+
rights in the material. A licensor may make special requests,
|
| 209 |
+
such as asking that all changes be marked or described.
|
| 210 |
+
Although not required by our licenses, you are encouraged to
|
| 211 |
+
respect those requests where reasonable. More considerations
|
| 212 |
+
for the public:
|
| 213 |
+
wiki.creativecommons.org/Considerations_for_licensees
|
| 214 |
+
|
| 215 |
+
=======================================================================
|
| 216 |
+
|
| 217 |
+
Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
|
| 218 |
+
Public License
|
| 219 |
+
|
| 220 |
+
By exercising the Licensed Rights (defined below), You accept and agree
|
| 221 |
+
to be bound by the terms and conditions of this Creative Commons
|
| 222 |
+
Attribution-NonCommercial-ShareAlike 4.0 International Public License
|
| 223 |
+
("Public License"). To the extent this Public License may be
|
| 224 |
+
interpreted as a contract, You are granted the Licensed Rights in
|
| 225 |
+
consideration of Your acceptance of these terms and conditions, and the
|
| 226 |
+
Licensor grants You such rights in consideration of benefits the
|
| 227 |
+
Licensor receives from making the Licensed Material available under
|
| 228 |
+
these terms and conditions.
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
Section 1 -- Definitions.
|
| 232 |
+
|
| 233 |
+
a. Adapted Material means material subject to Copyright and Similar
|
| 234 |
+
Rights that is derived from or based upon the Licensed Material
|
| 235 |
+
and in which the Licensed Material is translated, altered,
|
| 236 |
+
arranged, transformed, or otherwise modified in a manner requiring
|
| 237 |
+
permission under the Copyright and Similar Rights held by the
|
| 238 |
+
Licensor. For purposes of this Public License, where the Licensed
|
| 239 |
+
Material is a musical work, performance, or sound recording,
|
| 240 |
+
Adapted Material is always produced where the Licensed Material is
|
| 241 |
+
synched in timed relation with a moving image.
|
| 242 |
+
|
| 243 |
+
b. Adapter's License means the license You apply to Your Copyright
|
| 244 |
+
and Similar Rights in Your contributions to Adapted Material in
|
| 245 |
+
accordance with the terms and conditions of this Public License.
|
| 246 |
+
|
| 247 |
+
c. BY-NC-SA Compatible License means a license listed at
|
| 248 |
+
creativecommons.org/compatiblelicenses, approved by Creative
|
| 249 |
+
Commons as essentially the equivalent of this Public License.
|
| 250 |
+
|
| 251 |
+
d. Copyright and Similar Rights means copyright and/or similar rights
|
| 252 |
+
closely related to copyright including, without limitation,
|
| 253 |
+
performance, broadcast, sound recording, and Sui Generis Database
|
| 254 |
+
Rights, without regard to how the rights are labeled or
|
| 255 |
+
categorized. For purposes of this Public License, the rights
|
| 256 |
+
specified in Section 2(b)(1)-(2) are not Copyright and Similar
|
| 257 |
+
Rights.
|
| 258 |
+
|
| 259 |
+
e. Effective Technological Measures means those measures that, in the
|
| 260 |
+
absence of proper authority, may not be circumvented under laws
|
| 261 |
+
fulfilling obligations under Article 11 of the WIPO Copyright
|
| 262 |
+
Treaty adopted on December 20, 1996, and/or similar international
|
| 263 |
+
agreements.
|
| 264 |
+
|
| 265 |
+
f. Exceptions and Limitations means fair use, fair dealing, and/or
|
| 266 |
+
any other exception or limitation to Copyright and Similar Rights
|
| 267 |
+
that applies to Your use of the Licensed Material.
|
| 268 |
+
|
| 269 |
+
g. License Elements means the license attributes listed in the name
|
| 270 |
+
of a Creative Commons Public License. The License Elements of this
|
| 271 |
+
Public License are Attribution, NonCommercial, and ShareAlike.
|
| 272 |
+
|
| 273 |
+
h. Licensed Material means the artistic or literary work, database,
|
| 274 |
+
or other material to which the Licensor applied this Public
|
| 275 |
+
License.
|
| 276 |
+
|
| 277 |
+
i. Licensed Rights means the rights granted to You subject to the
|
| 278 |
+
terms and conditions of this Public License, which are limited to
|
| 279 |
+
all Copyright and Similar Rights that apply to Your use of the
|
| 280 |
+
Licensed Material and that the Licensor has authority to license.
|
| 281 |
+
|
| 282 |
+
j. Licensor means the individual(s) or entity(ies) granting rights
|
| 283 |
+
under this Public License.
|
| 284 |
+
|
| 285 |
+
k. NonCommercial means not primarily intended for or directed towards
|
| 286 |
+
commercial advantage or monetary compensation. For purposes of
|
| 287 |
+
this Public License, the exchange of the Licensed Material for
|
| 288 |
+
other material subject to Copyright and Similar Rights by digital
|
| 289 |
+
file-sharing or similar means is NonCommercial provided there is
|
| 290 |
+
no payment of monetary compensation in connection with the
|
| 291 |
+
exchange.
|
| 292 |
+
|
| 293 |
+
l. Share means to provide material to the public by any means or
|
| 294 |
+
process that requires permission under the Licensed Rights, such
|
| 295 |
+
as reproduction, public display, public performance, distribution,
|
| 296 |
+
dissemination, communication, or importation, and to make material
|
| 297 |
+
available to the public including in ways that members of the
|
| 298 |
+
public may access the material from a place and at a time
|
| 299 |
+
individually chosen by them.
|
| 300 |
+
|
| 301 |
+
m. Sui Generis Database Rights means rights other than copyright
|
| 302 |
+
resulting from Directive 96/9/EC of the European Parliament and of
|
| 303 |
+
the Council of 11 March 1996 on the legal protection of databases,
|
| 304 |
+
as amended and/or succeeded, as well as other essentially
|
| 305 |
+
equivalent rights anywhere in the world.
|
| 306 |
+
|
| 307 |
+
n. You means the individual or entity exercising the Licensed Rights
|
| 308 |
+
under this Public License. Your has a corresponding meaning.
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
Section 2 -- Scope.
|
| 312 |
+
|
| 313 |
+
a. License grant.
|
| 314 |
+
|
| 315 |
+
1. Subject to the terms and conditions of this Public License,
|
| 316 |
+
the Licensor hereby grants You a worldwide, royalty-free,
|
| 317 |
+
non-sublicensable, non-exclusive, irrevocable license to
|
| 318 |
+
exercise the Licensed Rights in the Licensed Material to:
|
| 319 |
+
|
| 320 |
+
a. reproduce and Share the Licensed Material, in whole or
|
| 321 |
+
in part, for NonCommercial purposes only; and
|
| 322 |
+
|
| 323 |
+
b. produce, reproduce, and Share Adapted Material for
|
| 324 |
+
NonCommercial purposes only.
|
| 325 |
+
|
| 326 |
+
2. Exceptions and Limitations. For the avoidance of doubt, where
|
| 327 |
+
Exceptions and Limitations apply to Your use, this Public
|
| 328 |
+
License does not apply, and You do not need to comply with
|
| 329 |
+
its terms and conditions.
|
| 330 |
+
|
| 331 |
+
3. Term. The term of this Public License is specified in Section
|
| 332 |
+
6(a).
|
| 333 |
+
|
| 334 |
+
4. Media and formats; technical modifications allowed. The
|
| 335 |
+
Licensor authorizes You to exercise the Licensed Rights in
|
| 336 |
+
all media and formats whether now known or hereafter created,
|
| 337 |
+
and to make technical modifications necessary to do so. The
|
| 338 |
+
Licensor waives and/or agrees not to assert any right or
|
| 339 |
+
authority to forbid You from making technical modifications
|
| 340 |
+
necessary to exercise the Licensed Rights, including
|
| 341 |
+
technical modifications necessary to circumvent Effective
|
| 342 |
+
Technological Measures. For purposes of this Public License,
|
| 343 |
+
simply making modifications authorized by this Section 2(a)
|
| 344 |
+
(4) never produces Adapted Material.
|
| 345 |
+
|
| 346 |
+
5. Downstream recipients.
|
| 347 |
+
|
| 348 |
+
a. Offer from the Licensor -- Licensed Material. Every
|
| 349 |
+
recipient of the Licensed Material automatically
|
| 350 |
+
receives an offer from the Licensor to exercise the
|
| 351 |
+
Licensed Rights under the terms and conditions of this
|
| 352 |
+
Public License.
|
| 353 |
+
|
| 354 |
+
b. Additional offer from the Licensor -- Adapted Material.
|
| 355 |
+
Every recipient of Adapted Material from You
|
| 356 |
+
automatically receives an offer from the Licensor to
|
| 357 |
+
exercise the Licensed Rights in the Adapted Material
|
| 358 |
+
under the conditions of the Adapter's License You apply.
|
| 359 |
+
|
| 360 |
+
c. No downstream restrictions. You may not offer or impose
|
| 361 |
+
any additional or different terms or conditions on, or
|
| 362 |
+
apply any Effective Technological Measures to, the
|
| 363 |
+
Licensed Material if doing so restricts exercise of the
|
| 364 |
+
Licensed Rights by any recipient of the Licensed
|
| 365 |
+
Material.
|
| 366 |
+
|
| 367 |
+
6. No endorsement. Nothing in this Public License constitutes or
|
| 368 |
+
may be construed as permission to assert or imply that You
|
| 369 |
+
are, or that Your use of the Licensed Material is, connected
|
| 370 |
+
with, or sponsored, endorsed, or granted official status by,
|
| 371 |
+
the Licensor or others designated to receive attribution as
|
| 372 |
+
provided in Section 3(a)(1)(A)(i).
|
| 373 |
+
|
| 374 |
+
b. Other rights.
|
| 375 |
+
|
| 376 |
+
1. Moral rights, such as the right of integrity, are not
|
| 377 |
+
licensed under this Public License, nor are publicity,
|
| 378 |
+
privacy, and/or other similar personality rights; however, to
|
| 379 |
+
the extent possible, the Licensor waives and/or agrees not to
|
| 380 |
+
assert any such rights held by the Licensor to the limited
|
| 381 |
+
extent necessary to allow You to exercise the Licensed
|
| 382 |
+
Rights, but not otherwise.
|
| 383 |
+
|
| 384 |
+
2. Patent and trademark rights are not licensed under this
|
| 385 |
+
Public License.
|
| 386 |
+
|
| 387 |
+
3. To the extent possible, the Licensor waives any right to
|
| 388 |
+
collect royalties from You for the exercise of the Licensed
|
| 389 |
+
Rights, whether directly or through a collecting society
|
| 390 |
+
under any voluntary or waivable statutory or compulsory
|
| 391 |
+
licensing scheme. In all other cases the Licensor expressly
|
| 392 |
+
reserves any right to collect such royalties, including when
|
| 393 |
+
the Licensed Material is used other than for NonCommercial
|
| 394 |
+
purposes.
|
| 395 |
+
|
| 396 |
+
|
| 397 |
+
Section 3 -- License Conditions.
|
| 398 |
+
|
| 399 |
+
Your exercise of the Licensed Rights is expressly made subject to the
|
| 400 |
+
following conditions.
|
| 401 |
+
|
| 402 |
+
a. Attribution.
|
| 403 |
+
|
| 404 |
+
1. If You Share the Licensed Material (including in modified
|
| 405 |
+
form), You must:
|
| 406 |
+
|
| 407 |
+
a. retain the following if it is supplied by the Licensor
|
| 408 |
+
with the Licensed Material:
|
| 409 |
+
|
| 410 |
+
i. identification of the creator(s) of the Licensed
|
| 411 |
+
Material and any others designated to receive
|
| 412 |
+
attribution, in any reasonable manner requested by
|
| 413 |
+
the Licensor (including by pseudonym if
|
| 414 |
+
designated);
|
| 415 |
+
|
| 416 |
+
ii. a copyright notice;
|
| 417 |
+
|
| 418 |
+
iii. a notice that refers to this Public License;
|
| 419 |
+
|
| 420 |
+
iv. a notice that refers to the disclaimer of
|
| 421 |
+
warranties;
|
| 422 |
+
|
| 423 |
+
v. a URI or hyperlink to the Licensed Material to the
|
| 424 |
+
extent reasonably practicable;
|
| 425 |
+
|
| 426 |
+
b. indicate if You modified the Licensed Material and
|
| 427 |
+
retain an indication of any previous modifications; and
|
| 428 |
+
|
| 429 |
+
c. indicate the Licensed Material is licensed under this
|
| 430 |
+
Public License, and include the text of, or the URI or
|
| 431 |
+
hyperlink to, this Public License.
|
| 432 |
+
|
| 433 |
+
2. You may satisfy the conditions in Section 3(a)(1) in any
|
| 434 |
+
reasonable manner based on the medium, means, and context in
|
| 435 |
+
which You Share the Licensed Material. For example, it may be
|
| 436 |
+
reasonable to satisfy the conditions by providing a URI or
|
| 437 |
+
hyperlink to a resource that includes the required
|
| 438 |
+
information.
|
| 439 |
+
3. If requested by the Licensor, You must remove any of the
|
| 440 |
+
information required by Section 3(a)(1)(A) to the extent
|
| 441 |
+
reasonably practicable.
|
| 442 |
+
|
| 443 |
+
b. ShareAlike.
|
| 444 |
+
|
| 445 |
+
In addition to the conditions in Section 3(a), if You Share
|
| 446 |
+
Adapted Material You produce, the following conditions also apply.
|
| 447 |
+
|
| 448 |
+
1. The Adapter's License You apply must be a Creative Commons
|
| 449 |
+
license with the same License Elements, this version or
|
| 450 |
+
later, or a BY-NC-SA Compatible License.
|
| 451 |
+
|
| 452 |
+
2. You must include the text of, or the URI or hyperlink to, the
|
| 453 |
+
Adapter's License You apply. You may satisfy this condition
|
| 454 |
+
in any reasonable manner based on the medium, means, and
|
| 455 |
+
context in which You Share Adapted Material.
|
| 456 |
+
|
| 457 |
+
3. You may not offer or impose any additional or different terms
|
| 458 |
+
or conditions on, or apply any Effective Technological
|
| 459 |
+
Measures to, Adapted Material that restrict exercise of the
|
| 460 |
+
rights granted under the Adapter's License You apply.
|
| 461 |
+
|
| 462 |
+
|
| 463 |
+
Section 4 -- Sui Generis Database Rights.
|
| 464 |
+
|
| 465 |
+
Where the Licensed Rights include Sui Generis Database Rights that
|
| 466 |
+
apply to Your use of the Licensed Material:
|
| 467 |
+
|
| 468 |
+
a. for the avoidance of doubt, Section 2(a)(1) grants You the right
|
| 469 |
+
to extract, reuse, reproduce, and Share all or a substantial
|
| 470 |
+
portion of the contents of the database for NonCommercial purposes
|
| 471 |
+
only;
|
| 472 |
+
|
| 473 |
+
b. if You include all or a substantial portion of the database
|
| 474 |
+
contents in a database in which You have Sui Generis Database
|
| 475 |
+
Rights, then the database in which You have Sui Generis Database
|
| 476 |
+
Rights (but not its individual contents) is Adapted Material,
|
| 477 |
+
including for purposes of Section 3(b); and
|
| 478 |
+
|
| 479 |
+
c. You must comply with the conditions in Section 3(a) if You Share
|
| 480 |
+
all or a substantial portion of the contents of the database.
|
| 481 |
+
|
| 482 |
+
For the avoidance of doubt, this Section 4 supplements and does not
|
| 483 |
+
replace Your obligations under this Public License where the Licensed
|
| 484 |
+
Rights include other Copyright and Similar Rights.
|
| 485 |
+
|
| 486 |
+
|
| 487 |
+
Section 5 -- Disclaimer of Warranties and Limitation of Liability.
|
| 488 |
+
|
| 489 |
+
a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
|
| 490 |
+
EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
|
| 491 |
+
AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
|
| 492 |
+
ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
|
| 493 |
+
IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
|
| 494 |
+
WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
|
| 495 |
+
PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
|
| 496 |
+
ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
|
| 497 |
+
KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
|
| 498 |
+
ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
|
| 499 |
+
|
| 500 |
+
b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
|
| 501 |
+
TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
|
| 502 |
+
NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
|
| 503 |
+
INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
|
| 504 |
+
COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
|
| 505 |
+
USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
|
| 506 |
+
ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
|
| 507 |
+
DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
|
| 508 |
+
IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
|
| 509 |
+
|
| 510 |
+
c. The disclaimer of warranties and limitation of liability provided
|
| 511 |
+
above shall be interpreted in a manner that, to the extent
|
| 512 |
+
possible, most closely approximates an absolute disclaimer and
|
| 513 |
+
waiver of all liability.
|
| 514 |
+
|
| 515 |
+
|
| 516 |
+
Section 6 -- Term and Termination.
|
| 517 |
+
|
| 518 |
+
a. This Public License applies for the term of the Copyright and
|
| 519 |
+
Similar Rights licensed here. However, if You fail to comply with
|
| 520 |
+
this Public License, then Your rights under this Public License
|
| 521 |
+
terminate automatically.
|
| 522 |
+
|
| 523 |
+
b. Where Your right to use the Licensed Material has terminated under
|
| 524 |
+
Section 6(a), it reinstates:
|
| 525 |
+
|
| 526 |
+
1. automatically as of the date the violation is cured, provided
|
| 527 |
+
it is cured within 30 days of Your discovery of the
|
| 528 |
+
violation; or
|
| 529 |
+
|
| 530 |
+
2. upon express reinstatement by the Licensor.
|
| 531 |
+
|
| 532 |
+
For the avoidance of doubt, this Section 6(b) does not affect any
|
| 533 |
+
right the Licensor may have to seek remedies for Your violations
|
| 534 |
+
of this Public License.
|
| 535 |
+
|
| 536 |
+
c. For the avoidance of doubt, the Licensor may also offer the
|
| 537 |
+
Licensed Material under separate terms or conditions or stop
|
| 538 |
+
distributing the Licensed Material at any time; however, doing so
|
| 539 |
+
will not terminate this Public License.
|
| 540 |
+
|
| 541 |
+
d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
|
| 542 |
+
License.
|
| 543 |
+
|
| 544 |
+
|
| 545 |
+
Section 7 -- Other Terms and Conditions.
|
| 546 |
+
|
| 547 |
+
a. The Licensor shall not be bound by any additional or different
|
| 548 |
+
terms or conditions communicated by You unless expressly agreed.
|
| 549 |
+
|
| 550 |
+
b. Any arrangements, understandings, or agreements regarding the
|
| 551 |
+
Licensed Material not stated herein are separate from and
|
| 552 |
+
independent of the terms and conditions of this Public License.
|
| 553 |
+
|
| 554 |
+
|
| 555 |
+
Section 8 -- Interpretation.
|
| 556 |
+
|
| 557 |
+
a. For the avoidance of doubt, this Public License does not, and
|
| 558 |
+
shall not be interpreted to, reduce, limit, restrict, or impose
|
| 559 |
+
conditions on any use of the Licensed Material that could lawfully
|
| 560 |
+
be made without permission under this Public License.
|
| 561 |
+
|
| 562 |
+
b. To the extent possible, if any provision of this Public License is
|
| 563 |
+
deemed unenforceable, it shall be automatically reformed to the
|
| 564 |
+
minimum extent necessary to make it enforceable. If the provision
|
| 565 |
+
cannot be reformed, it shall be severed from this Public License
|
| 566 |
+
without affecting the enforceability of the remaining terms and
|
| 567 |
+
conditions.
|
| 568 |
+
|
| 569 |
+
c. No term or condition of this Public License will be waived and no
|
| 570 |
+
failure to comply consented to unless expressly agreed to by the
|
| 571 |
+
Licensor.
|
| 572 |
+
|
| 573 |
+
d. Nothing in this Public License constitutes or may be interpreted
|
| 574 |
+
as a limitation upon, or waiver of, any privileges and immunities
|
| 575 |
+
that apply to the Licensor or You, including from the legal
|
| 576 |
+
processes of any jurisdiction or authority.
|
| 577 |
+
|
| 578 |
+
=======================================================================
|
| 579 |
+
|
| 580 |
+
Creative Commons is not a party to its public
|
| 581 |
+
licenses. Notwithstanding, Creative Commons may elect to apply one of
|
| 582 |
+
its public licenses to material it publishes and in those instances
|
| 583 |
+
will be considered the “Licensor.” The text of the Creative Commons
|
| 584 |
+
public licenses is dedicated to the public domain under the CC0 Public
|
| 585 |
+
Domain Dedication. Except for the limited purpose of indicating that
|
| 586 |
+
material is shared under a Creative Commons public license or as
|
| 587 |
+
otherwise permitted by the Creative Commons policies published at
|
| 588 |
+
creativecommons.org/policies, Creative Commons does not authorize the
|
| 589 |
+
use of the trademark "Creative Commons" or any other trademark or logo
|
| 590 |
+
of Creative Commons without its prior written consent including,
|
| 591 |
+
without limitation, in connection with any unauthorized modifications
|
| 592 |
+
to any of its public licenses or any other arrangements,
|
| 593 |
+
understandings, or agreements concerning use of licensed material. For
|
| 594 |
+
the avoidance of doubt, this paragraph does not form part of the
|
| 595 |
+
public licenses.
|
| 596 |
+
|
| 597 |
+
Creative Commons may be contacted at creativecommons.org.
|
README.md
CHANGED
|
@@ -8,6 +8,10 @@ sdk_version: 6.9.0
|
|
| 8 |
app_file: app.py
|
| 9 |
python_version: "3.10"
|
| 10 |
pinned: false
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
tags:
|
| 12 |
- green-screen
|
| 13 |
- background-removal
|
|
@@ -27,32 +31,86 @@ short_description: Remove green/blue screen from video, even glass & hair
|
|
| 27 |
|
| 28 |
Remove green or blue screen backgrounds from video. Handles transparent objects (glass, water, cloth) that traditional chroma key cannot.
|
| 29 |
|
| 30 |
-
|
| 31 |
|
| 32 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
|
| 34 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
- **CPU (fallback)**: ONNX Runtime sequential inference (batch 1)
|
| 36 |
|
| 37 |
-
##
|
| 38 |
|
| 39 |
1. **BiRefNet** - Generates coarse foreground mask (ONNX, or fast classical HSV for green/blue screens)
|
| 40 |
2. **CorridorKey GreenFormer** - Refines alpha matte + extracts clean foreground (PyTorch on GPU, ONNX on CPU)
|
| 41 |
-
3. **GPU Postprocessing** - Despill, despeckle (connected components), resize
|
| 42 |
|
| 43 |
-
## GPU
|
| 44 |
|
| 45 |
-
- **Full GPU pipeline**: preprocessing (resize + normalize) and postprocessing (despill, clean_matte, resize) stay on device
|
| 46 |
- **TF32 tensor cores**: `torch.set_float32_matmul_precision('high')` for FP32 postprocessing ops
|
| 47 |
-
- **AOTI compilation with torch.inductor + triton cudagraphs**
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
|
| 49 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
|
| 51 |
## API
|
| 52 |
|
|
|
|
|
|
|
| 53 |
### REST API
|
| 54 |
|
| 55 |
-
**Step 1: Submit request**
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
```bash
|
| 57 |
curl -X POST "https://luminia-corridorkey.hf.space/gradio_api/call/process_video" \
|
| 58 |
-H "Content-Type: application/json" \
|
|
@@ -61,7 +119,7 @@ curl -X POST "https://luminia-corridorkey.hf.space/gradio_api/call/process_video
|
|
| 61 |
|
| 62 |
**Step 2: Get result**
|
| 63 |
```bash
|
| 64 |
-
curl "https://luminia-corridorkey.hf.space/gradio_api/call/
|
| 65 |
```
|
| 66 |
|
| 67 |
### MCP (Model Context Protocol)
|
|
@@ -77,8 +135,14 @@ curl "https://luminia-corridorkey.hf.space/gradio_api/call/process_video/{event_
|
|
| 77 |
}
|
| 78 |
```
|
| 79 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
## Credits
|
| 81 |
|
| 82 |
-
- [CorridorKey](https://github.com/nikopueringer/
|
|
|
|
| 83 |
- [EZ-CorridorKey](https://github.com/edenaion/EZ-CorridorKey) UI reference by edenaion (synced to `888e032`, 2026-04-25)
|
| 84 |
- [BiRefNet](https://github.com/ZhengPeng7/BiRefNet) by ZhengPeng7
|
|
|
|
|
|
| 8 |
app_file: app.py
|
| 9 |
python_version: "3.10"
|
| 10 |
pinned: false
|
| 11 |
+
startup_duration_timeout: 1h
|
| 12 |
+
license: other
|
| 13 |
+
license_name: corridor-key-licence-2.0
|
| 14 |
+
license_link: LICENSE_CorridorKey2.0
|
| 15 |
tags:
|
| 16 |
- green-screen
|
| 17 |
- background-removal
|
|
|
|
| 31 |
|
| 32 |
Remove green or blue screen backgrounds from video. Handles transparent objects (glass, water, cloth) that traditional chroma key cannot.
|
| 33 |
|
| 34 |
+
Pick the model version in the UI. **v2 is the default.**
|
| 35 |
|
| 36 |
+
| | v2 (default) | v1 |
|
| 37 |
+
|---|---|---|
|
| 38 |
+
| Model | [CorridorKey 2.0](https://github.com/nikopueringer/CorridorKey2.0) | [CorridorKey v1.0](https://github.com/nikopueringer/CorridorKey) |
|
| 39 |
+
| Runs on | GPU only (ZeroGPU) | GPU, or CPU through ONNX |
|
| 40 |
+
| Frames | Temporal: sees a window of frames together | One frame at a time |
|
| 41 |
+
| Mask | None needed, the model finds the subject | BiRefNet / classical pre-mask |
|
| 42 |
+
| Input size | Clip is downscaled on CPU to 540p, 720p or 1080p first | Model runs at a fixed 1024 or 2048 |
|
| 43 |
|
| 44 |
+
## v2
|
| 45 |
+
|
| 46 |
+
CorridorKey 2.0 is a two-stage transformer. Stage 1 sees frames together (full spatio-temporal attention at 1/64 resolution, Swin-3D at 1/16) using frozen features from three foundation models:
|
| 47 |
+
C-RADIO (semantics), MoGe-2 (geometry) and RVM (motion). Stage 2 turns that context plus the raw RGB into a full-resolution alpha and foreground, one frame at a time.
|
| 48 |
+
|
| 49 |
+
How this Space runs it:
|
| 50 |
+
|
| 51 |
+
1. The clip is downscaled on CPU with ffmpeg to the chosen quality.
|
| 52 |
+
2. One ZeroGPU call runs the unmodified v2 runtime.
|
| 53 |
+
3. Previews are encoded on CPU. The zip holds the alpha, foreground and composite videos at the working resolution.
|
| 54 |
+
|
| 55 |
+
Measured end to end on an RTX 5090 (GPU phase, warm, 89 frames): 540p 19s, 720p 28s, 1080p 48-54s, and 720p with 267 frames 68s. The first call after the model is loaded takes longer.
|
| 56 |
+
The GPU time requested from ZeroGPU is estimated from those numbers, so a clip that does not fit gets an error naming the limit instead of being cut off.
|
| 57 |
+
The status line reports the GPU that actually ran.
|
| 58 |
+
|
| 59 |
+
**Temporal context**: Auto picks a window of 80 frames up to 720p and 24 frames at 1080p. An 80-frame window costs about the same as 8 frames up to 720p, but at 1080p it is about 2.5x slower, which is why Auto narrows it there. Peak VRAM depends on resolution and window size, not clip length (about 6 GB at 720p with 8 frames, 13 GB with 80, 25 GB at 1080p with 80).
|
| 60 |
+
|
| 61 |
+
Native 4K is not offered: the example clip is 4K and takes about 160s on an RTX 5090, over the 120s limit.
|
| 62 |
+
|
| 63 |
+
## v1 inference paths
|
| 64 |
+
|
| 65 |
+
- **GPU (ZeroGPU)**: PyTorch GreenFormer with batched inference (batch 32 at 1024, batch 16 at 2048)
|
| 66 |
- **CPU (fallback)**: ONNX Runtime sequential inference (batch 1)
|
| 67 |
|
| 68 |
+
### v1 pipeline
|
| 69 |
|
| 70 |
1. **BiRefNet** - Generates coarse foreground mask (ONNX, or fast classical HSV for green/blue screens)
|
| 71 |
2. **CorridorKey GreenFormer** - Refines alpha matte + extracts clean foreground (PyTorch on GPU, ONNX on CPU)
|
| 72 |
+
3. **GPU Postprocessing** - Despill, despeckle (connected components), resize, all on GPU via torchvision, single CPU transfer at the end
|
| 73 |
|
| 74 |
+
### v1 GPU optimizations
|
| 75 |
|
| 76 |
+
- **Full GPU pipeline**: preprocessing (resize + normalize) and postprocessing (despill, clean_matte, resize) stay on device, avoiding CPU↔GPU round-trips per batch
|
| 77 |
- **TF32 tensor cores**: `torch.set_float32_matmul_precision('high')` for FP32 postprocessing ops
|
| 78 |
+
- **AOTI compilation with torch.inductor + triton cudagraphs** do not benefit GreenFormer: tested max-autotune (118s, 0 triton kernels) and reduce-overhead (36s compile + 48s graph recording = 84s for 5% speedup). Small feature maps (112-896ch) are cublas-optimal, not triton-friendly. Disabled on ZeroGPU: eager at 0.32s/frame beats 84s+ overhead. torch.compile still available for a local GPU
|
| 79 |
+
|
| 80 |
+
**v1 pipeline timing** (89 frames, batch 32 @ 1024px model res): CPU mask 22s → GPU load 5s → inference 29s → write 15s → stitch 9s ≈ 80s total, 49s GPU. The model always processes at 1024x1024 or 2048x2048 regardless of input resolution
|
| 81 |
+
|
| 82 |
+
## CLI
|
| 83 |
+
|
| 84 |
+
v2 is the default. v1 needs `--version v1`, and v1-only options are rejected without it.
|
| 85 |
|
| 86 |
+
```bash
|
| 87 |
+
python app.py --input clip.mp4 # v2
|
| 88 |
+
python app.py --input clip.mp4 --quality 1080p --temporal 24 # v2 options
|
| 89 |
+
python app.py --input clip.mp4 --version v1 --resolution 2048 # v1
|
| 90 |
+
python app.py --input clip.mp4 --version v1 --device cpu # v1 on CPU
|
| 91 |
+
```
|
| 92 |
+
|
| 93 |
+
v2 options: `--quality {540p,720p,1080p}`, `--temporal {auto,1,8,24}`, `--upscale`.
|
| 94 |
+
v1 options: `--device`, `--resolution`, `--mask-mode`, `--no-despeckle`, `--despeckle-size`, `--screen-color`.
|
| 95 |
+
Shared: `--despill`, `--output`.
|
| 96 |
+
|
| 97 |
+
A `--version v1` run does not download the v2 weights.
|
| 98 |
|
| 99 |
## API
|
| 100 |
|
| 101 |
+
The v1 endpoint is unchanged. v2 is a separate endpoint.
|
| 102 |
+
|
| 103 |
### REST API
|
| 104 |
|
| 105 |
+
**Step 1: Submit request (v2)**
|
| 106 |
+
```bash
|
| 107 |
+
curl -X POST "https://luminia-corridorkey.hf.space/gradio_api/call/process_video_v2" \
|
| 108 |
+
-H "Content-Type: application/json" \
|
| 109 |
+
-d '{"data": [{"path": "https://example.com/clip.mp4", "meta": {"_type": "gradio.FileData"}}, "720p", "auto", 5, false]}'
|
| 110 |
+
```
|
| 111 |
+
Arguments: video, quality (`540p`/`720p`/`1080p`), temporal (`auto`/`1`/`8`/`24`), despill 0-10, upscale previews.
|
| 112 |
+
|
| 113 |
+
**v1 (unchanged)**
|
| 114 |
```bash
|
| 115 |
curl -X POST "https://luminia-corridorkey.hf.space/gradio_api/call/process_video" \
|
| 116 |
-H "Content-Type: application/json" \
|
|
|
|
| 119 |
|
| 120 |
**Step 2: Get result**
|
| 121 |
```bash
|
| 122 |
+
curl "https://luminia-corridorkey.hf.space/gradio_api/call/process_video_v2/{event_id}"
|
| 123 |
```
|
| 124 |
|
| 125 |
### MCP (Model Context Protocol)
|
|
|
|
| 135 |
}
|
| 136 |
```
|
| 137 |
|
| 138 |
+
## Licence
|
| 139 |
+
|
| 140 |
+
CorridorKey 2.0 is released under the Corridor Key Licence (a variation of CC BY-NC-SA 4.0 with additional terms), copied in [LICENSE_CorridorKey2.0](LICENSE_CorridorKey2.0). This Space is free to use, keeps the CorridorKey name, and is not a paid service. Organizations above US $20 million annual revenue need a seat of the official plugin ([corridorkey.com](https://corridorkey.com)). Third-party components are listed in [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md): C-RADIOv4 (NVIDIA Open Model License), MoGe-2 (MIT), RVM (CC-BY 4.0).
|
| 141 |
+
|
| 142 |
## Credits
|
| 143 |
|
| 144 |
+
- [CorridorKey 2.0](https://github.com/nikopueringer/CorridorKey2.0) by Corridor Digital, designed and trained by Zachary Harris with Niko Pueringer. The v2 runtime in `corridorkey/` and `third_party/MoGe/` is vendored unmodified from it.
|
| 145 |
+
- [CorridorKey](https://github.com/nikopueringer/CorridorKey) v1 by Niko Pueringer / Corridor Digital (synced to `8a4e4b4`, 2026-05-01)
|
| 146 |
- [EZ-CorridorKey](https://github.com/edenaion/EZ-CorridorKey) UI reference by edenaion (synced to `888e032`, 2026-04-25)
|
| 147 |
- [BiRefNet](https://github.com/ZhengPeng7/BiRefNet) by ZhengPeng7
|
| 148 |
+
- [C-RADIO](https://github.com/NVlabs/RADIO) (NVIDIA), [MoGe](https://github.com/microsoft/MoGe) (Microsoft), [RVM](https://github.com/google-deepmind/representations4d) (Google DeepMind)
|
THIRD_PARTY_NOTICES.md
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Third-Party Notices
|
| 2 |
+
|
| 3 |
+
CorridorKey v2 is licensed under the Corridor Key Licence (see `LICENSE`).
|
| 4 |
+
It depends on the third-party components below, which are not owned by
|
| 5 |
+
Corridor Digital LLC and remain under their own licenses.
|
| 6 |
+
|
| 7 |
+
The three foundation models are not redistributed in this repository or in the
|
| 8 |
+
CorridorKey Hugging Face repository. `scripts/download_weights.py` fetches them
|
| 9 |
+
from their original publishers.
|
| 10 |
+
|
| 11 |
+
## C-RADIOv4-SO400M
|
| 12 |
+
|
| 13 |
+
- Publisher: NVIDIA Corporation
|
| 14 |
+
- Source: https://huggingface.co/nvidia/C-RADIOv4-SO400M
|
| 15 |
+
- License: [NVIDIA Open Model License](https://developer.download.nvidia.com/licenses/nvidia-open-model-license-agreement-june-2024.pdf)
|
| 16 |
+
- Use: frozen visual feature extractor, downloaded to `vfms/cradio/`
|
| 17 |
+
|
| 18 |
+
Licensed by NVIDIA Corporation under the NVIDIA Open Model License.
|
| 19 |
+
|
| 20 |
+
## MoGe-2 ViT-B
|
| 21 |
+
|
| 22 |
+
- Publisher: Microsoft Corporation
|
| 23 |
+
- Source: https://github.com/microsoft/MoGe, weights at https://huggingface.co/Ruicheng/moge-2-vitb-normal
|
| 24 |
+
- License: MIT (code and weights)
|
| 25 |
+
- Use: frozen geometry feature extractor. The model code is vendored in
|
| 26 |
+
`third_party/MoGe/` with its license at `third_party/MoGe/LICENSE`; weights
|
| 27 |
+
are downloaded to `vfms/moge/`.
|
| 28 |
+
|
| 29 |
+
The vendored MoGe code includes DINOv2 (Meta Platforms, Inc.), licensed under
|
| 30 |
+
the Apache License 2.0, also reproduced in `third_party/MoGe/LICENSE`.
|
| 31 |
+
|
| 32 |
+
## RVM (Recurrent Video MAE), small
|
| 33 |
+
|
| 34 |
+
- Publisher: Google DeepMind
|
| 35 |
+
- Source: https://github.com/google-deepmind/representations4d
|
| 36 |
+
- License: Apache License 2.0 (code), Creative Commons Attribution 4.0 (weights)
|
| 37 |
+
- Use: frozen recurrent video feature extractor. `corridorkey/rvm.py` is a
|
| 38 |
+
PyTorch port of the reference architecture; weights are downloaded to
|
| 39 |
+
`vfms/rvm/`.
|
app.py
CHANGED
|
@@ -24,6 +24,9 @@ import logging
|
|
| 24 |
os.environ["OMP_NUM_THREADS"] = "2"
|
| 25 |
os.environ["OPENBLAS_NUM_THREADS"] = "2"
|
| 26 |
os.environ["MKL_NUM_THREADS"] = "2"
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
import numpy as np
|
| 29 |
import cv2
|
|
@@ -104,6 +107,26 @@ except Exception:
|
|
| 104 |
except Exception as e:
|
| 105 |
logger.warning("CorridorKey green preload failed: %s", e)
|
| 106 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
# Batch sizes for GPU inference (conservative for H200 80GB)
|
| 108 |
GPU_BATCH_SIZES = {"1024": 32, "2048": 16} # 2048 uses only 5.7GB/batch=2, so 16 easily fits in 69.8GB
|
| 109 |
|
|
@@ -1014,9 +1037,47 @@ def process_video(video_path, resolution, despill_val, mask_mode,
|
|
| 1014 |
def process_example(video_path, resolution, despill, mask_mode, despeckle, despeckle_size, screen_color="Auto"):
|
| 1015 |
return process_video(video_path, resolution, despill, mask_mode, despeckle, despeckle_size, screen_color)
|
| 1016 |
|
| 1017 |
-
|
| 1018 |
-
|
| 1019 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1020 |
|
| 1021 |
with gr.Blocks(title="CorridorKey") as demo:
|
| 1022 |
gr.Markdown(DESCRIPTION)
|
|
@@ -1024,53 +1085,104 @@ with gr.Blocks(title="CorridorKey") as demo:
|
|
| 1024 |
with gr.Row():
|
| 1025 |
with gr.Column(scale=1):
|
| 1026 |
input_video = gr.Video(label="Upload Green/Blue Screen Video")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1027 |
with gr.Accordion("Settings", open=True):
|
| 1028 |
-
with gr.Row():
|
| 1029 |
-
resolution = gr.Radio(
|
| 1030 |
-
choices=["1024", "2048"], value="1024",
|
| 1031 |
-
label="Resolution", info="1024 = fast, 2048 = quality"
|
| 1032 |
-
)
|
| 1033 |
-
screen_color = gr.Radio(
|
| 1034 |
-
choices=["Auto", "Green", "Blue"], value="Auto",
|
| 1035 |
-
label="Screen Color", info="Auto-detects. Blue requires GPU"
|
| 1036 |
-
)
|
| 1037 |
-
mask_mode = gr.Radio(
|
| 1038 |
-
choices=["Hybrid (auto)", "AI (BiRefNet)", "Fast (classical)"],
|
| 1039 |
-
value="Hybrid (auto)", label="Mask Mode",
|
| 1040 |
-
info="Hybrid = fast detection + AI fallback. Fast = classical only. AI = always BiRefNet"
|
| 1041 |
-
)
|
| 1042 |
despill_slider = gr.Slider(
|
| 1043 |
0, 10, value=5, step=1, label="Despill Strength",
|
| 1044 |
info="Remove screen reflections (0=off, 10=max)"
|
| 1045 |
)
|
| 1046 |
-
with gr.
|
| 1047 |
-
|
| 1048 |
-
|
| 1049 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1050 |
|
| 1051 |
with gr.Column(scale=1):
|
| 1052 |
with gr.Row():
|
| 1053 |
comp_video = gr.Video(label="Composite Preview")
|
| 1054 |
matte_video = gr.Video(label="Alpha Matte")
|
| 1055 |
-
download_zip = gr.File(label="Download Full Package
|
| 1056 |
status_text = gr.Textbox(label="Status", interactive=False)
|
| 1057 |
|
| 1058 |
-
|
| 1059 |
-
|
| 1060 |
-
|
| 1061 |
-
|
| 1062 |
-
|
| 1063 |
-
|
| 1064 |
-
|
| 1065 |
-
|
| 1066 |
-
|
| 1067 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1068 |
)
|
| 1069 |
|
| 1070 |
process_btn.click(
|
| 1071 |
fn=process_video,
|
| 1072 |
inputs=[input_video, resolution, despill_slider, mask_mode, despeckle_check, despeckle_size, screen_color],
|
| 1073 |
-
outputs=
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1074 |
)
|
| 1075 |
|
| 1076 |
|
|
@@ -1079,35 +1191,72 @@ with gr.Blocks(title="CorridorKey") as demo:
|
|
| 1079 |
# ---------------------------------------------------------------------------
|
| 1080 |
def cli_main():
|
| 1081 |
import argparse
|
| 1082 |
-
parser = argparse.ArgumentParser(
|
|
|
|
| 1083 |
parser.add_argument("--input", required=True)
|
| 1084 |
parser.add_argument("--output", default="output")
|
| 1085 |
-
parser.add_argument("--
|
| 1086 |
-
|
| 1087 |
-
parser.add_argument("--mask-mode", default="Hybrid (auto)",
|
| 1088 |
-
choices=["Hybrid (auto)", "AI (BiRefNet)", "Fast (classical)"])
|
| 1089 |
parser.add_argument("--despill", type=int, default=5)
|
| 1090 |
-
parser.
|
| 1091 |
-
|
| 1092 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1093 |
args = parser.parse_args()
|
| 1094 |
|
| 1095 |
-
|
| 1096 |
-
|
| 1097 |
-
|
| 1098 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1099 |
|
| 1100 |
class CLIProgress:
|
| 1101 |
def __call__(self, val, desc=""):
|
| 1102 |
if desc: print(f" [{val:.0%}] {desc}")
|
| 1103 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1104 |
comp, matte, zipf, status = process_video(
|
| 1105 |
-
args.input, args.resolution, args.despill, args.mask_mode,
|
| 1106 |
-
not args.no_despeckle, args.despeckle_size
|
| 1107 |
-
progress=CLIProgress()
|
| 1108 |
)
|
| 1109 |
print(f"\n{status}")
|
| 1110 |
-
os.makedirs(args.output, exist_ok=True)
|
| 1111 |
if zipf:
|
| 1112 |
dst = os.path.join(args.output, os.path.basename(zipf))
|
| 1113 |
shutil.copy2(zipf, dst)
|
|
@@ -1119,4 +1268,4 @@ if __name__ == "__main__":
|
|
| 1119 |
cli_main()
|
| 1120 |
else:
|
| 1121 |
demo.queue(default_concurrency_limit=1)
|
| 1122 |
-
demo.launch(ssr_mode=False, mcp_server=True)
|
|
|
|
| 24 |
os.environ["OMP_NUM_THREADS"] = "2"
|
| 25 |
os.environ["OPENBLAS_NUM_THREADS"] = "2"
|
| 26 |
os.environ["MKL_NUM_THREADS"] = "2"
|
| 27 |
+
# Bound HF downloads so a stalled connection fails and retries instead of hanging startup.
|
| 28 |
+
os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "60")
|
| 29 |
+
os.environ.setdefault("HF_HUB_ETAG_TIMEOUT", "30")
|
| 30 |
|
| 31 |
import numpy as np
|
| 32 |
import cv2
|
|
|
|
| 107 |
except Exception as e:
|
| 108 |
logger.warning("CorridorKey green preload failed: %s", e)
|
| 109 |
|
| 110 |
+
# CorridorKey v2 (GPU only). v1 keeps working if v2 cannot load.
|
| 111 |
+
import ck2
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def _v1_cli_requested() -> bool:
|
| 115 |
+
"""A CLI run that asks for v1 explicitly must not pay for the v2 download."""
|
| 116 |
+
argv = sys.argv[1:]
|
| 117 |
+
if "--input" not in argv:
|
| 118 |
+
return False
|
| 119 |
+
for i, arg in enumerate(argv):
|
| 120 |
+
if arg == "--version" and i + 1 < len(argv):
|
| 121 |
+
return argv[i + 1] == "v1"
|
| 122 |
+
if arg.startswith("--version="):
|
| 123 |
+
return arg.split("=", 1)[1] == "v1"
|
| 124 |
+
return False
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
if not _v1_cli_requested():
|
| 128 |
+
ck2.init()
|
| 129 |
+
|
| 130 |
# Batch sizes for GPU inference (conservative for H200 80GB)
|
| 131 |
GPU_BATCH_SIZES = {"1024": 32, "2048": 16} # 2048 uses only 5.7GB/batch=2, so 16 easily fits in 69.8GB
|
| 132 |
|
|
|
|
| 1037 |
def process_example(video_path, resolution, despill, mask_mode, despeckle, despeckle_size, screen_color="Auto"):
|
| 1038 |
return process_video(video_path, resolution, despill, mask_mode, despeckle, despeckle_size, screen_color)
|
| 1039 |
|
| 1040 |
+
|
| 1041 |
+
def process_video_v2(video_path: str, quality: str = ck2.DEFAULT_QUALITY,
|
| 1042 |
+
temporal: str = ck2.DEFAULT_TEMPORAL, despill_val: int = 5,
|
| 1043 |
+
upscale: bool = False, progress=gr.Progress()):
|
| 1044 |
+
"""CorridorKey v2 green/blue screen matting (GPU only).
|
| 1045 |
+
|
| 1046 |
+
Args:
|
| 1047 |
+
video_path: Green or blue screen video.
|
| 1048 |
+
quality: Working resolution as short side, "540p", "720p" or "1080p".
|
| 1049 |
+
temporal: Frames of temporal context, "auto", "1", "8" or "24".
|
| 1050 |
+
despill_val: Despill strength 0-10 for the composite preview.
|
| 1051 |
+
upscale: Upscale the previews to the source resolution.
|
| 1052 |
+
|
| 1053 |
+
Returns:
|
| 1054 |
+
Composite preview, alpha matte preview, full-package zip, status text.
|
| 1055 |
+
"""
|
| 1056 |
+
try:
|
| 1057 |
+
return ck2.process(video_path, quality, temporal, despill_val, upscale, progress)
|
| 1058 |
+
except gr.Error:
|
| 1059 |
+
raise
|
| 1060 |
+
except RuntimeError as e:
|
| 1061 |
+
raise gr.Error(str(e))
|
| 1062 |
+
except Exception as e:
|
| 1063 |
+
logger.exception("v2 failed")
|
| 1064 |
+
raise gr.Error(f"CorridorKey v2 failed: {e}")
|
| 1065 |
+
|
| 1066 |
+
|
| 1067 |
+
def process_example_v2(video_path, quality, temporal, despill, upscale):
|
| 1068 |
+
return process_video_v2(video_path, quality, temporal, despill, upscale)
|
| 1069 |
+
|
| 1070 |
+
|
| 1071 |
+
V2_OK = ck2.is_ready()
|
| 1072 |
+
VERSION_CHOICES = ([("v2 (GPU, newer)", "v2")] if V2_OK else []) + [("v1 (GPU or CPU)", "v1")]
|
| 1073 |
+
DEFAULT_VERSION = "v2" if V2_OK else "v1"
|
| 1074 |
+
|
| 1075 |
+
DESCRIPTION = (
|
| 1076 |
+
f"CorridorKey | v2 (default): GPU only, about {ck2.seconds_per_frame(1280, 720):.2f}s per frame at 720p, "
|
| 1077 |
+
f"up to {ck2.max_frames(1280, 720)} frames at 720p | v1: GPU or CPU (ONNX) at a fixed model resolution"
|
| 1078 |
+
)
|
| 1079 |
+
if not V2_OK:
|
| 1080 |
+
DESCRIPTION += f" | v2 unavailable: {(ck2.load_error() or 'not loaded')[:120]}"
|
| 1081 |
|
| 1082 |
with gr.Blocks(title="CorridorKey") as demo:
|
| 1083 |
gr.Markdown(DESCRIPTION)
|
|
|
|
| 1085 |
with gr.Row():
|
| 1086 |
with gr.Column(scale=1):
|
| 1087 |
input_video = gr.Video(label="Upload Green/Blue Screen Video")
|
| 1088 |
+
version = gr.Radio(
|
| 1089 |
+
choices=VERSION_CHOICES, value=DEFAULT_VERSION, label="Model version",
|
| 1090 |
+
info="v2: temporal model, GPU only. v1: single frame, also runs on CPU"
|
| 1091 |
+
)
|
| 1092 |
with gr.Accordion("Settings", open=True):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1093 |
despill_slider = gr.Slider(
|
| 1094 |
0, 10, value=5, step=1, label="Despill Strength",
|
| 1095 |
info="Remove screen reflections (0=off, 10=max)"
|
| 1096 |
)
|
| 1097 |
+
with gr.Column(visible=DEFAULT_VERSION == "v2") as v2_controls:
|
| 1098 |
+
with gr.Row():
|
| 1099 |
+
quality = gr.Radio(
|
| 1100 |
+
choices=ck2.QUALITY_CHOICES, value=ck2.DEFAULT_QUALITY,
|
| 1101 |
+
label="Quality", info="Working resolution (short side). Higher is slower"
|
| 1102 |
+
)
|
| 1103 |
+
temporal = gr.Radio(
|
| 1104 |
+
choices=ck2.TEMPORAL_CHOICES, value=ck2.DEFAULT_TEMPORAL,
|
| 1105 |
+
label="Temporal context", info="Frames seen together. Auto picks the widest cheap window"
|
| 1106 |
+
)
|
| 1107 |
+
upscale_check = gr.Checkbox(value=False, label="Upscale previews to source resolution")
|
| 1108 |
+
with gr.Column(visible=DEFAULT_VERSION == "v1") as v1_controls:
|
| 1109 |
+
with gr.Row():
|
| 1110 |
+
resolution = gr.Radio(
|
| 1111 |
+
choices=["1024", "2048"], value="1024",
|
| 1112 |
+
label="Resolution", info="1024 = fast, 2048 = quality"
|
| 1113 |
+
)
|
| 1114 |
+
screen_color = gr.Radio(
|
| 1115 |
+
choices=["Auto", "Green", "Blue"], value="Auto",
|
| 1116 |
+
label="Screen Color", info="Auto-detects. Blue requires GPU"
|
| 1117 |
+
)
|
| 1118 |
+
mask_mode = gr.Radio(
|
| 1119 |
+
choices=["Hybrid (auto)", "AI (BiRefNet)", "Fast (classical)"],
|
| 1120 |
+
value="Hybrid (auto)", label="Mask Mode",
|
| 1121 |
+
info="Hybrid = fast detection + AI fallback. Fast = classical only. AI = always BiRefNet"
|
| 1122 |
+
)
|
| 1123 |
+
with gr.Row():
|
| 1124 |
+
despeckle_check = gr.Checkbox(value=True, label="Auto Despeckle")
|
| 1125 |
+
despeckle_size = gr.Number(value=400, precision=0, label="Min Area", info="Pixels to keep")
|
| 1126 |
+
process_btn_v2 = gr.Button("Process Video", variant="primary", size="lg",
|
| 1127 |
+
visible=DEFAULT_VERSION == "v2")
|
| 1128 |
+
process_btn = gr.Button("Process Video", variant="primary", size="lg",
|
| 1129 |
+
visible=DEFAULT_VERSION == "v1")
|
| 1130 |
|
| 1131 |
with gr.Column(scale=1):
|
| 1132 |
with gr.Row():
|
| 1133 |
comp_video = gr.Video(label="Composite Preview")
|
| 1134 |
matte_video = gr.Video(label="Alpha Matte")
|
| 1135 |
+
download_zip = gr.File(label="Download Full Package")
|
| 1136 |
status_text = gr.Textbox(label="Status", interactive=False)
|
| 1137 |
|
| 1138 |
+
_outputs = [comp_video, matte_video, download_zip, status_text]
|
| 1139 |
+
|
| 1140 |
+
with gr.Column(visible=DEFAULT_VERSION == "v2") as v2_examples:
|
| 1141 |
+
gr.Examples(
|
| 1142 |
+
examples=[["examples/corridor_greenscreen_demo.mp4", "720p", "auto", 5, False]],
|
| 1143 |
+
inputs=[input_video, quality, temporal, despill_slider, upscale_check],
|
| 1144 |
+
outputs=_outputs,
|
| 1145 |
+
fn=process_example_v2,
|
| 1146 |
+
cache_examples=True,
|
| 1147 |
+
cache_mode="lazy",
|
| 1148 |
+
label="Examples v2 (click to load)"
|
| 1149 |
+
)
|
| 1150 |
+
with gr.Column(visible=DEFAULT_VERSION == "v1") as v1_examples:
|
| 1151 |
+
gr.Examples(
|
| 1152 |
+
examples=[
|
| 1153 |
+
["examples/corridor_greenscreen_demo.mp4", "1024", 5, "Hybrid (auto)", True, 400, "Auto"],
|
| 1154 |
+
],
|
| 1155 |
+
inputs=[input_video, resolution, despill_slider, mask_mode, despeckle_check, despeckle_size, screen_color],
|
| 1156 |
+
outputs=_outputs,
|
| 1157 |
+
fn=process_example,
|
| 1158 |
+
cache_examples=True,
|
| 1159 |
+
cache_mode="lazy",
|
| 1160 |
+
label="Examples v1 (click to load)"
|
| 1161 |
+
)
|
| 1162 |
+
|
| 1163 |
+
def _on_version(value):
|
| 1164 |
+
is_v2 = value == "v2"
|
| 1165 |
+
return (gr.update(visible=is_v2), gr.update(visible=not is_v2),
|
| 1166 |
+
gr.update(visible=is_v2), gr.update(visible=not is_v2),
|
| 1167 |
+
gr.update(visible=is_v2), gr.update(visible=not is_v2))
|
| 1168 |
+
|
| 1169 |
+
version.change(
|
| 1170 |
+
_on_version, inputs=[version],
|
| 1171 |
+
outputs=[v2_controls, v1_controls, process_btn_v2, process_btn, v2_examples, v1_examples],
|
| 1172 |
+
api_name=False,
|
| 1173 |
)
|
| 1174 |
|
| 1175 |
process_btn.click(
|
| 1176 |
fn=process_video,
|
| 1177 |
inputs=[input_video, resolution, despill_slider, mask_mode, despeckle_check, despeckle_size, screen_color],
|
| 1178 |
+
outputs=_outputs,
|
| 1179 |
+
api_name="process_video",
|
| 1180 |
+
)
|
| 1181 |
+
process_btn_v2.click(
|
| 1182 |
+
fn=process_video_v2,
|
| 1183 |
+
inputs=[input_video, quality, temporal, despill_slider, upscale_check],
|
| 1184 |
+
outputs=_outputs,
|
| 1185 |
+
api_name="process_video_v2",
|
| 1186 |
)
|
| 1187 |
|
| 1188 |
|
|
|
|
| 1191 |
# ---------------------------------------------------------------------------
|
| 1192 |
def cli_main():
|
| 1193 |
import argparse
|
| 1194 |
+
parser = argparse.ArgumentParser(
|
| 1195 |
+
description="CorridorKey Green/Blue Screen Matting. v2 by default; add --version v1 for v1.")
|
| 1196 |
parser.add_argument("--input", required=True)
|
| 1197 |
parser.add_argument("--output", default="output")
|
| 1198 |
+
parser.add_argument("--version", default="v2", choices=["v2", "v1"],
|
| 1199 |
+
help="model version (default: v2, GPU only)")
|
|
|
|
|
|
|
| 1200 |
parser.add_argument("--despill", type=int, default=5)
|
| 1201 |
+
g1 = parser.add_argument_group("v1 only (needs --version v1)")
|
| 1202 |
+
g1.add_argument("--device", default=None, choices=["auto", "cpu", "cuda"])
|
| 1203 |
+
g1.add_argument("--resolution", default=None, choices=["1024", "2048"])
|
| 1204 |
+
g1.add_argument("--mask-mode", default=None,
|
| 1205 |
+
choices=["Hybrid (auto)", "AI (BiRefNet)", "Fast (classical)"])
|
| 1206 |
+
g1.add_argument("--no-despeckle", action="store_true")
|
| 1207 |
+
g1.add_argument("--despeckle-size", type=int, default=None)
|
| 1208 |
+
g1.add_argument("--screen-color", default=None, choices=["Auto", "Green", "Blue"])
|
| 1209 |
+
g2 = parser.add_argument_group("v2 only")
|
| 1210 |
+
g2.add_argument("--quality", default=None, choices=ck2.QUALITY_CHOICES)
|
| 1211 |
+
g2.add_argument("--temporal", default=None, choices=[v for _, v in ck2.TEMPORAL_CHOICES])
|
| 1212 |
+
g2.add_argument("--upscale", action="store_true", help="upscale previews to the source resolution")
|
| 1213 |
args = parser.parse_args()
|
| 1214 |
|
| 1215 |
+
v1_given = [flag for flag, value in (
|
| 1216 |
+
("--device", args.device), ("--resolution", args.resolution), ("--mask-mode", args.mask_mode),
|
| 1217 |
+
("--no-despeckle", args.no_despeckle or None), ("--despeckle-size", args.despeckle_size),
|
| 1218 |
+
("--screen-color", args.screen_color)) if value is not None]
|
| 1219 |
+
v2_given = [flag for flag, value in (
|
| 1220 |
+
("--quality", args.quality), ("--temporal", args.temporal),
|
| 1221 |
+
("--upscale", args.upscale or None)) if value is not None]
|
| 1222 |
+
if args.version == "v2" and v1_given:
|
| 1223 |
+
parser.error(f"{', '.join(v1_given)}: v1-only option(s). Add --version v1 "
|
| 1224 |
+
"(v2 needs a CUDA GPU and has no CPU mode).")
|
| 1225 |
+
if args.version == "v1" and v2_given:
|
| 1226 |
+
parser.error(f"{', '.join(v2_given)}: v2-only option(s). Drop them or use --version v2.")
|
| 1227 |
|
| 1228 |
class CLIProgress:
|
| 1229 |
def __call__(self, val, desc=""):
|
| 1230 |
if desc: print(f" [{val:.0%}] {desc}")
|
| 1231 |
|
| 1232 |
+
os.makedirs(args.output, exist_ok=True)
|
| 1233 |
+
if args.version == "v2":
|
| 1234 |
+
if not ck2.is_ready():
|
| 1235 |
+
parser.exit(1, f"v2 is unavailable: {ck2.load_error() or 'not loaded'}\n"
|
| 1236 |
+
"Use --version v1 for the v1 pipeline.\n")
|
| 1237 |
+
comp, matte, zipf, status = ck2.process(
|
| 1238 |
+
args.input, args.quality or ck2.DEFAULT_QUALITY, args.temporal or ck2.DEFAULT_TEMPORAL,
|
| 1239 |
+
args.despill, args.upscale, progress=CLIProgress())
|
| 1240 |
+
print(f"\n{status}")
|
| 1241 |
+
for path in (comp, matte, zipf):
|
| 1242 |
+
if path:
|
| 1243 |
+
dst = os.path.join(args.output, os.path.basename(path))
|
| 1244 |
+
shutil.copy2(path, dst)
|
| 1245 |
+
print(f"Output: {dst}")
|
| 1246 |
+
return
|
| 1247 |
+
|
| 1248 |
+
global HAS_CUDA
|
| 1249 |
+
device = args.device or "auto"
|
| 1250 |
+
if device == "cpu": HAS_CUDA = False
|
| 1251 |
+
elif device == "cuda": HAS_CUDA = True
|
| 1252 |
+
print(f"Device: {'CUDA' if HAS_CUDA else 'CPU'}")
|
| 1253 |
+
|
| 1254 |
comp, matte, zipf, status = process_video(
|
| 1255 |
+
args.input, args.resolution or "1024", args.despill, args.mask_mode or "Hybrid (auto)",
|
| 1256 |
+
not args.no_despeckle, args.despeckle_size if args.despeckle_size is not None else 400,
|
| 1257 |
+
args.screen_color or "Auto", progress=CLIProgress()
|
| 1258 |
)
|
| 1259 |
print(f"\n{status}")
|
|
|
|
| 1260 |
if zipf:
|
| 1261 |
dst = os.path.join(args.output, os.path.basename(zipf))
|
| 1262 |
shutil.copy2(zipf, dst)
|
|
|
|
| 1268 |
cli_main()
|
| 1269 |
else:
|
| 1270 |
demo.queue(default_concurrency_limit=1)
|
| 1271 |
+
demo.launch(ssr_mode=False, mcp_server=True, show_error=True)
|
ck2.py
ADDED
|
@@ -0,0 +1,337 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CorridorKey v2 adapter: assets, CPU-resident model, CPU pre/post, one GPU call.
|
| 2 |
+
|
| 3 |
+
The vendored `corridorkey` package is the unmodified v2 runtime. This module only
|
| 4 |
+
decides what runs where, so the time spent inside the ZeroGPU window is model work:
|
| 5 |
+
|
| 6 |
+
CPU probe + downscale the clip with ffmpeg
|
| 7 |
+
GPU run_inference (RVM prepass, Hann stage 1, stage 2, ffmpeg writers)
|
| 8 |
+
CPU browser-safe previews + zip
|
| 9 |
+
|
| 10 |
+
v2 needs CUDA. There is no CPU path; v1 in app.py keeps that role.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import base64
|
| 16 |
+
import hashlib
|
| 17 |
+
import logging
|
| 18 |
+
import math
|
| 19 |
+
import os
|
| 20 |
+
import shutil
|
| 21 |
+
import subprocess
|
| 22 |
+
import sys
|
| 23 |
+
import tempfile
|
| 24 |
+
import time
|
| 25 |
+
import urllib.request
|
| 26 |
+
import zipfile
|
| 27 |
+
from pathlib import Path
|
| 28 |
+
|
| 29 |
+
import cv2
|
| 30 |
+
|
| 31 |
+
try:
|
| 32 |
+
import spaces
|
| 33 |
+
HAS_SPACES = True
|
| 34 |
+
except ImportError:
|
| 35 |
+
HAS_SPACES = False
|
| 36 |
+
|
| 37 |
+
logger = logging.getLogger("ck2")
|
| 38 |
+
|
| 39 |
+
ROOT = Path(__file__).resolve().parent
|
| 40 |
+
if str(ROOT) not in sys.path:
|
| 41 |
+
sys.path.insert(0, str(ROOT))
|
| 42 |
+
# Weights and foundation models live here. CK2_ASSET_ROOT lets a local run reuse an existing download.
|
| 43 |
+
ASSETS = Path(os.environ.get("CK2_ASSET_ROOT") or ROOT)
|
| 44 |
+
|
| 45 |
+
CK2_REPO = "nikopueringer/CorridorKey2.0"
|
| 46 |
+
CK2_FILES = ("corridorkey_v2.safetensors", "foundation_stats.safetensors")
|
| 47 |
+
CRADIO_REPO = "nvidia/C-RADIOv4-SO400M"
|
| 48 |
+
CRADIO_REVISION = "c0457f5dc26ca145f954cd4fc5bb6114e5705ad8"
|
| 49 |
+
MOGE_REPO = "Ruicheng/moge-2-vitb-normal"
|
| 50 |
+
MOGE_REVISION = "ca5f0e07ff01d3e5a364c1d954ed12ee1814b368"
|
| 51 |
+
RVM_FILE = "pretrain_rvm_small16_256_204031069.npz"
|
| 52 |
+
RVM_URL = f"https://storage.googleapis.com/representations4d/checkpoints/{RVM_FILE}"
|
| 53 |
+
RVM_MD5_B64 = "ZJb6kj8+tasiJbVeplXu7Q=="
|
| 54 |
+
|
| 55 |
+
QUALITY_CHOICES = ["540p", "720p", "1080p"]
|
| 56 |
+
DEFAULT_QUALITY = "720p"
|
| 57 |
+
TEMPORAL_CHOICES = [("Auto", "auto"), ("Off (1 frame)", "1"), ("8 frames", "8"), ("24 frames", "24")]
|
| 58 |
+
DEFAULT_TEMPORAL = "auto"
|
| 59 |
+
STAGE2_BATCH = 4
|
| 60 |
+
|
| 61 |
+
MAX_GPU_SECONDS = int(os.environ.get("CK2_MAX_GPU_SECONDS", "120"))
|
| 62 |
+
GPU_OVERHEAD_S = 25.0
|
| 63 |
+
GPU_SAFETY = 1.35
|
| 64 |
+
|
| 65 |
+
_model = None
|
| 66 |
+
_load_error = ""
|
| 67 |
+
_ready = False
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def seconds_per_frame(width: int, height: int) -> float:
|
| 71 |
+
"""Measured on an RTX 5090 (89 frames, 540p/720p/1080p): about 0.05 + 0.22 s per megapixel."""
|
| 72 |
+
return 0.05 + 0.22 * (width * height / 1e6)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def estimate_gpu_seconds(frames: int, width: int, height: int) -> int:
|
| 76 |
+
estimate = GPU_OVERHEAD_S + GPU_SAFETY * frames * seconds_per_frame(width, height)
|
| 77 |
+
return int(max(30, min(MAX_GPU_SECONDS, math.ceil(estimate))))
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def max_frames(width: int, height: int) -> int:
|
| 81 |
+
return max(1, int((MAX_GPU_SECONDS - GPU_OVERHEAD_S) / (GPU_SAFETY * seconds_per_frame(width, height))))
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def working_size(src_w: int, src_h: int, quality: str) -> tuple[int, int]:
|
| 85 |
+
"""Largest size not above the quality's short side, even dimensions, source aspect kept."""
|
| 86 |
+
target = int(quality.rstrip("p"))
|
| 87 |
+
short = min(src_w, src_h)
|
| 88 |
+
if short <= target:
|
| 89 |
+
return src_w - src_w % 2, src_h - src_h % 2
|
| 90 |
+
scale = target / short
|
| 91 |
+
return max(2, round(src_w * scale / 2) * 2), max(2, round(src_h * scale / 2) * 2)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def resolve_window(choice: str, width: int, height: int) -> tuple[int, int]:
|
| 95 |
+
"""Hann window (chunk, stride). Auto takes the widest window that stays close to the cost of 8
|
| 96 |
+
frames: 80 up to 720p, 24 above (an 80-frame window at 1080p is about 2.5x slower)."""
|
| 97 |
+
if str(choice) == "auto":
|
| 98 |
+
chunk = 80 if width * height <= 1_000_000 else 24
|
| 99 |
+
elif str(choice) in ("1", "8", "24"):
|
| 100 |
+
chunk = int(choice)
|
| 101 |
+
else:
|
| 102 |
+
raise RuntimeError(f"Unsupported temporal context {choice!r}; choose auto, 1, 8 or 24.")
|
| 103 |
+
return chunk, max(1, chunk // 2)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def _md5_b64(path: Path) -> str:
|
| 107 |
+
digest = hashlib.md5()
|
| 108 |
+
with open(path, "rb") as f:
|
| 109 |
+
for chunk in iter(lambda: f.read(1 << 20), b""):
|
| 110 |
+
digest.update(chunk)
|
| 111 |
+
return base64.b64encode(digest.digest()).decode()
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def _retry(label: str, fn, attempts: int = 3):
|
| 115 |
+
for attempt in range(1, attempts + 1):
|
| 116 |
+
try:
|
| 117 |
+
return fn()
|
| 118 |
+
except Exception as exc:
|
| 119 |
+
logger.warning("[ck2] %s failed (%d/%d): %s", label, attempt, attempts, exc)
|
| 120 |
+
if attempt == attempts:
|
| 121 |
+
raise
|
| 122 |
+
time.sleep(3 * attempt)
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def ensure_assets() -> None:
|
| 126 |
+
"""Fetch the v2 weights and the three frozen foundation models, skipping anything present."""
|
| 127 |
+
from huggingface_hub import hf_hub_download, snapshot_download
|
| 128 |
+
|
| 129 |
+
for name in CK2_FILES:
|
| 130 |
+
_retry(name, lambda n=name: hf_hub_download(CK2_REPO, n, local_dir=ASSETS / "weights"))
|
| 131 |
+
cradio = ASSETS / "vfms/cradio/C-RADIOv4-SO400M"
|
| 132 |
+
_retry("C-RADIO", lambda: snapshot_download(
|
| 133 |
+
CRADIO_REPO, revision=CRADIO_REVISION, local_dir=cradio, ignore_patterns=["*.pth.tar"]))
|
| 134 |
+
_retry("MoGe", lambda: hf_hub_download(
|
| 135 |
+
MOGE_REPO, "model.pt", revision=MOGE_REVISION, local_dir=ASSETS / "vfms/moge"))
|
| 136 |
+
|
| 137 |
+
dst = ASSETS / "vfms/rvm" / RVM_FILE
|
| 138 |
+
if dst.is_file() and _md5_b64(dst) == RVM_MD5_B64:
|
| 139 |
+
return
|
| 140 |
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
| 141 |
+
tmp = dst.with_suffix(dst.suffix + ".part")
|
| 142 |
+
|
| 143 |
+
def fetch():
|
| 144 |
+
with urllib.request.urlopen(RVM_URL, timeout=60) as response, open(tmp, "wb") as f:
|
| 145 |
+
shutil.copyfileobj(response, f, length=1 << 20)
|
| 146 |
+
if _md5_b64(tmp) != RVM_MD5_B64:
|
| 147 |
+
tmp.unlink()
|
| 148 |
+
raise RuntimeError("checksum mismatch downloading RVM weights")
|
| 149 |
+
tmp.replace(dst)
|
| 150 |
+
|
| 151 |
+
_retry("RVM", fetch)
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def init() -> bool:
|
| 155 |
+
"""Download assets and build the model on CPU. Never raises: v1 must keep working."""
|
| 156 |
+
global _model, _ready, _load_error
|
| 157 |
+
try:
|
| 158 |
+
t0 = time.time()
|
| 159 |
+
ensure_assets()
|
| 160 |
+
t1 = time.time()
|
| 161 |
+
saved = {k: os.environ.get(k) for k in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE")}
|
| 162 |
+
try:
|
| 163 |
+
from corridorkey import runtime as rt
|
| 164 |
+
_model = rt.load_model(rt.RuntimePaths.from_package(ASSETS), device="cpu")
|
| 165 |
+
finally:
|
| 166 |
+
for key, value in saved.items():
|
| 167 |
+
if value is None:
|
| 168 |
+
os.environ.pop(key, None)
|
| 169 |
+
else:
|
| 170 |
+
os.environ[key] = value
|
| 171 |
+
_ready = True
|
| 172 |
+
logger.info("[ck2] ready: assets %.0fs, model load %.0fs", t1 - t0, time.time() - t1)
|
| 173 |
+
except Exception as exc:
|
| 174 |
+
_load_error = f"{type(exc).__name__}: {exc}"
|
| 175 |
+
logger.exception("[ck2] v2 unavailable")
|
| 176 |
+
return _ready
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def is_ready() -> bool:
|
| 180 |
+
return _ready
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def load_error() -> str:
|
| 184 |
+
return _load_error
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def _run(cmd: list[str], what: str, timeout: int = 600) -> None:
|
| 188 |
+
try:
|
| 189 |
+
subprocess.run(cmd, capture_output=True, timeout=timeout, check=True)
|
| 190 |
+
except subprocess.CalledProcessError as exc:
|
| 191 |
+
raise RuntimeError(f"{what} failed: {exc.stderr.decode(errors='ignore')[-300:]}") from exc
|
| 192 |
+
except (FileNotFoundError, subprocess.TimeoutExpired) as exc:
|
| 193 |
+
raise RuntimeError(f"{what} failed: {exc}") from exc
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def _probe(path: str) -> tuple[int, int, float, int]:
|
| 197 |
+
cap = cv2.VideoCapture(path)
|
| 198 |
+
try:
|
| 199 |
+
w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
| 200 |
+
h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
| 201 |
+
fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
|
| 202 |
+
frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
| 203 |
+
finally:
|
| 204 |
+
cap.release()
|
| 205 |
+
if frames <= 0 or w <= 0 or h <= 0:
|
| 206 |
+
raise RuntimeError("Could not read video frames.")
|
| 207 |
+
return w, h, fps, frames
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def prepare_input(video_path: str, quality: str, workdir: str) -> dict:
|
| 211 |
+
"""CPU stage: validate against the GPU budget and downscale the clip once."""
|
| 212 |
+
src_w, src_h, fps, frames = _probe(video_path)
|
| 213 |
+
w, h = working_size(src_w, src_h, quality)
|
| 214 |
+
limit = max_frames(w, h)
|
| 215 |
+
if frames > limit:
|
| 216 |
+
raise RuntimeError(
|
| 217 |
+
f"Video has {frames} frames ({frames / fps:.1f}s); at {quality} the limit is "
|
| 218 |
+
f"{limit} frames (~{limit / fps:.1f}s). Pick a lower quality or a shorter clip.")
|
| 219 |
+
work = os.path.join(workdir, "clip.mp4")
|
| 220 |
+
if (w, h) == (src_w, src_h):
|
| 221 |
+
cmd = ["ffmpeg", "-y", "-i", video_path, "-an", "-c:v", "libx264", "-preset", "veryfast",
|
| 222 |
+
"-crf", "8", "-pix_fmt", "yuv420p", work]
|
| 223 |
+
else:
|
| 224 |
+
cmd = ["ffmpeg", "-y", "-i", video_path, "-an", "-vf", f"scale={w}:{h}:flags=lanczos",
|
| 225 |
+
"-c:v", "libx264", "-preset", "veryfast", "-crf", "8", "-pix_fmt", "yuv420p", work]
|
| 226 |
+
_run(cmd, "Downscale")
|
| 227 |
+
return {"work": work, "w": w, "h": h, "src_w": src_w, "src_h": src_h, "fps": fps, "frames": frames}
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def _gpu_duration(work_video, out_dir, frames, width, height, *args, **kwargs):
|
| 231 |
+
return estimate_gpu_seconds(int(frames), int(width), int(height))
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def _gpu_decorator(fn):
|
| 235 |
+
if HAS_SPACES:
|
| 236 |
+
return spaces.GPU(duration=_gpu_duration)(fn)
|
| 237 |
+
return fn
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
@_gpu_decorator
|
| 241 |
+
def _gpu_phase(work_video, out_dir, frames, width, height, chunk, stride, despill_strength):
|
| 242 |
+
"""All GPU work. The model is moved in and moved back out so every call starts clean."""
|
| 243 |
+
import torch
|
| 244 |
+
from corridorkey import runtime as rt
|
| 245 |
+
|
| 246 |
+
if not torch.cuda.is_available():
|
| 247 |
+
raise RuntimeError("CorridorKey v2 requires a CUDA GPU.")
|
| 248 |
+
t0 = time.time()
|
| 249 |
+
props = torch.cuda.get_device_properties(0)
|
| 250 |
+
gpu = f"{props.name} {props.total_memory / 2**30:.0f}GB"
|
| 251 |
+
tmp = Path(tempfile.mkdtemp(prefix="ck2_tmp_"))
|
| 252 |
+
try:
|
| 253 |
+
_model.to("cuda")
|
| 254 |
+
cfg = rt.InferenceConfig(
|
| 255 |
+
hann_chunk=int(chunk), hann_stride=int(stride), auto_vram=False,
|
| 256 |
+
stage2_batch=STAGE2_BATCH, outputs={"alpha", "fg", "checker"},
|
| 257 |
+
despill_strength=float(despill_strength), temp_dir=tmp)
|
| 258 |
+
writer = rt.WriterConfig(preset="veryfast", threads=4)
|
| 259 |
+
rt.run_inference(_model, work_video, out_dir, cfg, writer,
|
| 260 |
+
initial_hint_path=None, hint_video_path=None)
|
| 261 |
+
finally:
|
| 262 |
+
_model.to("cpu")
|
| 263 |
+
torch.cuda.empty_cache()
|
| 264 |
+
shutil.rmtree(tmp, ignore_errors=True)
|
| 265 |
+
return time.time() - t0, gpu
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def _find(out_dir: str, suffix: str) -> str | None:
|
| 269 |
+
for name in sorted(os.listdir(out_dir)):
|
| 270 |
+
if name.endswith(".mp4") and suffix in name:
|
| 271 |
+
return os.path.join(out_dir, name)
|
| 272 |
+
return None
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
def _preview(src: str, dst: str, size: tuple[int, int] | None) -> str:
|
| 276 |
+
vf = "format=yuv420p" if size is None else f"scale={size[0]}:{size[1]}:flags=lanczos,format=yuv420p"
|
| 277 |
+
_run(["ffmpeg", "-y", "-i", src, "-vf", vf, "-c:v", "libx264", "-preset", "fast", "-crf", "18",
|
| 278 |
+
"-movflags", "+faststart", dst], "Preview encode")
|
| 279 |
+
return dst
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
def process(video_path, quality=DEFAULT_QUALITY, temporal=DEFAULT_TEMPORAL, despill_val=5,
|
| 283 |
+
upscale=False, progress=None):
|
| 284 |
+
"""Run CorridorKey v2. Returns (composite, matte, zip, status) like the v1 pipeline."""
|
| 285 |
+
if not _ready:
|
| 286 |
+
raise RuntimeError(f"CorridorKey v2 is not available ({_load_error or 'still loading'}). "
|
| 287 |
+
"Switch to v1 or retry in a minute.")
|
| 288 |
+
if video_path is None:
|
| 289 |
+
raise RuntimeError("Please upload a video.")
|
| 290 |
+
if quality not in QUALITY_CHOICES:
|
| 291 |
+
raise RuntimeError(f"Unsupported quality {quality!r}; choose one of {QUALITY_CHOICES}.")
|
| 292 |
+
def step(frac, text):
|
| 293 |
+
if progress is not None:
|
| 294 |
+
progress(frac, desc=text)
|
| 295 |
+
|
| 296 |
+
t_total = time.time()
|
| 297 |
+
workdir = tempfile.mkdtemp(prefix="ck2_job_")
|
| 298 |
+
out_dir = os.path.join(workdir, "out")
|
| 299 |
+
os.makedirs(out_dir)
|
| 300 |
+
|
| 301 |
+
step(0.05, "Preparing clip (CPU)...")
|
| 302 |
+
t0 = time.time()
|
| 303 |
+
info = prepare_input(video_path, quality, workdir)
|
| 304 |
+
t_prep = time.time() - t0
|
| 305 |
+
|
| 306 |
+
chunk, stride = resolve_window(temporal, info["w"], info["h"])
|
| 307 |
+
step(0.2, f"CorridorKey v2 on GPU ({info['w']}x{info['h']}, {info['frames']} frames)...")
|
| 308 |
+
t_gpu, gpu = _gpu_phase(info["work"], out_dir, info["frames"], info["w"], info["h"],
|
| 309 |
+
chunk, stride, float(despill_val) / 10.0)
|
| 310 |
+
|
| 311 |
+
step(0.85, "Encoding previews (CPU)...")
|
| 312 |
+
t0 = time.time()
|
| 313 |
+
alpha = _find(out_dir, "_alpha")
|
| 314 |
+
fg = _find(out_dir, "_fg")
|
| 315 |
+
comp = _find(out_dir, "_checker")
|
| 316 |
+
if not (alpha and fg and comp):
|
| 317 |
+
raise RuntimeError("CorridorKey v2 finished without all expected outputs.")
|
| 318 |
+
size = None
|
| 319 |
+
if upscale and (info["src_w"], info["src_h"]) != (info["w"], info["h"]):
|
| 320 |
+
size = (info["src_w"] - info["src_w"] % 2, info["src_h"] - info["src_h"] % 2)
|
| 321 |
+
comp_preview = _preview(comp, os.path.join(workdir, "comp_preview.mp4"), size)
|
| 322 |
+
matte_preview = _preview(alpha, os.path.join(workdir, "matte_preview.mp4"), size)
|
| 323 |
+
zip_path = os.path.join(workdir, "CorridorKey2_Output.zip")
|
| 324 |
+
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_STORED) as zf:
|
| 325 |
+
for path in (alpha, fg, comp):
|
| 326 |
+
zf.write(path, f"Output/{os.path.basename(path)}")
|
| 327 |
+
settings = [p for p in os.listdir(out_dir) if p.endswith("_settings.txt")]
|
| 328 |
+
for name in settings:
|
| 329 |
+
zf.write(os.path.join(out_dir, name), f"Output/{name}")
|
| 330 |
+
t_post = time.time() - t0
|
| 331 |
+
|
| 332 |
+
step(1.0, "Done!")
|
| 333 |
+
out_res = f"{size[0]}x{size[1]}" if size else f"{info['w']}x{info['h']}"
|
| 334 |
+
status = (f"CorridorKey v2 | {info['frames']} frames at {quality} ({out_res}) | "
|
| 335 |
+
f"temporal {chunk} | prep {t_prep:.0f}s, GPU {t_gpu:.0f}s ({gpu}), post {t_post:.0f}s, "
|
| 336 |
+
f"{time.time() - t_total:.0f}s total")
|
| 337 |
+
return comp_preview, matte_preview, zip_path, status
|
corridorkey/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .model import CorridorKeyV3
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
class CorridorKeyV2(CorridorKeyV3):
|
| 5 |
+
"""Public CorridorKey v2 runtime alias for the exported model."""
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
__all__ = ["CorridorKeyV2"]
|
corridorkey/blocks.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
corridorkey/color_utils.py
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import numpy as np
|
| 3 |
+
|
| 4 |
+
HDR_ASINH_KNEE = 0.008
|
| 5 |
+
HDR_ASINH_REF = 16.0
|
| 6 |
+
|
| 7 |
+
def _is_tensor(x):
|
| 8 |
+
return isinstance(x, torch.Tensor)
|
| 9 |
+
|
| 10 |
+
def _signed_pow(x, gamma):
|
| 11 |
+
"""Sign-preserving power. For x ≥ 0 this is identical to `x.pow(gamma)`.
|
| 12 |
+
For x < 0 it returns -|x|^gamma — keeps the operation well-defined (no
|
| 13 |
+
NaN) and lets gradient flow through negative inputs.
|
| 14 |
+
|
| 15 |
+
For the physically-valid [0, 1] inputs produced by the dataset the
|
| 16 |
+
behavior is bit-for-bit identical to the previous clamp-based form.
|
| 17 |
+
`to_srgb` uses its own epsilon-shifted branch because gamma < 1 has an
|
| 18 |
+
infinite mathematical slope at zero.
|
| 19 |
+
"""
|
| 20 |
+
if _is_tensor(x):
|
| 21 |
+
pos = x.clamp(min=0.0).pow(gamma)
|
| 22 |
+
neg = -((-x).clamp(min=0.0).pow(gamma))
|
| 23 |
+
return torch.where(x >= 0, pos, neg)
|
| 24 |
+
if isinstance(x, np.ndarray):
|
| 25 |
+
t = torch.from_numpy(x)
|
| 26 |
+
pos = t.clamp(min=0.0).pow(gamma)
|
| 27 |
+
neg = -((-t).clamp(min=0.0).pow(gamma))
|
| 28 |
+
return torch.where(t >= 0, pos, neg).numpy()
|
| 29 |
+
return np.sign(x) * np.power(np.abs(x), gamma)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def linear_to_srgb(x):
|
| 33 |
+
"""Scene-linear RGB → sRGB (IEC 61966-2-1). Numpy or torch input.
|
| 34 |
+
|
| 35 |
+
Uses standard IEC 61966-2-1 equations:
|
| 36 |
+
- x <= 0.0031308: 12.92 * x
|
| 37 |
+
- x > 0.0031308: 1.055 * x^(1/2.4) - 0.055
|
| 38 |
+
|
| 39 |
+
To prevent gradient NaN at zero during backpropagation in PyTorch,
|
| 40 |
+
we clamp the power branch input to a minimum positive threshold.
|
| 41 |
+
"""
|
| 42 |
+
if _is_tensor(x):
|
| 43 |
+
pos_x = x.clamp(min=0.0)
|
| 44 |
+
pos_pow = pos_x.clamp(min=0.0031308).pow(1.0 / 2.4)
|
| 45 |
+
pos = torch.where(pos_x <= 0.0031308, pos_x * 12.92, 1.055 * pos_pow - 0.055)
|
| 46 |
+
|
| 47 |
+
neg_x = (-x).clamp(min=0.0)
|
| 48 |
+
neg_pow = neg_x.clamp(min=0.0031308).pow(1.0 / 2.4)
|
| 49 |
+
neg = -torch.where(neg_x <= 0.0031308, neg_x * 12.92, 1.055 * neg_pow - 0.055)
|
| 50 |
+
return torch.where(x >= 0, pos, neg)
|
| 51 |
+
|
| 52 |
+
t = torch.from_numpy(x)
|
| 53 |
+
pos_t = t.clamp(min=0.0)
|
| 54 |
+
pos_pow = pos_t.clamp(min=0.0031308).pow(1.0 / 2.4)
|
| 55 |
+
pos = torch.where(pos_t <= 0.0031308, pos_t * 12.92, 1.055 * pos_pow - 0.055)
|
| 56 |
+
|
| 57 |
+
neg_t = (-t).clamp(min=0.0)
|
| 58 |
+
neg_pow = neg_t.clamp(min=0.0031308).pow(1.0 / 2.4)
|
| 59 |
+
neg = -torch.where(neg_t <= 0.0031308, neg_t * 12.92, 1.055 * neg_pow - 0.055)
|
| 60 |
+
t = torch.where(t >= 0, pos, neg)
|
| 61 |
+
return t.numpy()
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def srgb_to_linear(x):
|
| 65 |
+
"""sRGB → Scene-linear RGB (IEC 61966-2-1). Numpy or torch input.
|
| 66 |
+
|
| 67 |
+
Uses standard IEC 61966-2-1 equations:
|
| 68 |
+
- x <= 0.04045: x / 12.92
|
| 69 |
+
- x > 0.04045: ((x + 0.055) / 1.055)^2.4
|
| 70 |
+
"""
|
| 71 |
+
if _is_tensor(x):
|
| 72 |
+
pos_x = x.clamp(min=0.0)
|
| 73 |
+
pos_pow = ((pos_x + 0.055) / 1.055).pow(2.4)
|
| 74 |
+
pos = torch.where(pos_x <= 0.04045, pos_x / 12.92, pos_pow)
|
| 75 |
+
|
| 76 |
+
neg_x = (-x).clamp(min=0.0)
|
| 77 |
+
neg_pow = ((neg_x + 0.055) / 1.055).pow(2.4)
|
| 78 |
+
neg = -torch.where(neg_x <= 0.04045, neg_x / 12.92, neg_pow)
|
| 79 |
+
return torch.where(x >= 0, pos, neg)
|
| 80 |
+
|
| 81 |
+
t = torch.from_numpy(x)
|
| 82 |
+
pos_t = t.clamp(min=0.0)
|
| 83 |
+
pos_pow = ((pos_t + 0.055) / 1.055).pow(2.4)
|
| 84 |
+
pos = torch.where(pos_t <= 0.04045, pos_t / 12.92, pos_pow)
|
| 85 |
+
|
| 86 |
+
neg_t = (-t).clamp(min=0.0)
|
| 87 |
+
neg_pow = ((neg_t + 0.055) / 1.055).pow(2.4)
|
| 88 |
+
neg = -torch.where(neg_t <= 0.04045, neg_t / 12.92, neg_pow)
|
| 89 |
+
t = torch.where(t >= 0, pos, neg)
|
| 90 |
+
return t.numpy()
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
# Backwards compatibility aliases
|
| 94 |
+
to_srgb = linear_to_srgb
|
| 95 |
+
to_linear = srgb_to_linear
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def linear_to_asinh(x, knee=HDR_ASINH_KNEE, ref=HDR_ASINH_REF):
|
| 99 |
+
"""Signed asinh encoding for scene-linear RGB.
|
| 100 |
+
|
| 101 |
+
`ref` is a normalization reference, not a clamp. Values above `ref`
|
| 102 |
+
encode above 1.0 and remain reversible. This is the model RGB contract for
|
| 103 |
+
HDR-capable training: physical compositing stays in decoded linear light,
|
| 104 |
+
while the network sees a compressed but exact signed representation.
|
| 105 |
+
|
| 106 |
+
Native asinh is already odd and has finite nonzero derivative at zero, so
|
| 107 |
+
this intentionally avoids sign/abs branch construction.
|
| 108 |
+
"""
|
| 109 |
+
knee = float(knee)
|
| 110 |
+
ref = float(ref)
|
| 111 |
+
denom = np.arcsinh(ref / knee)
|
| 112 |
+
if _is_tensor(x):
|
| 113 |
+
scale = x.new_tensor(denom)
|
| 114 |
+
k = x.new_tensor(knee)
|
| 115 |
+
return torch.asinh(x / k) / scale
|
| 116 |
+
return np.arcsinh(x / knee) / denom
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def asinh_to_linear(y, knee=HDR_ASINH_KNEE, ref=HDR_ASINH_REF):
|
| 120 |
+
"""Inverse of `linear_to_asinh`."""
|
| 121 |
+
knee = float(knee)
|
| 122 |
+
ref = float(ref)
|
| 123 |
+
denom = np.arcsinh(ref / knee)
|
| 124 |
+
if _is_tensor(y):
|
| 125 |
+
scale = y.new_tensor(denom)
|
| 126 |
+
k = y.new_tensor(knee)
|
| 127 |
+
return torch.sinh(y * scale) * k
|
| 128 |
+
return np.sinh(y * denom) * knee
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def linear_to_vfm_sdr(x):
|
| 132 |
+
"""Tone-limited proxy for foundation models that expect SDR RGB [0, 1]."""
|
| 133 |
+
return np.clip(linear_to_srgb(np.maximum(x, 0.0)), 0.0, 1.0)
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def linear_to_vfm_sdr_torch(x):
|
| 137 |
+
"""Torch version of `linear_to_vfm_sdr`."""
|
| 138 |
+
return linear_to_srgb(x.clamp(min=0.0)).clamp(0.0, 1.0)
|
| 139 |
+
|
| 140 |
+
def premultiply(fg, alpha):
|
| 141 |
+
"""
|
| 142 |
+
Premultiplies foreground by alpha.
|
| 143 |
+
fg: Color [..., C] or [C, ...]
|
| 144 |
+
alpha: Alpha [..., 1] or [1, ...]
|
| 145 |
+
"""
|
| 146 |
+
return fg * alpha
|
| 147 |
+
|
| 148 |
+
def unpremultiply(fg, alpha, eps=1e-6):
|
| 149 |
+
"""
|
| 150 |
+
Un-premultiplies foreground by alpha.
|
| 151 |
+
Ref: fg_straight = fg_premul / (alpha + eps)
|
| 152 |
+
|
| 153 |
+
Uses max(alpha, 0) as the denominator floor so slightly-negative alpha
|
| 154 |
+
(e.g. from Lanczos ringing) can't land on alpha == -eps → denom == 0.
|
| 155 |
+
"""
|
| 156 |
+
if _is_tensor(fg):
|
| 157 |
+
return fg / (alpha.clamp(min=0.0) + eps)
|
| 158 |
+
return fg / (np.maximum(alpha, 0.0) + eps)
|
| 159 |
+
|
| 160 |
+
def composite_straight(fg, bg, alpha):
|
| 161 |
+
"""
|
| 162 |
+
Composites Straight FG over BG.
|
| 163 |
+
Formula: FG * Alpha + BG * (1 - Alpha)
|
| 164 |
+
"""
|
| 165 |
+
return fg * alpha + bg * (1.0 - alpha)
|
| 166 |
+
|
| 167 |
+
def composite_premul(fg, bg, alpha):
|
| 168 |
+
"""
|
| 169 |
+
Composites Premultiplied FG over BG.
|
| 170 |
+
Formula: FG + BG * (1 - Alpha)
|
| 171 |
+
"""
|
| 172 |
+
return fg + bg * (1.0 - alpha)
|
| 173 |
+
|
| 174 |
+
def rgb_to_yuv(image):
|
| 175 |
+
"""
|
| 176 |
+
Converts RGB to YUV (Rec. 601).
|
| 177 |
+
Input: [..., 3, H, W] or [..., 3] depending on layout.
|
| 178 |
+
Supports standard PyTorch BCHW.
|
| 179 |
+
"""
|
| 180 |
+
if not _is_tensor(image):
|
| 181 |
+
raise TypeError("rgb_to_yuv only supports dict/tensor inputs currently")
|
| 182 |
+
|
| 183 |
+
# Weights for RGB -> Y
|
| 184 |
+
# Rec. 601: 0.299, 0.587, 0.114
|
| 185 |
+
|
| 186 |
+
# Assume BCHW layout if 4 dims
|
| 187 |
+
if image.dim() == 4:
|
| 188 |
+
r = image[:, 0:1, :, :]
|
| 189 |
+
g = image[:, 1:2, :, :]
|
| 190 |
+
b = image[:, 2:3, :, :]
|
| 191 |
+
elif image.dim() == 3 and image.shape[0] == 3: # CHW
|
| 192 |
+
r = image[0:1, :, :]
|
| 193 |
+
g = image[1:2, :, :]
|
| 194 |
+
b = image[2:3, :, :]
|
| 195 |
+
else:
|
| 196 |
+
# Last dim conversion
|
| 197 |
+
r = image[..., 0]
|
| 198 |
+
g = image[..., 1]
|
| 199 |
+
b = image[..., 2]
|
| 200 |
+
|
| 201 |
+
y = 0.299 * r + 0.587 * g + 0.114 * b
|
| 202 |
+
u = 0.492 * (b - y)
|
| 203 |
+
v = 0.877 * (r - y)
|
| 204 |
+
|
| 205 |
+
if image.dim() >= 3 and image.shape[-3] == 3: # Concatenate along Channel dim
|
| 206 |
+
return torch.cat([y, u, v], dim=-3)
|
| 207 |
+
else:
|
| 208 |
+
return torch.stack([y, u, v], dim=-1)
|
corridorkey/foundations.py
ADDED
|
@@ -0,0 +1,327 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Frozen foundation-model wrappers for CorridorKey v2.
|
| 2 |
+
|
| 3 |
+
Produces per-token features at /64 of the input resolution:
|
| 4 |
+
• C-RADIOv4-SO400M: input (H/4, W/4), patch 16 → /64 grid.
|
| 5 |
+
• MoGe-ViT-B: input (7H/32, 7W/32), patch 14 → /64 grid.
|
| 6 |
+
|
| 7 |
+
Outputs are normalized per-channel via precomputed mean/std (loaded via
|
| 8 |
+
`load_stats`); when stats aren't loaded, the normalization is pass-through
|
| 9 |
+
(mean=0, std=1).
|
| 10 |
+
|
| 11 |
+
Stats are precomputed and shipped alongside the model weights.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
import importlib.util
|
| 15 |
+
import os
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
import sys
|
| 18 |
+
import types
|
| 19 |
+
|
| 20 |
+
import torch
|
| 21 |
+
import torch.nn as nn
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
# ---------------------------------------------------------------------------
|
| 25 |
+
# MoGe loader
|
| 26 |
+
# ---------------------------------------------------------------------------
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def build_moge_vitb(device='cuda', checkpoint='Ruicheng/moge-2-vitb-normal'):
|
| 30 |
+
from moge.model.v2 import MoGeModel
|
| 31 |
+
m = MoGeModel.from_pretrained(checkpoint)
|
| 32 |
+
m = m.to(device).eval()
|
| 33 |
+
try:
|
| 34 |
+
m.enable_pytorch_native_sdpa()
|
| 35 |
+
except Exception:
|
| 36 |
+
pass
|
| 37 |
+
for p in m.parameters():
|
| 38 |
+
p.requires_grad_(False)
|
| 39 |
+
return m
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
# ---------------------------------------------------------------------------
|
| 43 |
+
# C-RADIO loader
|
| 44 |
+
# ---------------------------------------------------------------------------
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _load_local_cradio_so400m(repo_path: Path):
|
| 48 |
+
"""Load bundled C-RADIO remote-code files without HF dynamic-module cache.
|
| 49 |
+
|
| 50 |
+
The exported runtime ships a full local HuggingFace snapshot. Loading the
|
| 51 |
+
class directly avoids any dependency on the user's global
|
| 52 |
+
~/.cache/huggingface/modules state while still using the official C-RADIO
|
| 53 |
+
implementation and safetensors weights from the snapshot.
|
| 54 |
+
"""
|
| 55 |
+
hf_model_path = repo_path / "hf_model.py"
|
| 56 |
+
if not hf_model_path.is_file():
|
| 57 |
+
raise FileNotFoundError(f"C-RADIO snapshot is missing {hf_model_path}")
|
| 58 |
+
|
| 59 |
+
package_name = "_corridorkey_cradio_snapshot"
|
| 60 |
+
package = sys.modules.get(package_name)
|
| 61 |
+
if package is None:
|
| 62 |
+
package = types.ModuleType(package_name)
|
| 63 |
+
package.__file__ = str(repo_path / "__init__.py")
|
| 64 |
+
package.__path__ = [str(repo_path)]
|
| 65 |
+
sys.modules[package_name] = package
|
| 66 |
+
|
| 67 |
+
module_name = f"{package_name}.hf_model"
|
| 68 |
+
module = sys.modules.get(module_name)
|
| 69 |
+
if module is None:
|
| 70 |
+
spec = importlib.util.spec_from_file_location(module_name, hf_model_path)
|
| 71 |
+
if spec is None or spec.loader is None:
|
| 72 |
+
raise RuntimeError(f"Could not import bundled C-RADIO module at {hf_model_path}")
|
| 73 |
+
module = importlib.util.module_from_spec(spec)
|
| 74 |
+
sys.modules[module_name] = module
|
| 75 |
+
spec.loader.exec_module(module)
|
| 76 |
+
|
| 77 |
+
config = module.RADIOConfig.from_pretrained(
|
| 78 |
+
str(repo_path), local_files_only=True,
|
| 79 |
+
)
|
| 80 |
+
return module.RADIOModel.from_pretrained(
|
| 81 |
+
str(repo_path), config=config, local_files_only=True,
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def build_cradio_so400m(repo: str = 'nvidia/C-RADIOv4-SO400M', device='cuda'):
|
| 86 |
+
"""Build frozen C-RADIOv4-SO400M.
|
| 87 |
+
|
| 88 |
+
C-RADIO's internal input conditioner expects RGB in [0, 1] unless
|
| 89 |
+
`make_preprocessor_external()` has been called. We keep the conditioner
|
| 90 |
+
internal so callers can feed the same range used by the rest of this
|
| 91 |
+
pipeline.
|
| 92 |
+
"""
|
| 93 |
+
repo_path = Path(repo)
|
| 94 |
+
if repo_path.is_dir():
|
| 95 |
+
m = _load_local_cradio_so400m(repo_path)
|
| 96 |
+
else:
|
| 97 |
+
from transformers import AutoModel
|
| 98 |
+
m = AutoModel.from_pretrained(
|
| 99 |
+
repo, trust_remote_code=True, local_files_only=True,
|
| 100 |
+
)
|
| 101 |
+
for p in m.parameters():
|
| 102 |
+
p.requires_grad_(False)
|
| 103 |
+
return m.eval().to(device)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
# ---------------------------------------------------------------------------
|
| 107 |
+
# Unified wrapper
|
| 108 |
+
# ---------------------------------------------------------------------------
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
class FrozenFoundations(nn.Module):
|
| 112 |
+
"""Frozen C-RADIO + MoGe/RVM feature extractor producing /64 features.
|
| 113 |
+
|
| 114 |
+
Stats normalization: outputs are normalized channel-wise with
|
| 115 |
+
normalized = (x - mean) / std
|
| 116 |
+
using precomputed per-channel buffers (`cradio_mean`, `cradio_std`,
|
| 117 |
+
`moge_mean`, `moge_std`, `rvm_mean`, `rvm_std`). When stats file hasn't been loaded the
|
| 118 |
+
buffers hold (0, 1) so normalization is the identity.
|
| 119 |
+
"""
|
| 120 |
+
|
| 121 |
+
def __init__(self, device='cuda',
|
| 122 |
+
use_moge=True,
|
| 123 |
+
use_cradio: bool = False,
|
| 124 |
+
cradio_repo: str = 'nvidia/C-RADIOv4-SO400M',
|
| 125 |
+
moge_checkpoint: str = 'Ruicheng/moge-2-vitb-normal',
|
| 126 |
+
rvm_variant: str = 'small',
|
| 127 |
+
rvm_ckpt: str = None,
|
| 128 |
+
stats_path: str = None):
|
| 129 |
+
super().__init__()
|
| 130 |
+
self.use_moge = use_moge
|
| 131 |
+
self.use_cradio = use_cradio
|
| 132 |
+
self.cradio_repo = cradio_repo if use_cradio else None
|
| 133 |
+
# RVM is implicitly enabled by passing --rvm_ckpt.
|
| 134 |
+
self.use_rvm = rvm_ckpt is not None and os.path.isfile(str(rvm_ckpt))
|
| 135 |
+
self.rvm_variant = rvm_variant if self.use_rvm else None
|
| 136 |
+
|
| 137 |
+
self.moge_checkpoint = str(moge_checkpoint) if use_moge else None
|
| 138 |
+
self.moge = (
|
| 139 |
+
build_moge_vitb(device=device, checkpoint=self.moge_checkpoint)
|
| 140 |
+
if use_moge else None
|
| 141 |
+
)
|
| 142 |
+
self.cradio = build_cradio_so400m(
|
| 143 |
+
repo=cradio_repo, device=device,
|
| 144 |
+
) if use_cradio else None
|
| 145 |
+
|
| 146 |
+
# RVM (Recurrent Video MAE from google-deepmind/representations4d).
|
| 147 |
+
# Stateful across frames; input in [0, 1] (no ImageNet norm).
|
| 148 |
+
if self.use_rvm:
|
| 149 |
+
from .rvm import RVMVideoSiamMAE, load_rvm_npz
|
| 150 |
+
self.rvm = RVMVideoSiamMAE(variant=rvm_variant).to(device)
|
| 151 |
+
load_rvm_npz(self.rvm, rvm_ckpt, strict_load=True, verbose=True)
|
| 152 |
+
for p in self.rvm.parameters():
|
| 153 |
+
p.requires_grad_(False)
|
| 154 |
+
self.rvm.eval()
|
| 155 |
+
else:
|
| 156 |
+
self.rvm = None
|
| 157 |
+
|
| 158 |
+
# Per-channel output stats (buffers for state-dict persistence).
|
| 159 |
+
# Shapes: (C,). Default 0/1 → pass-through.
|
| 160 |
+
moge_c = 768 if use_moge else 0
|
| 161 |
+
cradio_c = self._infer_cradio_dim() if use_cradio else 0
|
| 162 |
+
rvm_c = self.rvm.embed_dim if (self.use_rvm and self.rvm is not None) else 0
|
| 163 |
+
self.register_buffer('moge_mean', torch.zeros(max(moge_c, 1)))
|
| 164 |
+
self.register_buffer('moge_std', torch.ones(max(moge_c, 1)))
|
| 165 |
+
self.register_buffer('rvm_mean', torch.zeros(max(rvm_c, 1)))
|
| 166 |
+
self.register_buffer('rvm_std', torch.ones(max(rvm_c, 1)))
|
| 167 |
+
if use_cradio:
|
| 168 |
+
self.register_buffer('cradio_mean', torch.zeros(cradio_c))
|
| 169 |
+
self.register_buffer('cradio_std', torch.ones(cradio_c))
|
| 170 |
+
|
| 171 |
+
if stats_path is not None and os.path.isfile(stats_path):
|
| 172 |
+
self.load_stats(stats_path)
|
| 173 |
+
|
| 174 |
+
# Make sure the wrapper's own buffers land on the right device.
|
| 175 |
+
self.to(device)
|
| 176 |
+
|
| 177 |
+
def train(self, mode: bool = True):
|
| 178 |
+
"""Keep frozen feature extractors in eval mode at all times."""
|
| 179 |
+
super().train(mode) # sets self.training = mode
|
| 180 |
+
if self.moge is not None:
|
| 181 |
+
self.moge.eval()
|
| 182 |
+
for p in self.moge.parameters():
|
| 183 |
+
p.requires_grad_(False)
|
| 184 |
+
if self.rvm is not None:
|
| 185 |
+
self.rvm.eval()
|
| 186 |
+
for p in self.rvm.parameters():
|
| 187 |
+
p.requires_grad_(False)
|
| 188 |
+
if self.cradio is not None:
|
| 189 |
+
self.cradio.eval()
|
| 190 |
+
for p in self.cradio.parameters():
|
| 191 |
+
p.requires_grad_(False)
|
| 192 |
+
return self
|
| 193 |
+
|
| 194 |
+
def load_stats(self, path: str):
|
| 195 |
+
"""Load per-channel mean/std from a .safetensors (or legacy .pt) file.
|
| 196 |
+
File format:
|
| 197 |
+
{'cradio_mean': (C_c,), 'cradio_std': (C_c,),
|
| 198 |
+
'moge_mean': (C_m,), 'moge_std': (C_m,),
|
| 199 |
+
'rvm_mean': (C_r,), 'rvm_std': (C_r,)}
|
| 200 |
+
Silently skips any key we don't have a buffer for."""
|
| 201 |
+
if str(path).endswith('.safetensors'):
|
| 202 |
+
from safetensors.torch import load_file
|
| 203 |
+
stats = load_file(str(path), device='cpu')
|
| 204 |
+
else:
|
| 205 |
+
stats = torch.load(path, map_location='cpu', weights_only=False)
|
| 206 |
+
for key in ('moge_mean', 'moge_std', 'rvm_mean', 'rvm_std',
|
| 207 |
+
'cradio_mean', 'cradio_std'):
|
| 208 |
+
if key in stats and hasattr(self, key):
|
| 209 |
+
buf = getattr(self, key)
|
| 210 |
+
val = stats[key].to(buf.device, buf.dtype)
|
| 211 |
+
if val.shape != buf.shape:
|
| 212 |
+
print(f"[FrozenFoundations] {key}: stats shape "
|
| 213 |
+
f"{tuple(val.shape)} != buffer shape {tuple(buf.shape)} — skipping")
|
| 214 |
+
continue
|
| 215 |
+
buf.copy_(val)
|
| 216 |
+
print(f"[FrozenFoundations] loaded stats from {path}")
|
| 217 |
+
|
| 218 |
+
@torch.no_grad()
|
| 219 |
+
def forward(self, rgb_moge=None, rgb_cradio=None, rgb_rvm=None,
|
| 220 |
+
rvm_state=None):
|
| 221 |
+
"""C-RADIO and MoGe consume (B·T, 3, ...) 4D tensors (per-frame IID).
|
| 222 |
+
RVM consumes (B, T, 3, H_rvm, W_rvm) 5D because it is recurrent across T.
|
| 223 |
+
|
| 224 |
+
Args:
|
| 225 |
+
rgb_cradio: (B·T, 3, H/4, W/4) — if C-RADIO enabled.
|
| 226 |
+
rgb_moge: (B·T, 3, 7H/32, 7W/32) — if MoGe enabled.
|
| 227 |
+
rgb_rvm: (B, T, 3, H_rvm, W_rvm) — if RVM enabled. Typically
|
| 228 |
+
the same H/4 input used by C-RADIO.
|
| 229 |
+
rvm_state: optional (B, 1+N_rvm, D_rvm) carry state for RVM.
|
| 230 |
+
At training time, pass None (fresh per batch).
|
| 231 |
+
|
| 232 |
+
Returns:
|
| 233 |
+
Dict with channel-normalized features:
|
| 234 |
+
'moge': (B·T, C_m, H/64, W/64)
|
| 235 |
+
'cradio': (B·T, C_c, H/64, W/64)
|
| 236 |
+
'rvm': (B·T, C_r, h_rvm, w_rvm) (grid at RVM input / 16)
|
| 237 |
+
'rvm_state': (B, 1+N_rvm, D_rvm) final state, for streaming
|
| 238 |
+
inference across windows (ignored in training).
|
| 239 |
+
"""
|
| 240 |
+
out = {}
|
| 241 |
+
|
| 242 |
+
if self.cradio is not None:
|
| 243 |
+
assert rgb_cradio is not None, "C-RADIO enabled but rgb_cradio not provided"
|
| 244 |
+
try:
|
| 245 |
+
cradio_out = self.cradio(rgb_cradio, feature_fmt='NCHW')
|
| 246 |
+
except TypeError:
|
| 247 |
+
cradio_out = self.cradio(rgb_cradio)
|
| 248 |
+
cradio_feat = cradio_out.features
|
| 249 |
+
if cradio_feat.dim() == 3:
|
| 250 |
+
Bc, n_tok, Cc = cradio_feat.shape
|
| 251 |
+
patch = int(getattr(self.cradio, 'patch_size', 16))
|
| 252 |
+
hc, wc = rgb_cradio.shape[-2] // patch, rgb_cradio.shape[-1] // patch
|
| 253 |
+
if n_tok != hc * wc:
|
| 254 |
+
raise RuntimeError(
|
| 255 |
+
f"C-RADIO token count {n_tok} != expected grid {hc}x{wc}")
|
| 256 |
+
cradio_feat = cradio_feat.transpose(1, 2).reshape(Bc, Cc, hc, wc)
|
| 257 |
+
cradio_feat = cradio_feat.float()
|
| 258 |
+
mean = self.cradio_mean.to(
|
| 259 |
+
cradio_feat.device, cradio_feat.dtype,
|
| 260 |
+
).view(1, -1, 1, 1)
|
| 261 |
+
std = self.cradio_std.to(
|
| 262 |
+
cradio_feat.device, cradio_feat.dtype,
|
| 263 |
+
).view(1, -1, 1, 1)
|
| 264 |
+
cradio_feat = (cradio_feat - mean) / std.clamp(min=1e-6)
|
| 265 |
+
out['cradio'] = cradio_feat
|
| 266 |
+
|
| 267 |
+
if self.moge is not None:
|
| 268 |
+
assert rgb_moge is not None, "MoGe enabled but rgb_moge not provided"
|
| 269 |
+
Bm, _, Hm, Wm = rgb_moge.shape
|
| 270 |
+
base_h, base_w = Hm // 14, Wm // 14
|
| 271 |
+
moge_feat, _ = self.moge.encoder(
|
| 272 |
+
rgb_moge, base_h, base_w, return_class_token=True,
|
| 273 |
+
)
|
| 274 |
+
moge_feat = moge_feat.float()
|
| 275 |
+
mean = self.moge_mean.to(moge_feat.device, moge_feat.dtype).view(1, -1, 1, 1)
|
| 276 |
+
std = self.moge_std.to(moge_feat.device, moge_feat.dtype).view(1, -1, 1, 1)
|
| 277 |
+
moge_feat = (moge_feat - mean) / std.clamp(min=1e-6)
|
| 278 |
+
out['moge'] = moge_feat
|
| 279 |
+
|
| 280 |
+
# RVM is allowed to be skipped when rgb_rvm is None even if the
|
| 281 |
+
# model is enabled — useful when the caller pre-computes RVM
|
| 282 |
+
# features externally (e.g. inference's full-clip stateful pass)
|
| 283 |
+
# and supplies them to stage1 directly. In that case foundations
|
| 284 |
+
# shouldn't also run RVM inline.
|
| 285 |
+
if self.rvm is not None and rgb_rvm is not None:
|
| 286 |
+
# rgb_rvm: (B, T, 3, H, W). No mean/std norm — RVM wants [0, 1].
|
| 287 |
+
rvm_feat, new_state = self.rvm(rgb_rvm, state=rvm_state) # (B, T, D, h, w)
|
| 288 |
+
B_r, T_r, D_r, h_r, w_r = rvm_feat.shape
|
| 289 |
+
rvm_feat = rvm_feat.reshape(B_r * T_r, D_r, h_r, w_r).float()
|
| 290 |
+
mean = self.rvm_mean.to(rvm_feat.device, rvm_feat.dtype).view(1, -1, 1, 1)
|
| 291 |
+
std = self.rvm_std.to(rvm_feat.device, rvm_feat.dtype).view(1, -1, 1, 1)
|
| 292 |
+
rvm_feat = (rvm_feat - mean) / std.clamp(min=1e-6)
|
| 293 |
+
out['rvm'] = rvm_feat
|
| 294 |
+
out['rvm_state'] = new_state
|
| 295 |
+
|
| 296 |
+
return out
|
| 297 |
+
|
| 298 |
+
def _infer_cradio_dim(self):
|
| 299 |
+
if self.cradio is None:
|
| 300 |
+
return 0
|
| 301 |
+
for obj in (self.cradio, getattr(self.cradio, 'radio_model', None),
|
| 302 |
+
getattr(self.cradio, 'model', None)):
|
| 303 |
+
dim = getattr(obj, 'embed_dim', None) if obj is not None else None
|
| 304 |
+
if dim is not None:
|
| 305 |
+
return int(dim)
|
| 306 |
+
raise AttributeError("Unable to infer C-RADIO embed_dim")
|
| 307 |
+
|
| 308 |
+
@property
|
| 309 |
+
def moge_dim(self):
|
| 310 |
+
return 768 if self.use_moge else 0
|
| 311 |
+
|
| 312 |
+
@property
|
| 313 |
+
def cradio_dim(self):
|
| 314 |
+
if not self.use_cradio:
|
| 315 |
+
return 0
|
| 316 |
+
return self._infer_cradio_dim()
|
| 317 |
+
|
| 318 |
+
@property
|
| 319 |
+
def rvm_dim(self):
|
| 320 |
+
if not self.use_rvm or self.rvm is None:
|
| 321 |
+
return 0
|
| 322 |
+
return self.rvm.embed_dim
|
| 323 |
+
|
| 324 |
+
@property
|
| 325 |
+
def vfm_dim(self):
|
| 326 |
+
"""Concatenated foundation channel count."""
|
| 327 |
+
return self.cradio_dim + self.moge_dim + self.rvm_dim
|
corridorkey/model.py
ADDED
|
@@ -0,0 +1,307 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Top-level model assembly for CorridorKey v2.
|
| 2 |
+
|
| 3 |
+
Forward contract:
|
| 4 |
+
rgb: (B, T, 3, H, W) signed asinh encoded scene-linear RGB
|
| 5 |
+
hint: (B, T, 1, H, W) in [0, 1]
|
| 6 |
+
rgb_cradio:(B, T, 3, H/4, W/4) for C-RADIO/RVM (patch 16 → /64 grid)
|
| 7 |
+
rgb_moge: (B, T, 3, 7H/32, 7W/32) for MoGe (patch 14 → /64 grid)
|
| 8 |
+
t_stride: float — temporal RoPE scaling factor
|
| 9 |
+
|
| 10 |
+
H and W must be multiples of 64 (shape_quantum=64). The dataset
|
| 11 |
+
guarantees this.
|
| 12 |
+
|
| 13 |
+
Output dict:
|
| 14 |
+
'alpha': (B, T, 1, H, W) — stage-2 refined
|
| 15 |
+
'fg': (B, T, 3, H, W) asinh encoded — stage-2 refined
|
| 16 |
+
'alpha_aux': (B, T, 1, H/2, W/2) — stage-1 auxiliary prediction
|
| 17 |
+
'fg_aux': (B, T, 3, H/2, W/2) — stage-1 auxiliary prediction
|
| 18 |
+
'alpha_base': optional (B, T, 1, H, W) — pre-/4-refine stage-2 output
|
| 19 |
+
'fg_base': optional (B, T, 3, H, W) — pre-/4-refine stage-2 output
|
| 20 |
+
'alpha_bottleneck': optional (B, T, 1, H/8, W/8) — /64 supervision head
|
| 21 |
+
'fg_bottleneck': optional (B, T, 3, H/8, W/8) — /64 supervision head
|
| 22 |
+
'baton': (B, T, d_baton, H/16, W/16) — debug/intermediate
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
import torch
|
| 26 |
+
import torch.nn as nn
|
| 27 |
+
|
| 28 |
+
from .foundations import FrozenFoundations
|
| 29 |
+
from .stage1 import Stage1
|
| 30 |
+
from .stage2 import Stage2
|
| 31 |
+
from .rope import refresh_all_augs
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class CorridorKeyV3(nn.Module):
|
| 35 |
+
"""Transformer-native two-stage video matting model.
|
| 36 |
+
|
| 37 |
+
Stage 1 produces a /16 baton (d=512) plus an auxiliary /2 prediction
|
| 38 |
+
for deep supervision. Stage 2 refines the baton + raw RGB to a /1
|
| 39 |
+
alpha+FG output via pure-transformer Swin-2D + depatchify.
|
| 40 |
+
|
| 41 |
+
Temporal mixing lives primarily in stage 1: full spatiotemporal attention
|
| 42 |
+
at /64, Swin-3D at /16, and optional /64 recurrent memory sites for
|
| 43 |
+
streaming identity/decision carry.
|
| 44 |
+
"""
|
| 45 |
+
|
| 46 |
+
def __init__(
|
| 47 |
+
self,
|
| 48 |
+
# Stage 1
|
| 49 |
+
d_16: int = 512,
|
| 50 |
+
d_64: int = 1536,
|
| 51 |
+
n_enc_16: int = 2,
|
| 52 |
+
n_body_64: int = 10,
|
| 53 |
+
n_dec_16: int = 6,
|
| 54 |
+
num_heads_16: int = 8,
|
| 55 |
+
num_heads_64: int = 16,
|
| 56 |
+
window_HW: int = 16,
|
| 57 |
+
stage1_local_mix_kernel: int = 7,
|
| 58 |
+
stage1_local_mix_init_std: float | None = None,
|
| 59 |
+
stage1_body64_local_mix_kernel: int = 7,
|
| 60 |
+
stage1_body64_local_mix_init_std: float | None = None,
|
| 61 |
+
stage1_body64_mlp_hidden: int | None = None,
|
| 62 |
+
stage1_body64_direct_concat: bool = False,
|
| 63 |
+
stage1_recurrent_64_sites: tuple[int, ...] | None = None,
|
| 64 |
+
stage1_hint_adaln_blocks: int = 2,
|
| 65 |
+
stage1_mid_16_blocks: int = 4,
|
| 66 |
+
# Stage 2
|
| 67 |
+
d_stage2: int = 1280,
|
| 68 |
+
n_blocks_stage2: int = 10,
|
| 69 |
+
n_final_blocks_stage2: int = 2,
|
| 70 |
+
num_heads_stage2: int = 16,
|
| 71 |
+
window_size_stage2: int = 16,
|
| 72 |
+
stage2_local_mix_kernel: int = 7,
|
| 73 |
+
stage2_local_mix_init_std: float | None = None,
|
| 74 |
+
refine4_blocks: int = 0,
|
| 75 |
+
refine4_width: int = 96,
|
| 76 |
+
refine4_token_channels: int = 48,
|
| 77 |
+
refine4_kernel: int = 7,
|
| 78 |
+
refine4_expansion: float = 2.0,
|
| 79 |
+
# Shared
|
| 80 |
+
mlp_mult: float = 2.5,
|
| 81 |
+
mlp_dw_kernel: int = 0,
|
| 82 |
+
qkv_dw_kernel: int = 7,
|
| 83 |
+
qkv_dw_fraction: float = 0.25,
|
| 84 |
+
# Foundations
|
| 85 |
+
use_moge: bool = True,
|
| 86 |
+
use_cradio: bool = False,
|
| 87 |
+
cradio_repo: str = 'nvidia/C-RADIOv4-SO400M',
|
| 88 |
+
moge_checkpoint: str = 'Ruicheng/moge-2-vitb-normal',
|
| 89 |
+
# RVM is enabled by passing a ckpt path. RVM has patch 16, so
|
| 90 |
+
# we feed it the same H/4-sized input C-RADIO sees; features land at
|
| 91 |
+
# the /64 grid naturally, no resize or aspect-distortion needed.
|
| 92 |
+
rvm_variant: str = 'small',
|
| 93 |
+
rvm_ckpt: str = None,
|
| 94 |
+
foundation_stats_path: str = None,
|
| 95 |
+
device: str = 'cuda',
|
| 96 |
+
# Training
|
| 97 |
+
gradient_checkpoint: bool = True,
|
| 98 |
+
# Architecture ablation knob: overlap patchify/depatchify.
|
| 99 |
+
# overlap_patchify is a legacy bool alias (True → mult=3).
|
| 100 |
+
# overlap_mult: 1 (no overlap), 2 (half), 3 (full 3×3 neighborhood).
|
| 101 |
+
overlap_patchify: bool = False,
|
| 102 |
+
overlap_mult: int = 2,
|
| 103 |
+
depatch_fourier_features: int = 0,
|
| 104 |
+
depatch_fourier_kernel: int = 3,
|
| 105 |
+
# MAE-style pretraining hooks plus the /64 bottleneck supervision head.
|
| 106 |
+
# The bottleneck head default matches phase-2 training.
|
| 107 |
+
enable_mae_mask_tokens: bool = False,
|
| 108 |
+
bottleneck_head_stride: int | None = 8,
|
| 109 |
+
# DINOv3-style RoPE augmentations (training-time, per-forward coherent)
|
| 110 |
+
rope_shift_coords: float = None,
|
| 111 |
+
rope_jitter_coords: float = None,
|
| 112 |
+
rope_rescale_coords: float = None,
|
| 113 |
+
rope_base: float = 100.0,
|
| 114 |
+
):
|
| 115 |
+
super().__init__()
|
| 116 |
+
self.gradient_checkpoint = gradient_checkpoint
|
| 117 |
+
self.d_16 = d_16
|
| 118 |
+
|
| 119 |
+
self.foundations = FrozenFoundations(
|
| 120 |
+
device=device,
|
| 121 |
+
use_moge=use_moge,
|
| 122 |
+
use_cradio=use_cradio, cradio_repo=cradio_repo,
|
| 123 |
+
moge_checkpoint=moge_checkpoint,
|
| 124 |
+
rvm_variant=rvm_variant,
|
| 125 |
+
rvm_ckpt=rvm_ckpt,
|
| 126 |
+
stats_path=foundation_stats_path,
|
| 127 |
+
)
|
| 128 |
+
|
| 129 |
+
self.stage1 = Stage1(
|
| 130 |
+
d_16=d_16, d_64=d_64,
|
| 131 |
+
cradio_dim=self.foundations.cradio_dim,
|
| 132 |
+
moge_dim=self.foundations.moge_dim,
|
| 133 |
+
rvm_dim=self.foundations.rvm_dim,
|
| 134 |
+
n_enc_16=n_enc_16, n_body_64=n_body_64, n_dec_16=n_dec_16,
|
| 135 |
+
num_heads_16=num_heads_16, num_heads_64=num_heads_64,
|
| 136 |
+
window_HW=window_HW, mlp_mult=mlp_mult,
|
| 137 |
+
local_mix_kernel=stage1_local_mix_kernel,
|
| 138 |
+
local_mix_init_std=stage1_local_mix_init_std,
|
| 139 |
+
body64_local_mix_kernel=stage1_body64_local_mix_kernel,
|
| 140 |
+
body64_local_mix_init_std=stage1_body64_local_mix_init_std,
|
| 141 |
+
body64_mlp_hidden=stage1_body64_mlp_hidden,
|
| 142 |
+
body64_direct_concat=stage1_body64_direct_concat,
|
| 143 |
+
recurrent_64_sites=stage1_recurrent_64_sites,
|
| 144 |
+
hint_adaln_blocks=stage1_hint_adaln_blocks,
|
| 145 |
+
mid_16_blocks=stage1_mid_16_blocks,
|
| 146 |
+
mlp_dw_kernel=mlp_dw_kernel,
|
| 147 |
+
qkv_dw_kernel=qkv_dw_kernel,
|
| 148 |
+
qkv_dw_fraction=qkv_dw_fraction,
|
| 149 |
+
gradient_checkpoint=gradient_checkpoint,
|
| 150 |
+
overlap_patchify=overlap_patchify,
|
| 151 |
+
overlap_mult=overlap_mult,
|
| 152 |
+
depatch_fourier_features=depatch_fourier_features,
|
| 153 |
+
depatch_fourier_kernel=depatch_fourier_kernel,
|
| 154 |
+
enable_mae_mask_tokens=enable_mae_mask_tokens,
|
| 155 |
+
bottleneck_head_stride=bottleneck_head_stride,
|
| 156 |
+
rope_shift_coords=rope_shift_coords,
|
| 157 |
+
rope_jitter_coords=rope_jitter_coords,
|
| 158 |
+
rope_rescale_coords=rope_rescale_coords,
|
| 159 |
+
rope_base=rope_base,
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
self.stage2 = Stage2(
|
| 163 |
+
d=d_stage2, d_baton=d_16,
|
| 164 |
+
num_heads=num_heads_stage2,
|
| 165 |
+
num_blocks=n_blocks_stage2,
|
| 166 |
+
num_final_blocks=n_final_blocks_stage2,
|
| 167 |
+
window_size=window_size_stage2,
|
| 168 |
+
mlp_mult=mlp_mult,
|
| 169 |
+
gradient_checkpoint=gradient_checkpoint,
|
| 170 |
+
overlap_patchify=overlap_patchify,
|
| 171 |
+
overlap_mult=overlap_mult,
|
| 172 |
+
depatch_fourier_features=depatch_fourier_features,
|
| 173 |
+
depatch_fourier_kernel=depatch_fourier_kernel,
|
| 174 |
+
enable_mae_mask_tokens=enable_mae_mask_tokens,
|
| 175 |
+
local_mix_kernel=stage2_local_mix_kernel,
|
| 176 |
+
local_mix_init_std=stage2_local_mix_init_std,
|
| 177 |
+
mlp_dw_kernel=mlp_dw_kernel,
|
| 178 |
+
qkv_dw_kernel=qkv_dw_kernel,
|
| 179 |
+
qkv_dw_fraction=qkv_dw_fraction,
|
| 180 |
+
refine4_blocks=refine4_blocks,
|
| 181 |
+
refine4_width=refine4_width,
|
| 182 |
+
refine4_token_channels=refine4_token_channels,
|
| 183 |
+
refine4_kernel=refine4_kernel,
|
| 184 |
+
refine4_expansion=refine4_expansion,
|
| 185 |
+
rope_shift_coords=rope_shift_coords,
|
| 186 |
+
rope_jitter_coords=rope_jitter_coords,
|
| 187 |
+
rope_rescale_coords=rope_rescale_coords,
|
| 188 |
+
rope_base=rope_base,
|
| 189 |
+
)
|
| 190 |
+
|
| 191 |
+
def forward(
|
| 192 |
+
self,
|
| 193 |
+
rgb: torch.Tensor,
|
| 194 |
+
hint: torch.Tensor,
|
| 195 |
+
rgb_cradio: 'torch.Tensor | None',
|
| 196 |
+
rgb_moge: torch.Tensor,
|
| 197 |
+
rgb_rvm: 'torch.Tensor | None' = None,
|
| 198 |
+
rvm_state: 'torch.Tensor | None' = None,
|
| 199 |
+
t_stride: float = 1.0,
|
| 200 |
+
mae_mask_16: 'torch.Tensor | None' = None,
|
| 201 |
+
mae_mask_64: 'torch.Tensor | None' = None,
|
| 202 |
+
mae_vfm_mask_64: 'torch.Tensor | None' = None,
|
| 203 |
+
hint_quality: 'torch.Tensor | None' = None,
|
| 204 |
+
stage1_recurrent_states: 'tuple[torch.Tensor | None, ...] | None' = None,
|
| 205 |
+
) -> dict:
|
| 206 |
+
B, T, _, H, W = rgb.shape
|
| 207 |
+
assert H % 64 == 0 and W % 64 == 0, (
|
| 208 |
+
f"H, W must be multiples of 64 (shape_quantum); got {(H, W)}.")
|
| 209 |
+
if (mae_mask_16 is None) != (mae_mask_64 is None):
|
| 210 |
+
raise ValueError("mae_mask_16 and mae_mask_64 must be passed together")
|
| 211 |
+
if mae_vfm_mask_64 is not None and mae_mask_64 is None:
|
| 212 |
+
raise ValueError("mae_vfm_mask_64 requires mae_mask_64")
|
| 213 |
+
|
| 214 |
+
# Refresh RoPE augmentations ONCE per forward so every RoPE call
|
| 215 |
+
# inside stage1/stage2 (across all blocks) uses the same sample.
|
| 216 |
+
# In eval mode this clears the state to identity (deterministic).
|
| 217 |
+
refresh_all_augs(self, device=rgb.device)
|
| 218 |
+
|
| 219 |
+
# Foundations — per-frame, no gradients.
|
| 220 |
+
BT = B * T
|
| 221 |
+
rgb_moge_flat = rgb_moge.reshape(BT, 3, *rgb_moge.shape[-2:])
|
| 222 |
+
rgb_cradio_flat = (
|
| 223 |
+
rgb_cradio.reshape(BT, 3, *rgb_cradio.shape[-2:])
|
| 224 |
+
if rgb_cradio is not None else None
|
| 225 |
+
)
|
| 226 |
+
# RVM input: feed the same H/4-sized tensor C-RADIO sees. RVM has
|
| 227 |
+
# patch 16, so the patch grid lands on /64 directly —
|
| 228 |
+
# no resize, no aspect distortion. If caller supplied a custom
|
| 229 |
+
# rgb_rvm explicitly, honor that instead.
|
| 230 |
+
if self.foundations.rvm is not None and rgb_rvm is None:
|
| 231 |
+
assert rgb_cradio is not None, (
|
| 232 |
+
"RVM inline extraction requires an H/4 C-RADIO/RVM input")
|
| 233 |
+
H_e, W_e = rgb_cradio.shape[-2:]
|
| 234 |
+
rgb_rvm = rgb_cradio.reshape(B, T, 3, H_e, W_e)
|
| 235 |
+
with torch.no_grad():
|
| 236 |
+
foundation_feats = self.foundations(
|
| 237 |
+
rgb_moge=rgb_moge_flat,
|
| 238 |
+
rgb_cradio=rgb_cradio_flat,
|
| 239 |
+
rgb_rvm=rgb_rvm, rvm_state=rvm_state,
|
| 240 |
+
)
|
| 241 |
+
cradio_feat = foundation_feats.get('cradio')
|
| 242 |
+
moge_feat = foundation_feats.get('moge')
|
| 243 |
+
rvm_feat = foundation_feats.get('rvm')
|
| 244 |
+
rvm_state_out = foundation_feats.get('rvm_state')
|
| 245 |
+
if cradio_feat is not None:
|
| 246 |
+
cradio_feat = cradio_feat.reshape(B, T, -1, H // 64, W // 64)
|
| 247 |
+
else:
|
| 248 |
+
cradio_feat = rgb.new_zeros(B, T, 0, H // 64, W // 64)
|
| 249 |
+
if moge_feat is not None:
|
| 250 |
+
moge_feat = moge_feat.reshape(B, T, -1, H // 64, W // 64)
|
| 251 |
+
else:
|
| 252 |
+
moge_feat = rgb.new_zeros(B, T, 0, H // 64, W // 64)
|
| 253 |
+
if rvm_feat is not None:
|
| 254 |
+
rvm_feat = rvm_feat.reshape(B, T, rvm_feat.shape[-3],
|
| 255 |
+
rvm_feat.shape[-2], rvm_feat.shape[-1])
|
| 256 |
+
|
| 257 |
+
# Stage 1 & 2 — per-block gradient checkpointing is done INSIDE each
|
| 258 |
+
# stage's forward (see Stage1._ckpt_block, Stage2.run_block). No outer
|
| 259 |
+
# stage-level wrap — per-block peak activations are much smaller than
|
| 260 |
+
# stage-level, and double-wrapping would just cause nested recompute.
|
| 261 |
+
stage1_out = self.stage1(
|
| 262 |
+
rgb, hint, moge_feat,
|
| 263 |
+
cradio_feat=cradio_feat,
|
| 264 |
+
rvm_feat=rvm_feat, t_stride=t_stride,
|
| 265 |
+
mae_mask_16=mae_mask_16, mae_mask_64=mae_mask_64,
|
| 266 |
+
mae_vfm_mask_64=mae_vfm_mask_64,
|
| 267 |
+
hint_quality=hint_quality,
|
| 268 |
+
recurrent_states=stage1_recurrent_states,
|
| 269 |
+
)
|
| 270 |
+
baton = stage1_out['baton'] # (B, T, d_16, H/16, W/16)
|
| 271 |
+
aux = stage1_out['aux'] # (B, T, 4, H/2, W/2)
|
| 272 |
+
bottleneck = stage1_out.get('bottleneck')
|
| 273 |
+
recurrent_states_out = stage1_out.get('recurrent_states')
|
| 274 |
+
del stage1_out, foundation_feats
|
| 275 |
+
del cradio_feat, moge_feat, rvm_feat
|
| 276 |
+
del rgb_moge_flat, rgb_cradio_flat, rgb_rvm
|
| 277 |
+
|
| 278 |
+
logits_out = self.stage2(rgb, baton, mae_mask_16=mae_mask_16)
|
| 279 |
+
if isinstance(logits_out, tuple):
|
| 280 |
+
logits, logits_base = logits_out
|
| 281 |
+
else:
|
| 282 |
+
logits, logits_base = logits_out, None
|
| 283 |
+
|
| 284 |
+
# Split outputs
|
| 285 |
+
alpha = logits[:, :, 0:1]
|
| 286 |
+
fg = logits[:, :, 1:4]
|
| 287 |
+
alpha_aux = aux[:, :, 0:1]
|
| 288 |
+
fg_aux = aux[:, :, 1:4]
|
| 289 |
+
|
| 290 |
+
out = {
|
| 291 |
+
'alpha': alpha,
|
| 292 |
+
'fg': fg,
|
| 293 |
+
'alpha_aux': alpha_aux,
|
| 294 |
+
'fg_aux': fg_aux,
|
| 295 |
+
'baton': baton,
|
| 296 |
+
}
|
| 297 |
+
if logits_base is not None:
|
| 298 |
+
out['alpha_base'] = logits_base[:, :, 0:1]
|
| 299 |
+
out['fg_base'] = logits_base[:, :, 1:4]
|
| 300 |
+
if bottleneck is not None:
|
| 301 |
+
out['alpha_bottleneck'] = bottleneck[:, :, 0:1]
|
| 302 |
+
out['fg_bottleneck'] = bottleneck[:, :, 1:4]
|
| 303 |
+
if recurrent_states_out is not None:
|
| 304 |
+
out['stage1_recurrent_states'] = recurrent_states_out
|
| 305 |
+
if rvm_state_out is not None:
|
| 306 |
+
out['rvm_state'] = rvm_state_out
|
| 307 |
+
return out
|
corridorkey/rope.py
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Rotary Position Embeddings — 2D (stage 2) and 3D (stage 1) variants.
|
| 2 |
+
|
| 3 |
+
DINOv3/EUPE-style: normalized [-1, 1] cell-center coordinates with
|
| 4 |
+
training-time augmentations (shift, per-axis jitter, global rescale).
|
| 5 |
+
|
| 6 |
+
CorridorKey v2 uses:
|
| 7 |
+
• 3D RoPE (T, H, W) inside full-attention body at /64 and inside Swin-3D
|
| 8 |
+
windows at /16 (stage 1).
|
| 9 |
+
• 2D RoPE (H, W) inside Swin-2D windows at /16 (stage 2; no temporal).
|
| 10 |
+
|
| 11 |
+
Periods (lengths in normalized-coord units):
|
| 12 |
+
periods[i] = base ** (2i / (D_axis)) for i in [0, D_axis // 2)
|
| 13 |
+
angles = 2π · coord / period
|
| 14 |
+
where `coord ∈ [-1, 1]` is the cell-center coord of a token in the
|
| 15 |
+
(sub-)grid and `D_axis` is the portion of head_dim assigned to this axis.
|
| 16 |
+
|
| 17 |
+
Head-dim split:
|
| 18 |
+
2D: head_dim/4 periods per axis × 2 axes → head_dim/2 cos/sin pairs
|
| 19 |
+
3D: head_dim/2 cos/sin pairs split as evenly as possible across T/H/W
|
| 20 |
+
(2D asserts divisibility by 4; 3D only requires an even head_dim)
|
| 21 |
+
|
| 22 |
+
Application format (kept from original):
|
| 23 |
+
x[0::2] / x[1::2] interleave — cos/sin tensors are (N, head_dim//2),
|
| 24 |
+
one value per pair.
|
| 25 |
+
|
| 26 |
+
Training augmentations, sampled once per forward pass via refresh_augs()
|
| 27 |
+
so every RoPE call during the same forward gets a CONSISTENT positional
|
| 28 |
+
encoding:
|
| 29 |
+
• shift_coords (s): per-axis additive shift ~ U[-s, s]
|
| 30 |
+
• jitter_coords (J): per-axis multiplicative log-uniform in [1/J, J]
|
| 31 |
+
(aspect-ratio BREAKING — each axis jitters
|
| 32 |
+
independently)
|
| 33 |
+
• rescale_coords (R): global multiplicative log-uniform in [1/R, R]
|
| 34 |
+
(aspect-ratio PRESERVING — same scale on every
|
| 35 |
+
axis)
|
| 36 |
+
All disabled at eval.
|
| 37 |
+
|
| 38 |
+
Convention:
|
| 39 |
+
• Token ordering: (t, h, w) with w varying fastest for 3D;
|
| 40 |
+
(h, w) with w fastest for 2D.
|
| 41 |
+
• All three axes use the same `base` (default 100).
|
| 42 |
+
• Coord normalization per EUPE's "separate" mode: coord_a = (i+0.5)/N_a.
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
import math
|
| 46 |
+
from typing import List, Optional, Tuple
|
| 47 |
+
|
| 48 |
+
import torch
|
| 49 |
+
import torch.nn as nn
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _compute_periods(dim: int, base: float) -> torch.Tensor:
|
| 53 |
+
"""Period lengths in normalized-coord units. Returns (dim//2,) tensor."""
|
| 54 |
+
assert dim % 2 == 0, f"_compute_periods dim={dim} must be even"
|
| 55 |
+
exps = torch.arange(0, dim // 2, dtype=torch.float32) * 2.0 / dim
|
| 56 |
+
return base ** exps # Range: [1, base^((dim-2)/dim)]
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _normalized_coords(n: int, device=None, dtype=torch.float32) -> torch.Tensor:
|
| 60 |
+
"""Cell-center normalized coords in [-1, 1] for n tokens along an axis.
|
| 61 |
+
Matches DINOv3 / EUPE 'separate' normalization: (i+0.5)/n then 2x-1."""
|
| 62 |
+
x = (torch.arange(n, device=device, dtype=dtype) + 0.5) / n # [0.5/n, (n-0.5)/n]
|
| 63 |
+
return 2.0 * x - 1.0
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _apply_axis_rotary(coord: torch.Tensor, periods: torch.Tensor
|
| 67 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 68 |
+
"""Given coord (N,) in normalized units and periods (D_axis//2,),
|
| 69 |
+
return (cos, sin) of shape (N, D_axis//2)."""
|
| 70 |
+
angles = 2.0 * math.pi * coord.unsqueeze(1) / periods.unsqueeze(0)
|
| 71 |
+
return torch.cos(angles), torch.sin(angles)
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def apply_rotary(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
|
| 75 |
+
) -> torch.Tensor:
|
| 76 |
+
"""Apply rotation to the head dimension of `x`.
|
| 77 |
+
|
| 78 |
+
Args:
|
| 79 |
+
x: (..., N, head_dim) — queries or keys.
|
| 80 |
+
cos/sin: (N, head_dim // 2) — frequency tables.
|
| 81 |
+
|
| 82 |
+
Returns rotated tensor of same shape as `x`.
|
| 83 |
+
"""
|
| 84 |
+
while cos.dim() < x.dim():
|
| 85 |
+
cos = cos.unsqueeze(0)
|
| 86 |
+
sin = sin.unsqueeze(0)
|
| 87 |
+
cos = cos.to(x.dtype)
|
| 88 |
+
sin = sin.to(x.dtype)
|
| 89 |
+
x1 = x[..., 0::2]
|
| 90 |
+
x2 = x[..., 1::2]
|
| 91 |
+
out1 = x1 * cos - x2 * sin
|
| 92 |
+
out2 = x1 * sin + x2 * cos
|
| 93 |
+
return torch.stack([out1, out2], dim=-1).flatten(-2)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
# ---------------------------------------------------------------------------
|
| 97 |
+
# Augmentation state (shared logic between RoPE2D and RoPE3D)
|
| 98 |
+
# ---------------------------------------------------------------------------
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
class _RopeAugBase(nn.Module):
|
| 102 |
+
"""Holds training-time augmentation params and a per-forward cached
|
| 103 |
+
sample. Subclasses fill in axis geometry + coord construction."""
|
| 104 |
+
|
| 105 |
+
def __init__(self, base: float = 100.0,
|
| 106 |
+
shift_coords: Optional[float] = None,
|
| 107 |
+
jitter_coords: Optional[float] = None,
|
| 108 |
+
rescale_coords: Optional[float] = None,
|
| 109 |
+
n_axes: int = 2):
|
| 110 |
+
super().__init__()
|
| 111 |
+
self.base = base
|
| 112 |
+
self.shift_coords = shift_coords
|
| 113 |
+
self.jitter_coords = jitter_coords
|
| 114 |
+
self.rescale_coords = rescale_coords
|
| 115 |
+
self.n_axes = n_axes
|
| 116 |
+
# Cached per-forward augmentation: (shift_vec, jitter_log_vec, rescale_log_scalar).
|
| 117 |
+
# `None` means "use identity" (eval-time or uninitialized).
|
| 118 |
+
self._aug_shift: Optional[List[float]] = None
|
| 119 |
+
self._aug_jitter_log: Optional[List[float]] = None
|
| 120 |
+
self._aug_rescale_log: Optional[float] = None
|
| 121 |
+
|
| 122 |
+
def refresh_augs(self, device=None, force_eval: bool = False) -> None:
|
| 123 |
+
"""Call at the start of each model forward during training; clears
|
| 124 |
+
the aug state (identity) during eval."""
|
| 125 |
+
if force_eval or not self.training:
|
| 126 |
+
self._aug_shift = None
|
| 127 |
+
self._aug_jitter_log = None
|
| 128 |
+
self._aug_rescale_log = None
|
| 129 |
+
return
|
| 130 |
+
if self.shift_coords is not None and self.shift_coords > 0:
|
| 131 |
+
s = float(self.shift_coords)
|
| 132 |
+
self._aug_shift = (torch.empty(self.n_axes)
|
| 133 |
+
.uniform_(-s, s).tolist())
|
| 134 |
+
else:
|
| 135 |
+
self._aug_shift = None
|
| 136 |
+
if self.jitter_coords is not None and self.jitter_coords > 1.0:
|
| 137 |
+
lg = math.log(float(self.jitter_coords))
|
| 138 |
+
self._aug_jitter_log = (torch.empty(self.n_axes)
|
| 139 |
+
.uniform_(-lg, lg).tolist())
|
| 140 |
+
else:
|
| 141 |
+
self._aug_jitter_log = None
|
| 142 |
+
if self.rescale_coords is not None and self.rescale_coords > 1.0:
|
| 143 |
+
lg = math.log(float(self.rescale_coords))
|
| 144 |
+
self._aug_rescale_log = float(torch.empty(1)
|
| 145 |
+
.uniform_(-lg, lg).item())
|
| 146 |
+
else:
|
| 147 |
+
self._aug_rescale_log = None
|
| 148 |
+
|
| 149 |
+
def _aug_coord(self, coord: torch.Tensor, axis_idx: int) -> torch.Tensor:
|
| 150 |
+
"""Apply current-forward augs to a 1D coord tensor for axis `axis_idx`."""
|
| 151 |
+
out = coord
|
| 152 |
+
if self._aug_shift is not None:
|
| 153 |
+
out = out + self._aug_shift[axis_idx]
|
| 154 |
+
if self._aug_jitter_log is not None:
|
| 155 |
+
out = out * math.exp(self._aug_jitter_log[axis_idx])
|
| 156 |
+
if self._aug_rescale_log is not None:
|
| 157 |
+
out = out * math.exp(self._aug_rescale_log)
|
| 158 |
+
return out
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
# ---------------------------------------------------------------------------
|
| 162 |
+
# RoPE3D — used by stage 1 (Swin-3D enc/dec at /16, Full-ST body at /64)
|
| 163 |
+
# ---------------------------------------------------------------------------
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
class RoPE3D(_RopeAugBase):
|
| 167 |
+
"""3D Rotary Position Embedding for spatiotemporal tokens.
|
| 168 |
+
|
| 169 |
+
head_dim split: rotary pairs are split nearly evenly across T/H/W.
|
| 170 |
+
Exact thirds are used when available; otherwise the extra pairs are given
|
| 171 |
+
to H/W first, which lets power-of-two head dims such as 64 work cleanly.
|
| 172 |
+
"""
|
| 173 |
+
|
| 174 |
+
def __init__(self, head_dim: int,
|
| 175 |
+
base: float = 100.0,
|
| 176 |
+
shift_coords: Optional[float] = None,
|
| 177 |
+
jitter_coords: Optional[float] = None,
|
| 178 |
+
rescale_coords: Optional[float] = None):
|
| 179 |
+
super().__init__(base=base,
|
| 180 |
+
shift_coords=shift_coords,
|
| 181 |
+
jitter_coords=jitter_coords,
|
| 182 |
+
rescale_coords=rescale_coords,
|
| 183 |
+
n_axes=3)
|
| 184 |
+
assert head_dim % 2 == 0, (
|
| 185 |
+
f"3D RoPE head_dim={head_dim} must be even for cos/sin pairs.")
|
| 186 |
+
self.head_dim = head_dim
|
| 187 |
+
total_pairs = head_dim // 2
|
| 188 |
+
base_pairs = total_pairs // 3
|
| 189 |
+
rem = total_pairs - base_pairs * 3
|
| 190 |
+
pairs_t = base_pairs
|
| 191 |
+
pairs_h = base_pairs + (1 if rem >= 2 else 0)
|
| 192 |
+
pairs_w = base_pairs + (1 if rem >= 1 else 0)
|
| 193 |
+
self.dim_t = pairs_t * 2
|
| 194 |
+
self.dim_h = pairs_h * 2
|
| 195 |
+
self.dim_w = pairs_w * 2
|
| 196 |
+
|
| 197 |
+
self.register_buffer('periods_t',
|
| 198 |
+
_compute_periods(self.dim_t, base),
|
| 199 |
+
persistent=False)
|
| 200 |
+
self.register_buffer('periods_h',
|
| 201 |
+
_compute_periods(self.dim_h, base),
|
| 202 |
+
persistent=False)
|
| 203 |
+
self.register_buffer('periods_w',
|
| 204 |
+
_compute_periods(self.dim_w, base),
|
| 205 |
+
persistent=False)
|
| 206 |
+
|
| 207 |
+
def get_cos_sin(
|
| 208 |
+
self,
|
| 209 |
+
num_t: int, num_h: int, num_w: int,
|
| 210 |
+
device: torch.device = None,
|
| 211 |
+
t_stride: float = 1.0,
|
| 212 |
+
# Legacy args kept for call-site compat; ignored in the normalized
|
| 213 |
+
# formulation (Swin windows use local-to-window normalization).
|
| 214 |
+
t_offset: int = 0, h_offset: int = 0, w_offset: int = 0,
|
| 215 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 216 |
+
"""Build (num_t * num_h * num_w, head_dim // 2) cos and sin tables.
|
| 217 |
+
|
| 218 |
+
Positions are cell-center normalized to [-1, 1] per axis, optionally
|
| 219 |
+
augmented by the cached refresh_augs() sample. The `t_stride` scales
|
| 220 |
+
the temporal axis post-normalization (dataset-level frame-stride /
|
| 221 |
+
frame-rate knob, orthogonal to the training jitter).
|
| 222 |
+
"""
|
| 223 |
+
device = device or self.periods_t.device
|
| 224 |
+
coord_t = _normalized_coords(num_t, device=device) * t_stride
|
| 225 |
+
coord_h = _normalized_coords(num_h, device=device)
|
| 226 |
+
coord_w = _normalized_coords(num_w, device=device)
|
| 227 |
+
coord_t = self._aug_coord(coord_t, axis_idx=0)
|
| 228 |
+
coord_h = self._aug_coord(coord_h, axis_idx=1)
|
| 229 |
+
coord_w = self._aug_coord(coord_w, axis_idx=2)
|
| 230 |
+
|
| 231 |
+
cos_t, sin_t = _apply_axis_rotary(coord_t, self.periods_t.to(device))
|
| 232 |
+
cos_h, sin_h = _apply_axis_rotary(coord_h, self.periods_h.to(device))
|
| 233 |
+
cos_w, sin_w = _apply_axis_rotary(coord_w, self.periods_w.to(device))
|
| 234 |
+
|
| 235 |
+
N = num_t * num_h * num_w
|
| 236 |
+
cos_t_exp = cos_t[:, None, None, :].expand(num_t, num_h, num_w, -1).reshape(N, -1)
|
| 237 |
+
sin_t_exp = sin_t[:, None, None, :].expand(num_t, num_h, num_w, -1).reshape(N, -1)
|
| 238 |
+
cos_h_exp = cos_h[None, :, None, :].expand(num_t, num_h, num_w, -1).reshape(N, -1)
|
| 239 |
+
sin_h_exp = sin_h[None, :, None, :].expand(num_t, num_h, num_w, -1).reshape(N, -1)
|
| 240 |
+
cos_w_exp = cos_w[None, None, :, :].expand(num_t, num_h, num_w, -1).reshape(N, -1)
|
| 241 |
+
sin_w_exp = sin_w[None, None, :, :].expand(num_t, num_h, num_w, -1).reshape(N, -1)
|
| 242 |
+
|
| 243 |
+
cos = torch.cat([cos_t_exp, cos_h_exp, cos_w_exp], dim=-1)
|
| 244 |
+
sin = torch.cat([sin_t_exp, sin_h_exp, sin_w_exp], dim=-1)
|
| 245 |
+
return cos, sin
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
# ---------------------------------------------------------------------------
|
| 249 |
+
# RoPE2D — used by stage 2 (Swin-2D at /16)
|
| 250 |
+
# ---------------------------------------------------------------------------
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
class RoPE2D(_RopeAugBase):
|
| 254 |
+
"""2D Rotary Position Embedding for spatial-only tokens (stage 2).
|
| 255 |
+
|
| 256 |
+
head_dim split evenly between H and W. head_dim must be divisible by 4.
|
| 257 |
+
"""
|
| 258 |
+
|
| 259 |
+
def __init__(self, head_dim: int,
|
| 260 |
+
base: float = 100.0,
|
| 261 |
+
shift_coords: Optional[float] = None,
|
| 262 |
+
jitter_coords: Optional[float] = None,
|
| 263 |
+
rescale_coords: Optional[float] = None):
|
| 264 |
+
super().__init__(base=base,
|
| 265 |
+
shift_coords=shift_coords,
|
| 266 |
+
jitter_coords=jitter_coords,
|
| 267 |
+
rescale_coords=rescale_coords,
|
| 268 |
+
n_axes=2)
|
| 269 |
+
assert head_dim % 4 == 0, (
|
| 270 |
+
f"2D RoPE head_dim={head_dim} must be divisible by 4.")
|
| 271 |
+
self.head_dim = head_dim
|
| 272 |
+
dim_each = head_dim // 2
|
| 273 |
+
self.dim_h = dim_each
|
| 274 |
+
self.dim_w = dim_each
|
| 275 |
+
self.register_buffer('periods_h',
|
| 276 |
+
_compute_periods(dim_each, base),
|
| 277 |
+
persistent=False)
|
| 278 |
+
self.register_buffer('periods_w',
|
| 279 |
+
_compute_periods(dim_each, base),
|
| 280 |
+
persistent=False)
|
| 281 |
+
|
| 282 |
+
def get_cos_sin(
|
| 283 |
+
self, num_h: int, num_w: int,
|
| 284 |
+
device: torch.device = None,
|
| 285 |
+
# Legacy kwargs kept for call-site compat; unused in normalized form.
|
| 286 |
+
h_offset: int = 0, w_offset: int = 0,
|
| 287 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 288 |
+
"""Build (num_h * num_w, head_dim // 2) cos/sin tables."""
|
| 289 |
+
device = device or self.periods_h.device
|
| 290 |
+
coord_h = _normalized_coords(num_h, device=device)
|
| 291 |
+
coord_w = _normalized_coords(num_w, device=device)
|
| 292 |
+
coord_h = self._aug_coord(coord_h, axis_idx=0)
|
| 293 |
+
coord_w = self._aug_coord(coord_w, axis_idx=1)
|
| 294 |
+
|
| 295 |
+
cos_h, sin_h = _apply_axis_rotary(coord_h, self.periods_h.to(device))
|
| 296 |
+
cos_w, sin_w = _apply_axis_rotary(coord_w, self.periods_w.to(device))
|
| 297 |
+
N = num_h * num_w
|
| 298 |
+
cos_h_exp = cos_h[:, None, :].expand(num_h, num_w, -1).reshape(N, -1)
|
| 299 |
+
sin_h_exp = sin_h[:, None, :].expand(num_h, num_w, -1).reshape(N, -1)
|
| 300 |
+
cos_w_exp = cos_w[None, :, :].expand(num_h, num_w, -1).reshape(N, -1)
|
| 301 |
+
sin_w_exp = sin_w[None, :, :].expand(num_h, num_w, -1).reshape(N, -1)
|
| 302 |
+
cos = torch.cat([cos_h_exp, cos_w_exp], dim=-1)
|
| 303 |
+
sin = torch.cat([sin_h_exp, sin_w_exp], dim=-1)
|
| 304 |
+
return cos, sin
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
# ---------------------------------------------------------------------------
|
| 308 |
+
# Convenience: refresh all RoPE modules' augs in one call
|
| 309 |
+
# ---------------------------------------------------------------------------
|
| 310 |
+
|
| 311 |
+
|
| 312 |
+
def refresh_all_augs(module: nn.Module, device=None, force_eval: bool = False) -> None:
|
| 313 |
+
"""Walk a module tree and call refresh_augs on every RoPE instance.
|
| 314 |
+
|
| 315 |
+
Invoke once at the top of each model forward pass so all RoPE modules
|
| 316 |
+
in the model get a COHERENT augmentation sample for this step.
|
| 317 |
+
"""
|
| 318 |
+
for m in module.modules():
|
| 319 |
+
if isinstance(m, _RopeAugBase):
|
| 320 |
+
m.refresh_augs(device=device, force_eval=force_eval)
|
corridorkey/runtime.py
ADDED
|
@@ -0,0 +1,937 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import os
|
| 5 |
+
import sys
|
| 6 |
+
import tempfile
|
| 7 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 8 |
+
from dataclasses import dataclass, field
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
import torch
|
| 13 |
+
import torch.nn.functional as F
|
| 14 |
+
|
| 15 |
+
from . import color_utils as cu
|
| 16 |
+
from .model import CorridorKeyV3
|
| 17 |
+
from .video_io import (
|
| 18 |
+
ExrSequenceWriter,
|
| 19 |
+
ThreadedVideoWriter,
|
| 20 |
+
WriterConfig,
|
| 21 |
+
apply_linear_adjust,
|
| 22 |
+
frame_paths,
|
| 23 |
+
has_linear_adjust,
|
| 24 |
+
iter_frame_dir_chunks,
|
| 25 |
+
iter_video_frame_chunks,
|
| 26 |
+
load_initial_hint_frame,
|
| 27 |
+
make_checker,
|
| 28 |
+
make_hint_reader,
|
| 29 |
+
none_path,
|
| 30 |
+
probe_frame_dir_meta,
|
| 31 |
+
probe_video_meta,
|
| 32 |
+
raw_clip_name,
|
| 33 |
+
safe_remove,
|
| 34 |
+
taper_weights,
|
| 35 |
+
window_starts,
|
| 36 |
+
SequentialFrameDirWindowReader,
|
| 37 |
+
SequentialVideoWindowReader,
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
@dataclass
|
| 42 |
+
class RuntimePaths:
|
| 43 |
+
package_root: Path
|
| 44 |
+
weights: Path
|
| 45 |
+
foundation_stats: Path
|
| 46 |
+
rvm_ckpt: Path
|
| 47 |
+
moge_checkpoint: Path
|
| 48 |
+
cradio_dir: Path
|
| 49 |
+
|
| 50 |
+
@staticmethod
|
| 51 |
+
def _resolve_user_or_package_path(root: Path, value, default: str) -> Path:
|
| 52 |
+
if value is None:
|
| 53 |
+
return (root / default).resolve()
|
| 54 |
+
path = Path(value)
|
| 55 |
+
if path.is_absolute():
|
| 56 |
+
return path.resolve()
|
| 57 |
+
cwd_path = (Path.cwd() / path).resolve()
|
| 58 |
+
if cwd_path.exists():
|
| 59 |
+
return cwd_path
|
| 60 |
+
return (root / path).resolve()
|
| 61 |
+
|
| 62 |
+
@staticmethod
|
| 63 |
+
def _prefer_safetensors(root: Path, stem: str) -> str:
|
| 64 |
+
# Legacy pickle .pt files are still accepted if that is all that is present.
|
| 65 |
+
if not (root / f"{stem}.safetensors").exists() and (root / f"{stem}.pt").exists():
|
| 66 |
+
return f"{stem}.pt"
|
| 67 |
+
return f"{stem}.safetensors"
|
| 68 |
+
|
| 69 |
+
@classmethod
|
| 70 |
+
def from_package(cls, package_root: str | os.PathLike,
|
| 71 |
+
weights: str | os.PathLike | None = None):
|
| 72 |
+
root = Path(package_root).resolve()
|
| 73 |
+
return cls(
|
| 74 |
+
package_root=root,
|
| 75 |
+
weights=cls._resolve_user_or_package_path(
|
| 76 |
+
root, weights, cls._prefer_safetensors(root, "weights/corridorkey_v2")),
|
| 77 |
+
foundation_stats=(
|
| 78 |
+
root / cls._prefer_safetensors(root, "weights/foundation_stats")).resolve(),
|
| 79 |
+
rvm_ckpt=(root / "vfms/rvm/pretrain_rvm_small16_256_204031069.npz").resolve(),
|
| 80 |
+
moge_checkpoint=(root / "vfms/moge/model.pt").resolve(),
|
| 81 |
+
cradio_dir=(root / "vfms/cradio/C-RADIOv4-SO400M").resolve(),
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
def validate(self) -> None:
|
| 85 |
+
missing = [
|
| 86 |
+
p for p in (
|
| 87 |
+
self.weights, self.foundation_stats, self.rvm_ckpt,
|
| 88 |
+
self.moge_checkpoint, self.cradio_dir,
|
| 89 |
+
)
|
| 90 |
+
if not p.exists()
|
| 91 |
+
]
|
| 92 |
+
if missing:
|
| 93 |
+
raise FileNotFoundError(
|
| 94 |
+
"missing runtime asset(s): " + ", ".join(str(p) for p in missing)
|
| 95 |
+
+ "\nRun `python scripts/download_weights.py` to fetch them.")
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
@dataclass
|
| 99 |
+
class InferenceConfig:
|
| 100 |
+
hann_chunk: int = 80
|
| 101 |
+
hann_stride: int = 40
|
| 102 |
+
carry_hint: bool = True
|
| 103 |
+
hint_quality: float = 0.95
|
| 104 |
+
carry_hint_quality: float = 0.95
|
| 105 |
+
shape_quantum: int = 128
|
| 106 |
+
model_scale: float = 1.0
|
| 107 |
+
num_frames: int = -1
|
| 108 |
+
start_mode: str = "begin"
|
| 109 |
+
seed: int = 0
|
| 110 |
+
low_vram: bool = False
|
| 111 |
+
# Pick hann_chunk/hann_stride/low_vram per clip from free GPU memory.
|
| 112 |
+
auto_vram: bool = False
|
| 113 |
+
low_vram_foundation_chunk: int = 4
|
| 114 |
+
low_vram_mlp_chunk_tokens: int = 65536
|
| 115 |
+
low_vram_enc_frame_chunk: int = 1
|
| 116 |
+
low_vram_swin_window_batch: int = 16
|
| 117 |
+
low_vram_full_attn_query_chunk_tokens: int = 2048
|
| 118 |
+
stage2_batch: int = 1
|
| 119 |
+
outputs: set[str] = field(default_factory=lambda: {"alpha", "fg"})
|
| 120 |
+
temp_dir: Path = Path(tempfile.gettempdir()) / "corridorkey_v2"
|
| 121 |
+
linear_brightness: float = 1.0
|
| 122 |
+
linear_contrast: float = 1.0
|
| 123 |
+
linear_contrast_pivot: float = 0.18
|
| 124 |
+
frame_dir_fps: float = 24.0
|
| 125 |
+
frame_dir_linear: bool = False
|
| 126 |
+
cutout_linear: bool = False
|
| 127 |
+
despill_strength: float = 0.5
|
| 128 |
+
output_format: str = "video"
|
| 129 |
+
exr_dtype: str = "half"
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
# Peak GPU memory in GiB by megapixels of model input, measured at each Hann
|
| 133 |
+
# chunk size. Keys are low_vram False/True.
|
| 134 |
+
_VRAM_PEAK_GB = {
|
| 135 |
+
False: ((8, 24, 48, 80), (
|
| 136 |
+
(0.98, (6.19, 7.52, 9.77, 12.89)),
|
| 137 |
+
(2.21, (7.50, 11.33, 18.66, 24.89)),
|
| 138 |
+
(3.93, (9.06, 14.78, 23.92, 33.19)),
|
| 139 |
+
(8.36, (13.94, 25.96, 45.15, 70.45)),
|
| 140 |
+
)),
|
| 141 |
+
True: ((1, 2, 4, 8, 24, 48, 80), (
|
| 142 |
+
(0.98, (2.71, 2.78, 2.94, 3.25, 4.54, 6.57, 9.50)),
|
| 143 |
+
(2.21, (2.80, 2.97, 3.35, 4.12, 7.69, 13.20, 20.56)),
|
| 144 |
+
(3.93, (2.92, 3.23, 3.85, 5.09, 10.09, 17.57, 27.75)),
|
| 145 |
+
(8.36, (3.25, 3.92, 5.28, 8.38, 20.78, 39.66, 64.65)),
|
| 146 |
+
)),
|
| 147 |
+
}
|
| 148 |
+
_VRAM_MARGIN_SCALE = 1.1
|
| 149 |
+
_VRAM_MARGIN_GB = 0.3
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def _interp(x: float, points) -> float:
|
| 153 |
+
"""Piecewise-linear through (x, y) points, extrapolating past the ends."""
|
| 154 |
+
points = list(points)
|
| 155 |
+
for (x0, y0), (x1, y1) in zip(points, points[1:]):
|
| 156 |
+
if x <= x1 or (x1, y1) == points[-1]:
|
| 157 |
+
return y0 + (y1 - y0) * (x - x0) / (x1 - x0)
|
| 158 |
+
return points[0][1]
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def estimate_peak_vram_gb(megapixels: float, frames: int, low_vram: bool) -> float:
|
| 162 |
+
"""Estimated peak GPU memory for a Hann window of `frames` frames."""
|
| 163 |
+
chunks, rows = _VRAM_PEAK_GB[low_vram]
|
| 164 |
+
frames = max(frames, chunks[0])
|
| 165 |
+
per_chunk = [
|
| 166 |
+
max(_interp(megapixels, [(mp, peaks[i]) for mp, peaks in rows]), rows[0][1][i])
|
| 167 |
+
for i in range(len(chunks))
|
| 168 |
+
]
|
| 169 |
+
return _interp(frames, zip(chunks, per_chunk))
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def select_vram_profile(shape_hw, num_frames: int, budget_gb: float):
|
| 173 |
+
"""Largest Hann window that fits `budget_gb`, as (chunk, stride, low_vram, estimate).
|
| 174 |
+
|
| 175 |
+
Bigger windows give better temporal consistency, and low-VRAM mode is
|
| 176 |
+
exact, so a larger low-VRAM window is preferred over a smaller normal one.
|
| 177 |
+
Falls back to a single-frame window when nothing is estimated to fit.
|
| 178 |
+
"""
|
| 179 |
+
megapixels = shape_hw[0] * shape_hw[1] / 1e6
|
| 180 |
+
for chunk in reversed(_VRAM_PEAK_GB[True][0]):
|
| 181 |
+
for low_vram in (False, True):
|
| 182 |
+
if chunk not in _VRAM_PEAK_GB[low_vram][0]:
|
| 183 |
+
continue
|
| 184 |
+
estimate = estimate_peak_vram_gb(megapixels, min(chunk, num_frames), low_vram)
|
| 185 |
+
if estimate * _VRAM_MARGIN_SCALE + _VRAM_MARGIN_GB <= budget_gb:
|
| 186 |
+
return chunk, max(1, chunk // 2), low_vram, estimate
|
| 187 |
+
return 1, 1, True, estimate
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def apply_auto_vram(cfg: InferenceConfig, shape_hw, num_frames: int) -> None:
|
| 191 |
+
free_bytes, _ = torch.cuda.mem_get_info()
|
| 192 |
+
# Memory this process already holds (the loaded model) is part of the budget.
|
| 193 |
+
budget_gb = (free_bytes + torch.cuda.memory_reserved()) / 2**30
|
| 194 |
+
chunk, stride, low_vram, estimate = select_vram_profile(shape_hw, num_frames, budget_gb)
|
| 195 |
+
cfg.hann_chunk, cfg.hann_stride, cfg.low_vram = chunk, stride, low_vram
|
| 196 |
+
print(f" Auto VRAM: {budget_gb:.1f} GB available at {shape_hw[1]}x{shape_hw[0]} -> "
|
| 197 |
+
f"hann_chunk={chunk}, hann_stride={stride}, low_vram={low_vram} "
|
| 198 |
+
f"(estimated peak {estimate:.1f} GB)")
|
| 199 |
+
if estimate * _VRAM_MARGIN_SCALE + _VRAM_MARGIN_GB > budget_gb:
|
| 200 |
+
print(" [WARNING] Even a single-frame window may not fit this GPU at this "
|
| 201 |
+
"resolution. Inference may run out of memory.")
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def prepare_runtime_environment(paths: RuntimePaths) -> None:
|
| 205 |
+
os.environ.setdefault("HF_HUB_OFFLINE", "1")
|
| 206 |
+
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
|
| 207 |
+
moge_root = paths.package_root / "third_party/MoGe"
|
| 208 |
+
if str(moge_root) not in sys.path:
|
| 209 |
+
sys.path.insert(0, str(moge_root))
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
def _resolve_package_path(paths: RuntimePaths, value: str | None) -> str | None:
|
| 213 |
+
if value is None:
|
| 214 |
+
return None
|
| 215 |
+
p = Path(value)
|
| 216 |
+
if not p.is_absolute():
|
| 217 |
+
p = paths.package_root / p
|
| 218 |
+
return str(p.resolve())
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def load_runtime_checkpoint(path: Path) -> dict:
|
| 222 |
+
"""Read a runtime checkpoint from .safetensors (preferred) or legacy .pt."""
|
| 223 |
+
path = Path(path)
|
| 224 |
+
if path.suffix == ".safetensors":
|
| 225 |
+
from safetensors import safe_open
|
| 226 |
+
from safetensors.torch import load_file
|
| 227 |
+
with safe_open(str(path), framework="pt") as f:
|
| 228 |
+
header = f.metadata() or {}
|
| 229 |
+
return {
|
| 230 |
+
"format": header.get("format"),
|
| 231 |
+
"model_config": json.loads(header.get("model_config", "{}")),
|
| 232 |
+
"metadata": json.loads(header.get("metadata", "{}")),
|
| 233 |
+
"state_dict": load_file(str(path), device="cpu"),
|
| 234 |
+
}
|
| 235 |
+
return torch.load(path, map_location="cpu", weights_only=False, mmap=True)
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def load_model(paths: RuntimePaths, device: str = "cuda") -> CorridorKeyV3:
|
| 239 |
+
paths.validate()
|
| 240 |
+
prepare_runtime_environment(paths)
|
| 241 |
+
ckpt = load_runtime_checkpoint(paths.weights)
|
| 242 |
+
if ckpt.get("format") != "corridorkey_v2_runtime_checkpoint":
|
| 243 |
+
raise RuntimeError(f"{paths.weights} is not a CorridorKey v2 runtime checkpoint")
|
| 244 |
+
cfg = dict(ckpt["model_config"])
|
| 245 |
+
cfg["cradio_repo"] = str(paths.cradio_dir)
|
| 246 |
+
cfg["moge_checkpoint"] = str(paths.moge_checkpoint)
|
| 247 |
+
cfg["rvm_ckpt"] = str(paths.rvm_ckpt)
|
| 248 |
+
cfg["foundation_stats_path"] = str(paths.foundation_stats)
|
| 249 |
+
cfg["gradient_checkpoint"] = False
|
| 250 |
+
cfg["device"] = device
|
| 251 |
+
cfg["stage1_recurrent_64_sites"] = ()
|
| 252 |
+
|
| 253 |
+
print(f"Loading CorridorKey v2 runtime weights: {paths.weights}")
|
| 254 |
+
model = CorridorKeyV3(**cfg)
|
| 255 |
+
missing, unexpected = model.load_state_dict(ckpt["state_dict"], strict=False)
|
| 256 |
+
train_missing = [k for k in missing if not k.startswith("foundations.")]
|
| 257 |
+
train_unexpected = [k for k in unexpected if not k.startswith("foundations.")]
|
| 258 |
+
if train_missing or train_unexpected:
|
| 259 |
+
raise RuntimeError(
|
| 260 |
+
"runtime checkpoint did not match model: "
|
| 261 |
+
f"missing={train_missing[:8]} unexpected={train_unexpected[:8]}")
|
| 262 |
+
print(f" loaded weights; omitted frozen foundation tensors={len(missing)}")
|
| 263 |
+
model.eval()
|
| 264 |
+
del ckpt
|
| 265 |
+
return model
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def configure_low_vram_inference(model, cfg: InferenceConfig):
|
| 269 |
+
model.stage1.low_vram_enc_frame_chunk = max(0, int(cfg.low_vram_enc_frame_chunk))
|
| 270 |
+
for module in model.modules():
|
| 271 |
+
name = module.__class__.__name__
|
| 272 |
+
if name in ("SwiGLU", "FusionLinSwiglu"):
|
| 273 |
+
module.inference_chunk_tokens = max(0, int(cfg.low_vram_mlp_chunk_tokens))
|
| 274 |
+
if name == "SwinAttention3D":
|
| 275 |
+
module.inference_window_batch = max(0, int(cfg.low_vram_swin_window_batch))
|
| 276 |
+
if name == "CSwinAttention3D":
|
| 277 |
+
module.inference_strip_batch = max(0, int(cfg.low_vram_swin_window_batch))
|
| 278 |
+
if name in ("FullSpatialAttention2D", "FullAttention3D"):
|
| 279 |
+
module.inference_query_chunk_tokens = max(
|
| 280 |
+
0, int(cfg.low_vram_full_attn_query_chunk_tokens))
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
def _build_foundation_inputs(rgb_bt: torch.Tensor):
|
| 284 |
+
B, T, _, H, W = rgb_bt.shape
|
| 285 |
+
flat = rgb_bt.reshape(B * T, 3, H, W)
|
| 286 |
+
rgb_radio = F.interpolate(flat, size=(H // 4, W // 4), mode="area")
|
| 287 |
+
H_moge, W_moge = (H * 7) // 32, (W * 7) // 32
|
| 288 |
+
rgb_moge = F.interpolate(flat, size=(H_moge, W_moge), mode="area")
|
| 289 |
+
return (
|
| 290 |
+
rgb_radio.reshape(B, T, 3, H // 4, W // 4),
|
| 291 |
+
rgb_moge.reshape(B, T, 3, H_moge, W_moge),
|
| 292 |
+
)
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def _next_or_none(iterator):
|
| 296 |
+
try:
|
| 297 |
+
return next(iterator)
|
| 298 |
+
except StopIteration:
|
| 299 |
+
return None
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
@torch.no_grad()
|
| 303 |
+
def precompute_rvm_to_memmap(model, source, meta, rvm_path: Path,
|
| 304 |
+
*, chunk: int, reader_kind: str):
|
| 305 |
+
assert model.foundations.rvm is not None
|
| 306 |
+
device = next(model.foundations.rvm.parameters()).device
|
| 307 |
+
T_all = int(meta["num_frames"])
|
| 308 |
+
H, W = meta["shape_hw"]
|
| 309 |
+
H_e, W_e = H // 4, W // 4
|
| 310 |
+
D = model.foundations.rvm.embed_dim
|
| 311 |
+
h_e, w_e = H_e // 16, W_e // 16
|
| 312 |
+
print(f" RVM continuous prepass: T={T_all}, chunk={chunk}, "
|
| 313 |
+
f"input={W_e}x{H_e}, grid={w_e}x{h_e}")
|
| 314 |
+
out = np.memmap(rvm_path, dtype=np.float16, mode="w+",
|
| 315 |
+
shape=(T_all, D, h_e, w_e))
|
| 316 |
+
mean = model.foundations.rvm_mean.view(1, 1, -1, 1, 1).float().cpu()
|
| 317 |
+
std = model.foundations.rvm_std.view(1, 1, -1, 1, 1).float().cpu().clamp(min=1e-6)
|
| 318 |
+
if reader_kind == "video":
|
| 319 |
+
chunks = iter_video_frame_chunks(str(source), meta, chunk_size=chunk, need_model=False)
|
| 320 |
+
else:
|
| 321 |
+
chunks = iter_frame_dir_chunks(str(source), meta, chunk_size=chunk, need_model=False)
|
| 322 |
+
state = None
|
| 323 |
+
with ThreadPoolExecutor(max_workers=1) as pool:
|
| 324 |
+
future = pool.submit(_next_or_none, chunks)
|
| 325 |
+
while True:
|
| 326 |
+
item = future.result()
|
| 327 |
+
if item is None:
|
| 328 |
+
break
|
| 329 |
+
future = pool.submit(_next_or_none, chunks)
|
| 330 |
+
t0, _model, display = item
|
| 331 |
+
Tc = display.shape[0]
|
| 332 |
+
rgb = display.unsqueeze(0).to(device, non_blocking=True).float()
|
| 333 |
+
rgb_rvm = F.interpolate(
|
| 334 |
+
rgb.reshape(Tc, 3, H, W), size=(H_e, W_e),
|
| 335 |
+
mode="bilinear", align_corners=False,
|
| 336 |
+
).reshape(1, Tc, 3, H_e, W_e).clamp(0.0, 1.0)
|
| 337 |
+
with torch.autocast("cuda", dtype=torch.bfloat16):
|
| 338 |
+
feats, state = model.foundations.rvm(rgb_rvm, state=state)
|
| 339 |
+
feats_cpu = (feats.float().cpu() - mean) / std
|
| 340 |
+
out[t0:t0 + Tc] = feats_cpu[0].numpy().astype(np.float16)
|
| 341 |
+
del rgb, rgb_rvm, feats, feats_cpu, display
|
| 342 |
+
out.flush()
|
| 343 |
+
del out
|
| 344 |
+
torch.cuda.empty_cache()
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
@torch.no_grad()
|
| 348 |
+
def precompute_foundations_to_memmap(model, source, meta, prefix: Path,
|
| 349 |
+
*, chunk: int, reader_kind: str):
|
| 350 |
+
device = next(model.foundations.parameters()).device
|
| 351 |
+
T_all = int(meta["num_frames"])
|
| 352 |
+
H, W = meta["shape_hw"]
|
| 353 |
+
h64, w64 = H // 64, W // 64
|
| 354 |
+
C_cradio = int(getattr(model.stage1, "cradio_dim", 0) or 0)
|
| 355 |
+
C_moge = int(getattr(model.stage1, "moge_dim", 0) or 0)
|
| 356 |
+
if C_moge <= 0:
|
| 357 |
+
raise RuntimeError("MoGe is required for this model")
|
| 358 |
+
paths = {
|
| 359 |
+
"moge": str(prefix) + "_moge.f32",
|
| 360 |
+
"moge_shape": (T_all, C_moge, h64, w64),
|
| 361 |
+
}
|
| 362 |
+
moge_out = np.memmap(paths["moge"], dtype=np.float32, mode="w+",
|
| 363 |
+
shape=paths["moge_shape"])
|
| 364 |
+
cradio_out = None
|
| 365 |
+
if C_cradio > 0:
|
| 366 |
+
paths["cradio"] = str(prefix) + "_cradio.f32"
|
| 367 |
+
paths["cradio_shape"] = (T_all, C_cradio, h64, w64)
|
| 368 |
+
cradio_out = np.memmap(paths["cradio"], dtype=np.float32, mode="w+",
|
| 369 |
+
shape=paths["cradio_shape"])
|
| 370 |
+
print(f" frozen VFM prepass: T={T_all}, grid={w64}x{h64}, chunk={chunk}")
|
| 371 |
+
if reader_kind == "video":
|
| 372 |
+
chunks = iter_video_frame_chunks(str(source), meta, chunk_size=chunk, need_model=False)
|
| 373 |
+
else:
|
| 374 |
+
chunks = iter_frame_dir_chunks(str(source), meta, chunk_size=chunk, need_model=False)
|
| 375 |
+
with ThreadPoolExecutor(max_workers=1) as pool:
|
| 376 |
+
future = pool.submit(_next_or_none, chunks)
|
| 377 |
+
while True:
|
| 378 |
+
item = future.result()
|
| 379 |
+
if item is None:
|
| 380 |
+
break
|
| 381 |
+
future = pool.submit(_next_or_none, chunks)
|
| 382 |
+
t0, _model, display = item
|
| 383 |
+
Tc = display.shape[0]
|
| 384 |
+
rgb = display.unsqueeze(0).to(device, non_blocking=True)
|
| 385 |
+
rgb_radio, rgb_moge = _build_foundation_inputs(rgb)
|
| 386 |
+
with torch.autocast("cuda", dtype=torch.bfloat16):
|
| 387 |
+
ff = model.foundations(
|
| 388 |
+
rgb_cradio=rgb_radio.reshape(Tc, 3, *rgb_radio.shape[-2:]),
|
| 389 |
+
rgb_moge=rgb_moge.reshape(Tc, 3, *rgb_moge.shape[-2:]),
|
| 390 |
+
)
|
| 391 |
+
moge_out[t0:t0 + Tc] = ff["moge"].float().cpu().numpy()
|
| 392 |
+
if cradio_out is not None:
|
| 393 |
+
cradio_out[t0:t0 + Tc] = ff["cradio"].float().cpu().numpy()
|
| 394 |
+
del rgb, rgb_radio, rgb_moge, ff, display
|
| 395 |
+
moge_out.flush()
|
| 396 |
+
del moge_out
|
| 397 |
+
if cradio_out is not None:
|
| 398 |
+
cradio_out.flush()
|
| 399 |
+
del cradio_out
|
| 400 |
+
torch.cuda.empty_cache()
|
| 401 |
+
return paths
|
| 402 |
+
|
| 403 |
+
|
| 404 |
+
def _phase1_setup(model, *, stage1_on_gpu: bool = True):
|
| 405 |
+
model.foundations.cuda()
|
| 406 |
+
# Low-VRAM mode precomputes foundation features first, so stage 1 waits on CPU.
|
| 407 |
+
if stage1_on_gpu:
|
| 408 |
+
model.stage1.cuda()
|
| 409 |
+
else:
|
| 410 |
+
model.stage1.cpu()
|
| 411 |
+
model.stage2.cpu()
|
| 412 |
+
torch.cuda.empty_cache()
|
| 413 |
+
|
| 414 |
+
|
| 415 |
+
def _phase2_setup(model):
|
| 416 |
+
model.foundations.cpu()
|
| 417 |
+
model.stage1.cpu()
|
| 418 |
+
torch.cuda.empty_cache()
|
| 419 |
+
model.stage2.cuda()
|
| 420 |
+
torch.cuda.synchronize()
|
| 421 |
+
|
| 422 |
+
|
| 423 |
+
@torch.no_grad()
|
| 424 |
+
def hann_stage1_stream(model, source, meta, *, reader_cls, reader_kind: str,
|
| 425 |
+
chunk: int, stride: int, baton_path: Path,
|
| 426 |
+
initial_hint, initial_hint_quality: float,
|
| 427 |
+
full_hint_video_path: str | None,
|
| 428 |
+
full_hint_quality: float,
|
| 429 |
+
rvm_path: Path | None,
|
| 430 |
+
foundation_paths,
|
| 431 |
+
carry_hint_from_aux: bool,
|
| 432 |
+
carry_hint_quality: float,
|
| 433 |
+
on_baton_frames):
|
| 434 |
+
device = next(model.stage1.parameters()).device
|
| 435 |
+
T_all = int(meta["num_frames"])
|
| 436 |
+
H, W = meta["shape_hw"]
|
| 437 |
+
d_16 = model.stage1.d_16
|
| 438 |
+
h16, w16 = H // 16, W // 16
|
| 439 |
+
starts = window_starts(T_all, chunk, stride)
|
| 440 |
+
overlap = max(0, chunk - stride)
|
| 441 |
+
baton_fp16 = None
|
| 442 |
+
if on_baton_frames is None:
|
| 443 |
+
baton_fp16 = np.memmap(baton_path, dtype=np.float16, mode="w+",
|
| 444 |
+
shape=(T_all, d_16, h16, w16))
|
| 445 |
+
live_acc = {}
|
| 446 |
+
live_w = {}
|
| 447 |
+
print(f" Hann stage1: chunk={chunk}, stride={stride}, "
|
| 448 |
+
f"windows={len(starts)}, baton_grid={w16}x{h16}")
|
| 449 |
+
|
| 450 |
+
rvm_map = None
|
| 451 |
+
if rvm_path is not None:
|
| 452 |
+
D = model.foundations.rvm.embed_dim
|
| 453 |
+
rvm_map = np.memmap(rvm_path, dtype=np.float16, mode="r",
|
| 454 |
+
shape=(T_all, D, H // 64, W // 64))
|
| 455 |
+
cradio_map = moge_map = None
|
| 456 |
+
if foundation_paths is not None:
|
| 457 |
+
moge_map = np.memmap(foundation_paths["moge"], dtype=np.float32,
|
| 458 |
+
mode="r", shape=tuple(foundation_paths["moge_shape"]))
|
| 459 |
+
if model.stage1.cradio_dim > 0:
|
| 460 |
+
cradio_map = np.memmap(foundation_paths["cradio"], dtype=np.float32,
|
| 461 |
+
mode="r", shape=tuple(foundation_paths["cradio_shape"]))
|
| 462 |
+
|
| 463 |
+
reader = reader_cls(str(source), meta)
|
| 464 |
+
hint_reader = make_hint_reader(full_hint_video_path, meta)
|
| 465 |
+
carry_cache = {}
|
| 466 |
+
stage2_frame_cache = {}
|
| 467 |
+
|
| 468 |
+
def flush_before(t_limit: int):
|
| 469 |
+
ready = []
|
| 470 |
+
for k in sorted(k for k in live_acc if k < t_limit):
|
| 471 |
+
frame = (live_acc.pop(k) / max(live_w.pop(k), 1e-6)).astype(np.float16)
|
| 472 |
+
if on_baton_frames is None:
|
| 473 |
+
baton_fp16[k] = frame
|
| 474 |
+
else:
|
| 475 |
+
model_frame = stage2_frame_cache.pop(k)
|
| 476 |
+
ready.append((k, frame, model_frame))
|
| 477 |
+
if ready:
|
| 478 |
+
on_baton_frames(ready)
|
| 479 |
+
|
| 480 |
+
try:
|
| 481 |
+
for i, t0 in enumerate(starts):
|
| 482 |
+
t1 = min(t0 + chunk, T_all)
|
| 483 |
+
Tc = t1 - t0
|
| 484 |
+
model_chunk, display_chunk, review_chunk = reader.window(t0, t1)
|
| 485 |
+
hint_cpu = torch.zeros(1, Tc, 1, H, W, dtype=torch.bfloat16)
|
| 486 |
+
hint_quality_cpu = torch.zeros(1, Tc, dtype=torch.bfloat16)
|
| 487 |
+
hint_present = torch.zeros(1, Tc, dtype=torch.bool)
|
| 488 |
+
if hint_reader is not None:
|
| 489 |
+
hint_cpu[0] = hint_reader.window(t0, t1)
|
| 490 |
+
hint_quality_cpu.fill_(float(full_hint_quality))
|
| 491 |
+
hint_present.fill_(True)
|
| 492 |
+
for j, t_abs in enumerate(range(t0, t1)):
|
| 493 |
+
if t_abs in carry_cache:
|
| 494 |
+
hint_cpu[0, j] = carry_cache[t_abs]
|
| 495 |
+
hint_quality_cpu[0, j] = float(carry_hint_quality)
|
| 496 |
+
hint_present[0, j] = True
|
| 497 |
+
if initial_hint is not None and t0 == 0:
|
| 498 |
+
hint_cpu[0, 0] = initial_hint.to(dtype=hint_cpu.dtype)
|
| 499 |
+
hint_quality_cpu[0, 0] = float(initial_hint_quality)
|
| 500 |
+
hint_present[0, 0] = True
|
| 501 |
+
if not bool(hint_present.any().item()):
|
| 502 |
+
hint_quality_cpu.fill_(-1.0)
|
| 503 |
+
|
| 504 |
+
if on_baton_frames is not None:
|
| 505 |
+
for j, t_abs in enumerate(range(t0, t1)):
|
| 506 |
+
stage2_frame_cache[t_abs] = model_chunk[j]
|
| 507 |
+
|
| 508 |
+
rgb_c = model_chunk.unsqueeze(0).to(device, non_blocking=True)
|
| 509 |
+
hint_c = hint_cpu.to(device, non_blocking=True)
|
| 510 |
+
hint_quality_c = hint_quality_cpu.to(device, non_blocking=True)
|
| 511 |
+
ff = rgb_radio = rgb_moge = None
|
| 512 |
+
if foundation_paths is None:
|
| 513 |
+
rgb_found = display_chunk.unsqueeze(0).to(device, non_blocking=True)
|
| 514 |
+
rgb_radio, rgb_moge = _build_foundation_inputs(rgb_found)
|
| 515 |
+
with torch.autocast("cuda", dtype=torch.bfloat16):
|
| 516 |
+
ff = model.foundations(
|
| 517 |
+
rgb_cradio=rgb_radio.reshape(Tc, 3, *rgb_radio.shape[-2:]),
|
| 518 |
+
rgb_moge=rgb_moge.reshape(Tc, 3, *rgb_moge.shape[-2:]),
|
| 519 |
+
)
|
| 520 |
+
cradio_feat = ff["cradio"].reshape(1, Tc, -1, H // 64, W // 64)
|
| 521 |
+
moge_feat = ff["moge"].reshape(1, Tc, -1, H // 64, W // 64)
|
| 522 |
+
else:
|
| 523 |
+
moge_feat = torch.from_numpy(np.asarray(moge_map[t0:t1]).copy())
|
| 524 |
+
moge_feat = moge_feat.unsqueeze(0).to(device, non_blocking=True).float()
|
| 525 |
+
cradio_feat = torch.from_numpy(np.asarray(cradio_map[t0:t1]).copy())
|
| 526 |
+
cradio_feat = cradio_feat.unsqueeze(0).to(device, non_blocking=True).float()
|
| 527 |
+
rvm_feat = None
|
| 528 |
+
if rvm_map is not None:
|
| 529 |
+
rvm_feat = torch.from_numpy(np.asarray(rvm_map[t0:t1]).copy())
|
| 530 |
+
rvm_feat = rvm_feat.unsqueeze(0).to(device, non_blocking=True).float()
|
| 531 |
+
with torch.autocast("cuda", dtype=torch.bfloat16):
|
| 532 |
+
s1 = model.stage1(
|
| 533 |
+
rgb_c, hint_c, moge_feat, cradio_feat=cradio_feat,
|
| 534 |
+
rvm_feat=rvm_feat, t_stride=1.0,
|
| 535 |
+
hint_quality=hint_quality_c,
|
| 536 |
+
)
|
| 537 |
+
baton_gpu = s1["baton"][0].float()
|
| 538 |
+
if carry_hint_from_aux:
|
| 539 |
+
aux_alpha = s1["aux"][0, :, 0:1].float()
|
| 540 |
+
aux_full = F.interpolate(
|
| 541 |
+
aux_alpha, size=(H, W), mode="bilinear",
|
| 542 |
+
align_corners=False).clamp(0.0, 1.0)
|
| 543 |
+
aux_cpu = aux_full.to("cpu", dtype=torch.bfloat16)
|
| 544 |
+
for j, t_abs in enumerate(range(t0, t1)):
|
| 545 |
+
carry_cache[t_abs] = aux_cpu[j].clone()
|
| 546 |
+
next_start = starts[i + 1] if i + 1 < len(starts) else T_all
|
| 547 |
+
for old_t in [k for k in carry_cache if k < next_start]:
|
| 548 |
+
del carry_cache[old_t]
|
| 549 |
+
del aux_alpha, aux_full, aux_cpu
|
| 550 |
+
w = taper_weights(Tc, overlap, is_first=(i == 0),
|
| 551 |
+
is_last=(i == len(starts) - 1),
|
| 552 |
+
device="cpu", dtype=torch.float32)
|
| 553 |
+
weighted = (baton_gpu * w.to(device).view(Tc, 1, 1, 1)).to("cpu").numpy()
|
| 554 |
+
w_np = w.numpy()
|
| 555 |
+
for j, t_abs in enumerate(range(t0, t1)):
|
| 556 |
+
if t_abs not in live_acc:
|
| 557 |
+
live_acc[t_abs] = weighted[j].copy()
|
| 558 |
+
live_w[t_abs] = float(w_np[j])
|
| 559 |
+
else:
|
| 560 |
+
live_acc[t_abs] += weighted[j]
|
| 561 |
+
live_w[t_abs] += float(w_np[j])
|
| 562 |
+
next_start = starts[i + 1] if i + 1 < len(starts) else T_all
|
| 563 |
+
del s1, baton_gpu, weighted, ff, cradio_feat, moge_feat, rvm_feat
|
| 564 |
+
del rgb_c, hint_c, hint_quality_c, hint_cpu, hint_quality_cpu
|
| 565 |
+
del model_chunk, display_chunk, review_chunk
|
| 566 |
+
flush_before(next_start)
|
| 567 |
+
print(f" window {i + 1}/{len(starts)} [{t0}:{t1}]", flush=True)
|
| 568 |
+
torch.cuda.synchronize()
|
| 569 |
+
flush_before(T_all + 1)
|
| 570 |
+
finally:
|
| 571 |
+
reader.close()
|
| 572 |
+
if hint_reader is not None:
|
| 573 |
+
hint_reader.close()
|
| 574 |
+
|
| 575 |
+
if baton_fp16 is not None:
|
| 576 |
+
baton_fp16.flush()
|
| 577 |
+
del baton_fp16, live_acc, live_w, rvm_map, cradio_map, moge_map
|
| 578 |
+
torch.cuda.empty_cache()
|
| 579 |
+
|
| 580 |
+
|
| 581 |
+
def _crop_native_torch(x: torch.Tensor, meta: dict) -> torch.Tensor:
|
| 582 |
+
h, w = meta["model_native_hw"]
|
| 583 |
+
pad_l, _, pad_t, _ = meta["pad"]
|
| 584 |
+
return x[..., pad_t:pad_t + h, pad_l:pad_l + w]
|
| 585 |
+
|
| 586 |
+
|
| 587 |
+
def _resize_output_torch(x: torch.Tensor, meta: dict) -> torch.Tensor:
|
| 588 |
+
if tuple(meta["model_native_hw"]) == tuple(meta["output_hw"]):
|
| 589 |
+
return x
|
| 590 |
+
raise RuntimeError(
|
| 591 |
+
"optimized runtime render path requires native model_scale=1.0; "
|
| 592 |
+
"non-native scaling would need explicit Lanczos output rendering")
|
| 593 |
+
|
| 594 |
+
|
| 595 |
+
def _linear_adjust_inverse_torch(linear: torch.Tensor, meta: dict) -> torch.Tensor:
|
| 596 |
+
gain = float(meta.get("linear_brightness", 1.0))
|
| 597 |
+
contrast = float(meta.get("linear_contrast", 1.0))
|
| 598 |
+
if abs(gain - 1.0) <= 1e-6 and abs(contrast - 1.0) <= 1e-6:
|
| 599 |
+
return linear
|
| 600 |
+
pivot = float(meta.get("linear_contrast_pivot", 0.18))
|
| 601 |
+
return (linear / gain - pivot) / contrast + pivot
|
| 602 |
+
|
| 603 |
+
|
| 604 |
+
def _despill_average_srgb_torch(fg: torch.Tensor, strength: float = 0.5) -> torch.Tensor:
|
| 605 |
+
if strength <= 0.0:
|
| 606 |
+
return fg
|
| 607 |
+
r = fg[:, 0:1]
|
| 608 |
+
g = fg[:, 1:2]
|
| 609 |
+
b = fg[:, 2:3]
|
| 610 |
+
spill = (g - (r + b) * 0.5).clamp(min=0.0)
|
| 611 |
+
full = torch.cat([r + spill * 0.5, g - spill, b + spill * 0.5], dim=1)
|
| 612 |
+
if strength >= 1.0:
|
| 613 |
+
return full
|
| 614 |
+
return fg * (1.0 - strength) + full * strength
|
| 615 |
+
|
| 616 |
+
|
| 617 |
+
def _srgb_to_bgr_uint16_cpu(x: torch.Tensor) -> np.ndarray:
|
| 618 |
+
x = x.clamp(0.0, 1.0)
|
| 619 |
+
x = x[:, [2, 1, 0]].permute(0, 2, 3, 1).contiguous().cpu().numpy()
|
| 620 |
+
return (x * 65535.0).astype(np.uint16)
|
| 621 |
+
|
| 622 |
+
|
| 623 |
+
def _srgb_to_bgra_uint16_cpu(rgb: torch.Tensor, alpha: torch.Tensor) -> np.ndarray:
|
| 624 |
+
rgb = rgb.clamp(0.0, 1.0)
|
| 625 |
+
alpha = alpha.clamp(0.0, 1.0)
|
| 626 |
+
bgra = torch.cat([rgb[:, [2, 1, 0]], alpha], dim=1)
|
| 627 |
+
bgra = bgra.permute(0, 2, 3, 1).contiguous().cpu().numpy()
|
| 628 |
+
return (bgra * 65535.0).astype(np.uint16)
|
| 629 |
+
|
| 630 |
+
|
| 631 |
+
def _chw_to_bgr_f32_cpu(x: torch.Tensor) -> np.ndarray:
|
| 632 |
+
return (x[:, [2, 1, 0]].permute(0, 2, 3, 1)
|
| 633 |
+
.contiguous().cpu().numpy().astype(np.float32, copy=False))
|
| 634 |
+
|
| 635 |
+
|
| 636 |
+
def _chw_to_bgra_f32_cpu(rgb: torch.Tensor, alpha: torch.Tensor) -> np.ndarray:
|
| 637 |
+
bgra = torch.cat([rgb[:, [2, 1, 0]], alpha], dim=1)
|
| 638 |
+
return (bgra.permute(0, 2, 3, 1)
|
| 639 |
+
.contiguous().cpu().numpy().astype(np.float32, copy=False))
|
| 640 |
+
|
| 641 |
+
|
| 642 |
+
def _render_predictions_exr_torch(meta: dict, outputs: set[str],
|
| 643 |
+
checker: torch.Tensor,
|
| 644 |
+
logits: torch.Tensor) -> dict[str, np.ndarray]:
|
| 645 |
+
"""Render a stage-2 batch as unclamped scene-linear float32 NHWC arrays.
|
| 646 |
+
|
| 647 |
+
HDR contract: `fg`, `cutout`, and `checker` color stays in decoded linear
|
| 648 |
+
light with no [0, 1] clamp and no quantization, so values above 1.0 (and
|
| 649 |
+
negative values the model may emit) survive into the EXR files. Alpha is a
|
| 650 |
+
matte and stays clamped to [0, 1].
|
| 651 |
+
"""
|
| 652 |
+
alpha_model = logits[:, 0:1].float()
|
| 653 |
+
pa_model = _crop_native_torch(alpha_model, meta).clamp(0.0, 1.0)
|
| 654 |
+
despill_strength = float(meta.get("despill_strength", 0.5))
|
| 655 |
+
|
| 656 |
+
pa = _resize_output_torch(pa_model, meta).clamp(0.0, 1.0)
|
| 657 |
+
rendered: dict[str, np.ndarray] = {}
|
| 658 |
+
if "alpha" in outputs:
|
| 659 |
+
rendered["alpha"] = (pa.permute(0, 2, 3, 1)
|
| 660 |
+
.contiguous().cpu().numpy()
|
| 661 |
+
.astype(np.float32, copy=False))
|
| 662 |
+
if not (outputs & {"fg", "cutout", "checker"}):
|
| 663 |
+
return rendered
|
| 664 |
+
|
| 665 |
+
fg_model = logits[:, 1:4].float()
|
| 666 |
+
pf_model = _crop_native_torch(fg_model, meta)
|
| 667 |
+
pf_linear = _linear_adjust_inverse_torch(cu.asinh_to_linear(pf_model), meta)
|
| 668 |
+
pf_linear = _resize_output_torch(pf_linear, meta)
|
| 669 |
+
if "fg" in outputs:
|
| 670 |
+
rendered["fg"] = _chw_to_bgr_f32_cpu(pf_linear)
|
| 671 |
+
if "cutout" in outputs:
|
| 672 |
+
pf_linear_despilled = _despill_average_srgb_torch(
|
| 673 |
+
pf_linear, strength=despill_strength)
|
| 674 |
+
rendered["cutout"] = _chw_to_bgra_f32_cpu(pf_linear_despilled, pa)
|
| 675 |
+
if "checker" in outputs:
|
| 676 |
+
checker_b = checker.to(device=pf_linear.device,
|
| 677 |
+
dtype=pf_linear.dtype).unsqueeze(0)
|
| 678 |
+
bg_lin = cu.srgb_to_linear(checker_b)
|
| 679 |
+
pf_despill = _despill_average_srgb_torch(
|
| 680 |
+
pf_linear, strength=despill_strength)
|
| 681 |
+
comp_lin = pf_despill * pa + bg_lin * (1.0 - pa)
|
| 682 |
+
rendered["checker"] = _chw_to_bgr_f32_cpu(comp_lin)
|
| 683 |
+
return rendered
|
| 684 |
+
|
| 685 |
+
|
| 686 |
+
def _render_predictions_torch(meta: dict, outputs: set[str],
|
| 687 |
+
checker: torch.Tensor,
|
| 688 |
+
logits: torch.Tensor) -> dict[str, np.ndarray]:
|
| 689 |
+
"""Render a stage-2 batch on GPU, returning uint16 BGR/BGRA NHWC arrays.
|
| 690 |
+
|
| 691 |
+
This removes the previous per-frame numpy HDR decode, sRGB encode,
|
| 692 |
+
composition, channel shuffle, and uint8 conversion hot path.
|
| 693 |
+
"""
|
| 694 |
+
alpha_model = logits[:, 0:1].float()
|
| 695 |
+
pa_model = _crop_native_torch(alpha_model, meta).clamp(0.0, 1.0)
|
| 696 |
+
despill_strength = float(meta.get("despill_strength", 0.5))
|
| 697 |
+
|
| 698 |
+
pa = _resize_output_torch(pa_model, meta).clamp(0.0, 1.0)
|
| 699 |
+
rendered: dict[str, np.ndarray] = {}
|
| 700 |
+
if "alpha" in outputs:
|
| 701 |
+
rendered["alpha"] = _srgb_to_bgr_uint16_cpu(pa.expand(-1, 3, -1, -1))
|
| 702 |
+
if not (outputs & {"fg", "cutout", "checker"}):
|
| 703 |
+
return rendered
|
| 704 |
+
|
| 705 |
+
fg_model = logits[:, 1:4].float()
|
| 706 |
+
pf_model = _crop_native_torch(fg_model, meta)
|
| 707 |
+
pf_linear = _linear_adjust_inverse_torch(cu.asinh_to_linear(pf_model), meta)
|
| 708 |
+
pf_linear_pos = pf_linear.clamp(min=0.0)
|
| 709 |
+
pf = _resize_output_torch(cu.linear_to_srgb(pf_linear_pos), meta).clamp(0.0, 1.0)
|
| 710 |
+
if "fg" in outputs:
|
| 711 |
+
rendered["fg"] = _srgb_to_bgr_uint16_cpu(pf)
|
| 712 |
+
if "cutout" in outputs:
|
| 713 |
+
pf_linear_despilled = _despill_average_srgb_torch(pf_linear_pos, strength=despill_strength)
|
| 714 |
+
if meta.get("cutout_linear", False):
|
| 715 |
+
pf_linear_despilled_resized = _resize_output_torch(pf_linear_despilled, meta).clamp(0.0, 1.0)
|
| 716 |
+
rendered["cutout"] = _srgb_to_bgra_uint16_cpu(pf_linear_despilled_resized, pa)
|
| 717 |
+
else:
|
| 718 |
+
pf_despill = _despill_average_srgb_torch(pf, strength=despill_strength)
|
| 719 |
+
rendered["cutout"] = _srgb_to_bgra_uint16_cpu(pf_despill, pa)
|
| 720 |
+
if "checker" in outputs:
|
| 721 |
+
checker_b = checker.to(device=pf.device, dtype=pf.dtype).unsqueeze(0)
|
| 722 |
+
bg_lin = cu.srgb_to_linear(checker_b)
|
| 723 |
+
alpha = pa
|
| 724 |
+
pf_despill = _despill_average_srgb_torch(pf, strength=despill_strength)
|
| 725 |
+
comp_despill_lin = (
|
| 726 |
+
cu.srgb_to_linear(pf_despill) * alpha + bg_lin * (1.0 - alpha))
|
| 727 |
+
comp_despill = cu.linear_to_srgb(
|
| 728 |
+
comp_despill_lin.clamp(min=0.0)).clamp(0.0, 1.0)
|
| 729 |
+
rendered["checker"] = _srgb_to_bgr_uint16_cpu(comp_despill)
|
| 730 |
+
return rendered
|
| 731 |
+
|
| 732 |
+
|
| 733 |
+
def run_inference(model, input_path: str, output_dir: str, cfg: InferenceConfig,
|
| 734 |
+
writer_cfg: WriterConfig, *, initial_hint_path: str | None,
|
| 735 |
+
hint_video_path: str | None):
|
| 736 |
+
input_path = str(input_path)
|
| 737 |
+
output_dir = str(output_dir)
|
| 738 |
+
cfg.temp_dir.mkdir(parents=True, exist_ok=True)
|
| 739 |
+
os.makedirs(output_dir, exist_ok=True)
|
| 740 |
+
is_dir = os.path.isdir(input_path)
|
| 741 |
+
reader_kind = "frame_dir" if is_dir else "video"
|
| 742 |
+
reader_cls = SequentialFrameDirWindowReader if is_dir else SequentialVideoWindowReader
|
| 743 |
+
if is_dir:
|
| 744 |
+
meta = probe_frame_dir_meta(
|
| 745 |
+
input_path, num_frames=cfg.num_frames,
|
| 746 |
+
shape_quantum=cfg.shape_quantum, fps=cfg.frame_dir_fps,
|
| 747 |
+
exr_linear=cfg.frame_dir_linear, model_scale=cfg.model_scale)
|
| 748 |
+
if (not cfg.frame_dir_linear
|
| 749 |
+
and meta["frame_paths"]
|
| 750 |
+
and meta["frame_paths"][0].lower().endswith(".exr")):
|
| 751 |
+
print(" [WARNING] EXR input frames are being treated as "
|
| 752 |
+
"display-coded and clipped to [0, 1]. For scene-linear HDR "
|
| 753 |
+
"EXR plates, pass --frame_dir_linear.")
|
| 754 |
+
else:
|
| 755 |
+
meta = probe_video_meta(
|
| 756 |
+
input_path, num_frames=cfg.num_frames,
|
| 757 |
+
shape_quantum=cfg.shape_quantum, native_resolution=True,
|
| 758 |
+
start_mode=cfg.start_mode, seed=cfg.seed,
|
| 759 |
+
model_scale=cfg.model_scale,
|
| 760 |
+
linear_brightness=cfg.linear_brightness,
|
| 761 |
+
linear_contrast=cfg.linear_contrast,
|
| 762 |
+
linear_contrast_pivot=cfg.linear_contrast_pivot)
|
| 763 |
+
T_all = int(meta["num_frames"])
|
| 764 |
+
H, W = meta["shape_hw"]
|
| 765 |
+
if cfg.auto_vram:
|
| 766 |
+
apply_auto_vram(cfg, (H, W), T_all)
|
| 767 |
+
H_out, W_out = meta["output_hw"]
|
| 768 |
+
meta["cutout_linear"] = cfg.cutout_linear
|
| 769 |
+
meta["despill_strength"] = cfg.despill_strength
|
| 770 |
+
fps = float(meta["native_fps"])
|
| 771 |
+
clip_name = raw_clip_name(input_path)
|
| 772 |
+
checker = make_checker(H_out, W_out)
|
| 773 |
+
checker_t = torch.from_numpy(checker)
|
| 774 |
+
writers = {}
|
| 775 |
+
suffix = {
|
| 776 |
+
"alpha": "alpha",
|
| 777 |
+
"fg": "fg",
|
| 778 |
+
"checker": f"checker_comp_despill{int(cfg.despill_strength * 100):03d}",
|
| 779 |
+
"cutout": "cutout",
|
| 780 |
+
}
|
| 781 |
+
try:
|
| 782 |
+
use_exr = cfg.output_format == "exr"
|
| 783 |
+
if cfg.output_format not in ("video", "exr"):
|
| 784 |
+
raise ValueError(f"unsupported output format: {cfg.output_format}")
|
| 785 |
+
for name in sorted(cfg.outputs):
|
| 786 |
+
if name not in suffix:
|
| 787 |
+
raise ValueError(f"unsupported output choice: {name}")
|
| 788 |
+
if use_exr:
|
| 789 |
+
channels = 4 if name == "cutout" else (1 if name == "alpha" else 3)
|
| 790 |
+
writers[name] = ExrSequenceWriter(
|
| 791 |
+
os.path.join(output_dir, f"{clip_name}_{suffix[name]}_exr"),
|
| 792 |
+
fps, (W_out, H_out), channels=channels,
|
| 793 |
+
dtype=cfg.exr_dtype, frame_name=suffix[name])
|
| 794 |
+
else:
|
| 795 |
+
ext = ".mov" if name == "cutout" else ".mp4"
|
| 796 |
+
channels = 4 if name == "cutout" else 3
|
| 797 |
+
color_trc = "linear" if (name == "cutout" and cfg.cutout_linear) else "iec61966-2-1"
|
| 798 |
+
writers[name] = ThreadedVideoWriter(
|
| 799 |
+
os.path.join(output_dir, f"{clip_name}_{suffix[name]}{ext}"),
|
| 800 |
+
fps, (W_out, H_out), writer_cfg, channels=channels, color_trc=color_trc)
|
| 801 |
+
|
| 802 |
+
# Write settings log file in the output folder
|
| 803 |
+
settings_path = os.path.join(output_dir, f"{clip_name}_settings.txt")
|
| 804 |
+
try:
|
| 805 |
+
with open(settings_path, "w") as sf:
|
| 806 |
+
sf.write(f"CorridorKey v2 Inference Settings Log\n")
|
| 807 |
+
sf.write(f"====================================\n")
|
| 808 |
+
sf.write(f"Input: {input_path}\n")
|
| 809 |
+
sf.write(f"Output Directory: {output_dir}\n")
|
| 810 |
+
sf.write(f"Resolution: {W_out}x{H_out}\n")
|
| 811 |
+
sf.write(f"FPS: {fps}\n")
|
| 812 |
+
sf.write(f"Outputs: {', '.join(sorted(cfg.outputs))}\n")
|
| 813 |
+
sf.write(f"Output Format: {cfg.output_format}\n")
|
| 814 |
+
if cfg.output_format == "exr":
|
| 815 |
+
sf.write(f"EXR Pixel Type: {cfg.exr_dtype}\n")
|
| 816 |
+
sf.write(f"Hint First Frame: {initial_hint_path}\n")
|
| 817 |
+
sf.write(f"Hint Video: {hint_video_path}\n")
|
| 818 |
+
sf.write(f"Hint Quality (Trust): {cfg.hint_quality}\n")
|
| 819 |
+
sf.write(f"Carry Hint: {cfg.carry_hint}\n")
|
| 820 |
+
sf.write(f"Carry Hint Quality: {cfg.carry_hint_quality}\n")
|
| 821 |
+
sf.write(f"Despill Strength: {cfg.despill_strength}\n")
|
| 822 |
+
sf.write(f"Cutout Linear: {cfg.cutout_linear}\n")
|
| 823 |
+
sf.write(f"Frame Dir Linear: {cfg.frame_dir_linear}\n")
|
| 824 |
+
sf.write(f"Hann Chunk: {cfg.hann_chunk}\n")
|
| 825 |
+
sf.write(f"Hann Stride: {cfg.hann_stride}\n")
|
| 826 |
+
sf.write(f"Low VRAM: {cfg.low_vram}\n")
|
| 827 |
+
sf.write(f"Model Scale: {cfg.model_scale}\n")
|
| 828 |
+
sf.write(f"Num Frames: {cfg.num_frames}\n")
|
| 829 |
+
sf.write(f"Start Mode: {cfg.start_mode}\n")
|
| 830 |
+
sf.write(f"Seed: {cfg.seed}\n")
|
| 831 |
+
sf.write(f"Linear Brightness: {cfg.linear_brightness}\n")
|
| 832 |
+
sf.write(f"Linear Contrast: {cfg.linear_contrast}\n")
|
| 833 |
+
sf.write(f"Linear Contrast Pivot: {cfg.linear_contrast_pivot}\n")
|
| 834 |
+
except Exception as e:
|
| 835 |
+
print(f"[WARNING] Failed to write settings file: {e}")
|
| 836 |
+
|
| 837 |
+
initial_hint = (
|
| 838 |
+
load_initial_hint_frame(initial_hint_path, meta)
|
| 839 |
+
if none_path(initial_hint_path) is not None else None
|
| 840 |
+
)
|
| 841 |
+
_phase1_setup(model, stage1_on_gpu=not cfg.low_vram)
|
| 842 |
+
if cfg.low_vram:
|
| 843 |
+
configure_low_vram_inference(model, cfg)
|
| 844 |
+
rvm_path = cfg.temp_dir / f"{clip_name}_rvm_feat.f16"
|
| 845 |
+
baton_path = cfg.temp_dir / f"{clip_name}_baton.f16"
|
| 846 |
+
foundation_prefix = cfg.temp_dir / f"{clip_name}_foundation"
|
| 847 |
+
for p in (
|
| 848 |
+
rvm_path, baton_path, Path(str(foundation_prefix) + "_cradio.f32"),
|
| 849 |
+
Path(str(foundation_prefix) + "_moge.f32"),
|
| 850 |
+
):
|
| 851 |
+
safe_remove(p)
|
| 852 |
+
if model.foundations.rvm is not None:
|
| 853 |
+
precompute_rvm_to_memmap(
|
| 854 |
+
model, input_path, meta, rvm_path,
|
| 855 |
+
chunk=max(1, min(int(cfg.hann_chunk), 8)),
|
| 856 |
+
reader_kind=reader_kind)
|
| 857 |
+
rvm_feat_path = rvm_path
|
| 858 |
+
else:
|
| 859 |
+
rvm_feat_path = None
|
| 860 |
+
foundation_paths = None
|
| 861 |
+
if cfg.low_vram:
|
| 862 |
+
foundation_paths = precompute_foundations_to_memmap(
|
| 863 |
+
model, input_path, meta, foundation_prefix,
|
| 864 |
+
chunk=max(1, int(cfg.low_vram_foundation_chunk)),
|
| 865 |
+
reader_kind=reader_kind)
|
| 866 |
+
model.foundations.cpu()
|
| 867 |
+
model.stage2.cpu()
|
| 868 |
+
model.stage1.cuda()
|
| 869 |
+
torch.cuda.empty_cache()
|
| 870 |
+
else:
|
| 871 |
+
model.stage2.cuda()
|
| 872 |
+
model.stage2.eval()
|
| 873 |
+
torch.cuda.empty_cache()
|
| 874 |
+
|
| 875 |
+
stage2_batch = max(1, int(cfg.stage2_batch))
|
| 876 |
+
|
| 877 |
+
def write_prediction_batches(items):
|
| 878 |
+
device = next(model.stage2.parameters()).device
|
| 879 |
+
for offset in range(0, len(items), stage2_batch):
|
| 880 |
+
sub = items[offset:offset + stage2_batch]
|
| 881 |
+
rgb_t = torch.stack([item[2] for item in sub], dim=0)
|
| 882 |
+
rgb_t = rgb_t.unsqueeze(0).to(device, non_blocking=True)
|
| 883 |
+
baton_np = np.stack([item[1] for item in sub], axis=0)
|
| 884 |
+
baton_t = torch.from_numpy(baton_np).unsqueeze(0)
|
| 885 |
+
baton_t = baton_t.to(device, non_blocking=True).float()
|
| 886 |
+
with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16):
|
| 887 |
+
logits = model.stage2(rgb_t, baton_t)
|
| 888 |
+
if isinstance(logits, tuple):
|
| 889 |
+
logits = logits[0]
|
| 890 |
+
render_fn = (_render_predictions_exr_torch if use_exr
|
| 891 |
+
else _render_predictions_torch)
|
| 892 |
+
rendered = render_fn(meta, cfg.outputs, checker_t, logits[0])
|
| 893 |
+
for name, frames in rendered.items():
|
| 894 |
+
writers[name].write_many(frames)
|
| 895 |
+
del logits, baton_t, rgb_t, baton_np, rendered
|
| 896 |
+
|
| 897 |
+
hann_stage1_stream(
|
| 898 |
+
model, input_path, meta, reader_cls=reader_cls,
|
| 899 |
+
reader_kind=reader_kind, chunk=cfg.hann_chunk, stride=cfg.hann_stride,
|
| 900 |
+
baton_path=baton_path, initial_hint=initial_hint,
|
| 901 |
+
initial_hint_quality=cfg.hint_quality,
|
| 902 |
+
full_hint_video_path=hint_video_path,
|
| 903 |
+
full_hint_quality=cfg.hint_quality,
|
| 904 |
+
rvm_path=rvm_feat_path, foundation_paths=foundation_paths,
|
| 905 |
+
carry_hint_from_aux=cfg.carry_hint,
|
| 906 |
+
carry_hint_quality=cfg.carry_hint_quality,
|
| 907 |
+
on_baton_frames=None if cfg.low_vram else write_prediction_batches)
|
| 908 |
+
|
| 909 |
+
if cfg.low_vram:
|
| 910 |
+
_phase2_setup(model)
|
| 911 |
+
baton = np.memmap(
|
| 912 |
+
baton_path, dtype=np.float16, mode="r",
|
| 913 |
+
shape=(T_all, model.stage1.d_16, H // 16, W // 16))
|
| 914 |
+
reader = reader_cls(input_path, meta)
|
| 915 |
+
try:
|
| 916 |
+
for t0 in range(0, T_all, stage2_batch):
|
| 917 |
+
t1 = min(T_all, t0 + stage2_batch)
|
| 918 |
+
model_chunk, _display_chunk, _review_chunk = reader.window(t0, t1)
|
| 919 |
+
items = [
|
| 920 |
+
(t, np.asarray(baton[t]).copy(), model_chunk[t - t0])
|
| 921 |
+
for t in range(t0, t1)
|
| 922 |
+
]
|
| 923 |
+
write_prediction_batches(items)
|
| 924 |
+
del model_chunk, _display_chunk, _review_chunk, items
|
| 925 |
+
finally:
|
| 926 |
+
reader.close()
|
| 927 |
+
del baton
|
| 928 |
+
finally:
|
| 929 |
+
for writer in writers.values():
|
| 930 |
+
writer.release()
|
| 931 |
+
safe_remove(cfg.temp_dir / f"{raw_clip_name(input_path)}_rvm_feat.f16")
|
| 932 |
+
safe_remove(cfg.temp_dir / f"{raw_clip_name(input_path)}_baton.f16")
|
| 933 |
+
safe_remove(cfg.temp_dir / f"{raw_clip_name(input_path)}_foundation_cradio.f32")
|
| 934 |
+
safe_remove(cfg.temp_dir / f"{raw_clip_name(input_path)}_foundation_moge.f32")
|
| 935 |
+
torch.cuda.empty_cache()
|
| 936 |
+
print(f" wrote {len(writers)} output video(s) for {clip_name} "
|
| 937 |
+
f"({T_all} frames @ {W_out}x{H_out})")
|
corridorkey/rvm.py
ADDED
|
@@ -0,0 +1,562 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""RVM (Recurrent Video MAE) — PyTorch port of the representations4d RVM.
|
| 2 |
+
|
| 3 |
+
Architecture (from
|
| 4 |
+
https://github.com/google-deepmind/representations4d/blob/main/colabs/rvm_inference_demo.ipynb):
|
| 5 |
+
VideoSiamMAE(frame, state):
|
| 6 |
+
1. Patchify (2D 16×16 conv, T-kernel=1)
|
| 7 |
+
2. + 2D sinusoidal posenc, bicubic-resized from base 16×16
|
| 8 |
+
3. Prepend learned CLS token.
|
| 9 |
+
4. Run a plain ViT encoder (12 PreNorm blocks).
|
| 10 |
+
5. Final LayerNorm.
|
| 11 |
+
6. Feed to GatedTransformerCore:
|
| 12 |
+
- GRU gate: update = σ(W_iu·x + W_su·s); reset = σ(W_ir·x + W_sr·s)
|
| 13 |
+
- h = CrossAttnTransformer(queries=x, kv=reset·LN_noBias(s))
|
| 14 |
+
- state_new = (1 - update)·s + update·h
|
| 15 |
+
- return state_new, state_new
|
| 16 |
+
7. Features per frame = state at that time step (including CLS slot).
|
| 17 |
+
|
| 18 |
+
This port drops the MAE pretrain decoder — we only need the representation
|
| 19 |
+
path (encoder + rnn_core), which is 33M of the 67M param checkpoint for S.
|
| 20 |
+
|
| 21 |
+
Input contract:
|
| 22 |
+
video: (B, T, 3, H, W) fp32 in [0, 1].
|
| 23 |
+
IMPORTANT: RVM was pretrained on raw [0, 1] video frames; no imagenet
|
| 24 |
+
mean/std normalization applied.
|
| 25 |
+
|
| 26 |
+
Output:
|
| 27 |
+
features: (B, T, D, h, w) where h=H/16, w=W/16 and D=embed_dim.
|
| 28 |
+
final_state: (B, 1+h·w, D) — for continuing stateful inference across
|
| 29 |
+
clips (we carry this across Hann windows at inference time).
|
| 30 |
+
"""
|
| 31 |
+
from typing import Optional, Tuple
|
| 32 |
+
|
| 33 |
+
import math
|
| 34 |
+
|
| 35 |
+
import numpy as np
|
| 36 |
+
import torch
|
| 37 |
+
import torch.nn as nn
|
| 38 |
+
import torch.nn.functional as F
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# ---------------------------------------------------------------------------
|
| 42 |
+
# Model configs (from ckpt inspection)
|
| 43 |
+
# ---------------------------------------------------------------------------
|
| 44 |
+
|
| 45 |
+
RVM_CFG = {
|
| 46 |
+
# RVM-S — ViT-S encoder (D=384), 4 xa_blocks in rnn_core with 8×48 heads.
|
| 47 |
+
# Verified by inspection of pretrain_rvm_small16_256_204031069.npz.
|
| 48 |
+
'small': dict(
|
| 49 |
+
embed_dim=384,
|
| 50 |
+
enc_num_layers=12, enc_num_heads=6, enc_head_dim=64,
|
| 51 |
+
enc_mlp_hidden=1536,
|
| 52 |
+
rnn_num_xa_blocks=4, rnn_num_heads=8, rnn_head_dim=48,
|
| 53 |
+
rnn_mlp_hidden=2048,
|
| 54 |
+
),
|
| 55 |
+
# RVM-B — ViT-B encoder (D=768), 4 xa_blocks in rnn_core with 12×64 heads.
|
| 56 |
+
# Verified by inspection of pretrain_rvm_base16_256_203916225.npz.
|
| 57 |
+
'base': dict(
|
| 58 |
+
embed_dim=768,
|
| 59 |
+
enc_num_layers=12, enc_num_heads=12, enc_head_dim=64,
|
| 60 |
+
enc_mlp_hidden=3072,
|
| 61 |
+
rnn_num_xa_blocks=4, rnn_num_heads=12, rnn_head_dim=64,
|
| 62 |
+
rnn_mlp_hidden=3072,
|
| 63 |
+
),
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
PATCH_SIZE = 16
|
| 67 |
+
BASE_TOKEN_H = 16
|
| 68 |
+
BASE_TOKEN_W = 16
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
# ---------------------------------------------------------------------------
|
| 72 |
+
# 2D sinusoidal positional encoding (bicubic-resizable from base 16×16)
|
| 73 |
+
# ---------------------------------------------------------------------------
|
| 74 |
+
|
| 75 |
+
def _sincos_table_1d(n: int, d: int, dtype=torch.float32) -> torch.Tensor:
|
| 76 |
+
"""MAE-style 1D sinusoidal table of shape (n, d).
|
| 77 |
+
angles[i, j] = i / 10000^(2·⌊j/2⌋ / d). Even j → sin, odd j → cos."""
|
| 78 |
+
t = torch.zeros(n, d, dtype=dtype)
|
| 79 |
+
pos = torch.arange(n, dtype=dtype)[:, None]
|
| 80 |
+
j = torch.arange(d, dtype=dtype)[None, :]
|
| 81 |
+
angle = pos / torch.pow(10000.0, 2.0 * (j // 2) / d)
|
| 82 |
+
t[:, 0::2] = torch.sin(angle[:, 0::2])
|
| 83 |
+
t[:, 1::2] = torch.cos(angle[:, 1::2])
|
| 84 |
+
return t
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _build_base_posenc(d: int,
|
| 88 |
+
base_h: int = BASE_TOKEN_H,
|
| 89 |
+
base_w: int = BASE_TOKEN_W,
|
| 90 |
+
dtype=torch.float32) -> torch.Tensor:
|
| 91 |
+
"""(base_h, base_w, d) sinusoidal positional encoding, per the upstream
|
| 92 |
+
`SincosPosEmb` construction:
|
| 93 |
+
raw = sincos_1d(base_h·base_w, d) → reshape to (base_h, base_w, d)
|
| 94 |
+
"""
|
| 95 |
+
raw = _sincos_table_1d(base_h * base_w, d, dtype=dtype)
|
| 96 |
+
return raw.view(base_h, base_w, d)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def _resize_posenc(base: torch.Tensor, target_h: int, target_w: int) -> torch.Tensor:
|
| 100 |
+
"""Bicubic-resize a (base_h, base_w, D) posenc table to (target_h, target_w, D)."""
|
| 101 |
+
if base.shape[0] == target_h and base.shape[1] == target_w:
|
| 102 |
+
return base
|
| 103 |
+
# (H, W, D) → (1, D, H, W) for F.interpolate
|
| 104 |
+
x = base.permute(2, 0, 1).unsqueeze(0)
|
| 105 |
+
x = F.interpolate(x, size=(target_h, target_w), mode='bicubic', align_corners=False)
|
| 106 |
+
# Back to (target_h, target_w, D)
|
| 107 |
+
return x.squeeze(0).permute(1, 2, 0).contiguous()
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
# ---------------------------------------------------------------------------
|
| 111 |
+
# Attention / MLP / blocks (matching JAX layout for clean weight conversion)
|
| 112 |
+
# ---------------------------------------------------------------------------
|
| 113 |
+
|
| 114 |
+
class _MHA(nn.Module):
|
| 115 |
+
"""Multi-head dot-product attention.
|
| 116 |
+
|
| 117 |
+
Matches JAX `ImprovedMultiHeadDotProductAttention` layout: separate
|
| 118 |
+
`query`, `key`, `value`, and `out` linears, each with bias.
|
| 119 |
+
|
| 120 |
+
Allows cross-attention via (q, k, v) with k=v optional. head_dim may
|
| 121 |
+
differ from embed_dim//num_heads (e.g. rnn_core uses 8×48=384 on D=384).
|
| 122 |
+
"""
|
| 123 |
+
|
| 124 |
+
def __init__(self, d_model: int, num_heads: int, head_dim: int):
|
| 125 |
+
super().__init__()
|
| 126 |
+
self.d_model = d_model
|
| 127 |
+
self.num_heads = num_heads
|
| 128 |
+
self.head_dim = head_dim
|
| 129 |
+
self.inner_dim = num_heads * head_dim
|
| 130 |
+
self.query = nn.Linear(d_model, self.inner_dim, bias=True)
|
| 131 |
+
self.key = nn.Linear(d_model, self.inner_dim, bias=True)
|
| 132 |
+
self.value = nn.Linear(d_model, self.inner_dim, bias=True)
|
| 133 |
+
self.out = nn.Linear(self.inner_dim, d_model, bias=True)
|
| 134 |
+
|
| 135 |
+
def forward(self, q_in: torch.Tensor,
|
| 136 |
+
k_in: Optional[torch.Tensor] = None,
|
| 137 |
+
v_in: Optional[torch.Tensor] = None) -> torch.Tensor:
|
| 138 |
+
if k_in is None:
|
| 139 |
+
k_in = q_in
|
| 140 |
+
if v_in is None:
|
| 141 |
+
v_in = k_in
|
| 142 |
+
q = self.query(q_in) # (..., Lq, H·D)
|
| 143 |
+
k = self.key(k_in) # (..., Lk, H·D)
|
| 144 |
+
v = self.value(v_in) # (..., Lk, H·D)
|
| 145 |
+
# Reshape per head: (..., L, H, D) → (..., H, L, D).
|
| 146 |
+
def split_heads(x):
|
| 147 |
+
*lead, L, _ = x.shape
|
| 148 |
+
return x.view(*lead, L, self.num_heads, self.head_dim).transpose(-3, -2)
|
| 149 |
+
q = split_heads(q); k = split_heads(k); v = split_heads(v)
|
| 150 |
+
# Use flash SDPA for speed / memory.
|
| 151 |
+
attn = F.scaled_dot_product_attention(q, k, v, dropout_p=0.0, is_causal=False)
|
| 152 |
+
# (..., H, Lq, D) → (..., Lq, H, D) → (..., Lq, H·D)
|
| 153 |
+
attn = attn.transpose(-3, -2).contiguous()
|
| 154 |
+
attn = attn.reshape(*attn.shape[:-2], self.inner_dim)
|
| 155 |
+
return self.out(attn)
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
class _TransformerMLP(nn.Module):
|
| 159 |
+
"""GELU MLP: Dense_in → GELU → Dense_out. Matches JAX layout."""
|
| 160 |
+
|
| 161 |
+
def __init__(self, d_model: int, hidden: int):
|
| 162 |
+
super().__init__()
|
| 163 |
+
self.dense_in = nn.Linear(d_model, hidden, bias=True)
|
| 164 |
+
self.dense_out = nn.Linear(hidden, d_model, bias=True)
|
| 165 |
+
|
| 166 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 167 |
+
return self.dense_out(F.gelu(self.dense_in(x)))
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
class _PreNormBlock(nn.Module):
|
| 171 |
+
"""Encoder block: pre-LN MHA + pre-LN MLP, both residual."""
|
| 172 |
+
|
| 173 |
+
def __init__(self, d_model: int, num_heads: int, head_dim: int, mlp_hidden: int):
|
| 174 |
+
super().__init__()
|
| 175 |
+
self.attention_norm = nn.LayerNorm(d_model)
|
| 176 |
+
self.attention = _MHA(d_model, num_heads, head_dim)
|
| 177 |
+
self.mlp_norm = nn.LayerNorm(d_model)
|
| 178 |
+
self.mlp = _TransformerMLP(d_model, mlp_hidden)
|
| 179 |
+
|
| 180 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 181 |
+
x = x + self.attention(self.attention_norm(x))
|
| 182 |
+
x = x + self.mlp(self.mlp_norm(x))
|
| 183 |
+
return x
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
class _CrossAttentionBlock(nn.Module):
|
| 187 |
+
"""Cross-attention block used inside GatedTransformerCore.
|
| 188 |
+
|
| 189 |
+
Block order matches the upstream notebook's `CrossAttentionBlock.__call__`:
|
| 190 |
+
x = x + ca_attention(ca_norm(x), kv=kv) # cross-attn over kv
|
| 191 |
+
x = x + mlp(mlp_norm(x))
|
| 192 |
+
x = x + attention(attention_norm(x)) # self-attn
|
| 193 |
+
"""
|
| 194 |
+
|
| 195 |
+
def __init__(self, d_model: int, num_heads: int, head_dim: int, mlp_hidden: int):
|
| 196 |
+
super().__init__()
|
| 197 |
+
self.ca_attention_norm = nn.LayerNorm(d_model)
|
| 198 |
+
self.ca_attention = _MHA(d_model, num_heads, head_dim)
|
| 199 |
+
self.mlp_norm = nn.LayerNorm(d_model)
|
| 200 |
+
self.mlp = _TransformerMLP(d_model, mlp_hidden)
|
| 201 |
+
self.attention_norm = nn.LayerNorm(d_model)
|
| 202 |
+
self.attention = _MHA(d_model, num_heads, head_dim)
|
| 203 |
+
|
| 204 |
+
def forward(self, x: torch.Tensor, kv: torch.Tensor) -> torch.Tensor:
|
| 205 |
+
x = x + self.ca_attention(self.ca_attention_norm(x), k_in=kv, v_in=kv)
|
| 206 |
+
x = x + self.mlp(self.mlp_norm(x))
|
| 207 |
+
x = x + self.attention(self.attention_norm(x))
|
| 208 |
+
return x
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
class _CrossAttentionTransformer(nn.Module):
|
| 212 |
+
"""Stack of xa_blocks + final output_norm."""
|
| 213 |
+
|
| 214 |
+
def __init__(self, d_model: int, num_heads: int, head_dim: int,
|
| 215 |
+
mlp_hidden: int, num_blocks: int):
|
| 216 |
+
super().__init__()
|
| 217 |
+
self.xa_blocks = nn.ModuleList([
|
| 218 |
+
_CrossAttentionBlock(d_model, num_heads, head_dim, mlp_hidden)
|
| 219 |
+
for _ in range(num_blocks)
|
| 220 |
+
])
|
| 221 |
+
self.output_norm = nn.LayerNorm(d_model)
|
| 222 |
+
|
| 223 |
+
def forward(self, x: torch.Tensor, kv: torch.Tensor) -> torch.Tensor:
|
| 224 |
+
for blk in self.xa_blocks:
|
| 225 |
+
x = blk(x, kv)
|
| 226 |
+
return self.output_norm(x)
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
class _GatedTransformerCore(nn.Module):
|
| 230 |
+
"""GRU-gated cross-attention between current frame tokens and recurrent state.
|
| 231 |
+
|
| 232 |
+
update = σ(W_iu·x + W_su·s)
|
| 233 |
+
reset = σ(W_ir·x + W_sr·s)
|
| 234 |
+
h = CrossAttnTransformer(queries=x, kv = reset · state_LN(s))
|
| 235 |
+
s' = (1 - update)·s + update·h
|
| 236 |
+
return s', s'
|
| 237 |
+
|
| 238 |
+
All four gate linears are bias-free (matches upstream NPZ — no `bias` key
|
| 239 |
+
for input_update/input_reset/state_update/state_reset).
|
| 240 |
+
|
| 241 |
+
`state_layer_norm` has *only* the scale parameter (no bias), per the
|
| 242 |
+
upstream `nn.LayerNorm(use_bias=False)`. PyTorch LayerNorm has bias by
|
| 243 |
+
default; we zero-and-freeze the bias there.
|
| 244 |
+
"""
|
| 245 |
+
|
| 246 |
+
def __init__(self, d_model: int, num_heads: int, head_dim: int,
|
| 247 |
+
mlp_hidden: int, num_xa_blocks: int):
|
| 248 |
+
super().__init__()
|
| 249 |
+
self.d_model = d_model
|
| 250 |
+
self.input_update = nn.Linear(d_model, d_model, bias=False)
|
| 251 |
+
self.input_reset = nn.Linear(d_model, d_model, bias=False)
|
| 252 |
+
self.state_update = nn.Linear(d_model, d_model, bias=False)
|
| 253 |
+
self.state_reset = nn.Linear(d_model, d_model, bias=False)
|
| 254 |
+
# bias-free LN. PyTorch LayerNorm always has a bias Parameter; we
|
| 255 |
+
# just keep it at zero (and upstream ckpt has no corresponding bias).
|
| 256 |
+
self.state_layer_norm = nn.LayerNorm(d_model, eps=1e-4)
|
| 257 |
+
nn.init.zeros_(self.state_layer_norm.bias)
|
| 258 |
+
self.state_layer_norm.bias.requires_grad = False
|
| 259 |
+
|
| 260 |
+
self.transformer = _CrossAttentionTransformer(
|
| 261 |
+
d_model=d_model, num_heads=num_heads, head_dim=head_dim,
|
| 262 |
+
mlp_hidden=mlp_hidden, num_blocks=num_xa_blocks,
|
| 263 |
+
)
|
| 264 |
+
|
| 265 |
+
def forward(self, inputs: torch.Tensor, state: torch.Tensor
|
| 266 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 267 |
+
"""inputs: (B, N, D) current encoded frame tokens. state: (B, N, D).
|
| 268 |
+
Returns (features, new_state)."""
|
| 269 |
+
update = torch.sigmoid(self.input_update(inputs) + self.state_update(state))
|
| 270 |
+
reset = torch.sigmoid(self.input_reset(inputs) + self.state_reset(state))
|
| 271 |
+
gated_kv = reset * self.state_layer_norm(state)
|
| 272 |
+
h = self.transformer(inputs, gated_kv)
|
| 273 |
+
out = (1.0 - update) * state + update * h
|
| 274 |
+
return out, out
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
# ---------------------------------------------------------------------------
|
| 278 |
+
# Main model
|
| 279 |
+
# ---------------------------------------------------------------------------
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
class RVMVideoSiamMAE(nn.Module):
|
| 283 |
+
"""RVM encoder + rnn_core (MAE decoder dropped).
|
| 284 |
+
|
| 285 |
+
Processes video frame-by-frame. Encoder is fully batchable over B·T;
|
| 286 |
+
only the rnn_core is sequential across T (carrying state).
|
| 287 |
+
"""
|
| 288 |
+
|
| 289 |
+
def __init__(self, variant: str = 'small'):
|
| 290 |
+
super().__init__()
|
| 291 |
+
cfg = RVM_CFG[variant]
|
| 292 |
+
self.variant = variant
|
| 293 |
+
self.embed_dim = cfg['embed_dim']
|
| 294 |
+
|
| 295 |
+
# Tokenizer: 2D patch conv + learned CLS token + sincos posenc.
|
| 296 |
+
self.patch_embed = nn.Conv2d(3, self.embed_dim,
|
| 297 |
+
kernel_size=PATCH_SIZE,
|
| 298 |
+
stride=PATCH_SIZE, bias=True)
|
| 299 |
+
self.cls_token = nn.Parameter(torch.zeros(1, self.embed_dim))
|
| 300 |
+
|
| 301 |
+
# Pre-computed base posenc table (non-learnable). Registered as
|
| 302 |
+
# buffer so it moves with .cuda()/.to().
|
| 303 |
+
self.register_buffer(
|
| 304 |
+
'base_posenc',
|
| 305 |
+
_build_base_posenc(self.embed_dim, BASE_TOKEN_H, BASE_TOKEN_W),
|
| 306 |
+
persistent=False,
|
| 307 |
+
)
|
| 308 |
+
|
| 309 |
+
# Encoder.
|
| 310 |
+
self.encoder_layers = nn.ModuleList([
|
| 311 |
+
_PreNormBlock(self.embed_dim,
|
| 312 |
+
cfg['enc_num_heads'], cfg['enc_head_dim'],
|
| 313 |
+
cfg['enc_mlp_hidden'])
|
| 314 |
+
for _ in range(cfg['enc_num_layers'])
|
| 315 |
+
])
|
| 316 |
+
self.encoder_final_norm = nn.LayerNorm(self.embed_dim)
|
| 317 |
+
|
| 318 |
+
# Rnn core (stateful across T).
|
| 319 |
+
self.rnn_core = _GatedTransformerCore(
|
| 320 |
+
d_model=self.embed_dim,
|
| 321 |
+
num_heads=cfg['rnn_num_heads'],
|
| 322 |
+
head_dim=cfg['rnn_head_dim'],
|
| 323 |
+
mlp_hidden=cfg['rnn_mlp_hidden'],
|
| 324 |
+
num_xa_blocks=cfg['rnn_num_xa_blocks'],
|
| 325 |
+
)
|
| 326 |
+
|
| 327 |
+
# -- internals -------------------------------------------------------
|
| 328 |
+
|
| 329 |
+
def _tokenize(self, frames_bt: torch.Tensor) -> torch.Tensor:
|
| 330 |
+
"""(B·T, 3, H, W) → (B·T, 1 + h·w, D) with posenc applied to patches
|
| 331 |
+
(CLS token is not posenc'd)."""
|
| 332 |
+
# Patchify.
|
| 333 |
+
x = self.patch_embed(frames_bt) # (B·T, D, h, w)
|
| 334 |
+
BT, D, h, w = x.shape
|
| 335 |
+
# Channel-last token layout.
|
| 336 |
+
x = x.permute(0, 2, 3, 1) # (B·T, h, w, D)
|
| 337 |
+
# Add posenc (resized from base 16×16 if needed).
|
| 338 |
+
posenc = _resize_posenc(self.base_posenc, h, w) # (h, w, D)
|
| 339 |
+
x = x + posenc
|
| 340 |
+
# Flatten spatial and prepend CLS.
|
| 341 |
+
x = x.reshape(BT, h * w, D)
|
| 342 |
+
cls = self.cls_token.to(x.dtype).expand(BT, 1, D)
|
| 343 |
+
return torch.cat([cls, x], dim=1) # (B·T, 1 + h·w, D)
|
| 344 |
+
|
| 345 |
+
def _run_encoder(self, tokens: torch.Tensor) -> torch.Tensor:
|
| 346 |
+
for blk in self.encoder_layers:
|
| 347 |
+
tokens = blk(tokens)
|
| 348 |
+
return self.encoder_final_norm(tokens)
|
| 349 |
+
|
| 350 |
+
# -- forward ---------------------------------------------------------
|
| 351 |
+
|
| 352 |
+
def forward(self, video: torch.Tensor,
|
| 353 |
+
state: Optional[torch.Tensor] = None,
|
| 354 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 355 |
+
"""video: (B, T, 3, H, W) fp32 in [0, 1].
|
| 356 |
+
|
| 357 |
+
Returns:
|
| 358 |
+
features: (B, T, D, h, w) — CLS dropped, channel-first for
|
| 359 |
+
compatibility with our downstream pipeline.
|
| 360 |
+
final_state: (B, 1 + h·w, D) — carry across clip boundaries
|
| 361 |
+
at inference time.
|
| 362 |
+
"""
|
| 363 |
+
assert video.dim() == 5, f"expected (B,T,3,H,W); got {tuple(video.shape)}"
|
| 364 |
+
B, T, C, H, W = video.shape
|
| 365 |
+
assert C == 3 and H % PATCH_SIZE == 0 and W % PATCH_SIZE == 0, (
|
| 366 |
+
f"H, W must be multiples of {PATCH_SIZE}; got {(H, W)}")
|
| 367 |
+
h = H // PATCH_SIZE
|
| 368 |
+
w = W // PATCH_SIZE
|
| 369 |
+
N = 1 + h * w
|
| 370 |
+
|
| 371 |
+
# (1) Encoder — fully batched over B·T.
|
| 372 |
+
frames_bt = video.reshape(B * T, C, H, W)
|
| 373 |
+
tokens_bt = self._tokenize(frames_bt) # (B·T, N, D)
|
| 374 |
+
encoded_bt = self._run_encoder(tokens_bt) # (B·T, N, D)
|
| 375 |
+
encoded = encoded_bt.view(B, T, N, self.embed_dim)
|
| 376 |
+
|
| 377 |
+
# (2) Rnn core — sequential over T.
|
| 378 |
+
if state is None:
|
| 379 |
+
state = encoded.new_zeros(B, N, self.embed_dim)
|
| 380 |
+
features = encoded.new_empty(B, T, N, self.embed_dim)
|
| 381 |
+
for t in range(T):
|
| 382 |
+
out, state = self.rnn_core(encoded[:, t], state)
|
| 383 |
+
features[:, t] = out
|
| 384 |
+
|
| 385 |
+
# (3) Reshape: drop CLS, channel-first.
|
| 386 |
+
patch_feats = features[:, :, 1:, :] # (B, T, h·w, D)
|
| 387 |
+
patch_feats = patch_feats.view(B, T, h, w, self.embed_dim)
|
| 388 |
+
patch_feats = patch_feats.permute(0, 1, 4, 2, 3).contiguous()
|
| 389 |
+
return patch_feats, state
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
# ---------------------------------------------------------------------------
|
| 393 |
+
# Weight conversion: NPZ (JAX tree) → PyTorch state_dict
|
| 394 |
+
# ---------------------------------------------------------------------------
|
| 395 |
+
|
| 396 |
+
|
| 397 |
+
def _recover_tree(flat: dict) -> dict:
|
| 398 |
+
tree = {}
|
| 399 |
+
for k, v in flat.items():
|
| 400 |
+
node = tree
|
| 401 |
+
parts = k.split('/')
|
| 402 |
+
for part in parts[:-1]:
|
| 403 |
+
node = node.setdefault(part, {})
|
| 404 |
+
node[parts[-1]] = v
|
| 405 |
+
return tree
|
| 406 |
+
|
| 407 |
+
|
| 408 |
+
def _jax_dense_kernel_to_torch(k: np.ndarray) -> torch.Tensor:
|
| 409 |
+
"""JAX Dense kernel (in_dim, out_dim) → PyTorch Linear weight (out_dim, in_dim)."""
|
| 410 |
+
return torch.from_numpy(np.ascontiguousarray(k.T.copy())).float()
|
| 411 |
+
|
| 412 |
+
|
| 413 |
+
def _jax_qkv_kernel_to_torch(k: np.ndarray) -> torch.Tensor:
|
| 414 |
+
"""JAX DenseGeneral Q/K/V kernel (in_dim, num_heads, head_dim)
|
| 415 |
+
→ PyTorch flat Linear weight (num_heads·head_dim, in_dim)."""
|
| 416 |
+
in_dim, H, D = k.shape
|
| 417 |
+
flat = k.reshape(in_dim, H * D) # (in_dim, inner)
|
| 418 |
+
return torch.from_numpy(np.ascontiguousarray(flat.T.copy())).float()
|
| 419 |
+
|
| 420 |
+
|
| 421 |
+
def _jax_qkv_bias_to_torch(b: np.ndarray) -> torch.Tensor:
|
| 422 |
+
"""(num_heads, head_dim) → (num_heads·head_dim,)"""
|
| 423 |
+
return torch.from_numpy(np.ascontiguousarray(b.reshape(-1).copy())).float()
|
| 424 |
+
|
| 425 |
+
|
| 426 |
+
def _jax_out_kernel_to_torch(k: np.ndarray) -> torch.Tensor:
|
| 427 |
+
"""JAX DenseGeneral output kernel (num_heads, head_dim, out_dim)
|
| 428 |
+
→ PyTorch Linear weight (out_dim, num_heads·head_dim)."""
|
| 429 |
+
H, D, out_dim = k.shape
|
| 430 |
+
flat = k.reshape(H * D, out_dim)
|
| 431 |
+
return torch.from_numpy(np.ascontiguousarray(flat.T.copy())).float()
|
| 432 |
+
|
| 433 |
+
|
| 434 |
+
def _jax_out_bias_to_torch(b: np.ndarray) -> torch.Tensor:
|
| 435 |
+
return torch.from_numpy(np.ascontiguousarray(b.copy())).float()
|
| 436 |
+
|
| 437 |
+
|
| 438 |
+
def _convert_mha_block(sd_out: dict, prefix: str, src: dict):
|
| 439 |
+
"""Fill sd_out for a `_MHA` module under `prefix.` from src subtree."""
|
| 440 |
+
sd_out[prefix + 'query.weight'] = _jax_qkv_kernel_to_torch(src['query']['kernel'])
|
| 441 |
+
sd_out[prefix + 'query.bias'] = _jax_qkv_bias_to_torch(src['query']['bias'])
|
| 442 |
+
sd_out[prefix + 'key.weight'] = _jax_qkv_kernel_to_torch(src['key']['kernel'])
|
| 443 |
+
sd_out[prefix + 'key.bias'] = _jax_qkv_bias_to_torch(src['key']['bias'])
|
| 444 |
+
sd_out[prefix + 'value.weight'] = _jax_qkv_kernel_to_torch(src['value']['kernel'])
|
| 445 |
+
sd_out[prefix + 'value.bias'] = _jax_qkv_bias_to_torch(src['value']['bias'])
|
| 446 |
+
sd_out[prefix + 'out.weight'] = _jax_out_kernel_to_torch(src['out']['kernel'])
|
| 447 |
+
sd_out[prefix + 'out.bias'] = _jax_out_bias_to_torch(src['out']['bias'])
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
def _convert_ln(sd_out: dict, prefix: str, src: dict):
|
| 451 |
+
"""JAX LayerNorm (`scale`, `bias`) → PyTorch (`weight`, `bias`)."""
|
| 452 |
+
sd_out[prefix + 'weight'] = torch.from_numpy(np.ascontiguousarray(src['scale'].copy())).float()
|
| 453 |
+
sd_out[prefix + 'bias'] = torch.from_numpy(np.ascontiguousarray(src['bias'].copy())).float()
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
def _convert_mlp(sd_out: dict, prefix: str, src: dict):
|
| 457 |
+
sd_out[prefix + 'dense_in.weight'] = _jax_dense_kernel_to_torch(src['dense_in']['kernel'])
|
| 458 |
+
sd_out[prefix + 'dense_in.bias'] = torch.from_numpy(np.ascontiguousarray(src['dense_in']['bias'].copy())).float()
|
| 459 |
+
sd_out[prefix + 'dense_out.weight'] = _jax_dense_kernel_to_torch(src['dense_out']['kernel'])
|
| 460 |
+
sd_out[prefix + 'dense_out.bias'] = torch.from_numpy(np.ascontiguousarray(src['dense_out']['bias'].copy())).float()
|
| 461 |
+
|
| 462 |
+
|
| 463 |
+
def _convert_gate_kernel(sd_out: dict, key: str, src_kernel: np.ndarray):
|
| 464 |
+
"""GRU gate Linear(no bias). JAX kernel (D, D) → Linear weight (D, D) [transposed]."""
|
| 465 |
+
sd_out[key] = _jax_dense_kernel_to_torch(src_kernel)
|
| 466 |
+
|
| 467 |
+
|
| 468 |
+
def load_rvm_npz(model: RVMVideoSiamMAE, npz_path: str,
|
| 469 |
+
strict_load: bool = True, verbose: bool = True) -> None:
|
| 470 |
+
"""Load the NPZ ckpt into `model` (in-place).
|
| 471 |
+
|
| 472 |
+
Drops the MAE pretrain decoder, mask_token, and detokenizer paths —
|
| 473 |
+
only encoder + rnn_core + tokenizer + cls_token are consumed.
|
| 474 |
+
"""
|
| 475 |
+
raw = dict(np.load(npz_path, allow_pickle=False))
|
| 476 |
+
tree = _recover_tree(raw)
|
| 477 |
+
|
| 478 |
+
sd_out = {}
|
| 479 |
+
|
| 480 |
+
# -- Tokenizer (patch embed). ------------------------------------------
|
| 481 |
+
# JAX kernel shape (1, 16, 16, 3, D). T-kernel=1, so squeeze to (16, 16, 3, D)
|
| 482 |
+
# and permute to PyTorch Conv2d (D, 3, 16, 16).
|
| 483 |
+
conv_k = tree['tokenizer']['patch_embedding']['Conv_0']['kernel']
|
| 484 |
+
conv_k = np.squeeze(conv_k, axis=0) # (16, 16, 3, D)
|
| 485 |
+
conv_k = np.transpose(conv_k, (3, 2, 0, 1)) # (D, 3, 16, 16)
|
| 486 |
+
sd_out['patch_embed.weight'] = torch.from_numpy(np.ascontiguousarray(conv_k.copy())).float()
|
| 487 |
+
conv_b = tree['tokenizer']['patch_embedding']['Conv_0']['bias']
|
| 488 |
+
sd_out['patch_embed.bias'] = torch.from_numpy(np.ascontiguousarray(conv_b.copy())).float()
|
| 489 |
+
|
| 490 |
+
# -- CLS token. --------------------------------------------------------
|
| 491 |
+
sd_out['cls_token'] = torch.from_numpy(np.ascontiguousarray(tree['cls_token'].copy())).float()
|
| 492 |
+
|
| 493 |
+
# -- Encoder layers. ---------------------------------------------------
|
| 494 |
+
enc = tree['encoder']
|
| 495 |
+
for i in range(len(model.encoder_layers)):
|
| 496 |
+
src = enc[f'layers_{i}']
|
| 497 |
+
prefix = f'encoder_layers.{i}.'
|
| 498 |
+
_convert_ln(sd_out, prefix + 'attention_norm.', src['attention_norm'])
|
| 499 |
+
_convert_mha_block(sd_out, prefix + 'attention.', src['attention'])
|
| 500 |
+
_convert_ln(sd_out, prefix + 'mlp_norm.', src['mlp_norm'])
|
| 501 |
+
_convert_mlp(sd_out, prefix + 'mlp.', src['mlp'])
|
| 502 |
+
|
| 503 |
+
# Final encoder LayerNorm. In the npz it's named "LayerNorm_0".
|
| 504 |
+
_convert_ln(sd_out, 'encoder_final_norm.', enc['LayerNorm_0'])
|
| 505 |
+
|
| 506 |
+
# -- Rnn core. ---------------------------------------------------------
|
| 507 |
+
rc = tree['rnn_core']
|
| 508 |
+
_convert_gate_kernel(sd_out, 'rnn_core.input_update.weight', rc['input_update']['kernel'])
|
| 509 |
+
_convert_gate_kernel(sd_out, 'rnn_core.input_reset.weight', rc['input_reset']['kernel'])
|
| 510 |
+
_convert_gate_kernel(sd_out, 'rnn_core.state_update.weight', rc['state_update']['kernel'])
|
| 511 |
+
_convert_gate_kernel(sd_out, 'rnn_core.state_reset.weight', rc['state_reset']['kernel'])
|
| 512 |
+
# state_layer_norm — JAX has only `scale` (no bias). PyTorch LN has bias;
|
| 513 |
+
# we set weight=scale, bias=0 (and froze bias in __init__).
|
| 514 |
+
sd_out['rnn_core.state_layer_norm.weight'] = torch.from_numpy(
|
| 515 |
+
np.ascontiguousarray(rc['state_layer_norm']['scale'].copy())).float()
|
| 516 |
+
sd_out['rnn_core.state_layer_norm.bias'] = torch.zeros(model.embed_dim, dtype=torch.float32)
|
| 517 |
+
|
| 518 |
+
# Transformer (xa_blocks + output_norm).
|
| 519 |
+
tfm = rc['transformer']
|
| 520 |
+
for i in range(len(model.rnn_core.transformer.xa_blocks)):
|
| 521 |
+
src = tfm[f'xa_blocks_{i}']
|
| 522 |
+
p = f'rnn_core.transformer.xa_blocks.{i}.'
|
| 523 |
+
_convert_ln(sd_out, p + 'ca_attention_norm.', src['ca_attention_norm'])
|
| 524 |
+
_convert_mha_block(sd_out, p + 'ca_attention.', src['ca_attention'])
|
| 525 |
+
_convert_ln(sd_out, p + 'mlp_norm.', src['mlp_norm'])
|
| 526 |
+
_convert_mlp(sd_out, p + 'mlp.', src['mlp'])
|
| 527 |
+
_convert_ln(sd_out, p + 'attention_norm.', src['attention_norm'])
|
| 528 |
+
_convert_mha_block(sd_out, p + 'attention.', src['attention'])
|
| 529 |
+
_convert_ln(sd_out, 'rnn_core.transformer.output_norm.', tfm['output_norm'])
|
| 530 |
+
|
| 531 |
+
# -- Load. -------------------------------------------------------------
|
| 532 |
+
missing, unexpected = model.load_state_dict(sd_out, strict=False)
|
| 533 |
+
if verbose:
|
| 534 |
+
print(f"[RVM] loaded {npz_path} ({len(sd_out)} tensors)")
|
| 535 |
+
# base_posenc is a non-persistent buffer — expected "missing"
|
| 536 |
+
missing_keys = [k for k in missing if k != 'base_posenc']
|
| 537 |
+
if missing_keys:
|
| 538 |
+
print(f"[RVM] unexpected MISSING keys: {missing_keys[:8]}"
|
| 539 |
+
+ (f" ... (+{len(missing_keys)-8} more)" if len(missing_keys) > 8 else ""))
|
| 540 |
+
if unexpected:
|
| 541 |
+
print(f"[RVM] UNEXPECTED keys in state_dict: {unexpected[:8]}"
|
| 542 |
+
+ (f" ... (+{len(unexpected)-8} more)" if len(unexpected) > 8 else ""))
|
| 543 |
+
if strict_load and any(k != 'base_posenc' for k in missing):
|
| 544 |
+
raise RuntimeError(f"RVM ckpt load missing keys: {[k for k in missing if k != 'base_posenc']}")
|
| 545 |
+
|
| 546 |
+
|
| 547 |
+
# ---------------------------------------------------------------------------
|
| 548 |
+
# Convenience constructor
|
| 549 |
+
# ---------------------------------------------------------------------------
|
| 550 |
+
|
| 551 |
+
|
| 552 |
+
def build_rvm(variant: str = 'small', ckpt_path: Optional[str] = None,
|
| 553 |
+
device: str = 'cuda') -> RVMVideoSiamMAE:
|
| 554 |
+
"""Build a frozen RVM model and optionally load weights from NPZ."""
|
| 555 |
+
model = RVMVideoSiamMAE(variant=variant).to(device)
|
| 556 |
+
if ckpt_path is not None:
|
| 557 |
+
load_rvm_npz(model, ckpt_path)
|
| 558 |
+
# Freeze: RVM is a foundation model, not trained here.
|
| 559 |
+
for p in model.parameters():
|
| 560 |
+
p.requires_grad_(False)
|
| 561 |
+
model.eval()
|
| 562 |
+
return model
|
corridorkey/stage1.py
ADDED
|
@@ -0,0 +1,920 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Stage 1 — baton producer for CorridorKey v2.
|
| 2 |
+
|
| 3 |
+
Pure-transformer spatiotemporal encoder/decoder on the /16 ↔ /64 pyramid:
|
| 4 |
+
|
| 5 |
+
/1 RGB + hint
|
| 6 |
+
└ Patchify(k=16) → d=512 at /16
|
| 7 |
+
└ 2 × FullSpatialBlock3D (encoder side; per-frame full H×W)
|
| 8 |
+
└ Downsample(k=8, s=4) → d=512 aux at /64
|
| 9 |
+
└ cat frozen foundations + aux → d=1536 at /64
|
| 10 |
+
└ first half of FullSpatioTemporalBlock stack
|
| 11 |
+
└ zero-init upsample to /16 + learned skip from encoder /16
|
| 12 |
+
└ 4 × SwinBlock3D local /16 refinement
|
| 13 |
+
└ zero-init downsample add back to /64
|
| 14 |
+
└ remaining FullSpatioTemporalBlock stack
|
| 15 |
+
└ Upsample4ICNR → d=512 at /16
|
| 16 |
+
└ learned skip from latest /16 refinement
|
| 17 |
+
└ 8 × mixed Swin/CSwinBlock3D (decoder side)
|
| 18 |
+
with zero-init patch-stem skip before the final 2 blocks
|
| 19 |
+
→ baton (d=512 at /16)
|
| 20 |
+
→ aux head: Depatchify r=8 → 4-channel /2 output
|
| 21 |
+
initialized as half-res RGB passthrough when the 512d projected
|
| 22 |
+
RGB patch basis is available (pred_alpha = 0 init)
|
| 23 |
+
|
| 24 |
+
No recurrence. Temporal mixing lives entirely in attention: global at /64
|
| 25 |
+
(full spatiotemporal), windowed at /16 (Swin-3D with full-T windows and
|
| 26 |
+
balanced half-window spatial shift phases).
|
| 27 |
+
|
| 28 |
+
Windows cleanly divide the /16 grid when `shape_quantum=128` (the
|
| 29 |
+
dataset contract for this model), so no intra-window padding is needed.
|
| 30 |
+
"""
|
| 31 |
+
|
| 32 |
+
import torch
|
| 33 |
+
import torch.nn as nn
|
| 34 |
+
import torch.nn.functional as F
|
| 35 |
+
|
| 36 |
+
from .blocks import (
|
| 37 |
+
PatchifyConv, Downsample4, Upsample4ICNR, Depatchify,
|
| 38 |
+
FullSpatialBlock3D, SwinBlock3D, CSwinBlock3D,
|
| 39 |
+
FullSpatioTemporalBlock, FullResolutionRecurrentMemorySite,
|
| 40 |
+
FusionLinSwiglu,
|
| 41 |
+
init_orthogonal_rgb_passthrough_,
|
| 42 |
+
)
|
| 43 |
+
from .rope import RoPE2D, RoPE3D
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
@torch.no_grad()
|
| 47 |
+
def _init_center_area_downsample_(conv: nn.Conv2d) -> None:
|
| 48 |
+
"""Initialize k=8/s=4 downsample as exact per-cell area pooling.
|
| 49 |
+
|
| 50 |
+
The central stride×stride region averages the corresponding input cell;
|
| 51 |
+
all spill taps start at zero. When channel counts differ, a semi-orthogonal
|
| 52 |
+
channel projection is used before the same spatial average.
|
| 53 |
+
"""
|
| 54 |
+
out_ch, in_ch, kh, kw = conv.weight.shape
|
| 55 |
+
sy, sx = conv.stride
|
| 56 |
+
if kh < sy or kw < sx:
|
| 57 |
+
raise ValueError(
|
| 58 |
+
f"downsample kernel {(kh, kw)} must cover stride {(sy, sx)}")
|
| 59 |
+
y0 = (kh - sy) // 2
|
| 60 |
+
x0 = (kw - sx) // 2
|
| 61 |
+
if (kh - sy) % 2 != 0 or (kw - sx) % 2 != 0:
|
| 62 |
+
raise ValueError(
|
| 63 |
+
f"downsample kernel-stride delta must be even; "
|
| 64 |
+
f"kernel={(kh, kw)} stride={(sy, sx)}")
|
| 65 |
+
|
| 66 |
+
conv.weight.zero_()
|
| 67 |
+
if conv.bias is not None:
|
| 68 |
+
conv.bias.zero_()
|
| 69 |
+
|
| 70 |
+
if out_ch == in_ch:
|
| 71 |
+
proj = torch.eye(out_ch, device=conv.weight.device, dtype=conv.weight.dtype)
|
| 72 |
+
elif out_ch < in_ch:
|
| 73 |
+
q, r = torch.linalg.qr(
|
| 74 |
+
torch.randn(in_ch, out_ch, device=conv.weight.device,
|
| 75 |
+
dtype=torch.float32),
|
| 76 |
+
mode='reduced',
|
| 77 |
+
)
|
| 78 |
+
signs = torch.sign(torch.diagonal(r))
|
| 79 |
+
signs = torch.where(signs == 0, torch.ones_like(signs), signs)
|
| 80 |
+
proj = (q * signs.view(1, -1)).T.to(dtype=conv.weight.dtype)
|
| 81 |
+
else:
|
| 82 |
+
q, r = torch.linalg.qr(
|
| 83 |
+
torch.randn(out_ch, in_ch, device=conv.weight.device,
|
| 84 |
+
dtype=torch.float32),
|
| 85 |
+
mode='reduced',
|
| 86 |
+
)
|
| 87 |
+
signs = torch.sign(torch.diagonal(r))
|
| 88 |
+
signs = torch.where(signs == 0, torch.ones_like(signs), signs)
|
| 89 |
+
proj = (q * signs.view(1, -1)).to(dtype=conv.weight.dtype)
|
| 90 |
+
|
| 91 |
+
conv.weight[:, :, y0:y0 + sy, x0:x0 + sx].copy_(
|
| 92 |
+
proj[:, :, None, None] / float(sy * sx))
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _frontloaded_qkv_kernels(total: int, kernel: int,
|
| 96 |
+
fraction: float = 1.0) -> list[int]:
|
| 97 |
+
kernel = int(kernel or 0)
|
| 98 |
+
total = int(total)
|
| 99 |
+
if total <= 0 or kernel <= 0:
|
| 100 |
+
return [0 for _ in range(max(total, 0))]
|
| 101 |
+
frac = max(0.0, min(float(fraction), 1.0))
|
| 102 |
+
active = total if frac >= 1.0 else int(round(total * frac))
|
| 103 |
+
active = max(0, min(total, active))
|
| 104 |
+
return [kernel if i < active else 0 for i in range(total)]
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def _stage1_qkv_kernels(n_enc: int, n_body: int, n_mid: int, n_dec: int,
|
| 108 |
+
kernel: int, fraction: float = 1.0) -> list[int]:
|
| 109 |
+
plan = _frontloaded_qkv_kernels(
|
| 110 |
+
n_enc + n_body + n_mid + n_dec,
|
| 111 |
+
kernel,
|
| 112 |
+
fraction,
|
| 113 |
+
)
|
| 114 |
+
if not plan or int(kernel or 0) <= 0 or float(fraction) >= 1.0:
|
| 115 |
+
return plan
|
| 116 |
+
|
| 117 |
+
# The front-loaded plan would spend the first /16 slots on enc_16, which
|
| 118 |
+
# is now full-spatial attention. Move those /16 QKV-DW slots onto the
|
| 119 |
+
# local mid_16 Swin detour instead; keep the /64 body allocation unchanged.
|
| 120 |
+
enc_active = [i for i in range(n_enc) if plan[i] > 0]
|
| 121 |
+
if not enc_active or n_mid <= 0:
|
| 122 |
+
return plan
|
| 123 |
+
|
| 124 |
+
for i in enc_active:
|
| 125 |
+
plan[i] = 0
|
| 126 |
+
|
| 127 |
+
mid_start = n_enc + n_body
|
| 128 |
+
mid_slots = [
|
| 129 |
+
mid_start + i for i in range(n_mid)
|
| 130 |
+
if plan[mid_start + i] == 0
|
| 131 |
+
]
|
| 132 |
+
placed = 0
|
| 133 |
+
for i in mid_slots[:len(enc_active)]:
|
| 134 |
+
plan[i] = int(kernel)
|
| 135 |
+
placed += 1
|
| 136 |
+
|
| 137 |
+
# Preserve the requested active count if a nonstandard layout has fewer
|
| 138 |
+
# empty mid slots than relocated enc slots.
|
| 139 |
+
for i in enc_active[placed:]:
|
| 140 |
+
plan[i] = int(kernel)
|
| 141 |
+
return plan
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def _qkv_kernel_at(kernels, i: int) -> int:
|
| 145 |
+
if isinstance(kernels, (list, tuple)):
|
| 146 |
+
return int(kernels[i]) if i < len(kernels) else 0
|
| 147 |
+
return int(kernels or 0)
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def _stage1_dec_blocks(d_16, num_heads_16, window_HW, mlp_mult, rope_16,
|
| 151 |
+
n_blocks=8, local_mix_kernel=0,
|
| 152 |
+
local_mix_init_std=None, mlp_dw_kernel=0,
|
| 153 |
+
qkv_dw_kernel=0):
|
| 154 |
+
"""Build the decoder pattern.
|
| 155 |
+
|
| 156 |
+
An earlier layout used 6 blocks: 2 dense Swin → 2 CSwin → 2 dense Swin.
|
| 157 |
+
The current layout uses 8 blocks: 2 dense Swin → CSwin → 2 inserted dense Swin
|
| 158 |
+
→ CSwin → 2 dense Swin. The CSwin pair still gives two
|
| 159 |
+
full-global sweeps (each block mixes both H and W axes in parallel
|
| 160 |
+
heads), with a small local refinement stack between the two sweeps.
|
| 161 |
+
|
| 162 |
+
Dense Swin shifts expose all four half-window phases. The final pair is
|
| 163 |
+
the orthogonal H-only/W-only pair because the late patch-stem skip enters
|
| 164 |
+
immediately before those blocks.
|
| 165 |
+
"""
|
| 166 |
+
w = window_HW
|
| 167 |
+
hw = w // 2
|
| 168 |
+
if n_blocks == 6:
|
| 169 |
+
specs = [
|
| 170 |
+
("swin", (0, 0)),
|
| 171 |
+
("swin", (hw, hw)),
|
| 172 |
+
("cswin", None),
|
| 173 |
+
("cswin", None),
|
| 174 |
+
("swin", (hw, 0)),
|
| 175 |
+
("swin", (0, hw)),
|
| 176 |
+
]
|
| 177 |
+
elif n_blocks == 8:
|
| 178 |
+
specs = [
|
| 179 |
+
("swin", (0, 0)),
|
| 180 |
+
("swin", (hw, hw)),
|
| 181 |
+
("cswin", None),
|
| 182 |
+
("swin", (hw, 0)),
|
| 183 |
+
("swin", (0, hw)),
|
| 184 |
+
("cswin", None),
|
| 185 |
+
("swin", (hw, 0)),
|
| 186 |
+
("swin", (0, hw)),
|
| 187 |
+
]
|
| 188 |
+
else:
|
| 189 |
+
raise ValueError(
|
| 190 |
+
f"n_dec_16 must be 6 or 8 for the fixed CSwin pattern; got {n_blocks}")
|
| 191 |
+
blocks = []
|
| 192 |
+
for i, (kind, shift) in enumerate(specs):
|
| 193 |
+
k = _qkv_kernel_at(qkv_dw_kernel, i)
|
| 194 |
+
if kind == "swin":
|
| 195 |
+
blocks.append(
|
| 196 |
+
SwinBlock3D(d_16, num_heads_16, w, w, shift=shift,
|
| 197 |
+
mlp_mult=mlp_mult, rope=rope_16,
|
| 198 |
+
local_mix_kernel=local_mix_kernel,
|
| 199 |
+
local_mix_init_std=local_mix_init_std,
|
| 200 |
+
mlp_dw_kernel=mlp_dw_kernel,
|
| 201 |
+
qkv_dw_kernel=k)
|
| 202 |
+
)
|
| 203 |
+
else:
|
| 204 |
+
blocks.append(
|
| 205 |
+
CSwinBlock3D(d_16, num_heads_16,
|
| 206 |
+
mlp_mult=mlp_mult, rope=rope_16,
|
| 207 |
+
local_mix_kernel=local_mix_kernel,
|
| 208 |
+
local_mix_init_std=local_mix_init_std,
|
| 209 |
+
mlp_dw_kernel=mlp_dw_kernel,
|
| 210 |
+
qkv_dw_kernel=k)
|
| 211 |
+
)
|
| 212 |
+
return blocks
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def _stage1_mid16_blocks(d_16, num_heads_16, window_HW, mlp_mult, rope_16,
|
| 216 |
+
n_blocks=2, local_mix_kernel=0,
|
| 217 |
+
local_mix_init_std=None, mlp_dw_kernel=0,
|
| 218 |
+
qkv_dw_kernel=0):
|
| 219 |
+
w = window_HW
|
| 220 |
+
hw = w // 2
|
| 221 |
+
phases = [(0, 0), (hw, hw), (hw, 0), (0, hw)]
|
| 222 |
+
return [
|
| 223 |
+
SwinBlock3D(d_16, num_heads_16, w, w,
|
| 224 |
+
shift=phases[i % len(phases)],
|
| 225 |
+
mlp_mult=mlp_mult, rope=rope_16,
|
| 226 |
+
local_mix_kernel=local_mix_kernel,
|
| 227 |
+
local_mix_init_std=local_mix_init_std,
|
| 228 |
+
mlp_dw_kernel=mlp_dw_kernel,
|
| 229 |
+
qkv_dw_kernel=_qkv_kernel_at(qkv_dw_kernel, i))
|
| 230 |
+
for i in range(int(n_blocks))
|
| 231 |
+
]
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
class Stage1(nn.Module):
|
| 235 |
+
def __init__(
|
| 236 |
+
self,
|
| 237 |
+
# Channel widths
|
| 238 |
+
d_16: int = 512,
|
| 239 |
+
d_64: int = 1536,
|
| 240 |
+
cradio_dim: int = 0,
|
| 241 |
+
moge_dim: int = 768,
|
| 242 |
+
rvm_dim: int = 0, # 0 disables RVM feature fusion
|
| 243 |
+
# Depth
|
| 244 |
+
n_enc_16: int = 2,
|
| 245 |
+
n_body_64: int = 10,
|
| 246 |
+
n_dec_16: int = 6,
|
| 247 |
+
# Heads — d_16=512 with 8 heads gives head_dim=64; RoPE3D splits
|
| 248 |
+
# pairs nearly evenly across T/H/W.
|
| 249 |
+
num_heads_16: int = 8, # 512 / 8 = 64
|
| 250 |
+
num_heads_64: int = 16, # 1536 / 16 = 96
|
| 251 |
+
# Windowing (spatial only — temporal is always full). Window=16 at
|
| 252 |
+
# /16 means each Swin-3D block attends across 16×16 spatial tokens
|
| 253 |
+
# (256 pixel receptive field per block). The V+H axial pair in the
|
| 254 |
+
# middle of dec_16 gives full global mixing, so the windowed blocks
|
| 255 |
+
# only need local refinement at a smaller window. SwinAttention3D
|
| 256 |
+
# pads internally for /16 grids not divisible by 16 (shape_quantum=128
|
| 257 |
+
# guarantees /8 divisibility at /16 — w=16 may still need padding
|
| 258 |
+
# for oddly-sized crops).
|
| 259 |
+
window_HW: int = 16,
|
| 260 |
+
local_mix_kernel: int = 7,
|
| 261 |
+
local_mix_init_std: float | None = None,
|
| 262 |
+
body64_local_mix_kernel: int = 7,
|
| 263 |
+
body64_local_mix_init_std: float | None = None,
|
| 264 |
+
body64_mlp_hidden: int | None = None,
|
| 265 |
+
body64_direct_concat: bool = False,
|
| 266 |
+
recurrent_64_sites: tuple[int, ...] | None = None,
|
| 267 |
+
hint_adaln_blocks: int = 2,
|
| 268 |
+
mid_16_blocks: int = 4,
|
| 269 |
+
mlp_dw_kernel: int = 0,
|
| 270 |
+
qkv_dw_kernel: int = 0,
|
| 271 |
+
qkv_dw_fraction: float = 1.0,
|
| 272 |
+
# MLP
|
| 273 |
+
mlp_mult: float = 2.5,
|
| 274 |
+
# Per-block gradient checkpointing during training. Each Swin-3D /
|
| 275 |
+
# Full-ST block is individually checkpointed → peak activation memory
|
| 276 |
+
# during backward is ~one block's activations, not the whole stage.
|
| 277 |
+
gradient_checkpoint: bool = False,
|
| 278 |
+
# Overlap patchify/depatchify (stem + aux_depatchify).
|
| 279 |
+
# overlap_patchify is a legacy bool alias (True → mult=3).
|
| 280 |
+
# overlap_mult picks the receptive-field multiplier:
|
| 281 |
+
# 1 = no overlap; 2 = half-overlap (kernel=2·patch);
|
| 282 |
+
# 3 = full-overlap (kernel=3·patch, immediate + 8 neighbors).
|
| 283 |
+
overlap_patchify: bool = False,
|
| 284 |
+
overlap_mult: int = 2,
|
| 285 |
+
depatch_fourier_features: int = 0,
|
| 286 |
+
depatch_fourier_kernel: int = 3,
|
| 287 |
+
# MAE-style pretraining hooks. Disabled in the normal inference path.
|
| 288 |
+
enable_mae_mask_tokens: bool = False,
|
| 289 |
+
bottleneck_head_stride: int | None = 8,
|
| 290 |
+
# RoPE training-time augmentations (DINOv3-style, per-forward coherent).
|
| 291 |
+
# None / 0 / 1.0 respectively disables each.
|
| 292 |
+
rope_shift_coords: float = None,
|
| 293 |
+
rope_jitter_coords: float = None,
|
| 294 |
+
rope_rescale_coords: float = None,
|
| 295 |
+
rope_base: float = 100.0,
|
| 296 |
+
):
|
| 297 |
+
super().__init__()
|
| 298 |
+
self.d_16 = d_16
|
| 299 |
+
self.d_64 = d_64
|
| 300 |
+
self.window_HW = window_HW
|
| 301 |
+
self.gradient_checkpoint = gradient_checkpoint
|
| 302 |
+
self.enable_mae_mask_tokens = bool(enable_mae_mask_tokens)
|
| 303 |
+
self.bottleneck_head_stride = bottleneck_head_stride
|
| 304 |
+
self.hint_adaln_blocks = max(0, int(hint_adaln_blocks))
|
| 305 |
+
self.mid_16_blocks_count = max(0, int(mid_16_blocks))
|
| 306 |
+
self.recurrent_64_sites = tuple(
|
| 307 |
+
int(i) for i in (recurrent_64_sites or ())
|
| 308 |
+
)
|
| 309 |
+
if len(set(self.recurrent_64_sites)) != len(self.recurrent_64_sites):
|
| 310 |
+
raise ValueError(
|
| 311 |
+
f"recurrent_64_sites contains duplicates: "
|
| 312 |
+
f"{self.recurrent_64_sites}")
|
| 313 |
+
bad_sites = [
|
| 314 |
+
i for i in self.recurrent_64_sites
|
| 315 |
+
if i < 0 or i >= int(n_body_64)
|
| 316 |
+
]
|
| 317 |
+
if bad_sites:
|
| 318 |
+
raise ValueError(
|
| 319 |
+
f"recurrent_64_sites {bad_sites} outside body block range "
|
| 320 |
+
f"0..{int(n_body_64) - 1}")
|
| 321 |
+
qkv_plan = _stage1_qkv_kernels(
|
| 322 |
+
n_enc_16,
|
| 323 |
+
n_body_64,
|
| 324 |
+
self.mid_16_blocks_count,
|
| 325 |
+
n_dec_16,
|
| 326 |
+
qkv_dw_kernel,
|
| 327 |
+
qkv_dw_fraction,
|
| 328 |
+
)
|
| 329 |
+
qkv_pos = 0
|
| 330 |
+
qkv_enc = qkv_plan[qkv_pos:qkv_pos + n_enc_16]
|
| 331 |
+
qkv_pos += n_enc_16
|
| 332 |
+
qkv_body = qkv_plan[qkv_pos:qkv_pos + n_body_64]
|
| 333 |
+
qkv_pos += n_body_64
|
| 334 |
+
qkv_mid = qkv_plan[qkv_pos:qkv_pos + self.mid_16_blocks_count]
|
| 335 |
+
qkv_pos += self.mid_16_blocks_count
|
| 336 |
+
qkv_dec = qkv_plan[qkv_pos:qkv_pos + n_dec_16]
|
| 337 |
+
self.qkv_dw_active_blocks = sum(1 for k in qkv_plan if k > 0)
|
| 338 |
+
self.qkv_dw_total_blocks = len(qkv_plan)
|
| 339 |
+
|
| 340 |
+
# Stem: 4 input channels (RGB + hint) → d_16 at /16 via k=16 s=16.
|
| 341 |
+
# Normal Kaiming init (no zero-init here — we want the stem to
|
| 342 |
+
# actually produce features from t=0 since stage 1 has nothing
|
| 343 |
+
# else at /16 to take up the slack).
|
| 344 |
+
self.stem = PatchifyConv(4, d_16, patch=16, zero_init_first=0,
|
| 345 |
+
overlap=overlap_patchify,
|
| 346 |
+
overlap_mult=overlap_mult)
|
| 347 |
+
|
| 348 |
+
# Shared RoPE instances per attention scale — saves frequency tables
|
| 349 |
+
# and (negligibly) memory. The pre-down /16 blocks use spatial-only
|
| 350 |
+
# RoPE; later /16 and /64 blocks use spatiotemporal RoPE.
|
| 351 |
+
head_dim_16 = d_16 // num_heads_16
|
| 352 |
+
head_dim_64 = d_64 // num_heads_64
|
| 353 |
+
_rope_kw = dict(base=rope_base,
|
| 354 |
+
shift_coords=rope_shift_coords,
|
| 355 |
+
jitter_coords=rope_jitter_coords,
|
| 356 |
+
rescale_coords=rope_rescale_coords)
|
| 357 |
+
self.rope_16_spatial = RoPE2D(head_dim_16, **_rope_kw)
|
| 358 |
+
self.rope_16 = RoPE3D(head_dim_16, **_rope_kw)
|
| 359 |
+
self.rope_64 = RoPE3D(head_dim_64, **_rope_kw)
|
| 360 |
+
|
| 361 |
+
# /16 encoder side: per-frame full-spatial attention. This gives the
|
| 362 |
+
# high-res hint/RGB features global spatial context before downsample,
|
| 363 |
+
# while temporal mixing remains concentrated in the /64 body.
|
| 364 |
+
assert n_enc_16 >= 1
|
| 365 |
+
self.enc_16 = nn.ModuleList([
|
| 366 |
+
FullSpatialBlock3D(
|
| 367 |
+
d_16, num_heads_16,
|
| 368 |
+
mlp_mult=mlp_mult, rope=self.rope_16_spatial,
|
| 369 |
+
local_mix_kernel=local_mix_kernel,
|
| 370 |
+
local_mix_init_std=local_mix_init_std,
|
| 371 |
+
adaln_cond_dim=(1 if i < self.hint_adaln_blocks else 0),
|
| 372 |
+
mlp_dw_kernel=mlp_dw_kernel,
|
| 373 |
+
qkv_dw_kernel=qkv_enc[i],
|
| 374 |
+
)
|
| 375 |
+
for i in range(n_enc_16)
|
| 376 |
+
])
|
| 377 |
+
|
| 378 |
+
# /16 → /64 downsample — kernel-8 stride-4. Produces d_down_aux=d_16
|
| 379 |
+
# channels that get concat'd with VFM → FusionLinSwiglu
|
| 380 |
+
# → d_64. Init as exact center-cell area pooling so the aux stream has
|
| 381 |
+
# useful /16-derived content immediately; fusion still zeros aux
|
| 382 |
+
# contribution at t=0, so the model function remains VFM-only initially.
|
| 383 |
+
d_down_aux = d_16 # 512 by default
|
| 384 |
+
self.d_down_aux = d_down_aux
|
| 385 |
+
self.cradio_dim = cradio_dim
|
| 386 |
+
self.moge_dim = moge_dim
|
| 387 |
+
self.rvm_dim = rvm_dim
|
| 388 |
+
self.vfm_channels = cradio_dim + moge_dim + rvm_dim
|
| 389 |
+
self.down = Downsample4(d_16, d_down_aux, kernel=8, stride=4, norm=False)
|
| 390 |
+
_init_center_area_downsample_(self.down.conv)
|
| 391 |
+
|
| 392 |
+
# Fusion block: linear projection with SwiGLU wrapped AROUND it.
|
| 393 |
+
# in_dim = vfm + d_down_aux
|
| 394 |
+
# out_dim = d_64 (1536)
|
| 395 |
+
# hidden = in_dim · mlp_mult
|
| 396 |
+
# Linear shortcut: semi-orthogonal VFM projection, zero-for-aux.
|
| 397 |
+
# SwiGLU: up-projects from full in_dim before down-project, so the
|
| 398 |
+
# nonlinear gate sees the full input before compression. w_down is
|
| 399 |
+
# zero-inited → SwiGLU contributes 0 at t=0.
|
| 400 |
+
# Hidden size is pinned before RVM so retrofitting the recurrent
|
| 401 |
+
# stream does not expand an already-trained fusion MLP. C-RADIO is
|
| 402 |
+
# not a retrofit path for existing checkpoints; when explicitly enabled,
|
| 403 |
+
# it is treated as part of the base frozen-foundation budget.
|
| 404 |
+
pre_rvm_in = (
|
| 405 |
+
self.cradio_dim + self.moge_dim + d_down_aux
|
| 406 |
+
)
|
| 407 |
+
hidden_pinned = int(round(pre_rvm_in * mlp_mult))
|
| 408 |
+
self.body64_direct_concat = bool(body64_direct_concat)
|
| 409 |
+
fuse_in_dim = self.vfm_channels + d_down_aux
|
| 410 |
+
if self.body64_direct_concat:
|
| 411 |
+
if d_64 != fuse_in_dim:
|
| 412 |
+
raise ValueError(
|
| 413 |
+
f"body64_direct_concat requires d_64 == VFM+aux "
|
| 414 |
+
f"({fuse_in_dim}); got d_64={d_64}")
|
| 415 |
+
self.fuse = None
|
| 416 |
+
else:
|
| 417 |
+
self.fuse = FusionLinSwiglu(
|
| 418 |
+
vfm_dim=self.vfm_channels, aux_dim=d_down_aux,
|
| 419 |
+
out_dim=d_64, mult=mlp_mult, bias=True,
|
| 420 |
+
hidden=hidden_pinned,
|
| 421 |
+
)
|
| 422 |
+
|
| 423 |
+
# /64 body. Full spatiotemporal attention — no windowing. Token
|
| 424 |
+
# count (H/64 · W/64 · T) is small enough that full quadratic
|
| 425 |
+
# attention is cheap here.
|
| 426 |
+
self.body_64 = nn.ModuleList([
|
| 427 |
+
FullSpatioTemporalBlock(d_64, num_heads_64, mlp_mult=mlp_mult,
|
| 428 |
+
rope=self.rope_64,
|
| 429 |
+
local_mix_kernel=body64_local_mix_kernel,
|
| 430 |
+
local_mix_init_std=body64_local_mix_init_std,
|
| 431 |
+
mlp_dw_kernel=mlp_dw_kernel,
|
| 432 |
+
mlp_hidden=body64_mlp_hidden,
|
| 433 |
+
qkv_dw_kernel=qkv_body[i])
|
| 434 |
+
for i in range(n_body_64)
|
| 435 |
+
])
|
| 436 |
+
self.body_64_split = n_body_64 // 2
|
| 437 |
+
self.recurrent_64 = nn.ModuleList([
|
| 438 |
+
FullResolutionRecurrentMemorySite(
|
| 439 |
+
d_64, num_heads_64, rope=RoPE2D(head_dim_64, **_rope_kw))
|
| 440 |
+
for _ in self.recurrent_64_sites
|
| 441 |
+
])
|
| 442 |
+
self._recurrent_64_index = {
|
| 443 |
+
site: i for i, site in enumerate(self.recurrent_64_sites)
|
| 444 |
+
}
|
| 445 |
+
|
| 446 |
+
# Mid-stage /64 → /16 → /64 detour. The up/down projections are
|
| 447 |
+
# zero-inited so the loop is exact-identity at init while still
|
| 448 |
+
# allowing a local /16 refinement route to grow during training.
|
| 449 |
+
self.mid_up = Upsample4ICNR(d_64, d_16, r=4, post_smooth=True, norm=False)
|
| 450 |
+
self.mid_skip_scale = nn.Parameter(torch.ones(d_16))
|
| 451 |
+
self.mid_16 = nn.ModuleList(
|
| 452 |
+
_stage1_mid16_blocks(
|
| 453 |
+
d_16, num_heads_16, window_HW, mlp_mult, self.rope_16,
|
| 454 |
+
n_blocks=self.mid_16_blocks_count,
|
| 455 |
+
local_mix_kernel=local_mix_kernel,
|
| 456 |
+
local_mix_init_std=local_mix_init_std,
|
| 457 |
+
mlp_dw_kernel=mlp_dw_kernel,
|
| 458 |
+
qkv_dw_kernel=qkv_mid,
|
| 459 |
+
)
|
| 460 |
+
)
|
| 461 |
+
self.mid_down = Downsample4(d_16, d_64, kernel=8, stride=4, norm=False)
|
| 462 |
+
with torch.no_grad():
|
| 463 |
+
nn.init.zeros_(self.mid_up.proj.weight)
|
| 464 |
+
if self.mid_up.proj.bias is not None:
|
| 465 |
+
nn.init.zeros_(self.mid_up.proj.bias)
|
| 466 |
+
nn.init.zeros_(self.mid_down.conv.weight)
|
| 467 |
+
if self.mid_down.conv.bias is not None:
|
| 468 |
+
nn.init.zeros_(self.mid_down.conv.bias)
|
| 469 |
+
|
| 470 |
+
# /64 → /16 upsample. ICNR-inited + post-conv smoothing to mimic
|
| 471 |
+
# the overlap the kernel-8 downsample provided on the encoder side.
|
| 472 |
+
self.up = Upsample4ICNR(d_64, d_16, r=4, post_smooth=True, norm=False)
|
| 473 |
+
|
| 474 |
+
# Learned per-channel U-Net skip from /16 encoder → /16 decoder.
|
| 475 |
+
# Shape (d_16,). Initialized to 1.0 — at t=0 the decoder sees the
|
| 476 |
+
# encoder /16 features at full strength. The model can learn to
|
| 477 |
+
# scale this down if the /64 body's output is preferred. Alternative
|
| 478 |
+
# was init=0 (start with no skip, grow), but encoder features contain
|
| 479 |
+
# early-pipeline RGB-derived detail that's useful to have available
|
| 480 |
+
# from step 1.
|
| 481 |
+
self.uskip_scale = nn.Parameter(torch.ones(d_16))
|
| 482 |
+
self.patch_late_skip_scale = nn.Parameter(torch.zeros(d_16))
|
| 483 |
+
|
| 484 |
+
# /16 decoder side: mixed Swin/CSwin pattern. The 6-block default is
|
| 485 |
+
# the compact 2 Swin → 2 CSwin → 2 Swin layout; the optional 8-block
|
| 486 |
+
# layout inserts a local Swin pair between the two CSwin sweeps. The
|
| 487 |
+
# late patch-stem skip is injected before the final Swin pair.
|
| 488 |
+
assert n_dec_16 in (6, 8), (
|
| 489 |
+
f"n_dec_16 must be 6 or 8 for the CSwin-in-the-middle pattern; got {n_dec_16}.")
|
| 490 |
+
self.dec_16 = nn.ModuleList(
|
| 491 |
+
_stage1_dec_blocks(d_16, num_heads_16, window_HW, mlp_mult,
|
| 492 |
+
self.rope_16, n_blocks=n_dec_16,
|
| 493 |
+
local_mix_kernel=local_mix_kernel,
|
| 494 |
+
local_mix_init_std=local_mix_init_std,
|
| 495 |
+
mlp_dw_kernel=mlp_dw_kernel,
|
| 496 |
+
qkv_dw_kernel=qkv_dec)
|
| 497 |
+
)
|
| 498 |
+
|
| 499 |
+
# Aux head: baton at /16 (d=512) → 4-channel prediction at /2 via
|
| 500 |
+
# 8× depatchify. With production dims and non-overlap patching, the
|
| 501 |
+
# FG rows are initialized as area-downsampled inverse RGB patch basis
|
| 502 |
+
# and alpha rows remain zero.
|
| 503 |
+
self.aux_depatchify = Depatchify(d_16, out_channels=4, r=8,
|
| 504 |
+
zero_init_all=True, bias=True,
|
| 505 |
+
overlap=overlap_patchify,
|
| 506 |
+
overlap_mult=overlap_mult,
|
| 507 |
+
fourier_features=depatch_fourier_features,
|
| 508 |
+
fourier_kernel=depatch_fourier_kernel)
|
| 509 |
+
self.rgb_passthrough_init = init_orthogonal_rgb_passthrough_(
|
| 510 |
+
self.stem, self.aux_depatchify,
|
| 511 |
+
token_offset=0,
|
| 512 |
+
output_rgb_offset=1,
|
| 513 |
+
downsample=2,
|
| 514 |
+
)
|
| 515 |
+
if self.rgb_passthrough_init:
|
| 516 |
+
nn.init.zeros_(self.up.proj.weight)
|
| 517 |
+
if self.up.proj.bias is not None:
|
| 518 |
+
nn.init.zeros_(self.up.proj.bias)
|
| 519 |
+
if self.enable_mae_mask_tokens:
|
| 520 |
+
self.mae_token_16 = nn.Parameter(torch.zeros(d_16))
|
| 521 |
+
self.mae_token_64_aux = nn.Parameter(torch.zeros(d_down_aux))
|
| 522 |
+
if self.vfm_channels > 0:
|
| 523 |
+
self.mae_token_vfm = nn.Parameter(torch.zeros(self.vfm_channels))
|
| 524 |
+
if bottleneck_head_stride is not None:
|
| 525 |
+
if bottleneck_head_stride not in (4, 8):
|
| 526 |
+
raise ValueError(
|
| 527 |
+
"bottleneck_head_stride must be 4 or 8 when enabled")
|
| 528 |
+
r = 64 // int(bottleneck_head_stride)
|
| 529 |
+
self.bottleneck_depatchify = Depatchify(
|
| 530 |
+
d_64, out_channels=4, r=r,
|
| 531 |
+
zero_init_all=True, bias=True,
|
| 532 |
+
overlap=overlap_patchify,
|
| 533 |
+
overlap_mult=overlap_mult,
|
| 534 |
+
fourier_features=depatch_fourier_features,
|
| 535 |
+
fourier_kernel=depatch_fourier_kernel,
|
| 536 |
+
)
|
| 537 |
+
|
| 538 |
+
# -----------------------------------------------------------------
|
| 539 |
+
|
| 540 |
+
def _ckpt_block(
|
| 541 |
+
self,
|
| 542 |
+
blk,
|
| 543 |
+
x: torch.Tensor,
|
| 544 |
+
t_stride: float,
|
| 545 |
+
adaln_cond: torch.Tensor | None = None,
|
| 546 |
+
) -> torch.Tensor:
|
| 547 |
+
"""Call `blk(x, t_stride=t_stride)` — wrapped in `torch.utils.checkpoint`
|
| 548 |
+
when training with gradient_checkpoint enabled so only one block's
|
| 549 |
+
activations are resident at a time during backward."""
|
| 550 |
+
if adaln_cond is not None:
|
| 551 |
+
if self.gradient_checkpoint and self.training and torch.is_grad_enabled():
|
| 552 |
+
return torch.utils.checkpoint.checkpoint(
|
| 553 |
+
blk, x, t_stride, adaln_cond, use_reentrant=False,
|
| 554 |
+
)
|
| 555 |
+
return blk(x, t_stride, adaln_cond)
|
| 556 |
+
if self.gradient_checkpoint and self.training and torch.is_grad_enabled():
|
| 557 |
+
return torch.utils.checkpoint.checkpoint(
|
| 558 |
+
blk, x, t_stride, use_reentrant=False,
|
| 559 |
+
)
|
| 560 |
+
return blk(x, t_stride=t_stride)
|
| 561 |
+
|
| 562 |
+
def _swin3d_run(
|
| 563 |
+
self,
|
| 564 |
+
blocks,
|
| 565 |
+
x_5d: torch.Tensor,
|
| 566 |
+
t_stride: float,
|
| 567 |
+
hint_quality: torch.Tensor | None = None,
|
| 568 |
+
late_skip_5d: torch.Tensor | None = None,
|
| 569 |
+
late_skip_scale: torch.Tensor | None = None,
|
| 570 |
+
late_skip_index: int | None = None,
|
| 571 |
+
) -> torch.Tensor:
|
| 572 |
+
"""Run a list of Swin3D blocks on a 5D tensor (B, T, C, H, W).
|
| 573 |
+
|
| 574 |
+
Swin blocks accept (B, T, H, W, C), so permute in & out here.
|
| 575 |
+
"""
|
| 576 |
+
frame_chunk = int(getattr(self, "low_vram_enc_frame_chunk", 0) or 0)
|
| 577 |
+
if (frame_chunk > 0 and not self.training and late_skip_5d is None
|
| 578 |
+
and len(blocks) > 0
|
| 579 |
+
and all(isinstance(blk, FullSpatialBlock3D) for blk in blocks)
|
| 580 |
+
and x_5d.shape[1] > frame_chunk):
|
| 581 |
+
outs = []
|
| 582 |
+
for t0 in range(0, x_5d.shape[1], frame_chunk):
|
| 583 |
+
t1 = min(t0 + frame_chunk, x_5d.shape[1])
|
| 584 |
+
x = x_5d[:, t0:t1].permute(0, 1, 3, 4, 2).contiguous()
|
| 585 |
+
hq = (
|
| 586 |
+
hint_quality[:, t0:t1]
|
| 587 |
+
if hint_quality is not None else None
|
| 588 |
+
)
|
| 589 |
+
for blk in blocks:
|
| 590 |
+
cond = hq if getattr(blk, "adaln", None) is not None else None
|
| 591 |
+
x = self._ckpt_block(blk, x, t_stride, adaln_cond=cond)
|
| 592 |
+
outs.append(x.permute(0, 1, 4, 2, 3).contiguous())
|
| 593 |
+
return torch.cat(outs, dim=1)
|
| 594 |
+
|
| 595 |
+
# (B, T, C, H, W) → (B, T, H, W, C)
|
| 596 |
+
x = x_5d.permute(0, 1, 3, 4, 2).contiguous()
|
| 597 |
+
skip = None
|
| 598 |
+
if late_skip_5d is not None:
|
| 599 |
+
skip = late_skip_5d.permute(0, 1, 3, 4, 2).contiguous()
|
| 600 |
+
if late_skip_scale is None:
|
| 601 |
+
raise ValueError("late_skip_scale is required with late_skip_5d")
|
| 602 |
+
if late_skip_index is None:
|
| 603 |
+
late_skip_index = max(len(blocks) - 2, 0)
|
| 604 |
+
scale = late_skip_scale.to(device=x.device, dtype=x.dtype).view(
|
| 605 |
+
1, 1, 1, 1, -1)
|
| 606 |
+
else:
|
| 607 |
+
scale = None
|
| 608 |
+
for i, blk in enumerate(blocks):
|
| 609 |
+
if skip is not None and i == late_skip_index:
|
| 610 |
+
x = x + skip * scale
|
| 611 |
+
cond = hint_quality if getattr(blk, "adaln", None) is not None else None
|
| 612 |
+
x = self._ckpt_block(blk, x, t_stride, adaln_cond=cond)
|
| 613 |
+
return x.permute(0, 1, 4, 2, 3).contiguous()
|
| 614 |
+
|
| 615 |
+
def _full_run(self, blocks, x_5d: torch.Tensor, t_stride: float) -> torch.Tensor:
|
| 616 |
+
"""Run full-spatiotemporal blocks on (B, T, C, H, W)."""
|
| 617 |
+
x = x_5d.permute(0, 1, 3, 4, 2).contiguous()
|
| 618 |
+
for blk in blocks:
|
| 619 |
+
x = self._ckpt_block(blk, x, t_stride)
|
| 620 |
+
return x.permute(0, 1, 4, 2, 3).contiguous()
|
| 621 |
+
|
| 622 |
+
def _full_run_nthwc(
|
| 623 |
+
self,
|
| 624 |
+
blocks,
|
| 625 |
+
x: torch.Tensor,
|
| 626 |
+
t_stride: float,
|
| 627 |
+
) -> torch.Tensor:
|
| 628 |
+
"""Run full-spatiotemporal blocks on native (B, T, H, W, C) layout."""
|
| 629 |
+
for blk in blocks:
|
| 630 |
+
x = self._ckpt_block(blk, x, t_stride)
|
| 631 |
+
return x
|
| 632 |
+
|
| 633 |
+
def _ckpt_body64_with_recurrent(
|
| 634 |
+
self,
|
| 635 |
+
blk,
|
| 636 |
+
site,
|
| 637 |
+
x: torch.Tensor,
|
| 638 |
+
state: torch.Tensor | None,
|
| 639 |
+
t_stride: float,
|
| 640 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 641 |
+
if self.gradient_checkpoint and self.training and torch.is_grad_enabled():
|
| 642 |
+
if state is None:
|
| 643 |
+
def fn(inp):
|
| 644 |
+
return blk(
|
| 645 |
+
inp, t_stride,
|
| 646 |
+
recurrent_site=site,
|
| 647 |
+
recurrent_state=None,
|
| 648 |
+
)
|
| 649 |
+
return torch.utils.checkpoint.checkpoint(
|
| 650 |
+
fn, x, use_reentrant=False,
|
| 651 |
+
)
|
| 652 |
+
|
| 653 |
+
def fn(inp, st):
|
| 654 |
+
return blk(
|
| 655 |
+
inp, t_stride,
|
| 656 |
+
recurrent_site=site,
|
| 657 |
+
recurrent_state=st,
|
| 658 |
+
)
|
| 659 |
+
return torch.utils.checkpoint.checkpoint(
|
| 660 |
+
fn, x, state, use_reentrant=False,
|
| 661 |
+
)
|
| 662 |
+
return blk(
|
| 663 |
+
x, t_stride=t_stride,
|
| 664 |
+
recurrent_site=site,
|
| 665 |
+
recurrent_state=state,
|
| 666 |
+
)
|
| 667 |
+
|
| 668 |
+
def _run_body64_range(
|
| 669 |
+
self,
|
| 670 |
+
start: int,
|
| 671 |
+
end: int,
|
| 672 |
+
x: torch.Tensor,
|
| 673 |
+
t_stride: float,
|
| 674 |
+
recurrent_states: list[torch.Tensor | None] | None,
|
| 675 |
+
) -> tuple[torch.Tensor, list[torch.Tensor | None] | None]:
|
| 676 |
+
for block_idx in range(start, end):
|
| 677 |
+
site_idx = self._recurrent_64_index.get(block_idx)
|
| 678 |
+
if site_idx is not None:
|
| 679 |
+
if recurrent_states is None:
|
| 680 |
+
raise RuntimeError(
|
| 681 |
+
"internal error: recurrent site active without states")
|
| 682 |
+
x, recurrent_states[site_idx] = self._ckpt_body64_with_recurrent(
|
| 683 |
+
self.body_64[block_idx],
|
| 684 |
+
self.recurrent_64[site_idx],
|
| 685 |
+
x,
|
| 686 |
+
recurrent_states[site_idx],
|
| 687 |
+
t_stride,
|
| 688 |
+
)
|
| 689 |
+
else:
|
| 690 |
+
x = self._ckpt_block(self.body_64[block_idx], x, t_stride)
|
| 691 |
+
return x, recurrent_states
|
| 692 |
+
|
| 693 |
+
def _apply_mae_token(
|
| 694 |
+
self,
|
| 695 |
+
x: torch.Tensor,
|
| 696 |
+
mask: torch.Tensor | None,
|
| 697 |
+
token: torch.Tensor,
|
| 698 |
+
name: str,
|
| 699 |
+
) -> torch.Tensor:
|
| 700 |
+
if mask is None:
|
| 701 |
+
return x
|
| 702 |
+
if not self.enable_mae_mask_tokens:
|
| 703 |
+
raise RuntimeError(f"{name} received an MAE mask but MAE tokens are disabled")
|
| 704 |
+
expected = (x.shape[0], x.shape[1], 1, x.shape[-2], x.shape[-1])
|
| 705 |
+
if tuple(mask.shape) != expected:
|
| 706 |
+
raise ValueError(
|
| 707 |
+
f"{name} mask shape {tuple(mask.shape)} != expected {expected}")
|
| 708 |
+
mask = mask.to(device=x.device, dtype=torch.bool)
|
| 709 |
+
tok = token.to(device=x.device, dtype=x.dtype).view(1, 1, -1, 1, 1)
|
| 710 |
+
return torch.where(mask, tok, x)
|
| 711 |
+
|
| 712 |
+
@staticmethod
|
| 713 |
+
def _resolve_hint_quality(
|
| 714 |
+
hint: torch.Tensor,
|
| 715 |
+
hint_quality: torch.Tensor | None,
|
| 716 |
+
) -> torch.Tensor:
|
| 717 |
+
if hint_quality is None:
|
| 718 |
+
return (hint.detach().amax(dim=(-1, -2, -3)) > 1e-5).to(dtype=hint.dtype)
|
| 719 |
+
q = hint_quality.to(device=hint.device, dtype=hint.dtype)
|
| 720 |
+
while q.dim() > 2 and q.shape[-1] == 1:
|
| 721 |
+
q = q.squeeze(-1)
|
| 722 |
+
if q.dim() != 2:
|
| 723 |
+
raise ValueError(
|
| 724 |
+
f"hint_quality must be (B,T) or trailing-singleton equivalent; got {tuple(q.shape)}")
|
| 725 |
+
if tuple(q.shape) != tuple(hint.shape[:2]):
|
| 726 |
+
raise ValueError(
|
| 727 |
+
f"hint_quality shape {tuple(q.shape)} != hint B,T {tuple(hint.shape[:2])}")
|
| 728 |
+
return q.clamp_(-1.0, 1.0)
|
| 729 |
+
|
| 730 |
+
def forward(
|
| 731 |
+
self,
|
| 732 |
+
rgb: torch.Tensor, # (B, T, 3, H, W) signed asinh encoded scene-linear RGB
|
| 733 |
+
hint: torch.Tensor, # (B, T, 1, H, W) in [0, 1]
|
| 734 |
+
moge_feat: torch.Tensor, # (B, T, moge_dim, H/64, W/64)
|
| 735 |
+
cradio_feat: 'torch.Tensor | None' = None, # (B, T, cradio_dim, H/64, W/64)
|
| 736 |
+
rvm_feat: 'torch.Tensor | None' = None, # (B, T, rvm_dim, H/64, W/64) if enabled
|
| 737 |
+
t_stride: float = 1.0,
|
| 738 |
+
mae_mask_16: 'torch.Tensor | None' = None, # (B,T,1,H/16,W/16)
|
| 739 |
+
mae_mask_64: 'torch.Tensor | None' = None, # (B,T,1,H/64,W/64)
|
| 740 |
+
mae_vfm_mask_64: 'torch.Tensor | None' = None, # (B,T,1,H/64,W/64)
|
| 741 |
+
hint_quality: 'torch.Tensor | None' = None, # (B,T), -1=decide subject, 0=no frame hint, 1=perfect
|
| 742 |
+
recurrent_states: 'tuple[torch.Tensor | None, ...] | None' = None,
|
| 743 |
+
) -> dict:
|
| 744 |
+
"""Returns dict with:
|
| 745 |
+
baton: (B, T, d_16, H/16, W/16) — feeds stage 2
|
| 746 |
+
aux: (B, T, 4, H/2, W/2) — stage-1 supervised output
|
| 747 |
+
"""
|
| 748 |
+
B, T, _, H, W = rgb.shape
|
| 749 |
+
assert H % 64 == 0 and W % 64 == 0, (
|
| 750 |
+
f"Stage1: H,W must be multiples of 64 (shape_quantum); got {(H, W)}")
|
| 751 |
+
|
| 752 |
+
# Merge B·T for conv ops that don't care about T
|
| 753 |
+
BT = B * T
|
| 754 |
+
hint_quality = self._resolve_hint_quality(hint, hint_quality)
|
| 755 |
+
x_in = torch.cat([rgb, hint], dim=2).reshape(BT, 4, H, W)
|
| 756 |
+
|
| 757 |
+
# /1 → /16 patchify
|
| 758 |
+
f_16 = self.stem(x_in) # (BT, d_16, H/16, W/16)
|
| 759 |
+
del x_in
|
| 760 |
+
f_16 = f_16.reshape(B, T, self.d_16, H // 16, W // 16)
|
| 761 |
+
if mae_mask_16 is not None:
|
| 762 |
+
f_16 = self._apply_mae_token(f_16, mae_mask_16,
|
| 763 |
+
self.mae_token_16, "stage1 /16")
|
| 764 |
+
f_16_patch = f_16
|
| 765 |
+
|
| 766 |
+
# /16 encoder side (per-frame full-spatial). Keep a handle for the
|
| 767 |
+
# U-skip before the downsample eats the spatial detail.
|
| 768 |
+
f_16_enc = self._swin3d_run(
|
| 769 |
+
self.enc_16, f_16, t_stride, hint_quality=hint_quality)
|
| 770 |
+
|
| 771 |
+
# /16 → /64 downsample (center-cell area init; produces d_down_aux channels)
|
| 772 |
+
f_16_flat = f_16_enc.reshape(BT, self.d_16, H // 16, W // 16)
|
| 773 |
+
f_64_down = self.down(f_16_flat) # (BT, d_down_aux, H/64, W/64)
|
| 774 |
+
del f_16_flat
|
| 775 |
+
h_64, w_64 = H // 64, W // 64
|
| 776 |
+
f_64_down = f_64_down.reshape(B, T, self.d_down_aux, h_64, w_64)
|
| 777 |
+
if mae_mask_64 is not None:
|
| 778 |
+
f_64_down = self._apply_mae_token(
|
| 779 |
+
f_64_down, mae_mask_64, self.mae_token_64_aux,
|
| 780 |
+
"stage1 /64 aux")
|
| 781 |
+
|
| 782 |
+
# VFM features at /64 (already channel-normalized by FrozenFoundations).
|
| 783 |
+
# Concat order is fixed at [cradio | moge | rvm] — the fuse layer's
|
| 784 |
+
# linear_shortcut init relies on this ordering.
|
| 785 |
+
assert moge_feat.shape[-2:] == (h_64, w_64), (
|
| 786 |
+
f"moge_feat spatial {moge_feat.shape[-2:]} != /64 ({(h_64, w_64)})")
|
| 787 |
+
vfm_parts = []
|
| 788 |
+
if self.cradio_dim > 0:
|
| 789 |
+
assert cradio_feat is not None, (
|
| 790 |
+
"cradio_dim > 0 requires cradio_feat to be passed in")
|
| 791 |
+
assert cradio_feat.shape[-2:] == (h_64, w_64), (
|
| 792 |
+
f"cradio_feat spatial {cradio_feat.shape[-2:]} != /64 ({(h_64, w_64)})")
|
| 793 |
+
vfm_parts.append(cradio_feat)
|
| 794 |
+
vfm_parts.append(moge_feat)
|
| 795 |
+
if self.rvm_dim > 0:
|
| 796 |
+
assert rvm_feat is not None, (
|
| 797 |
+
"rvm_dim > 0 requires rvm_feat to be passed in")
|
| 798 |
+
# RVM shares C-RADIO's input (H/4) and patch size (16), so features
|
| 799 |
+
# land on the /64 grid directly — no resize needed.
|
| 800 |
+
assert rvm_feat.shape[-2:] == (h_64, w_64), (
|
| 801 |
+
f"rvm_feat spatial {rvm_feat.shape[-2:]} != /64 ({(h_64, w_64)}); "
|
| 802 |
+
f"RVM should be fed at C-RADIO's H/4 input size")
|
| 803 |
+
vfm_parts.append(rvm_feat)
|
| 804 |
+
vfm_cat = torch.cat(vfm_parts, dim=2) # (B, T, vfm_C, h, w)
|
| 805 |
+
|
| 806 |
+
# Pad VFM to match self.vfm_channels if something got disabled
|
| 807 |
+
# (e.g. one foundation off in smoke-test) so the SwiGLU input shape
|
| 808 |
+
# stays constant.
|
| 809 |
+
pad_c = self.vfm_channels - vfm_cat.shape[2]
|
| 810 |
+
if pad_c > 0:
|
| 811 |
+
vfm_cat = F.pad(vfm_cat, (0, 0, 0, 0, 0, pad_c))
|
| 812 |
+
if mae_vfm_mask_64 is not None:
|
| 813 |
+
if not hasattr(self, "mae_token_vfm"):
|
| 814 |
+
raise RuntimeError(
|
| 815 |
+
"received an MAE VFM mask but VFM MAE token is disabled")
|
| 816 |
+
vfm_cat = self._apply_mae_token(
|
| 817 |
+
vfm_cat, mae_vfm_mask_64, self.mae_token_vfm,
|
| 818 |
+
"stage1 /64 VFM")
|
| 819 |
+
|
| 820 |
+
# Full concat at /64 (vfm + downsampled aux) → either direct /64 body
|
| 821 |
+
# input or fusion (linear+SwiGLU).
|
| 822 |
+
combined = torch.cat([vfm_cat, f_64_down], dim=2) # (B, T, vfm+aux, h, w)
|
| 823 |
+
del vfm_cat, f_64_down, vfm_parts
|
| 824 |
+
tokens = combined.permute(0, 1, 3, 4, 2).contiguous() # (B, T, h, w, fuse_in_dim)
|
| 825 |
+
del combined
|
| 826 |
+
f_64 = tokens if self.fuse is None else self.fuse(tokens) # (B, T, h, w, d_64)
|
| 827 |
+
del tokens
|
| 828 |
+
|
| 829 |
+
# /64 body, with a mid-stage detour back through /16 for local
|
| 830 |
+
# refinement before returning to /64 for the remaining global blocks.
|
| 831 |
+
recurrent_states_out = None
|
| 832 |
+
if len(self.recurrent_64) > 0:
|
| 833 |
+
if recurrent_states is None:
|
| 834 |
+
recurrent_states_out = [None for _ in self.recurrent_64]
|
| 835 |
+
else:
|
| 836 |
+
if len(recurrent_states) != len(self.recurrent_64):
|
| 837 |
+
raise ValueError(
|
| 838 |
+
f"got {len(recurrent_states)} recurrent states for "
|
| 839 |
+
f"{len(self.recurrent_64)} recurrent sites")
|
| 840 |
+
recurrent_states_out = list(recurrent_states)
|
| 841 |
+
|
| 842 |
+
split = self.body_64_split
|
| 843 |
+
if recurrent_states_out is None:
|
| 844 |
+
f_64 = self._full_run_nthwc(self.body_64[:split], f_64, t_stride)
|
| 845 |
+
else:
|
| 846 |
+
f_64, recurrent_states_out = self._run_body64_range(
|
| 847 |
+
0, split, f_64, t_stride, recurrent_states_out)
|
| 848 |
+
|
| 849 |
+
f_64_chw = f_64.permute(0, 1, 4, 2, 3).contiguous()
|
| 850 |
+
f_64_mid_flat = f_64_chw.reshape(BT, self.d_64, H // 64, W // 64)
|
| 851 |
+
del f_64_chw
|
| 852 |
+
f_16_mid_up = self.mid_up(f_64_mid_flat).reshape(
|
| 853 |
+
B, T, self.d_16, H // 16, W // 16)
|
| 854 |
+
del f_64_mid_flat
|
| 855 |
+
mid_scale = self.mid_skip_scale.view(1, 1, self.d_16, 1, 1)
|
| 856 |
+
f_16_mid = f_16_mid_up + mid_scale * f_16_enc
|
| 857 |
+
del f_16_mid_up, f_16_enc
|
| 858 |
+
if len(self.mid_16) > 0:
|
| 859 |
+
f_16_mid = self._swin3d_run(self.mid_16, f_16_mid, t_stride)
|
| 860 |
+
f_16_mid_flat = f_16_mid.reshape(BT, self.d_16, H // 16, W // 16)
|
| 861 |
+
f_64_delta = self.mid_down(f_16_mid_flat).reshape(
|
| 862 |
+
B, T, self.d_64, H // 64, W // 64)
|
| 863 |
+
del f_16_mid_flat
|
| 864 |
+
f_64 = f_64 + f_64_delta.permute(0, 1, 3, 4, 2).contiguous()
|
| 865 |
+
del f_64_delta
|
| 866 |
+
|
| 867 |
+
if recurrent_states_out is None:
|
| 868 |
+
f_64 = self._full_run_nthwc(self.body_64[split:], f_64, t_stride)
|
| 869 |
+
else:
|
| 870 |
+
f_64, recurrent_states_out = self._run_body64_range(
|
| 871 |
+
split, len(self.body_64), f_64, t_stride,
|
| 872 |
+
recurrent_states_out)
|
| 873 |
+
f_64_chw = f_64.permute(0, 1, 4, 2, 3).contiguous()
|
| 874 |
+
del f_64
|
| 875 |
+
|
| 876 |
+
bottleneck_out = None
|
| 877 |
+
if self.bottleneck_head_stride is not None:
|
| 878 |
+
f_64_head = f_64_chw.reshape(BT, self.d_64, H // 64, W // 64)
|
| 879 |
+
raw = self.bottleneck_depatchify(f_64_head)
|
| 880 |
+
del f_64_head
|
| 881 |
+
s = int(self.bottleneck_head_stride)
|
| 882 |
+
bottleneck_out = raw.reshape(B, T, 4, H // s, W // s)
|
| 883 |
+
del raw
|
| 884 |
+
|
| 885 |
+
# /64 → /16 upsample
|
| 886 |
+
f_64_flat = f_64_chw.reshape(BT, self.d_64, H // 64, W // 64)
|
| 887 |
+
del f_64_chw
|
| 888 |
+
f_16_up = self.up(f_64_flat) # (BT, d_16, H/16, W/16)
|
| 889 |
+
del f_64_flat
|
| 890 |
+
f_16_up = f_16_up.reshape(B, T, self.d_16, H // 16, W // 16)
|
| 891 |
+
|
| 892 |
+
# Learned-weight U-Net skip: add encoder /16 features scaled by a
|
| 893 |
+
# per-channel parameter. Broadcasts (d_16,) over (B, T, C, H, W).
|
| 894 |
+
scale = self.uskip_scale.view(1, 1, self.d_16, 1, 1)
|
| 895 |
+
f_16_dec = f_16_up + scale * f_16_mid
|
| 896 |
+
del f_16_up, f_16_mid
|
| 897 |
+
|
| 898 |
+
# /16 decoder side (Swin-3D)
|
| 899 |
+
baton = self._swin3d_run(
|
| 900 |
+
self.dec_16, f_16_dec, t_stride,
|
| 901 |
+
late_skip_5d=f_16_patch,
|
| 902 |
+
late_skip_scale=self.patch_late_skip_scale,
|
| 903 |
+
late_skip_index=max(len(self.dec_16) - 2, 0),
|
| 904 |
+
) # (B, T, d_16, H/16, W/16)
|
| 905 |
+
del f_16_dec, f_16_patch
|
| 906 |
+
|
| 907 |
+
# Aux head: /16 baton → /2 pred (α, FG). With production dims and
|
| 908 |
+
# non-overlap patching, FG is the exact area-downsampled inverse of
|
| 909 |
+
# the stem RGB patch basis at init; alpha remains zero.
|
| 910 |
+
BT = B * T
|
| 911 |
+
baton_flat = baton.reshape(BT, self.d_16, H // 16, W // 16)
|
| 912 |
+
aux_raw = self.aux_depatchify(baton_flat) # (BT, 4, H/2, W/2)
|
| 913 |
+
del baton_flat
|
| 914 |
+
aux_out = aux_raw.reshape(B, T, 4, H // 2, W // 2)
|
| 915 |
+
del aux_raw
|
| 916 |
+
|
| 917 |
+
out = {'baton': baton, 'aux': aux_out, 'bottleneck': bottleneck_out}
|
| 918 |
+
if recurrent_states_out is not None:
|
| 919 |
+
out['recurrent_states'] = tuple(recurrent_states_out)
|
| 920 |
+
return out
|
corridorkey/stage2.py
ADDED
|
@@ -0,0 +1,532 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Stage 2 — pure-transformer refinement at /16, depatchify to /1.
|
| 2 |
+
|
| 3 |
+
Design:
|
| 4 |
+
• Patchify RGB to 768d at /16.
|
| 5 |
+
• Concatenate stage-1 baton (d=512) with the 768d RGB patch stream:
|
| 6 |
+
features = [baton | rgb_patchify].
|
| 7 |
+
• 8 × mixed FullSpatial/Swin/CSwinBlock2D spatial attention (d=1280):
|
| 8 |
+
full spatial at the first and middle refresh points, plus Swin/CSwin
|
| 9 |
+
local/axial refinement across the four half-window shift phases.
|
| 10 |
+
• In the main training phase, a SECOND zero-inited 768d patchify_2 value
|
| 11 |
+
branch of raw RGB is added one block before the end. A live full-rank
|
| 12 |
+
gate reads the mixed token stream and modulates that zero-init value,
|
| 13 |
+
so the branch is output-neutral at insertion while still becoming
|
| 14 |
+
context-dependent once it starts learning. MAE pretraining omits this
|
| 15 |
+
module; it is introduced zero-init when switching to the main phase.
|
| 16 |
+
• 2 more SwinBlock2D blocks.
|
| 17 |
+
• Depatchify 16× (Linear(d → 4·16²) + PixelShuffle(16)); FG rows are
|
| 18 |
+
initialized as the inverse of the RGB patch basis when the exact 768d
|
| 19 |
+
RGB basis is available, and alpha rows start at zero.
|
| 20 |
+
|
| 21 |
+
No temporal processing here — stage 1 has done all temporal work. Stage 2
|
| 22 |
+
runs per-frame (B·T batched), which also means windowed spatial attention
|
| 23 |
+
is the tightest-possible attention shape for a 16K-token-per-frame budget
|
| 24 |
+
(256 windows × 64 tokens squared = 1M entries per head per layer).
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
import torch
|
| 28 |
+
import torch.nn as nn
|
| 29 |
+
import torch.nn.functional as F
|
| 30 |
+
|
| 31 |
+
from .blocks import (
|
| 32 |
+
PatchifyConv, Depatchify, FullSpatialBlock2D, SwinBlock2D, CSwinBlock2D,
|
| 33 |
+
LayerNorm2d, icnr_init_, init_orthogonal_rgb_passthrough_,
|
| 34 |
+
)
|
| 35 |
+
from .rope import RoPE2D
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _frontloaded_qkv_kernels(total: int, kernel: int,
|
| 39 |
+
fraction: float = 1.0) -> list[int]:
|
| 40 |
+
kernel = int(kernel or 0)
|
| 41 |
+
total = int(total)
|
| 42 |
+
if total <= 0 or kernel <= 0:
|
| 43 |
+
return [0 for _ in range(max(total, 0))]
|
| 44 |
+
frac = max(0.0, min(float(fraction), 1.0))
|
| 45 |
+
active = total if frac >= 1.0 else int(round(total * frac))
|
| 46 |
+
active = max(0, min(total, active))
|
| 47 |
+
return [kernel if i < active else 0 for i in range(total)]
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _stage2_qkv_kernels(total: int, kernel: int,
|
| 51 |
+
fraction: float = 1.0) -> list[int]:
|
| 52 |
+
kernel = int(kernel or 0)
|
| 53 |
+
total = int(total)
|
| 54 |
+
if total <= 0 or kernel <= 0:
|
| 55 |
+
return [0 for _ in range(max(total, 0))]
|
| 56 |
+
frac = max(0.0, min(float(fraction), 1.0))
|
| 57 |
+
if frac <= 0.0:
|
| 58 |
+
return [0 for _ in range(total)]
|
| 59 |
+
|
| 60 |
+
if total != 10:
|
| 61 |
+
return _frontloaded_qkv_kernels(total, kernel, frac)
|
| 62 |
+
|
| 63 |
+
# Default stage-2 layout with full-spatial refresh blocks:
|
| 64 |
+
# 0 Full, 1 Swin(h,h), 2 CSwin, 3 Swin(h,0),
|
| 65 |
+
# 4 Full, 5 CSwin, 6 Swin(0,0), 7 Swin(h,h),
|
| 66 |
+
# 8 Swin(h,0), 9 Swin(0,h)
|
| 67 |
+
# Prefer QKV-DW on Swin blocks, one pass per shift phase first.
|
| 68 |
+
if abs(frac - 0.25) < 1e-12:
|
| 69 |
+
active = 4
|
| 70 |
+
else:
|
| 71 |
+
active = total if frac >= 1.0 else int(round(total * frac))
|
| 72 |
+
active = max(0, min(total, active))
|
| 73 |
+
|
| 74 |
+
priority = [1, 3, 6, 9, 7, 8, 0, 4, 2, 5]
|
| 75 |
+
plan = [0 for _ in range(total)]
|
| 76 |
+
for i in priority[:active]:
|
| 77 |
+
plan[i] = kernel
|
| 78 |
+
return plan
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _qkv_kernel_at(kernels, i: int) -> int:
|
| 82 |
+
if isinstance(kernels, (list, tuple)):
|
| 83 |
+
return int(kernels[i]) if i < len(kernels) else 0
|
| 84 |
+
return int(kernels or 0)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _stage2_pre_blocks(d, num_heads, window_size, mlp_mult, rope,
|
| 88 |
+
num_pre_blocks=8,
|
| 89 |
+
local_mix_kernel=0,
|
| 90 |
+
local_mix_init_std=None,
|
| 91 |
+
mlp_dw_kernel=0,
|
| 92 |
+
qkv_dw_kernel=0):
|
| 93 |
+
"""Build the pre-patch2 pattern.
|
| 94 |
+
|
| 95 |
+
An earlier layout used 6 blocks: 2 dense → 2 CSwin → 2 dense with orthogonal shifts.
|
| 96 |
+
The current layout uses 8 blocks: 2 dense → CSwin → 2 inserted dense → CSwin → 2 dense.
|
| 97 |
+
|
| 98 |
+
The CSwin pair in the middle gives two full-global sweeps (each block
|
| 99 |
+
mixes both H and W axes via parallel head groups), with a local Swin
|
| 100 |
+
refinement pair between those sweeps. Dense Swin shifts are arranged so
|
| 101 |
+
the 8 pre/final dense blocks cover the four half-window phases evenly.
|
| 102 |
+
"""
|
| 103 |
+
w = window_size
|
| 104 |
+
hw = w // 2
|
| 105 |
+
if num_pre_blocks == 6:
|
| 106 |
+
specs = [
|
| 107 |
+
("swin", (0, 0)),
|
| 108 |
+
("swin", (hw, hw)),
|
| 109 |
+
("cswin", None),
|
| 110 |
+
("cswin", None),
|
| 111 |
+
("swin", (hw, 0)),
|
| 112 |
+
("swin", (0, hw)),
|
| 113 |
+
]
|
| 114 |
+
elif num_pre_blocks == 8:
|
| 115 |
+
specs = [
|
| 116 |
+
("full", None),
|
| 117 |
+
("swin", (hw, hw)),
|
| 118 |
+
("cswin", None),
|
| 119 |
+
("swin", (hw, 0)),
|
| 120 |
+
("full", None),
|
| 121 |
+
("cswin", None),
|
| 122 |
+
("swin", (0, 0)),
|
| 123 |
+
("swin", (hw, hw)),
|
| 124 |
+
]
|
| 125 |
+
else:
|
| 126 |
+
raise ValueError(
|
| 127 |
+
f"num_pre_blocks must be 6 or 8 for the fixed CSwin pattern; got {num_pre_blocks}")
|
| 128 |
+
blocks = []
|
| 129 |
+
for i, (kind, shift) in enumerate(specs):
|
| 130 |
+
k = _qkv_kernel_at(qkv_dw_kernel, i)
|
| 131 |
+
if kind == "full":
|
| 132 |
+
blocks.append(
|
| 133 |
+
FullSpatialBlock2D(d, num_heads,
|
| 134 |
+
mlp_mult=mlp_mult, rope=rope,
|
| 135 |
+
local_mix_kernel=local_mix_kernel,
|
| 136 |
+
local_mix_init_std=local_mix_init_std,
|
| 137 |
+
mlp_dw_kernel=mlp_dw_kernel,
|
| 138 |
+
qkv_dw_kernel=k)
|
| 139 |
+
)
|
| 140 |
+
elif kind == "swin":
|
| 141 |
+
blocks.append(
|
| 142 |
+
SwinBlock2D(d, num_heads, w, shift=shift,
|
| 143 |
+
mlp_mult=mlp_mult, rope=rope,
|
| 144 |
+
local_mix_kernel=local_mix_kernel,
|
| 145 |
+
local_mix_init_std=local_mix_init_std,
|
| 146 |
+
mlp_dw_kernel=mlp_dw_kernel,
|
| 147 |
+
qkv_dw_kernel=k)
|
| 148 |
+
)
|
| 149 |
+
else:
|
| 150 |
+
blocks.append(
|
| 151 |
+
CSwinBlock2D(d, num_heads, mlp_mult=mlp_mult, rope=rope,
|
| 152 |
+
local_mix_kernel=local_mix_kernel,
|
| 153 |
+
local_mix_init_std=local_mix_init_std,
|
| 154 |
+
mlp_dw_kernel=mlp_dw_kernel,
|
| 155 |
+
qkv_dw_kernel=k)
|
| 156 |
+
)
|
| 157 |
+
return blocks
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def _stage2_final_blocks(d, num_heads, window_size, mlp_mult, rope,
|
| 161 |
+
local_mix_kernel=0,
|
| 162 |
+
local_mix_init_std=None,
|
| 163 |
+
mlp_dw_kernel=0,
|
| 164 |
+
qkv_dw_kernel=0):
|
| 165 |
+
"""Build the 2-block post-patch2 pattern: dense (w/2,0) → dense (0,w/2).
|
| 166 |
+
Runs AFTER the late zero-init RGB pixel-skip (patchify_2) is added, so
|
| 167 |
+
fresh pixel detail immediately crosses both seam axes."""
|
| 168 |
+
w = window_size
|
| 169 |
+
hw = w // 2
|
| 170 |
+
return [
|
| 171 |
+
SwinBlock2D(d, num_heads, w, shift=(hw, 0),
|
| 172 |
+
mlp_mult=mlp_mult, rope=rope,
|
| 173 |
+
local_mix_kernel=local_mix_kernel,
|
| 174 |
+
local_mix_init_std=local_mix_init_std,
|
| 175 |
+
mlp_dw_kernel=mlp_dw_kernel,
|
| 176 |
+
qkv_dw_kernel=_qkv_kernel_at(qkv_dw_kernel, 0)),
|
| 177 |
+
SwinBlock2D(d, num_heads, w, shift=(0, hw),
|
| 178 |
+
mlp_mult=mlp_mult, rope=rope,
|
| 179 |
+
local_mix_kernel=local_mix_kernel,
|
| 180 |
+
local_mix_init_std=local_mix_init_std,
|
| 181 |
+
mlp_dw_kernel=mlp_dw_kernel,
|
| 182 |
+
qkv_dw_kernel=_qkv_kernel_at(qkv_dw_kernel, 1)),
|
| 183 |
+
]
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
class Refine4ConvNeXtBlock(nn.Module):
|
| 187 |
+
"""Small ConvNeXt-style block over /4 feature maps.
|
| 188 |
+
|
| 189 |
+
NCHW is kept throughout so cuDNN sees ordinary conv tensors. The branch is
|
| 190 |
+
residual, but not zero-initialized; the enclosing residual-output projection
|
| 191 |
+
is zero-init, so enabling the head is still output-neutral at insertion.
|
| 192 |
+
"""
|
| 193 |
+
|
| 194 |
+
def __init__(self, width: int, kernel_size: int = 7,
|
| 195 |
+
expansion: float = 2.0):
|
| 196 |
+
super().__init__()
|
| 197 |
+
if kernel_size <= 0 or kernel_size % 2 != 1:
|
| 198 |
+
raise ValueError(
|
| 199 |
+
f"kernel_size must be a positive odd integer; got {kernel_size}")
|
| 200 |
+
hidden = int(round(width * float(expansion)))
|
| 201 |
+
self.pad = kernel_size // 2
|
| 202 |
+
self.dw = nn.Conv2d(width, width, kernel_size=kernel_size,
|
| 203 |
+
padding=0, groups=width, bias=True)
|
| 204 |
+
self.norm = LayerNorm2d(width)
|
| 205 |
+
self.pw1 = nn.Conv2d(width, hidden, kernel_size=1, bias=True)
|
| 206 |
+
self.pw2 = nn.Conv2d(hidden, width, kernel_size=1, bias=True)
|
| 207 |
+
|
| 208 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 209 |
+
y = x
|
| 210 |
+
if self.pad:
|
| 211 |
+
mode = 'reflect' if x.shape[-2] > self.pad and x.shape[-1] > self.pad else 'replicate'
|
| 212 |
+
y = F.pad(y, (self.pad, self.pad, self.pad, self.pad), mode=mode)
|
| 213 |
+
y = self.dw(y)
|
| 214 |
+
y = self.norm(y)
|
| 215 |
+
y = self.pw2(F.gelu(self.pw1(y), approximate='tanh'))
|
| 216 |
+
return x + y
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
class Stage2Refine4Head(nn.Module):
|
| 220 |
+
"""Optional /4 conv refinement head for the final stage-2 output.
|
| 221 |
+
|
| 222 |
+
Inputs:
|
| 223 |
+
* mixed /16 stage-2 tokens after the final transformer block;
|
| 224 |
+
* exact RGB pixel-unshuffle to /4 (3 * 4 * 4 = 48 channels).
|
| 225 |
+
|
| 226 |
+
Output is a full-res residual over the normal depatchified stage-2 logits.
|
| 227 |
+
The residual projection is zero-initialized, so the head is an exact no-op
|
| 228 |
+
until it learns.
|
| 229 |
+
"""
|
| 230 |
+
|
| 231 |
+
def __init__(
|
| 232 |
+
self,
|
| 233 |
+
token_dim: int,
|
| 234 |
+
token_channels: int = 48,
|
| 235 |
+
width: int = 96,
|
| 236 |
+
blocks: int = 2,
|
| 237 |
+
kernel_size: int = 7,
|
| 238 |
+
expansion: float = 2.0,
|
| 239 |
+
gradient_checkpoint: bool = False,
|
| 240 |
+
):
|
| 241 |
+
super().__init__()
|
| 242 |
+
if token_channels <= 0:
|
| 243 |
+
raise ValueError("token_channels must be positive")
|
| 244 |
+
if width <= 0:
|
| 245 |
+
raise ValueError("width must be positive")
|
| 246 |
+
if blocks <= 0:
|
| 247 |
+
raise ValueError("blocks must be positive")
|
| 248 |
+
self.token_channels = int(token_channels)
|
| 249 |
+
self.rgb_channels = 3 * 4 * 4
|
| 250 |
+
self.width = int(width)
|
| 251 |
+
self.gradient_checkpoint = bool(gradient_checkpoint)
|
| 252 |
+
|
| 253 |
+
self.token_up = nn.Conv2d(
|
| 254 |
+
token_dim, self.token_channels * 4 * 4,
|
| 255 |
+
kernel_size=1, bias=True,
|
| 256 |
+
)
|
| 257 |
+
icnr_init_(self.token_up, upscale_factor=4)
|
| 258 |
+
|
| 259 |
+
in_channels = self.token_channels + self.rgb_channels
|
| 260 |
+
self.in_proj = (
|
| 261 |
+
nn.Identity() if in_channels == self.width
|
| 262 |
+
else nn.Conv2d(in_channels, self.width, kernel_size=1, bias=True)
|
| 263 |
+
)
|
| 264 |
+
self.blocks = nn.ModuleList([
|
| 265 |
+
Refine4ConvNeXtBlock(
|
| 266 |
+
self.width, kernel_size=kernel_size, expansion=expansion)
|
| 267 |
+
for _ in range(int(blocks))
|
| 268 |
+
])
|
| 269 |
+
self.out = nn.Conv2d(self.width, 4 * 4 * 4, kernel_size=1, bias=True)
|
| 270 |
+
nn.init.zeros_(self.out.weight)
|
| 271 |
+
nn.init.zeros_(self.out.bias)
|
| 272 |
+
|
| 273 |
+
def forward(self, rgb_flat: torch.Tensor,
|
| 274 |
+
feat_ch_first: torch.Tensor) -> torch.Tensor:
|
| 275 |
+
token = F.pixel_shuffle(self.token_up(feat_ch_first), 4)
|
| 276 |
+
rgb = F.pixel_unshuffle(rgb_flat, 4)
|
| 277 |
+
x = torch.cat([token, rgb], dim=1)
|
| 278 |
+
x = self.in_proj(x)
|
| 279 |
+
for block in self.blocks:
|
| 280 |
+
if self.gradient_checkpoint and self.training and torch.is_grad_enabled():
|
| 281 |
+
x = torch.utils.checkpoint.checkpoint(
|
| 282 |
+
block, x, use_reentrant=False)
|
| 283 |
+
else:
|
| 284 |
+
x = block(x)
|
| 285 |
+
return F.pixel_shuffle(self.out(x), 4)
|
| 286 |
+
|
| 287 |
+
|
| 288 |
+
class Stage2(nn.Module):
|
| 289 |
+
def __init__(
|
| 290 |
+
self,
|
| 291 |
+
d: int = 1280,
|
| 292 |
+
d_baton: int = 512,
|
| 293 |
+
num_heads: int = 16, # 1280 / 16 = 80 (head_dim)
|
| 294 |
+
num_blocks: int = 10,
|
| 295 |
+
num_final_blocks: int = 2,
|
| 296 |
+
window_size: int = 16,
|
| 297 |
+
mlp_mult: float = 2.5,
|
| 298 |
+
# Per-block gradient checkpointing during training — each Swin-2D
|
| 299 |
+
# block individually wrapped so only one block's activations are
|
| 300 |
+
# resident at a time during backward.
|
| 301 |
+
gradient_checkpoint: bool = False,
|
| 302 |
+
# Overlap patchify/depatchify (see blocks.PatchifyConv / Depatchify).
|
| 303 |
+
# overlap_patchify is a legacy bool alias (True → mult=3).
|
| 304 |
+
# overlap_mult: 1 (no overlap) | 2 (half) | 3 (full, 3×3 neighborhood).
|
| 305 |
+
overlap_patchify: bool = False,
|
| 306 |
+
overlap_mult: int = 2,
|
| 307 |
+
depatch_fourier_features: int = 0,
|
| 308 |
+
depatch_fourier_kernel: int = 3,
|
| 309 |
+
# MAE-style pretraining hooks. Disabled in the normal inference path.
|
| 310 |
+
enable_mae_mask_tokens: bool = False,
|
| 311 |
+
# Optional depthwise local mixer inserted after attention and before
|
| 312 |
+
# MLP in every stage-2 Swin/CSwin block. kernel=0 disables.
|
| 313 |
+
local_mix_kernel: int = 0,
|
| 314 |
+
local_mix_init_std: float | None = None,
|
| 315 |
+
mlp_dw_kernel: int = 0,
|
| 316 |
+
qkv_dw_kernel: int = 0,
|
| 317 |
+
qkv_dw_fraction: float = 1.0,
|
| 318 |
+
refine4_blocks: int = 0,
|
| 319 |
+
refine4_width: int = 96,
|
| 320 |
+
refine4_token_channels: int = 48,
|
| 321 |
+
refine4_kernel: int = 7,
|
| 322 |
+
refine4_expansion: float = 2.0,
|
| 323 |
+
# RoPE augmentations (DINOv3-style) — see rope.py.
|
| 324 |
+
rope_shift_coords: float = None,
|
| 325 |
+
rope_jitter_coords: float = None,
|
| 326 |
+
rope_rescale_coords: float = None,
|
| 327 |
+
rope_base: float = 100.0,
|
| 328 |
+
):
|
| 329 |
+
super().__init__()
|
| 330 |
+
assert num_blocks >= 2, (
|
| 331 |
+
f"num_blocks={num_blocks} too few — need ≥2 so the zero-init "
|
| 332 |
+
f"pixel-skip can fire at least one block before the end.")
|
| 333 |
+
assert 1 <= num_final_blocks < num_blocks, (
|
| 334 |
+
f"num_final_blocks={num_final_blocks} must be in [1, num_blocks-1]; "
|
| 335 |
+
f"num_blocks={num_blocks}")
|
| 336 |
+
assert 0 < d_baton < d, (
|
| 337 |
+
f"d_baton={d_baton} must be in (0, d={d}) exclusive")
|
| 338 |
+
self.d = d
|
| 339 |
+
self.d_baton = d_baton
|
| 340 |
+
self.d_rgb = d - d_baton
|
| 341 |
+
self.num_blocks = num_blocks
|
| 342 |
+
self.num_final_blocks = num_final_blocks
|
| 343 |
+
self.num_pre_blocks = num_blocks - num_final_blocks
|
| 344 |
+
self.window_size = window_size
|
| 345 |
+
self.gradient_checkpoint = gradient_checkpoint
|
| 346 |
+
self.enable_mae_mask_tokens = bool(enable_mae_mask_tokens)
|
| 347 |
+
self.use_patch_2 = not self.enable_mae_mask_tokens
|
| 348 |
+
self.refine4_enabled = int(refine4_blocks) > 0
|
| 349 |
+
qkv_plan = _stage2_qkv_kernels(
|
| 350 |
+
num_blocks,
|
| 351 |
+
qkv_dw_kernel,
|
| 352 |
+
qkv_dw_fraction,
|
| 353 |
+
)
|
| 354 |
+
qkv_pre = qkv_plan[:self.num_pre_blocks]
|
| 355 |
+
qkv_final = qkv_plan[self.num_pre_blocks:]
|
| 356 |
+
self.qkv_dw_active_blocks = sum(1 for k in qkv_plan if k > 0)
|
| 357 |
+
self.qkv_dw_total_blocks = len(qkv_plan)
|
| 358 |
+
|
| 359 |
+
# Patchify_1: 3 → d_rgb at /16, concatenated after the baton.
|
| 360 |
+
self.patch_1 = PatchifyConv(3, self.d_rgb, patch=16,
|
| 361 |
+
zero_init_first=0,
|
| 362 |
+
overlap=overlap_patchify,
|
| 363 |
+
overlap_mult=overlap_mult)
|
| 364 |
+
|
| 365 |
+
# Patchify_2 (late pixel skip): main-training only. Pretraining
|
| 366 |
+
# omits it so full-RGB reconstruction cannot lean on a second raw-RGB
|
| 367 |
+
# tokenization path. In phase 2, the value branch starts at zero and
|
| 368 |
+
# a live gate reads the already-mixed token stream directly so token
|
| 369 |
+
# magnitude remains available as a precision/confidence signal.
|
| 370 |
+
if self.use_patch_2:
|
| 371 |
+
self.patch_2 = PatchifyConv(3, self.d_rgb, patch=16,
|
| 372 |
+
zero_init_all=True,
|
| 373 |
+
overlap=overlap_patchify,
|
| 374 |
+
overlap_mult=overlap_mult)
|
| 375 |
+
self.patch_2_gate = nn.Linear(d, self.d_rgb, bias=True)
|
| 376 |
+
nn.init.xavier_uniform_(self.patch_2_gate.weight, gain=0.25)
|
| 377 |
+
nn.init.zeros_(self.patch_2_gate.bias)
|
| 378 |
+
|
| 379 |
+
# Shared RoPE across Swin blocks.
|
| 380 |
+
head_dim = d // num_heads
|
| 381 |
+
self.rope = RoPE2D(head_dim,
|
| 382 |
+
base=rope_base,
|
| 383 |
+
shift_coords=rope_shift_coords,
|
| 384 |
+
jitter_coords=rope_jitter_coords,
|
| 385 |
+
rescale_coords=rope_rescale_coords)
|
| 386 |
+
|
| 387 |
+
# Block layout:
|
| 388 |
+
# pre_blocks (8): full-spatial, dense (w/2,w/2), CSwin,
|
| 389 |
+
# inserted dense (w/2,0), full-spatial,
|
| 390 |
+
# CSwin, dense (0,0), dense (w/2,w/2)
|
| 391 |
+
# → patchify_2 zero-init RGB skip added here
|
| 392 |
+
# final_blocks (2): dense (w/2,0), dense (0,w/2)
|
| 393 |
+
assert self.num_pre_blocks in (6, 8), (
|
| 394 |
+
f"num_pre_blocks must be 6 or 8 for the CSwin-in-the-middle pattern; "
|
| 395 |
+
f"got {self.num_pre_blocks} (num_blocks={num_blocks}, "
|
| 396 |
+
f"num_final_blocks={num_final_blocks})")
|
| 397 |
+
assert num_final_blocks == 2, (
|
| 398 |
+
f"num_final_blocks must be 2 for the 2-phase final stack; "
|
| 399 |
+
f"got {num_final_blocks}")
|
| 400 |
+
self.pre_blocks = nn.ModuleList(
|
| 401 |
+
_stage2_pre_blocks(d, num_heads, window_size, mlp_mult, self.rope,
|
| 402 |
+
num_pre_blocks=self.num_pre_blocks,
|
| 403 |
+
local_mix_kernel=local_mix_kernel,
|
| 404 |
+
local_mix_init_std=local_mix_init_std,
|
| 405 |
+
mlp_dw_kernel=mlp_dw_kernel,
|
| 406 |
+
qkv_dw_kernel=qkv_pre)
|
| 407 |
+
)
|
| 408 |
+
self.final_blocks = nn.ModuleList(
|
| 409 |
+
_stage2_final_blocks(d, num_heads, window_size, mlp_mult, self.rope,
|
| 410 |
+
local_mix_kernel=local_mix_kernel,
|
| 411 |
+
local_mix_init_std=local_mix_init_std,
|
| 412 |
+
mlp_dw_kernel=mlp_dw_kernel,
|
| 413 |
+
qkv_dw_kernel=qkv_final)
|
| 414 |
+
)
|
| 415 |
+
|
| 416 |
+
# Depatchify: d → 4 alpha/RGB channels at /1 via PixelShuffle(16).
|
| 417 |
+
# When dims are the production 768d RGB patch basis, FG rows invert
|
| 418 |
+
# patch_1 exactly and alpha/baton rows do not affect output at init.
|
| 419 |
+
self.depatchify = Depatchify(d, out_channels=4, r=16,
|
| 420 |
+
zero_init_all=True, bias=True,
|
| 421 |
+
overlap=overlap_patchify,
|
| 422 |
+
overlap_mult=overlap_mult,
|
| 423 |
+
fourier_features=depatch_fourier_features,
|
| 424 |
+
fourier_kernel=depatch_fourier_kernel)
|
| 425 |
+
self.rgb_passthrough_init = init_orthogonal_rgb_passthrough_(
|
| 426 |
+
self.patch_1, self.depatchify,
|
| 427 |
+
token_offset=self.d_baton,
|
| 428 |
+
output_rgb_offset=1,
|
| 429 |
+
downsample=1,
|
| 430 |
+
)
|
| 431 |
+
self.refine4 = (
|
| 432 |
+
Stage2Refine4Head(
|
| 433 |
+
d, token_channels=refine4_token_channels,
|
| 434 |
+
width=refine4_width, blocks=int(refine4_blocks),
|
| 435 |
+
kernel_size=refine4_kernel, expansion=refine4_expansion,
|
| 436 |
+
gradient_checkpoint=gradient_checkpoint,
|
| 437 |
+
)
|
| 438 |
+
if self.refine4_enabled else None
|
| 439 |
+
)
|
| 440 |
+
if self.enable_mae_mask_tokens:
|
| 441 |
+
self.mae_rgb_token = nn.Parameter(torch.zeros(self.d_rgb))
|
| 442 |
+
|
| 443 |
+
# -----------------------------------------------------------------
|
| 444 |
+
|
| 445 |
+
def _apply_mae_rgb_token(
|
| 446 |
+
self,
|
| 447 |
+
x: torch.Tensor,
|
| 448 |
+
mae_mask_16: torch.Tensor | None,
|
| 449 |
+
B: int,
|
| 450 |
+
T: int,
|
| 451 |
+
name: str,
|
| 452 |
+
) -> torch.Tensor:
|
| 453 |
+
if mae_mask_16 is None:
|
| 454 |
+
return x
|
| 455 |
+
if not self.enable_mae_mask_tokens:
|
| 456 |
+
raise RuntimeError(f"{name} received an MAE mask but MAE tokens are disabled")
|
| 457 |
+
BT, C, H16, W16 = x.shape
|
| 458 |
+
if C != self.d_rgb:
|
| 459 |
+
raise ValueError(f"{name} channel count {C} != d_rgb {self.d_rgb}")
|
| 460 |
+
expected = (B, T, 1, H16, W16)
|
| 461 |
+
if tuple(mae_mask_16.shape) != expected:
|
| 462 |
+
raise ValueError(
|
| 463 |
+
f"{name} mask shape {tuple(mae_mask_16.shape)} != expected {expected}")
|
| 464 |
+
mask = mae_mask_16.reshape(BT, 1, H16, W16).to(device=x.device,
|
| 465 |
+
dtype=torch.bool)
|
| 466 |
+
token = self.mae_rgb_token.to(device=x.device, dtype=x.dtype).view(1, C, 1, 1)
|
| 467 |
+
return torch.where(mask, token, x)
|
| 468 |
+
|
| 469 |
+
def forward(
|
| 470 |
+
self,
|
| 471 |
+
rgb: torch.Tensor, # (B, T, 3, H, W) signed asinh encoded scene-linear RGB
|
| 472 |
+
baton: torch.Tensor, # (B, T, d_baton, H/16, W/16) from stage 1
|
| 473 |
+
mae_mask_16: 'torch.Tensor | None' = None, # (B,T,1,H/16,W/16)
|
| 474 |
+
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
| 475 |
+
"""Returns logits (B, T, 4, H, W), or (refined, base) when refine4 is on."""
|
| 476 |
+
B, T, _, H, W = rgb.shape
|
| 477 |
+
assert H % 16 == 0 and W % 16 == 0, (
|
| 478 |
+
f"Stage2: H,W must be /16-aligned; got {(H, W)}")
|
| 479 |
+
assert baton.shape == (B, T, self.d_baton, H // 16, W // 16), (
|
| 480 |
+
f"baton shape mismatch: got {baton.shape}, "
|
| 481 |
+
f"expected {(B, T, self.d_baton, H // 16, W // 16)}")
|
| 482 |
+
|
| 483 |
+
BT = B * T
|
| 484 |
+
rgb_flat = rgb.reshape(BT, 3, H, W)
|
| 485 |
+
|
| 486 |
+
# Patchify_1 (RGB → d_rgb at /16), then concatenate after the baton.
|
| 487 |
+
rgb_feat = self.patch_1(rgb_flat) # (BT, d_rgb, H/16, W/16)
|
| 488 |
+
rgb_feat = self._apply_mae_rgb_token(
|
| 489 |
+
rgb_feat, mae_mask_16, B, T, "stage2 patch_1")
|
| 490 |
+
|
| 491 |
+
baton_flat = baton.reshape(BT, self.d_baton, H // 16, W // 16)
|
| 492 |
+
feat = torch.cat([baton_flat, rgb_feat], dim=1) # (BT, d, H/16, W/16)
|
| 493 |
+
|
| 494 |
+
# Swin-2D permute to (BT, H/16, W/16, d)
|
| 495 |
+
x = feat.permute(0, 2, 3, 1).contiguous()
|
| 496 |
+
|
| 497 |
+
# Per-block gradient checkpointing: only one block's activations
|
| 498 |
+
# are resident at a time during backward when gradient_checkpoint is on.
|
| 499 |
+
def run_block(blk, x_in):
|
| 500 |
+
if self.gradient_checkpoint and self.training and torch.is_grad_enabled():
|
| 501 |
+
return torch.utils.checkpoint.checkpoint(
|
| 502 |
+
blk, x_in, use_reentrant=False,
|
| 503 |
+
)
|
| 504 |
+
return blk(x_in)
|
| 505 |
+
|
| 506 |
+
# Pre-patch_2 blocks (num_pre_blocks)
|
| 507 |
+
for blk in self.pre_blocks:
|
| 508 |
+
x = run_block(blk, x)
|
| 509 |
+
|
| 510 |
+
if self.use_patch_2:
|
| 511 |
+
# Late pixel skip: zero-init patchify_2 of raw RGB, modulated by
|
| 512 |
+
# a full-rank gate over the mixed token stream. The value branch
|
| 513 |
+
# is exactly zero at insertion, so the live zero-bias gate only
|
| 514 |
+
# affects gradients until patch_2 wakes up.
|
| 515 |
+
gate = F.silu(self.patch_2_gate(x)) # (BT, H/16, W/16, d_rgb)
|
| 516 |
+
patch2 = self.patch_2(rgb_flat) # (BT, d_rgb, H/16, W/16)
|
| 517 |
+
patch2 = patch2.permute(0, 2, 3, 1).contiguous() * gate
|
| 518 |
+
x = x + F.pad(patch2, (self.d_baton, 0))
|
| 519 |
+
|
| 520 |
+
# Post-patch_2 final blocks (num_final_blocks)
|
| 521 |
+
for blk in self.final_blocks:
|
| 522 |
+
x = run_block(blk, x)
|
| 523 |
+
|
| 524 |
+
# Depatchify to /1. With production dims and non-overlap patching,
|
| 525 |
+
# FG rows are the exact inverse of patch_1 at init; alpha rows and
|
| 526 |
+
# baton columns are zero.
|
| 527 |
+
feat_ch_first = x.permute(0, 3, 1, 2).contiguous() # (BT, d, H/16, W/16)
|
| 528 |
+
out_raw = self.depatchify(feat_ch_first) # (BT, 4, H, W)
|
| 529 |
+
if self.refine4 is not None:
|
| 530 |
+
refined = out_raw + self.refine4(rgb_flat, feat_ch_first)
|
| 531 |
+
return refined.reshape(B, T, 4, H, W), out_raw.reshape(B, T, 4, H, W)
|
| 532 |
+
return out_raw.reshape(B, T, 4, H, W)
|
corridorkey/video_io.py
ADDED
|
@@ -0,0 +1,820 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import glob
|
| 4 |
+
import math
|
| 5 |
+
import os
|
| 6 |
+
import queue
|
| 7 |
+
import shutil
|
| 8 |
+
import subprocess
|
| 9 |
+
import threading
|
| 10 |
+
from dataclasses import dataclass
|
| 11 |
+
from pathlib import Path
|
| 12 |
+
|
| 13 |
+
os.environ.setdefault("OPENCV_IO_ENABLE_OPENEXR", "1")
|
| 14 |
+
|
| 15 |
+
import cv2
|
| 16 |
+
import numpy as np
|
| 17 |
+
import torch
|
| 18 |
+
|
| 19 |
+
from . import color_utils as cu
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@dataclass
|
| 23 |
+
class WriterConfig:
|
| 24 |
+
ffmpeg_bin: str = "ffmpeg"
|
| 25 |
+
codec: str = "libx264"
|
| 26 |
+
crf: int = 12
|
| 27 |
+
preset: str = "medium"
|
| 28 |
+
pix_fmt: str = "yuv444p10le"
|
| 29 |
+
bitrate: str | None = None
|
| 30 |
+
threads: int = 2
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def require_ffmpeg(ffmpeg_bin: str) -> None:
|
| 34 |
+
if shutil.which(ffmpeg_bin) is None:
|
| 35 |
+
raise FileNotFoundError(
|
| 36 |
+
f"ffmpeg binary not found: {ffmpeg_bin}. "
|
| 37 |
+
"Install ffmpeg or pass --ffmpeg_bin.")
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def safe_remove(path: str | os.PathLike | None) -> None:
|
| 41 |
+
if not path:
|
| 42 |
+
return
|
| 43 |
+
try:
|
| 44 |
+
os.remove(path)
|
| 45 |
+
except FileNotFoundError:
|
| 46 |
+
pass
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def none_path(path: str | None) -> str | None:
|
| 50 |
+
if path is None:
|
| 51 |
+
return None
|
| 52 |
+
text = str(path).strip()
|
| 53 |
+
if text.lower() in ("", "none", "null", "off", "false", "-"):
|
| 54 |
+
return None
|
| 55 |
+
return str(path)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def raw_clip_name(path: str | os.PathLike) -> str:
|
| 59 |
+
norm = os.path.normpath(str(path))
|
| 60 |
+
name = os.path.splitext(os.path.basename(norm))[0]
|
| 61 |
+
if name.lower() in {"input", "frames", "rgb"}:
|
| 62 |
+
return os.path.basename(os.path.dirname(norm)) or name
|
| 63 |
+
return name
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def frame_paths(frame_dir: str | os.PathLike) -> list[str]:
|
| 67 |
+
paths: list[str] = []
|
| 68 |
+
for ext in ("*.exr", "*.png", "*.jpg", "*.jpeg"):
|
| 69 |
+
paths.extend(glob.glob(os.path.join(str(frame_dir), ext)))
|
| 70 |
+
return sorted(paths)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def ceil_to_multiple(x: int, q: int) -> int:
|
| 74 |
+
return max(q, ((int(x) + q - 1) // q) * q)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def scaled_hw(h: int, w: int, scale: float) -> tuple[int, int]:
|
| 78 |
+
scale = float(scale)
|
| 79 |
+
if scale <= 0:
|
| 80 |
+
raise ValueError(f"scale must be > 0, got {scale}")
|
| 81 |
+
return max(1, int(round(h * scale))), max(1, int(round(w * scale)))
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def resize_hwc_lanczos(img: np.ndarray, hw: tuple[int, int]) -> np.ndarray:
|
| 85 |
+
h, w = hw
|
| 86 |
+
if img.shape[:2] == (h, w):
|
| 87 |
+
return img
|
| 88 |
+
return cv2.resize(img, (w, h), interpolation=cv2.INTER_LANCZOS4)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def resize_chw_lanczos(x: np.ndarray, hw: tuple[int, int]) -> np.ndarray:
|
| 92 |
+
h, w = hw
|
| 93 |
+
if x.shape[-2:] == (h, w):
|
| 94 |
+
return x
|
| 95 |
+
if x.shape[0] == 1:
|
| 96 |
+
y = cv2.resize(x[0], (w, h), interpolation=cv2.INTER_LANCZOS4)
|
| 97 |
+
return y[None].astype(x.dtype, copy=False)
|
| 98 |
+
y = cv2.resize(np.moveaxis(x, 0, -1), (w, h),
|
| 99 |
+
interpolation=cv2.INTER_LANCZOS4)
|
| 100 |
+
return np.moveaxis(y, -1, 0).astype(x.dtype, copy=False)
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def center_pad_to_shape(img: np.ndarray, h_out: int, w_out: int):
|
| 104 |
+
h, w = img.shape[:2]
|
| 105 |
+
pad_t = (h_out - h) // 2
|
| 106 |
+
pad_b = h_out - h - pad_t
|
| 107 |
+
pad_l = (w_out - w) // 2
|
| 108 |
+
pad_r = w_out - w - pad_l
|
| 109 |
+
if pad_t or pad_b or pad_l or pad_r:
|
| 110 |
+
img = cv2.copyMakeBorder(
|
| 111 |
+
img, pad_t, pad_b, pad_l, pad_r, cv2.BORDER_REFLECT_101)
|
| 112 |
+
return img, (pad_l, pad_r, pad_t, pad_b)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def crop_chw_to_native(x: np.ndarray, native_hw: tuple[int, int], pad):
|
| 116 |
+
h, w = native_hw
|
| 117 |
+
pad_l, _, pad_t, _ = pad
|
| 118 |
+
return x[..., pad_t:pad_t + h, pad_l:pad_l + w]
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def crop_model_chw_to_output(x: np.ndarray, model_native_hw, pad, output_hw):
|
| 122 |
+
cropped = crop_chw_to_native(x, model_native_hw, pad)
|
| 123 |
+
return resize_chw_lanczos(cropped, output_hw)
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def read_image_rgb(path: str | os.PathLike):
|
| 127 |
+
path = str(path)
|
| 128 |
+
if path.lower().endswith(".exr"):
|
| 129 |
+
img = cv2.imread(path, cv2.IMREAD_UNCHANGED)
|
| 130 |
+
if img is None:
|
| 131 |
+
raise FileNotFoundError(path)
|
| 132 |
+
img = img.astype(np.float32, copy=False)
|
| 133 |
+
if img.ndim == 3 and img.shape[2] == 4:
|
| 134 |
+
img = img[:, :, :3]
|
| 135 |
+
if img.ndim == 3:
|
| 136 |
+
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
| 137 |
+
return img
|
| 138 |
+
img = cv2.imread(path, cv2.IMREAD_COLOR)
|
| 139 |
+
if img is None:
|
| 140 |
+
raise FileNotFoundError(path)
|
| 141 |
+
return cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def read_hint_image(path: str | os.PathLike) -> np.ndarray:
|
| 145 |
+
img = cv2.imread(str(path), cv2.IMREAD_UNCHANGED)
|
| 146 |
+
if img is None:
|
| 147 |
+
raise FileNotFoundError(path)
|
| 148 |
+
if img.dtype == np.uint8:
|
| 149 |
+
img = img.astype(np.float32) / 255.0
|
| 150 |
+
elif img.dtype == np.uint16:
|
| 151 |
+
img = img.astype(np.float32) / 65535.0
|
| 152 |
+
else:
|
| 153 |
+
img = img.astype(np.float32)
|
| 154 |
+
if img.ndim == 3:
|
| 155 |
+
img = img[:, :, 3] if img.shape[2] == 4 else img[:, :, 0]
|
| 156 |
+
return np.clip(img, 0.0, 1.0)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def has_linear_adjust(meta: dict) -> bool:
|
| 160 |
+
return (
|
| 161 |
+
abs(float(meta.get("linear_brightness", 1.0)) - 1.0) > 1e-6
|
| 162 |
+
or abs(float(meta.get("linear_contrast", 1.0)) - 1.0) > 1e-6
|
| 163 |
+
)
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def apply_linear_adjust(linear: np.ndarray, meta: dict) -> np.ndarray:
|
| 167 |
+
gain = float(meta.get("linear_brightness", 1.0))
|
| 168 |
+
contrast = float(meta.get("linear_contrast", 1.0))
|
| 169 |
+
if abs(gain - 1.0) <= 1e-6 and abs(contrast - 1.0) <= 1e-6:
|
| 170 |
+
return linear
|
| 171 |
+
pivot = float(meta.get("linear_contrast_pivot", 0.18))
|
| 172 |
+
return gain * (pivot + contrast * (linear - pivot))
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def invert_linear_adjust(linear: np.ndarray, meta: dict) -> np.ndarray:
|
| 176 |
+
gain = float(meta.get("linear_brightness", 1.0))
|
| 177 |
+
contrast = float(meta.get("linear_contrast", 1.0))
|
| 178 |
+
if abs(gain - 1.0) <= 1e-6 and abs(contrast - 1.0) <= 1e-6:
|
| 179 |
+
return linear
|
| 180 |
+
pivot = float(meta.get("linear_contrast_pivot", 0.18))
|
| 181 |
+
return (linear / gain - pivot) / contrast + pivot
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def probe_video_meta(video_path: str, *, num_frames: int, shape_quantum: int,
|
| 185 |
+
native_resolution: bool, start_mode: str, seed: int,
|
| 186 |
+
model_scale: float, linear_brightness: float,
|
| 187 |
+
linear_contrast: float,
|
| 188 |
+
linear_contrast_pivot: float) -> dict:
|
| 189 |
+
cap = cv2.VideoCapture(video_path)
|
| 190 |
+
if not cap.isOpened():
|
| 191 |
+
raise RuntimeError(f"Could not open video: {video_path}")
|
| 192 |
+
native_fps = cap.get(cv2.CAP_PROP_FPS) or 24.0
|
| 193 |
+
w_native = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
| 194 |
+
h_native = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
| 195 |
+
n_total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
| 196 |
+
cap.release()
|
| 197 |
+
|
| 198 |
+
q = int(shape_quantum)
|
| 199 |
+
if native_resolution:
|
| 200 |
+
h_model, w_model = scaled_hw(h_native, w_native, model_scale)
|
| 201 |
+
h_out = ceil_to_multiple(h_model, q)
|
| 202 |
+
w_out = ceil_to_multiple(w_model, q)
|
| 203 |
+
pad_l = (w_out - w_model) // 2
|
| 204 |
+
pad_r = w_out - w_model - pad_l
|
| 205 |
+
pad_t = (h_out - h_model) // 2
|
| 206 |
+
pad_b = h_out - h_model - pad_t
|
| 207 |
+
output_hw = (h_native, w_native)
|
| 208 |
+
native_hw = (h_native, w_native)
|
| 209 |
+
else:
|
| 210 |
+
raise ValueError("non-native inference is intentionally not exposed")
|
| 211 |
+
|
| 212 |
+
n_to_load = n_total if num_frames <= 0 else min(num_frames, n_total)
|
| 213 |
+
start_frame = 0
|
| 214 |
+
max_start = max(n_total - n_to_load, 0)
|
| 215 |
+
if max_start > 0 and num_frames > 0:
|
| 216 |
+
if start_mode == "middle":
|
| 217 |
+
start_frame = max_start // 2
|
| 218 |
+
elif start_mode == "random_middle":
|
| 219 |
+
lo = max_start // 4
|
| 220 |
+
hi = (3 * max_start) // 4
|
| 221 |
+
start_frame = int(np.random.default_rng(int(seed)).integers(lo, hi + 1))
|
| 222 |
+
elif start_mode != "begin":
|
| 223 |
+
raise ValueError(f"unknown start mode {start_mode!r}")
|
| 224 |
+
|
| 225 |
+
meta = {
|
| 226 |
+
"native_fps": native_fps,
|
| 227 |
+
"native_hw": native_hw,
|
| 228 |
+
"output_hw": output_hw,
|
| 229 |
+
"model_native_hw": (h_model, w_model),
|
| 230 |
+
"model_scale": float(model_scale),
|
| 231 |
+
"source_hw": (h_native, w_native),
|
| 232 |
+
"shape_hw": (h_out, w_out),
|
| 233 |
+
"start_frame": start_frame,
|
| 234 |
+
"num_frames": n_to_load,
|
| 235 |
+
"pad": (pad_l, pad_r, pad_t, pad_b),
|
| 236 |
+
"native_resolution": True,
|
| 237 |
+
"linear_brightness": float(linear_brightness),
|
| 238 |
+
"linear_contrast": float(linear_contrast),
|
| 239 |
+
"linear_contrast_pivot": float(linear_contrast_pivot),
|
| 240 |
+
}
|
| 241 |
+
print(f" probed {n_to_load} frames @ {w_out}x{h_out} "
|
| 242 |
+
f"(native {w_native}x{h_native}, fps {native_fps:.2f}, "
|
| 243 |
+
f"start={start_frame}, pad={meta['pad']})")
|
| 244 |
+
return meta
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def _read_frame_dir_views(path: str, *, exr_linear: bool):
|
| 248 |
+
raw = read_image_rgb(path)
|
| 249 |
+
if exr_linear:
|
| 250 |
+
linear = raw.astype(np.float32, copy=False)
|
| 251 |
+
display = np.clip(cu.linear_to_srgb(np.maximum(linear, 0.0)), 0.0, 1.0)
|
| 252 |
+
else:
|
| 253 |
+
display = np.clip(raw.astype(np.float32, copy=False), 0.0, 1.0)
|
| 254 |
+
linear = cu.srgb_to_linear(display)
|
| 255 |
+
return display, linear
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
def probe_frame_dir_meta(frame_dir: str, *, num_frames: int,
|
| 259 |
+
shape_quantum: int, fps: float,
|
| 260 |
+
exr_linear: bool, model_scale: float) -> dict:
|
| 261 |
+
paths = frame_paths(frame_dir)
|
| 262 |
+
if not paths:
|
| 263 |
+
raise RuntimeError(f"No frames found under {frame_dir}")
|
| 264 |
+
if num_frames > 0:
|
| 265 |
+
paths = paths[:num_frames]
|
| 266 |
+
first_display, _ = _read_frame_dir_views(paths[0], exr_linear=exr_linear)
|
| 267 |
+
h_native, w_native = first_display.shape[:2]
|
| 268 |
+
h_model, w_model = scaled_hw(h_native, w_native, model_scale)
|
| 269 |
+
h_out = ceil_to_multiple(h_model, shape_quantum)
|
| 270 |
+
w_out = ceil_to_multiple(w_model, shape_quantum)
|
| 271 |
+
pad_l = (w_out - w_model) // 2
|
| 272 |
+
pad_r = w_out - w_model - pad_l
|
| 273 |
+
pad_t = (h_out - h_model) // 2
|
| 274 |
+
pad_b = h_out - h_model - pad_t
|
| 275 |
+
meta = {
|
| 276 |
+
"frame_paths": paths,
|
| 277 |
+
"native_fps": float(fps),
|
| 278 |
+
"native_hw": (h_native, w_native),
|
| 279 |
+
"output_hw": (h_native, w_native),
|
| 280 |
+
"model_native_hw": (h_model, w_model),
|
| 281 |
+
"model_scale": float(model_scale),
|
| 282 |
+
"source_hw": (h_native, w_native),
|
| 283 |
+
"shape_hw": (h_out, w_out),
|
| 284 |
+
"start_frame": 0,
|
| 285 |
+
"num_frames": len(paths),
|
| 286 |
+
"pad": (pad_l, pad_r, pad_t, pad_b),
|
| 287 |
+
"native_resolution": True,
|
| 288 |
+
"exr_linear": bool(exr_linear),
|
| 289 |
+
"linear_brightness": 1.0,
|
| 290 |
+
"linear_contrast": 1.0,
|
| 291 |
+
"linear_contrast_pivot": 0.18,
|
| 292 |
+
}
|
| 293 |
+
print(f" probed {len(paths)} frames from {frame_dir} @ {w_out}x{h_out} "
|
| 294 |
+
f"(native {w_native}x{h_native}, fps {fps:.2f}, "
|
| 295 |
+
f"exr_linear={exr_linear}, pad={meta['pad']})")
|
| 296 |
+
return meta
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
def preprocess_video_frame_views(frame_bgr, meta: dict, *, need_model: bool):
|
| 300 |
+
h_out, w_out = meta["shape_hw"]
|
| 301 |
+
h_model, w_model = meta["model_native_hw"]
|
| 302 |
+
frame = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0
|
| 303 |
+
frame = resize_hwc_lanczos(frame, (h_model, w_model))
|
| 304 |
+
frame, _ = center_pad_to_shape(frame, h_out, w_out)
|
| 305 |
+
review = torch.from_numpy(frame.transpose(2, 0, 1)).to(dtype=torch.bfloat16)
|
| 306 |
+
if not need_model and not has_linear_adjust(meta):
|
| 307 |
+
return None, review, review
|
| 308 |
+
linear = cu.srgb_to_linear(frame)
|
| 309 |
+
linear_model = apply_linear_adjust(linear, meta)
|
| 310 |
+
if has_linear_adjust(meta):
|
| 311 |
+
foundation_np = cu.linear_to_vfm_sdr(linear_model)
|
| 312 |
+
foundation = torch.from_numpy(
|
| 313 |
+
foundation_np.transpose(2, 0, 1)).to(dtype=torch.bfloat16)
|
| 314 |
+
else:
|
| 315 |
+
foundation = review
|
| 316 |
+
if not need_model:
|
| 317 |
+
return None, foundation, review
|
| 318 |
+
model = cu.linear_to_asinh(linear_model)
|
| 319 |
+
model_t = torch.from_numpy(model.transpose(2, 0, 1)).to(dtype=torch.bfloat16)
|
| 320 |
+
return model_t, foundation, review
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
def preprocess_frame_dir_path(path: str, meta: dict, *, need_model: bool):
|
| 324 |
+
h_out, w_out = meta["shape_hw"]
|
| 325 |
+
h_model, w_model = meta["model_native_hw"]
|
| 326 |
+
display, linear = _read_frame_dir_views(
|
| 327 |
+
path, exr_linear=bool(meta.get("exr_linear", False)))
|
| 328 |
+
display = resize_hwc_lanczos(display, (h_model, w_model))
|
| 329 |
+
linear = resize_hwc_lanczos(linear, (h_model, w_model))
|
| 330 |
+
display, _ = center_pad_to_shape(np.clip(display, 0.0, 1.0), h_out, w_out)
|
| 331 |
+
display_t = torch.from_numpy(
|
| 332 |
+
display.transpose(2, 0, 1)).to(dtype=torch.bfloat16)
|
| 333 |
+
if not need_model:
|
| 334 |
+
return None, display_t, display_t
|
| 335 |
+
linear, _ = center_pad_to_shape(linear, h_out, w_out)
|
| 336 |
+
model = cu.linear_to_asinh(linear)
|
| 337 |
+
model_t = torch.from_numpy(model.transpose(2, 0, 1)).to(dtype=torch.bfloat16)
|
| 338 |
+
return model_t, display_t, display_t
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
class SequentialVideoWindowReader:
|
| 342 |
+
def __init__(self, video_path: str, meta: dict):
|
| 343 |
+
self.video_path = video_path
|
| 344 |
+
self.meta = meta
|
| 345 |
+
self.cap = cv2.VideoCapture(video_path)
|
| 346 |
+
if not self.cap.isOpened():
|
| 347 |
+
raise RuntimeError(f"Could not open video: {video_path}")
|
| 348 |
+
self.start_frame = int(meta["start_frame"])
|
| 349 |
+
self.n_frames = int(meta["num_frames"])
|
| 350 |
+
if self.start_frame:
|
| 351 |
+
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.start_frame)
|
| 352 |
+
self.next_t = 0
|
| 353 |
+
self.cache: dict[int, tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = {}
|
| 354 |
+
|
| 355 |
+
def close(self):
|
| 356 |
+
self.cap.release()
|
| 357 |
+
|
| 358 |
+
def _read_until(self, t_end: int):
|
| 359 |
+
while self.next_t < t_end:
|
| 360 |
+
ret, frame = self.cap.read()
|
| 361 |
+
if not ret:
|
| 362 |
+
raise RuntimeError(
|
| 363 |
+
f"{self.video_path}: failed decoding frame "
|
| 364 |
+
f"{self.start_frame + self.next_t}")
|
| 365 |
+
self.cache[self.next_t] = preprocess_video_frame_views(
|
| 366 |
+
frame, self.meta, need_model=True)
|
| 367 |
+
self.next_t += 1
|
| 368 |
+
|
| 369 |
+
def window(self, t0: int, t1: int):
|
| 370 |
+
self._read_until(t1)
|
| 371 |
+
model = torch.stack([self.cache[t][0] for t in range(t0, t1)])
|
| 372 |
+
foundation = torch.stack([self.cache[t][1] for t in range(t0, t1)])
|
| 373 |
+
review = torch.stack([self.cache[t][2] for t in range(t0, t1)])
|
| 374 |
+
for old_t in [k for k in self.cache if k < t0]:
|
| 375 |
+
del self.cache[old_t]
|
| 376 |
+
return model, foundation, review
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
class SequentialFrameDirWindowReader:
|
| 380 |
+
def __init__(self, frame_dir: str, meta: dict):
|
| 381 |
+
self.frame_dir = frame_dir
|
| 382 |
+
self.meta = meta
|
| 383 |
+
self.paths = meta["frame_paths"]
|
| 384 |
+
self.next_t = 0
|
| 385 |
+
self.cache: dict[int, tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = {}
|
| 386 |
+
|
| 387 |
+
def close(self):
|
| 388 |
+
pass
|
| 389 |
+
|
| 390 |
+
def _read_until(self, t_end: int):
|
| 391 |
+
while self.next_t < t_end:
|
| 392 |
+
self.cache[self.next_t] = preprocess_frame_dir_path(
|
| 393 |
+
self.paths[self.next_t], self.meta, need_model=True)
|
| 394 |
+
self.next_t += 1
|
| 395 |
+
|
| 396 |
+
def window(self, t0: int, t1: int):
|
| 397 |
+
self._read_until(t1)
|
| 398 |
+
model = torch.stack([self.cache[t][0] for t in range(t0, t1)])
|
| 399 |
+
foundation = torch.stack([self.cache[t][1] for t in range(t0, t1)])
|
| 400 |
+
review = torch.stack([self.cache[t][2] for t in range(t0, t1)])
|
| 401 |
+
for old_t in [k for k in self.cache if k < t0]:
|
| 402 |
+
del self.cache[old_t]
|
| 403 |
+
return model, foundation, review
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
def iter_video_frame_chunks(video_path: str, meta: dict, *, chunk_size: int,
|
| 407 |
+
need_model: bool):
|
| 408 |
+
cap = cv2.VideoCapture(video_path)
|
| 409 |
+
if not cap.isOpened():
|
| 410 |
+
raise RuntimeError(f"Could not open video: {video_path}")
|
| 411 |
+
if int(meta["start_frame"]):
|
| 412 |
+
cap.set(cv2.CAP_PROP_POS_FRAMES, int(meta["start_frame"]))
|
| 413 |
+
t = 0
|
| 414 |
+
try:
|
| 415 |
+
while t < int(meta["num_frames"]):
|
| 416 |
+
model_list = [] if need_model else None
|
| 417 |
+
foundation_list = []
|
| 418 |
+
t0 = t
|
| 419 |
+
while t < int(meta["num_frames"]) and len(foundation_list) < chunk_size:
|
| 420 |
+
ret, frame = cap.read()
|
| 421 |
+
if not ret:
|
| 422 |
+
break
|
| 423 |
+
model, foundation, _review = preprocess_video_frame_views(
|
| 424 |
+
frame, meta, need_model=need_model)
|
| 425 |
+
if need_model:
|
| 426 |
+
model_list.append(model)
|
| 427 |
+
foundation_list.append(foundation)
|
| 428 |
+
t += 1
|
| 429 |
+
if not foundation_list:
|
| 430 |
+
break
|
| 431 |
+
yield t0, (torch.stack(model_list) if need_model else None), torch.stack(foundation_list)
|
| 432 |
+
finally:
|
| 433 |
+
cap.release()
|
| 434 |
+
if t < int(meta["num_frames"]):
|
| 435 |
+
raise RuntimeError(
|
| 436 |
+
f"{video_path}: decoded only {t}/{meta['num_frames']} requested frames")
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
def iter_frame_dir_chunks(frame_dir: str, meta: dict, *, chunk_size: int,
|
| 440 |
+
need_model: bool):
|
| 441 |
+
paths = meta["frame_paths"]
|
| 442 |
+
t = 0
|
| 443 |
+
while t < len(paths):
|
| 444 |
+
t0 = t
|
| 445 |
+
model_list = [] if need_model else None
|
| 446 |
+
foundation_list = []
|
| 447 |
+
while t < len(paths) and len(foundation_list) < chunk_size:
|
| 448 |
+
model, foundation, _review = preprocess_frame_dir_path(
|
| 449 |
+
paths[t], meta, need_model=need_model)
|
| 450 |
+
if need_model:
|
| 451 |
+
model_list.append(model)
|
| 452 |
+
foundation_list.append(foundation)
|
| 453 |
+
t += 1
|
| 454 |
+
yield t0, (torch.stack(model_list) if need_model else None), torch.stack(foundation_list)
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
def preprocess_hint_frame(frame_bgr, meta: dict) -> torch.Tensor:
|
| 458 |
+
h_out, w_out = meta["shape_hw"]
|
| 459 |
+
h_model, w_model = meta["model_native_hw"]
|
| 460 |
+
if frame_bgr is None:
|
| 461 |
+
raise RuntimeError("hint frame decode failed")
|
| 462 |
+
if frame_bgr.ndim == 3:
|
| 463 |
+
hint = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2GRAY)
|
| 464 |
+
else:
|
| 465 |
+
hint = frame_bgr
|
| 466 |
+
if hint.dtype == np.uint8:
|
| 467 |
+
hint = hint.astype(np.float32) / 255.0
|
| 468 |
+
elif hint.dtype == np.uint16:
|
| 469 |
+
hint = hint.astype(np.float32) / 65535.0
|
| 470 |
+
else:
|
| 471 |
+
hint = hint.astype(np.float32)
|
| 472 |
+
if hint.shape[:2] != (h_model, w_model):
|
| 473 |
+
hint = cv2.resize(hint, (w_model, h_model),
|
| 474 |
+
interpolation=cv2.INTER_LANCZOS4)
|
| 475 |
+
hint, _ = center_pad_to_shape(np.clip(hint, 0.0, 1.0), h_out, w_out)
|
| 476 |
+
return torch.from_numpy(hint[None]).to(dtype=torch.bfloat16)
|
| 477 |
+
|
| 478 |
+
|
| 479 |
+
def load_initial_hint_frame(path_or_dir: str, meta: dict) -> torch.Tensor:
|
| 480 |
+
paths = frame_paths(path_or_dir) if os.path.isdir(path_or_dir) else [path_or_dir]
|
| 481 |
+
paths = [p for p in paths if os.path.isfile(p)]
|
| 482 |
+
if not paths:
|
| 483 |
+
raise RuntimeError(f"No hint frame found at {path_or_dir}")
|
| 484 |
+
hint = read_hint_image(paths[0])
|
| 485 |
+
native_hw = meta["native_hw"]
|
| 486 |
+
model_hw = meta["model_native_hw"]
|
| 487 |
+
if hint.shape[:2] == native_hw and model_hw != native_hw:
|
| 488 |
+
hint = cv2.resize(hint, (model_hw[1], model_hw[0]),
|
| 489 |
+
interpolation=cv2.INTER_LANCZOS4)
|
| 490 |
+
elif hint.shape[:2] != model_hw:
|
| 491 |
+
raise ValueError(
|
| 492 |
+
f"{paths[0]}: hint shape {hint.shape[:2]} does not match "
|
| 493 |
+
f"native {native_hw} or model {model_hw}")
|
| 494 |
+
hint, pad_check = center_pad_to_shape(
|
| 495 |
+
hint, meta["shape_hw"][0], meta["shape_hw"][1])
|
| 496 |
+
if pad_check != meta["pad"]:
|
| 497 |
+
raise ValueError(f"{paths[0]}: hint pad {pad_check} != input pad {meta['pad']}")
|
| 498 |
+
print(f" loaded initial frame-1 hint from {paths[0]}")
|
| 499 |
+
return torch.from_numpy(hint[None]).to(dtype=torch.bfloat16)
|
| 500 |
+
|
| 501 |
+
|
| 502 |
+
class SequentialHintVideoWindowReader:
|
| 503 |
+
def __init__(self, hint_video_path: str, meta: dict):
|
| 504 |
+
self.hint_video_path = hint_video_path
|
| 505 |
+
self.meta = meta
|
| 506 |
+
self.cap = cv2.VideoCapture(hint_video_path)
|
| 507 |
+
if not self.cap.isOpened():
|
| 508 |
+
raise RuntimeError(f"Could not open hint video: {hint_video_path}")
|
| 509 |
+
self.start_frame = int(meta["start_frame"])
|
| 510 |
+
self.n_frames = int(meta["num_frames"])
|
| 511 |
+
hint_n_frames = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT) or 0)
|
| 512 |
+
required_frames = self.start_frame + self.n_frames
|
| 513 |
+
if hint_n_frames > 0 and hint_n_frames < required_frames:
|
| 514 |
+
raise ValueError(
|
| 515 |
+
f"Hint video {hint_video_path} has only {hint_n_frames} frames, "
|
| 516 |
+
f"but the source video requires {required_frames} frames "
|
| 517 |
+
f"(start={self.start_frame}, num={self.n_frames})")
|
| 518 |
+
if self.start_frame:
|
| 519 |
+
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.start_frame)
|
| 520 |
+
self.next_t = 0
|
| 521 |
+
self.cache: dict[int, torch.Tensor] = {}
|
| 522 |
+
|
| 523 |
+
def close(self):
|
| 524 |
+
self.cap.release()
|
| 525 |
+
|
| 526 |
+
def _read_until(self, t_end: int):
|
| 527 |
+
while self.next_t < t_end:
|
| 528 |
+
ret, frame = self.cap.read()
|
| 529 |
+
if not ret:
|
| 530 |
+
raise RuntimeError(
|
| 531 |
+
f"{self.hint_video_path}: failed decoding hint frame "
|
| 532 |
+
f"{self.start_frame + self.next_t}")
|
| 533 |
+
self.cache[self.next_t] = preprocess_hint_frame(frame, self.meta)
|
| 534 |
+
self.next_t += 1
|
| 535 |
+
|
| 536 |
+
def window(self, t0: int, t1: int):
|
| 537 |
+
self._read_until(t1)
|
| 538 |
+
hint = torch.stack([self.cache[t] for t in range(t0, t1)])
|
| 539 |
+
for old_t in [k for k in self.cache if k < t0]:
|
| 540 |
+
del self.cache[old_t]
|
| 541 |
+
return hint
|
| 542 |
+
|
| 543 |
+
|
| 544 |
+
class SequentialHintFrameDirWindowReader:
|
| 545 |
+
def __init__(self, hint_dir_path: str, meta: dict):
|
| 546 |
+
self.hint_dir_path = hint_dir_path
|
| 547 |
+
self.meta = meta
|
| 548 |
+
self.paths = frame_paths(hint_dir_path)
|
| 549 |
+
if len(self.paths) < int(meta["num_frames"]):
|
| 550 |
+
raise RuntimeError(
|
| 551 |
+
f"{hint_dir_path}: only {len(self.paths)} hint frames for "
|
| 552 |
+
f"{meta['num_frames']} source frames")
|
| 553 |
+
self.next_t = 0
|
| 554 |
+
self.cache: dict[int, torch.Tensor] = {}
|
| 555 |
+
|
| 556 |
+
def close(self):
|
| 557 |
+
pass
|
| 558 |
+
|
| 559 |
+
def _read_until(self, t_end: int):
|
| 560 |
+
while self.next_t < t_end:
|
| 561 |
+
frame = cv2.imread(self.paths[self.next_t], cv2.IMREAD_UNCHANGED)
|
| 562 |
+
self.cache[self.next_t] = preprocess_hint_frame(frame, self.meta)
|
| 563 |
+
self.next_t += 1
|
| 564 |
+
|
| 565 |
+
def window(self, t0: int, t1: int):
|
| 566 |
+
self._read_until(t1)
|
| 567 |
+
hint = torch.stack([self.cache[t] for t in range(t0, t1)])
|
| 568 |
+
for old_t in [k for k in self.cache if k < t0]:
|
| 569 |
+
del self.cache[old_t]
|
| 570 |
+
return hint
|
| 571 |
+
|
| 572 |
+
|
| 573 |
+
def make_hint_reader(path: str | None, meta: dict):
|
| 574 |
+
path = none_path(path)
|
| 575 |
+
if path is None:
|
| 576 |
+
return None
|
| 577 |
+
if os.path.isdir(path):
|
| 578 |
+
return SequentialHintFrameDirWindowReader(path, meta)
|
| 579 |
+
return SequentialHintVideoWindowReader(path, meta)
|
| 580 |
+
|
| 581 |
+
|
| 582 |
+
def window_starts(T_all: int, chunk: int, stride: int) -> list[int]:
|
| 583 |
+
if T_all <= chunk:
|
| 584 |
+
return [0]
|
| 585 |
+
starts = list(range(0, T_all - chunk + 1, stride))
|
| 586 |
+
if starts[-1] != T_all - chunk:
|
| 587 |
+
starts.append(T_all - chunk)
|
| 588 |
+
return starts
|
| 589 |
+
|
| 590 |
+
|
| 591 |
+
def taper_weights(Tc: int, overlap: int, *, is_first: bool, is_last: bool,
|
| 592 |
+
device=None, dtype=torch.float32) -> torch.Tensor:
|
| 593 |
+
w = torch.ones(Tc, device=device, dtype=dtype)
|
| 594 |
+
if overlap <= 0 or Tc <= 1:
|
| 595 |
+
return w
|
| 596 |
+
n = min(overlap, Tc)
|
| 597 |
+
ramp = 0.5 * (1.0 - torch.cos(
|
| 598 |
+
math.pi * (torch.arange(n, device=device, dtype=dtype) + 1) / (overlap + 1)
|
| 599 |
+
))
|
| 600 |
+
if not is_first:
|
| 601 |
+
w[:n] = ramp
|
| 602 |
+
if not is_last:
|
| 603 |
+
w[Tc - n:] = ramp.flip(0)
|
| 604 |
+
return w
|
| 605 |
+
|
| 606 |
+
|
| 607 |
+
class ThreadedVideoWriter:
|
| 608 |
+
def __init__(self, path: str, fps: float, size: tuple[int, int],
|
| 609 |
+
cfg: WriterConfig, *, maxsize: int = 2, channels: int = 3,
|
| 610 |
+
color_trc: str | None = None):
|
| 611 |
+
self.path = path
|
| 612 |
+
self.fps = float(fps)
|
| 613 |
+
self.size = size
|
| 614 |
+
self.cfg = cfg
|
| 615 |
+
self.maxsize = maxsize
|
| 616 |
+
self.channels = channels
|
| 617 |
+
self.color_trc = color_trc
|
| 618 |
+
self.proc = None
|
| 619 |
+
self.q = queue.Queue(maxsize=maxsize)
|
| 620 |
+
self.err = None
|
| 621 |
+
self.thread = threading.Thread(target=self._worker, daemon=True)
|
| 622 |
+
self._start()
|
| 623 |
+
|
| 624 |
+
def _start(self):
|
| 625 |
+
require_ffmpeg(self.cfg.ffmpeg_bin)
|
| 626 |
+
w, h = self.size
|
| 627 |
+
|
| 628 |
+
in_pix_fmt = "bgra64le" if self.channels == 4 else "bgr48le"
|
| 629 |
+
|
| 630 |
+
cmd = [
|
| 631 |
+
self.cfg.ffmpeg_bin, "-y", "-hide_banner", "-loglevel", "error",
|
| 632 |
+
"-nostats", "-f", "rawvideo", "-pix_fmt", in_pix_fmt,
|
| 633 |
+
"-s:v", f"{w}x{h}", "-r", f"{self.fps:.8f}", "-i", "pipe:0",
|
| 634 |
+
"-an"
|
| 635 |
+
]
|
| 636 |
+
|
| 637 |
+
if self.channels == 4:
|
| 638 |
+
cmd.extend([
|
| 639 |
+
"-c:v", "prores_ks",
|
| 640 |
+
"-profile:v", "4",
|
| 641 |
+
"-pix_fmt", "yuva444p10le"
|
| 642 |
+
])
|
| 643 |
+
else:
|
| 644 |
+
cmd.extend([
|
| 645 |
+
"-c:v", self.cfg.codec,
|
| 646 |
+
"-preset", self.cfg.preset,
|
| 647 |
+
"-pix_fmt", self.cfg.pix_fmt
|
| 648 |
+
])
|
| 649 |
+
if self.cfg.bitrate:
|
| 650 |
+
cmd.extend(["-b:v", self.cfg.bitrate])
|
| 651 |
+
else:
|
| 652 |
+
cmd.extend(["-crf", str(int(self.cfg.crf))])
|
| 653 |
+
|
| 654 |
+
trc = self.color_trc if self.color_trc is not None else "iec61966-2-1"
|
| 655 |
+
cmd.extend([
|
| 656 |
+
"-threads", str(max(1, self.cfg.threads)),
|
| 657 |
+
"-colorspace", "bt709", "-color_primaries", "bt709",
|
| 658 |
+
"-color_trc", trc, "-movflags", "+write_colr",
|
| 659 |
+
self.path
|
| 660 |
+
])
|
| 661 |
+
|
| 662 |
+
self.proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
|
| 663 |
+
self.thread.start()
|
| 664 |
+
|
| 665 |
+
def _worker(self):
|
| 666 |
+
try:
|
| 667 |
+
while True:
|
| 668 |
+
item = self.q.get()
|
| 669 |
+
if item is None:
|
| 670 |
+
self.q.task_done()
|
| 671 |
+
return
|
| 672 |
+
self.proc.stdin.write(item.tobytes())
|
| 673 |
+
self.q.task_done()
|
| 674 |
+
except Exception as exc:
|
| 675 |
+
self.err = exc
|
| 676 |
+
|
| 677 |
+
def write(self, frame: np.ndarray):
|
| 678 |
+
self.write_many(frame[None])
|
| 679 |
+
|
| 680 |
+
def write_many(self, frames: np.ndarray):
|
| 681 |
+
if self.err is not None:
|
| 682 |
+
raise self.err
|
| 683 |
+
arr = np.ascontiguousarray(frames)
|
| 684 |
+
if arr.ndim != 4 or arr.shape[-1] != self.channels:
|
| 685 |
+
raise ValueError(
|
| 686 |
+
f"expected batched frames as NHWC uint16 with {self.channels} channels, got {arr.shape}")
|
| 687 |
+
if arr.dtype != np.uint16:
|
| 688 |
+
raise ValueError(f"expected uint16 frames, got {arr.dtype}")
|
| 689 |
+
self.q.put(arr)
|
| 690 |
+
|
| 691 |
+
def release(self):
|
| 692 |
+
self.q.put(None)
|
| 693 |
+
self.thread.join()
|
| 694 |
+
if self.proc is not None and self.proc.stdin is not None:
|
| 695 |
+
self.proc.stdin.close()
|
| 696 |
+
rc = self.proc.wait() if self.proc is not None else 0
|
| 697 |
+
if rc != 0 and self.err is None:
|
| 698 |
+
self.err = RuntimeError(f"ffmpeg writer failed with exit code {rc}: {self.path}")
|
| 699 |
+
if self.err is not None:
|
| 700 |
+
raise self.err
|
| 701 |
+
|
| 702 |
+
|
| 703 |
+
class ExrSequenceWriter:
|
| 704 |
+
"""Writes float frames as an OpenEXR image sequence (scene-referred, unclamped).
|
| 705 |
+
|
| 706 |
+
Accepts NHWC float32 batches with `channels` channels in BGR(A) order,
|
| 707 |
+
matching the layout convention of ThreadedVideoWriter/cv2. Values are
|
| 708 |
+
written as-is: no clamping and no quantization. `dtype` selects the EXR
|
| 709 |
+
pixel type: "half" (float16, VFX standard) or "float" (full float32).
|
| 710 |
+
"""
|
| 711 |
+
|
| 712 |
+
def __init__(self, dir_path: str, fps: float, size: tuple[int, int], *,
|
| 713 |
+
channels: int = 3, dtype: str = "half", maxsize: int = 2,
|
| 714 |
+
frame_name: str = "frame"):
|
| 715 |
+
if dtype not in ("half", "float"):
|
| 716 |
+
raise ValueError(f"exr dtype must be 'half' or 'float', got {dtype!r}")
|
| 717 |
+
self.dir_path = str(dir_path)
|
| 718 |
+
self.path = self.dir_path # for logging parity with ThreadedVideoWriter
|
| 719 |
+
self.size = size
|
| 720 |
+
self.channels = channels
|
| 721 |
+
self.frame_name = frame_name
|
| 722 |
+
exr_type = (cv2.IMWRITE_EXR_TYPE_HALF if dtype == "half"
|
| 723 |
+
else cv2.IMWRITE_EXR_TYPE_FLOAT)
|
| 724 |
+
self.clamp_half = dtype == "half"
|
| 725 |
+
self.imwrite_params = [
|
| 726 |
+
cv2.IMWRITE_EXR_TYPE, exr_type,
|
| 727 |
+
cv2.IMWRITE_EXR_COMPRESSION, cv2.IMWRITE_EXR_COMPRESSION_ZIP,
|
| 728 |
+
]
|
| 729 |
+
os.makedirs(self.dir_path, exist_ok=True)
|
| 730 |
+
self.frame_idx = 0
|
| 731 |
+
self.q = queue.Queue(maxsize=maxsize)
|
| 732 |
+
self.err = None
|
| 733 |
+
self.thread = threading.Thread(target=self._worker, daemon=True)
|
| 734 |
+
self.thread.start()
|
| 735 |
+
|
| 736 |
+
def _worker(self):
|
| 737 |
+
try:
|
| 738 |
+
while True:
|
| 739 |
+
item = self.q.get()
|
| 740 |
+
if item is None:
|
| 741 |
+
self.q.task_done()
|
| 742 |
+
return
|
| 743 |
+
idx0, arr = item
|
| 744 |
+
if self.clamp_half:
|
| 745 |
+
# float16 overflows to inf above 65504; keep half EXRs finite
|
| 746 |
+
arr = np.clip(arr, -65504.0, 65504.0)
|
| 747 |
+
for j in range(arr.shape[0]):
|
| 748 |
+
out_path = os.path.join(
|
| 749 |
+
self.dir_path, f"{self.frame_name}_{idx0 + j:06d}.exr")
|
| 750 |
+
frame = np.ascontiguousarray(arr[j])
|
| 751 |
+
if frame.shape[-1] == 1:
|
| 752 |
+
frame = frame[..., 0]
|
| 753 |
+
if not cv2.imwrite(out_path, frame, self.imwrite_params):
|
| 754 |
+
raise RuntimeError(f"cv2.imwrite failed: {out_path}")
|
| 755 |
+
self.q.task_done()
|
| 756 |
+
except Exception as exc:
|
| 757 |
+
self.err = exc
|
| 758 |
+
|
| 759 |
+
def write_many(self, frames: np.ndarray):
|
| 760 |
+
if self.err is not None:
|
| 761 |
+
raise self.err
|
| 762 |
+
arr = np.ascontiguousarray(frames)
|
| 763 |
+
if arr.ndim != 4 or arr.shape[-1] != self.channels:
|
| 764 |
+
raise ValueError(
|
| 765 |
+
f"expected batched frames as NHWC float32 with "
|
| 766 |
+
f"{self.channels} channels, got {arr.shape}")
|
| 767 |
+
if arr.dtype != np.float32:
|
| 768 |
+
raise ValueError(f"expected float32 frames, got {arr.dtype}")
|
| 769 |
+
self.q.put((self.frame_idx, arr))
|
| 770 |
+
self.frame_idx += arr.shape[0]
|
| 771 |
+
|
| 772 |
+
def write(self, frame: np.ndarray):
|
| 773 |
+
self.write_many(frame[None])
|
| 774 |
+
|
| 775 |
+
def release(self):
|
| 776 |
+
self.q.put(None)
|
| 777 |
+
self.thread.join()
|
| 778 |
+
if self.err is not None:
|
| 779 |
+
raise self.err
|
| 780 |
+
|
| 781 |
+
|
| 782 |
+
def color_chw_to_bgr(x: np.ndarray) -> np.ndarray:
|
| 783 |
+
x = np.clip(np.moveaxis(x, 0, -1), 0, 1)
|
| 784 |
+
return cv2.cvtColor((x * 255).astype(np.uint8), cv2.COLOR_RGB2BGR)
|
| 785 |
+
|
| 786 |
+
|
| 787 |
+
def gray_1hw_to_bgr(x: np.ndarray) -> np.ndarray:
|
| 788 |
+
x = np.clip(x[0], 0, 1)
|
| 789 |
+
return cv2.cvtColor((x * 255).astype(np.uint8), cv2.COLOR_GRAY2BGR)
|
| 790 |
+
|
| 791 |
+
|
| 792 |
+
def make_checker(H: int, W: int, size: int = 64,
|
| 793 |
+
light: float = 0.75, dark: float = 0.5) -> np.ndarray:
|
| 794 |
+
y, x = np.indices((H, W))
|
| 795 |
+
mask = (((y // size) + (x // size)) % 2 == 0).astype(np.float32)
|
| 796 |
+
gray = mask * light + (1.0 - mask) * dark
|
| 797 |
+
return np.stack([gray, gray, gray], axis=0).astype(np.float32)
|
| 798 |
+
|
| 799 |
+
|
| 800 |
+
def composite_over_checker_srgb(fg_srgb_chw: np.ndarray, alpha_1hw: np.ndarray,
|
| 801 |
+
checker_chw: np.ndarray) -> np.ndarray:
|
| 802 |
+
fg_lin = cu.srgb_to_linear(fg_srgb_chw)
|
| 803 |
+
bg_lin = cu.srgb_to_linear(checker_chw)
|
| 804 |
+
alpha = np.clip(alpha_1hw, 0.0, 1.0)
|
| 805 |
+
comp_lin = fg_lin * alpha + bg_lin * (1.0 - alpha)
|
| 806 |
+
return np.clip(cu.linear_to_srgb(np.maximum(comp_lin, 0.0)), 0.0, 1.0)
|
| 807 |
+
|
| 808 |
+
|
| 809 |
+
def corridorkey_despill_average_srgb(fg_srgb_chw: np.ndarray,
|
| 810 |
+
strength: float = 0.5) -> np.ndarray:
|
| 811 |
+
if strength <= 0.0:
|
| 812 |
+
return fg_srgb_chw
|
| 813 |
+
r = fg_srgb_chw[0]
|
| 814 |
+
g = fg_srgb_chw[1]
|
| 815 |
+
b = fg_srgb_chw[2]
|
| 816 |
+
spill = np.maximum(g - (r + b) * 0.5, 0.0)
|
| 817 |
+
full = np.stack([r + spill * 0.5, g - spill, b + spill * 0.5], axis=0)
|
| 818 |
+
if strength >= 1.0:
|
| 819 |
+
return full.astype(np.float32, copy=False)
|
| 820 |
+
return fg_srgb_chw * (1.0 - strength) + full * strength
|
requirements.txt
CHANGED
|
@@ -10,3 +10,10 @@ torch
|
|
| 10 |
torchvision
|
| 11 |
timm
|
| 12 |
https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.0/flash_attn-2.8.3+cu126torch2.9-cp310-cp310-linux_x86_64.whl
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
torchvision
|
| 11 |
timm
|
| 12 |
https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.0/flash_attn-2.8.3+cu126torch2.9-cp310-cp310-linux_x86_64.whl
|
| 13 |
+
transformers>=4.57,<5
|
| 14 |
+
einops>=0.8
|
| 15 |
+
scipy
|
| 16 |
+
safetensors>=0.6
|
| 17 |
+
pillow
|
| 18 |
+
tqdm
|
| 19 |
+
utils3d @ git+https://github.com/EasternJournalist/utils3d.git@83bfa5bda008803f6521ce9547f94023b8afeddd
|
third_party/MoGe/LICENSE
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) Microsoft Corporation.
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
Apache License
|
| 25 |
+
Version 2.0, January 2004
|
| 26 |
+
http://www.apache.org/licenses/
|
| 27 |
+
|
| 28 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 29 |
+
|
| 30 |
+
1. Definitions.
|
| 31 |
+
|
| 32 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 33 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 34 |
+
|
| 35 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 36 |
+
the copyright owner that is granting the License.
|
| 37 |
+
|
| 38 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 39 |
+
other entities that control, are controlled by, or are under common
|
| 40 |
+
control with that entity. For the purposes of this definition,
|
| 41 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 42 |
+
direction or management of such entity, whether by contract or
|
| 43 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 44 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 45 |
+
|
| 46 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 47 |
+
exercising permissions granted by this License.
|
| 48 |
+
|
| 49 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 50 |
+
including but not limited to software source code, documentation
|
| 51 |
+
source, and configuration files.
|
| 52 |
+
|
| 53 |
+
"Object" form shall mean any form resulting from mechanical
|
| 54 |
+
transformation or translation of a Source form, including but
|
| 55 |
+
not limited to compiled object code, generated documentation,
|
| 56 |
+
and conversions to other media types.
|
| 57 |
+
|
| 58 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 59 |
+
Object form, made available under the License, as indicated by a
|
| 60 |
+
copyright notice that is included in or attached to the work
|
| 61 |
+
(an example is provided in the Appendix below).
|
| 62 |
+
|
| 63 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 64 |
+
form, that is based on (or derived from) the Work and for which the
|
| 65 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 66 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 67 |
+
of this License, Derivative Works shall not include works that remain
|
| 68 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 69 |
+
the Work and Derivative Works thereof.
|
| 70 |
+
|
| 71 |
+
"Contribution" shall mean any work of authorship, including
|
| 72 |
+
the original version of the Work and any modifications or additions
|
| 73 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 74 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 75 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 76 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 77 |
+
means any form of electronic, verbal, or written communication sent
|
| 78 |
+
to the Licensor or its representatives, including but not limited to
|
| 79 |
+
communication on electronic mailing lists, source code control systems,
|
| 80 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 81 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 82 |
+
excluding communication that is conspicuously marked or otherwise
|
| 83 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 84 |
+
|
| 85 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 86 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 87 |
+
subsequently incorporated within the Work.
|
| 88 |
+
|
| 89 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 90 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 91 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 92 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 93 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 94 |
+
Work and such Derivative Works in Source or Object form.
|
| 95 |
+
|
| 96 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 97 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 98 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 99 |
+
(except as stated in this section) patent license to make, have made,
|
| 100 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 101 |
+
where such license applies only to those patent claims licensable
|
| 102 |
+
by such Contributor that are necessarily infringed by their
|
| 103 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 104 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 105 |
+
institute patent litigation against any entity (including a
|
| 106 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 107 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 108 |
+
or contributory patent infringement, then any patent licenses
|
| 109 |
+
granted to You under this License for that Work shall terminate
|
| 110 |
+
as of the date such litigation is filed.
|
| 111 |
+
|
| 112 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 113 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 114 |
+
modifications, and in Source or Object form, provided that You
|
| 115 |
+
meet the following conditions:
|
| 116 |
+
|
| 117 |
+
(a) You must give any other recipients of the Work or
|
| 118 |
+
Derivative Works a copy of this License; and
|
| 119 |
+
|
| 120 |
+
(b) You must cause any modified files to carry prominent notices
|
| 121 |
+
stating that You changed the files; and
|
| 122 |
+
|
| 123 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 124 |
+
that You distribute, all copyright, patent, trademark, and
|
| 125 |
+
attribution notices from the Source form of the Work,
|
| 126 |
+
excluding those notices that do not pertain to any part of
|
| 127 |
+
the Derivative Works; and
|
| 128 |
+
|
| 129 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 130 |
+
distribution, then any Derivative Works that You distribute must
|
| 131 |
+
include a readable copy of the attribution notices contained
|
| 132 |
+
within such NOTICE file, excluding those notices that do not
|
| 133 |
+
pertain to any part of the Derivative Works, in at least one
|
| 134 |
+
of the following places: within a NOTICE text file distributed
|
| 135 |
+
as part of the Derivative Works; within the Source form or
|
| 136 |
+
documentation, if provided along with the Derivative Works; or,
|
| 137 |
+
within a display generated by the Derivative Works, if and
|
| 138 |
+
wherever such third-party notices normally appear. The contents
|
| 139 |
+
of the NOTICE file are for informational purposes only and
|
| 140 |
+
do not modify the License. You may add Your own attribution
|
| 141 |
+
notices within Derivative Works that You distribute, alongside
|
| 142 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 143 |
+
that such additional attribution notices cannot be construed
|
| 144 |
+
as modifying the License.
|
| 145 |
+
|
| 146 |
+
You may add Your own copyright statement to Your modifications and
|
| 147 |
+
may provide additional or different license terms and conditions
|
| 148 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 149 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 150 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 151 |
+
the conditions stated in this License.
|
| 152 |
+
|
| 153 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 154 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 155 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 156 |
+
this License, without any additional terms or conditions.
|
| 157 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 158 |
+
the terms of any separate license agreement you may have executed
|
| 159 |
+
with Licensor regarding such Contributions.
|
| 160 |
+
|
| 161 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 162 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 163 |
+
except as required for reasonable and customary use in describing the
|
| 164 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 165 |
+
|
| 166 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 167 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 168 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 169 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 170 |
+
implied, including, without limitation, any warranties or conditions
|
| 171 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 172 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 173 |
+
appropriateness of using or redistributing the Work and assume any
|
| 174 |
+
risks associated with Your exercise of permissions under this License.
|
| 175 |
+
|
| 176 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 177 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 178 |
+
unless required by applicable law (such as deliberate and grossly
|
| 179 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 180 |
+
liable to You for damages, including any direct, indirect, special,
|
| 181 |
+
incidental, or consequential damages of any character arising as a
|
| 182 |
+
result of this License or out of the use or inability to use the
|
| 183 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 184 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 185 |
+
other commercial damages or losses), even if such Contributor
|
| 186 |
+
has been advised of the possibility of such damages.
|
| 187 |
+
|
| 188 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 189 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 190 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 191 |
+
or other liability obligations and/or rights consistent with this
|
| 192 |
+
License. However, in accepting such obligations, You may act only
|
| 193 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 194 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 195 |
+
defend, and hold each Contributor harmless for any liability
|
| 196 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 197 |
+
of your accepting any such warranty or additional liability.
|
| 198 |
+
|
| 199 |
+
END OF TERMS AND CONDITIONS
|
| 200 |
+
|
| 201 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 202 |
+
|
| 203 |
+
To apply the Apache License to your work, attach the following
|
| 204 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 205 |
+
replaced with your own identifying information. (Don't include
|
| 206 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 207 |
+
comment syntax for the file format. We also recommend that a
|
| 208 |
+
file or class name and description of purpose be included on the
|
| 209 |
+
same "printed page" as the copyright notice for easier
|
| 210 |
+
identification within third-party archives.
|
| 211 |
+
|
| 212 |
+
Copyright [yyyy] [name of copyright owner]
|
| 213 |
+
|
| 214 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 215 |
+
you may not use this file except in compliance with the License.
|
| 216 |
+
You may obtain a copy of the License at
|
| 217 |
+
|
| 218 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 219 |
+
|
| 220 |
+
Unless required by applicable law or agreed to in writing, software
|
| 221 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 222 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 223 |
+
See the License for the specific language governing permissions and
|
| 224 |
+
limitations under the License.
|
third_party/MoGe/README.md
ADDED
|
@@ -0,0 +1,295 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# MoGe: Accurate Monocular Geometry Estimation
|
| 2 |
+
|
| 3 |
+
MoGe is a powerful model for recovering 3D geometry from monocular open-domain images, including metric point maps, metric depth maps, normal maps and camera FOV. ***Check our websites ([MoGe-1](https://wangrc.site/MoGePage), [MoGe-2](https://wangrc.site/MoGe2Page)) for videos and interactive results!***
|
| 4 |
+
|
| 5 |
+
## 📖 Publications
|
| 6 |
+
|
| 7 |
+
### MoGe-2: Accurate Monocular Geometry with Metric Scale and Sharp Details
|
| 8 |
+
|
| 9 |
+
<div align="center">
|
| 10 |
+
<a href="https://arxiv.org/abs/2507.02546"><img src='https://img.shields.io/badge/arXiv-Paper-red?logo=arxiv&logoColor=white' alt='arXiv'></a>
|
| 11 |
+
<a href='https://wangrc.site/MoGe2Page/'><img src='https://img.shields.io/badge/Project_Page-Website-green?logo=googlechrome&logoColor=white' alt='Project Page'></a>
|
| 12 |
+
<a href='https://huggingface.co/spaces/Ruicheng/MoGe-2'><img src='https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Demo_(MoGe_v2)-blue'></a>
|
| 13 |
+
|
| 14 |
+
https://github.com/user-attachments/assets/8f9ae680-659d-4f7f-82e2-b9ed9d6b988a
|
| 15 |
+
|
| 16 |
+
</div>
|
| 17 |
+
|
| 18 |
+
### MoGe: Unlocking Accurate Monocular Geometry Estimation for Open-Domain Images with Optimal Training Supervision
|
| 19 |
+
|
| 20 |
+
<div align="center">
|
| 21 |
+
<a href="https://arxiv.org/abs/2410.19115"><img src='https://img.shields.io/badge/arXiv-Paper-red?logo=arxiv&logoColor=white' alt='arXiv'></a>
|
| 22 |
+
<a href='https://wangrc.site/MoGePage/'><img src='https://img.shields.io/badge/Project_Page-Website-green?logo=googlechrome&logoColor=white' alt='Project Page'></a>
|
| 23 |
+
<a href='https://huggingface.co/spaces/Ruicheng/MoGe'><img src='https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Demo_(MoGe_v1)-blue'></a>
|
| 24 |
+
</div>
|
| 25 |
+
|
| 26 |
+
<img src="./assets/overview_simplified.png" width="100%" alt="Method overview" align="center">
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
## 🌟 Features
|
| 30 |
+
|
| 31 |
+
* **Accurate 3D geometry estimation**: Estimate point maps & depth maps & [normal maps](docs/normal.md) from open-domain single images with high precision -- all capabilities in one model, one forward pass.
|
| 32 |
+
* **Optional ground-truth FOV input**: Enhance model accuracy further by providing the true field of view.
|
| 33 |
+
* **Flexible resolution support**: Works seamlessly with various resolutions and aspect ratios, from 2:1 to 1:2.
|
| 34 |
+
* **Optimized for speed**: Achieves 60ms latency per image (A100 or RTX3090, FP16, ViT-L). Adjustable inference resolution for even faster speed.
|
| 35 |
+
|
| 36 |
+
## ✨ News
|
| 37 |
+
|
| 38 |
+
***(2025-10-16)***
|
| 39 |
+
* Updated training code for MoGe-2.
|
| 40 |
+
|
| 41 |
+
***(2025-06-10)***
|
| 42 |
+
|
| 43 |
+
* ❗**Released MoGe-2**, a state-of-the-art model for monocular geometry, with these new capabilities in one unified model:
|
| 44 |
+
* point map prediction in **metric scale**;
|
| 45 |
+
* comparable and even better performance over MoGe-1;
|
| 46 |
+
* significant improvement of **visual sharpness**;
|
| 47 |
+
* high-quality [**normal map** estimation](docs/normal.md);
|
| 48 |
+
* lower inference latency.
|
| 49 |
+
|
| 50 |
+
## 📦 Installation
|
| 51 |
+
|
| 52 |
+
### Install via pip
|
| 53 |
+
|
| 54 |
+
```bash
|
| 55 |
+
pip install git+https://github.com/microsoft/MoGe.git
|
| 56 |
+
```
|
| 57 |
+
|
| 58 |
+
### Or clone this repository
|
| 59 |
+
|
| 60 |
+
```bash
|
| 61 |
+
git clone https://github.com/microsoft/MoGe.git
|
| 62 |
+
cd MoGe
|
| 63 |
+
pip install -r requirements.txt # install the requirements
|
| 64 |
+
```
|
| 65 |
+
|
| 66 |
+
Note: MoGe should be compatible with most requirements versions. Please check the `requirements.txt` for more details if you encounter any dependency issues.
|
| 67 |
+
|
| 68 |
+
## 🤗 Pretrained Models
|
| 69 |
+
|
| 70 |
+
Our pretrained models are available on the huggingface hub:
|
| 71 |
+
|
| 72 |
+
<table>
|
| 73 |
+
<thead>
|
| 74 |
+
<tr>
|
| 75 |
+
<th>Version</th>
|
| 76 |
+
<th>Hugging Face Model</th>
|
| 77 |
+
<th>Metric scale</th>
|
| 78 |
+
<th>Normal</th>
|
| 79 |
+
<th>#Params</th>
|
| 80 |
+
</tr>
|
| 81 |
+
</thead>
|
| 82 |
+
<tbody>
|
| 83 |
+
<tr>
|
| 84 |
+
<td>MoGe-1</td>
|
| 85 |
+
<td><a href="https://huggingface.co/Ruicheng/moge-vitl" target="_blank"><code>Ruicheng/moge-vitl</code><a></td>
|
| 86 |
+
<td>-</td>
|
| 87 |
+
<td>-</td>
|
| 88 |
+
<td>314M</td>
|
| 89 |
+
</tr>
|
| 90 |
+
<tr>
|
| 91 |
+
<td rowspan="4">MoGe-2</td>
|
| 92 |
+
<td><a href="https://huggingface.co/Ruicheng/moge-2-vitl" target="_blank"><code>Ruicheng/moge-2-vitl</code></a></td>
|
| 93 |
+
<td>✅</td>
|
| 94 |
+
<td>-</td>
|
| 95 |
+
<td>326M</td>
|
| 96 |
+
</tr>
|
| 97 |
+
<tr>
|
| 98 |
+
<td><a href="https://huggingface.co/Ruicheng/moge-2-vitl-normal" target="_blank"><code>Ruicheng/moge-2-vitl-normal</code></a></td>
|
| 99 |
+
<td>✅</td>
|
| 100 |
+
<td>✅</td>
|
| 101 |
+
<td>331M</td>
|
| 102 |
+
</tr>
|
| 103 |
+
<tr>
|
| 104 |
+
<td><a href="https://huggingface.co/Ruicheng/moge-2-vitb-normal" target="_blank"><code>Ruicheng/moge-2-vitb-normal</code></a></td>
|
| 105 |
+
<td>✅</td>
|
| 106 |
+
<td>✅</td>
|
| 107 |
+
<td>104M</td>
|
| 108 |
+
</tr>
|
| 109 |
+
<tr>
|
| 110 |
+
<td><a href="https://huggingface.co/Ruicheng/moge-2-vits-normal" target="_blank"><code>Ruicheng/moge-2-vits-normal</code></a></td>
|
| 111 |
+
<td>✅</td>
|
| 112 |
+
<td>✅</td>
|
| 113 |
+
<td>35M</td>
|
| 114 |
+
</tr>
|
| 115 |
+
</tbody>
|
| 116 |
+
</table>
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
> NOTE: `moge-2-vitl-normal` has full capabilities, with almost the same level of performance as `moge-2-vitl` plus extra normal map estimation.
|
| 120 |
+
|
| 121 |
+
You may import the `MoGeModel` class of the matched version, then load the pretrained weights via `MoGeModel.from_pretrained("HUGGING_FACE_MODEL_REPO_NAME")` with automatic downloading.
|
| 122 |
+
If loading a local checkpoint, replace the model name with the local path.
|
| 123 |
+
|
| 124 |
+
For ONNX support, please refer to [docs/onnx.md](docs/onnx.md).
|
| 125 |
+
|
| 126 |
+
## 💡 Minimal Code Example
|
| 127 |
+
|
| 128 |
+
Here is a minimal example for loading the model and inferring on a single image.
|
| 129 |
+
|
| 130 |
+
```python
|
| 131 |
+
import cv2
|
| 132 |
+
import torch
|
| 133 |
+
# from moge.model.v1 import MoGeModel
|
| 134 |
+
from moge.model.v2 import MoGeModel # Let's try MoGe-2
|
| 135 |
+
|
| 136 |
+
device = torch.device("cuda")
|
| 137 |
+
|
| 138 |
+
# Load the model from huggingface hub (or load from local).
|
| 139 |
+
model = MoGeModel.from_pretrained("Ruicheng/moge-2-vitl-normal").to(device)
|
| 140 |
+
|
| 141 |
+
# Read the input image and convert to tensor (3, H, W) with RGB values normalized to [0, 1]
|
| 142 |
+
input_image = cv2.cvtColor(cv2.imread("PATH_TO_IMAGE.jpg"), cv2.COLOR_BGR2RGB)
|
| 143 |
+
input_image = torch.tensor(input_image / 255, dtype=torch.float32, device=device).permute(2, 0, 1)
|
| 144 |
+
|
| 145 |
+
# Infer
|
| 146 |
+
output = model.infer(input_image)
|
| 147 |
+
"""
|
| 148 |
+
`output` has keys "points", "depth", "mask", "normal" (optional) and "intrinsics",
|
| 149 |
+
The maps are in the same size as the input image.
|
| 150 |
+
{
|
| 151 |
+
"points": (H, W, 3), # point map in OpenCV camera coordinate system (x right, y down, z forward). For MoGe-2, the point map is in metric scale.
|
| 152 |
+
"depth": (H, W), # depth map
|
| 153 |
+
"normal": (H, W, 3) # normal map in OpenCV camera coordinate system. (available for MoGe-2-normal)
|
| 154 |
+
"mask": (H, W), # a binary mask for valid pixels.
|
| 155 |
+
"intrinsics": (3, 3), # normalized camera intrinsics
|
| 156 |
+
}
|
| 157 |
+
"""
|
| 158 |
+
```
|
| 159 |
+
For more usage details, see the `MoGeModel.infer()` docstring.
|
| 160 |
+
|
| 161 |
+
## 💡 Usage
|
| 162 |
+
|
| 163 |
+
### Gradio demo | `moge app`
|
| 164 |
+
|
| 165 |
+
> The demo for MoGe-1 is also available at our [Hugging Face Space](https://huggingface.co/spaces/Ruicheng/MoGe).
|
| 166 |
+
|
| 167 |
+
```bash
|
| 168 |
+
# Using the command line tool
|
| 169 |
+
moge app # will run MoGe-2 demo by default.
|
| 170 |
+
|
| 171 |
+
# In this repo
|
| 172 |
+
python moge/scripts/app.py # --share for Gradio public sharing
|
| 173 |
+
```
|
| 174 |
+
|
| 175 |
+
See also [`moge/scripts/app.py`](moge/scripts/app.py)
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
### Inference | `moge infer`
|
| 179 |
+
|
| 180 |
+
Run the script `moge/scripts/infer.py` via the following command:
|
| 181 |
+
|
| 182 |
+
```bash
|
| 183 |
+
# Save the output [maps], [glb] and [ply] files
|
| 184 |
+
moge infer -i IMAGES_FOLDER_OR_IMAGE_PATH --o OUTPUT_FOLDER --maps --glb --ply
|
| 185 |
+
|
| 186 |
+
# Show the result in a window (requires pyglet < 2.0, e.g. pip install pyglet==1.5.29)
|
| 187 |
+
moge infer -i IMAGES_FOLDER_OR_IMAGE_PATH --o OUTPUT_FOLDER --show
|
| 188 |
+
```
|
| 189 |
+
|
| 190 |
+
For detailed options, run `moge infer --help`:
|
| 191 |
+
|
| 192 |
+
```
|
| 193 |
+
Usage: moge infer [OPTIONS]
|
| 194 |
+
|
| 195 |
+
Inference script
|
| 196 |
+
|
| 197 |
+
Options:
|
| 198 |
+
-i, --input PATH Input image or folder path. "jpg" and "png" are
|
| 199 |
+
supported.
|
| 200 |
+
--fov_x FLOAT If camera parameters are known, set the
|
| 201 |
+
horizontal field of view in degrees. Otherwise,
|
| 202 |
+
MoGe will estimate it.
|
| 203 |
+
-o, --output PATH Output folder path
|
| 204 |
+
--pretrained TEXT Pretrained model name or path. If not provided,
|
| 205 |
+
the corresponding default model will be chosen.
|
| 206 |
+
--version [v1|v2] Model version. Defaults to "v2"
|
| 207 |
+
--device TEXT Device name (e.g. "cuda", "cuda:0", "cpu").
|
| 208 |
+
Defaults to "cuda"
|
| 209 |
+
--fp16 Use fp16 precision for much faster inference.
|
| 210 |
+
--resize INTEGER Resize the image(s) & output maps to a specific
|
| 211 |
+
size. Defaults to None (no resizing).
|
| 212 |
+
--resolution_level INTEGER An integer [0-9] for the resolution level for
|
| 213 |
+
inference. Higher value means more tokens and
|
| 214 |
+
the finer details will be captured, but
|
| 215 |
+
inference can be slower. Defaults to 9. Note
|
| 216 |
+
that it is irrelevant to the output size, which
|
| 217 |
+
is always the same as the input size.
|
| 218 |
+
`resolution_level` actually controls
|
| 219 |
+
`num_tokens`. See `num_tokens` for more details.
|
| 220 |
+
--num_tokens INTEGER number of tokens used for inference. A integer
|
| 221 |
+
in the (suggested) range of `[1200, 2500]`.
|
| 222 |
+
`resolution_level` will be ignored if
|
| 223 |
+
`num_tokens` is provided. Default: None
|
| 224 |
+
--threshold FLOAT Threshold for removing edges. Defaults to 0.01.
|
| 225 |
+
Smaller value removes more edges. "inf" means no
|
| 226 |
+
thresholding.
|
| 227 |
+
--maps Whether to save the output maps (image, point
|
| 228 |
+
map, depth map, normal map, mask) and fov.
|
| 229 |
+
--glb Whether to save the output as a.glb file. The
|
| 230 |
+
color will be saved as a texture.
|
| 231 |
+
--ply Whether to save the output as a.ply file. The
|
| 232 |
+
color will be saved as vertex colors.
|
| 233 |
+
--show Whether show the output in a window. Note that
|
| 234 |
+
this requires pyglet<2 installed as required by
|
| 235 |
+
trimesh.
|
| 236 |
+
--help Show this message and exit.
|
| 237 |
+
```
|
| 238 |
+
|
| 239 |
+
See also [`moge/scripts/infer.py`](moge/scripts/infer.py)
|
| 240 |
+
|
| 241 |
+
### 360° panorama images | `moge infer_panorama`
|
| 242 |
+
|
| 243 |
+
> *NOTE: This is an experimental extension of MoGe.*
|
| 244 |
+
|
| 245 |
+
The script will split the 360-degree panorama image into multiple perspective views and infer on each view separately.
|
| 246 |
+
The output maps will be combined to produce a panorama depth map and point map.
|
| 247 |
+
|
| 248 |
+
Note that the panorama image must have spherical parameterization (e.g., environment maps or equirectangular images). Other formats must be converted to spherical format before using this script. Run `moge infer_panorama --help` for detailed options.
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
<div align="center">
|
| 252 |
+
<img src="./assets/panorama_pipeline.png" width="80%">
|
| 253 |
+
|
| 254 |
+
The photo is from [this URL](https://commons.wikimedia.org/wiki/Category:360%C2%B0_panoramas_with_equirectangular_projection#/media/File:Braunschweig_Sankt-%C3%84gidien_Panorama_02.jpg)
|
| 255 |
+
</div>
|
| 256 |
+
|
| 257 |
+
See also [`moge/scripts/infer_panorama.py`](moge/scripts/infer_panorama.py)
|
| 258 |
+
|
| 259 |
+
## 🏋️♂️ Training & Finetuning
|
| 260 |
+
|
| 261 |
+
See [docs/train.md](docs/train.md)
|
| 262 |
+
|
| 263 |
+
## 🧪 Evaluation
|
| 264 |
+
|
| 265 |
+
See [docs/eval.md](docs/eval.md)
|
| 266 |
+
|
| 267 |
+
## ⚖️ License
|
| 268 |
+
|
| 269 |
+
MoGe code is released under the MIT license, except for DINOv2 code in `moge/model/dinov2` which is released by Meta AI under the Apache 2.0 license.
|
| 270 |
+
See [LICENSE](LICENSE) for more details.
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
## 📜 Citation
|
| 274 |
+
|
| 275 |
+
If you find our work useful in your research, we gratefully request that you consider citing our paper:
|
| 276 |
+
|
| 277 |
+
```
|
| 278 |
+
@inproceedings{wang2025moge,
|
| 279 |
+
title={Moge: Unlocking accurate monocular geometry estimation for open-domain images with optimal training supervision},
|
| 280 |
+
author={Wang, Ruicheng and Xu, Sicheng and Dai, Cassie and Xiang, Jianfeng and Deng, Yu and Tong, Xin and Yang, Jiaolong},
|
| 281 |
+
booktitle={Proceedings of the Computer Vision and Pattern Recognition Conference},
|
| 282 |
+
pages={5261--5271},
|
| 283 |
+
year={2025}
|
| 284 |
+
}
|
| 285 |
+
|
| 286 |
+
@misc{wang2025moge2,
|
| 287 |
+
title={MoGe-2: Accurate Monocular Geometry with Metric Scale and Sharp Details},
|
| 288 |
+
author={Ruicheng Wang and Sicheng Xu and Yue Dong and Yu Deng and Jianfeng Xiang and Zelong Lv and Guangzhong Sun and Xin Tong and Jiaolong Yang},
|
| 289 |
+
year={2025},
|
| 290 |
+
eprint={2507.02546},
|
| 291 |
+
archivePrefix={arXiv},
|
| 292 |
+
primaryClass={cs.CV},
|
| 293 |
+
url={https://arxiv.org/abs/2507.02546},
|
| 294 |
+
}
|
| 295 |
+
```
|
third_party/MoGe/moge/__init__.py
ADDED
|
File without changes
|
third_party/MoGe/moge/model/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import importlib
|
| 2 |
+
from typing import *
|
| 3 |
+
|
| 4 |
+
if TYPE_CHECKING:
|
| 5 |
+
from .v1 import MoGeModel as MoGeModelV1
|
| 6 |
+
from .v2 import MoGeModel as MoGeModelV2
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def import_model_class_by_version(version: str) -> Type[Union['MoGeModelV1', 'MoGeModelV2']]:
|
| 10 |
+
assert version in ['v1', 'v2'], f'Unsupported model version: {version}'
|
| 11 |
+
|
| 12 |
+
try:
|
| 13 |
+
module = importlib.import_module(f'.{version}', __package__)
|
| 14 |
+
except ModuleNotFoundError:
|
| 15 |
+
raise ValueError(f'Model version "{version}" not found.')
|
| 16 |
+
|
| 17 |
+
cls = getattr(module, 'MoGeModel')
|
| 18 |
+
return cls
|
third_party/MoGe/moge/model/dinov2/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
__version__ = "0.0.1"
|
third_party/MoGe/moge/model/dinov2/hub/__init__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
third_party/MoGe/moge/model/dinov2/hub/backbones.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
from enum import Enum
|
| 7 |
+
from typing import Union
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
|
| 11 |
+
from .utils import _DINOV2_BASE_URL, _make_dinov2_model_name
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class Weights(Enum):
|
| 15 |
+
LVD142M = "LVD142M"
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _make_dinov2_model(
|
| 19 |
+
*,
|
| 20 |
+
arch_name: str = "vit_large",
|
| 21 |
+
img_size: int = 518,
|
| 22 |
+
patch_size: int = 14,
|
| 23 |
+
init_values: float = 1.0,
|
| 24 |
+
ffn_layer: str = "mlp",
|
| 25 |
+
block_chunks: int = 0,
|
| 26 |
+
num_register_tokens: int = 0,
|
| 27 |
+
interpolate_antialias: bool = False,
|
| 28 |
+
interpolate_offset: float = 0.1,
|
| 29 |
+
pretrained: bool = True,
|
| 30 |
+
weights: Union[Weights, str] = Weights.LVD142M,
|
| 31 |
+
**kwargs,
|
| 32 |
+
):
|
| 33 |
+
from ..models import vision_transformer as vits
|
| 34 |
+
|
| 35 |
+
if isinstance(weights, str):
|
| 36 |
+
try:
|
| 37 |
+
weights = Weights[weights]
|
| 38 |
+
except KeyError:
|
| 39 |
+
raise AssertionError(f"Unsupported weights: {weights}")
|
| 40 |
+
|
| 41 |
+
model_base_name = _make_dinov2_model_name(arch_name, patch_size)
|
| 42 |
+
vit_kwargs = dict(
|
| 43 |
+
img_size=img_size,
|
| 44 |
+
patch_size=patch_size,
|
| 45 |
+
init_values=init_values,
|
| 46 |
+
ffn_layer=ffn_layer,
|
| 47 |
+
block_chunks=block_chunks,
|
| 48 |
+
num_register_tokens=num_register_tokens,
|
| 49 |
+
interpolate_antialias=interpolate_antialias,
|
| 50 |
+
interpolate_offset=interpolate_offset,
|
| 51 |
+
)
|
| 52 |
+
vit_kwargs.update(**kwargs)
|
| 53 |
+
model = vits.__dict__[arch_name](**vit_kwargs)
|
| 54 |
+
|
| 55 |
+
if pretrained:
|
| 56 |
+
model_full_name = _make_dinov2_model_name(arch_name, patch_size, num_register_tokens)
|
| 57 |
+
url = _DINOV2_BASE_URL + f"/{model_base_name}/{model_full_name}_pretrain.pth"
|
| 58 |
+
state_dict = torch.hub.load_state_dict_from_url(url, map_location="cpu")
|
| 59 |
+
model.load_state_dict(state_dict, strict=True)
|
| 60 |
+
|
| 61 |
+
return model
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def dinov2_vits14(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs):
|
| 65 |
+
"""
|
| 66 |
+
DINOv2 ViT-S/14 model (optionally) pretrained on the LVD-142M dataset.
|
| 67 |
+
"""
|
| 68 |
+
return _make_dinov2_model(arch_name="vit_small", pretrained=pretrained, weights=weights, **kwargs)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def dinov2_vitb14(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs):
|
| 72 |
+
"""
|
| 73 |
+
DINOv2 ViT-B/14 model (optionally) pretrained on the LVD-142M dataset.
|
| 74 |
+
"""
|
| 75 |
+
return _make_dinov2_model(arch_name="vit_base", pretrained=pretrained, weights=weights, **kwargs)
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def dinov2_vitl14(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs):
|
| 79 |
+
"""
|
| 80 |
+
DINOv2 ViT-L/14 model (optionally) pretrained on the LVD-142M dataset.
|
| 81 |
+
"""
|
| 82 |
+
return _make_dinov2_model(arch_name="vit_large", pretrained=pretrained, weights=weights, **kwargs)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def dinov2_vitg14(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs):
|
| 86 |
+
"""
|
| 87 |
+
DINOv2 ViT-g/14 model (optionally) pretrained on the LVD-142M dataset.
|
| 88 |
+
"""
|
| 89 |
+
return _make_dinov2_model(
|
| 90 |
+
arch_name="vit_giant2",
|
| 91 |
+
ffn_layer="swiglufused",
|
| 92 |
+
weights=weights,
|
| 93 |
+
pretrained=pretrained,
|
| 94 |
+
**kwargs,
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def dinov2_vits14_reg(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs):
|
| 99 |
+
"""
|
| 100 |
+
DINOv2 ViT-S/14 model with registers (optionally) pretrained on the LVD-142M dataset.
|
| 101 |
+
"""
|
| 102 |
+
return _make_dinov2_model(
|
| 103 |
+
arch_name="vit_small",
|
| 104 |
+
pretrained=pretrained,
|
| 105 |
+
weights=weights,
|
| 106 |
+
num_register_tokens=4,
|
| 107 |
+
interpolate_antialias=True,
|
| 108 |
+
interpolate_offset=0.0,
|
| 109 |
+
**kwargs,
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def dinov2_vitb14_reg(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs):
|
| 114 |
+
"""
|
| 115 |
+
DINOv2 ViT-B/14 model with registers (optionally) pretrained on the LVD-142M dataset.
|
| 116 |
+
"""
|
| 117 |
+
return _make_dinov2_model(
|
| 118 |
+
arch_name="vit_base",
|
| 119 |
+
pretrained=pretrained,
|
| 120 |
+
weights=weights,
|
| 121 |
+
num_register_tokens=4,
|
| 122 |
+
interpolate_antialias=True,
|
| 123 |
+
interpolate_offset=0.0,
|
| 124 |
+
**kwargs,
|
| 125 |
+
)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def dinov2_vitl14_reg(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs):
|
| 129 |
+
"""
|
| 130 |
+
DINOv2 ViT-L/14 model with registers (optionally) pretrained on the LVD-142M dataset.
|
| 131 |
+
"""
|
| 132 |
+
return _make_dinov2_model(
|
| 133 |
+
arch_name="vit_large",
|
| 134 |
+
pretrained=pretrained,
|
| 135 |
+
weights=weights,
|
| 136 |
+
num_register_tokens=4,
|
| 137 |
+
interpolate_antialias=True,
|
| 138 |
+
interpolate_offset=0.0,
|
| 139 |
+
**kwargs,
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def dinov2_vitg14_reg(*, pretrained: bool = True, weights: Union[Weights, str] = Weights.LVD142M, **kwargs):
|
| 144 |
+
"""
|
| 145 |
+
DINOv2 ViT-g/14 model with registers (optionally) pretrained on the LVD-142M dataset.
|
| 146 |
+
"""
|
| 147 |
+
return _make_dinov2_model(
|
| 148 |
+
arch_name="vit_giant2",
|
| 149 |
+
ffn_layer="swiglufused",
|
| 150 |
+
weights=weights,
|
| 151 |
+
pretrained=pretrained,
|
| 152 |
+
num_register_tokens=4,
|
| 153 |
+
interpolate_antialias=True,
|
| 154 |
+
interpolate_offset=0.0,
|
| 155 |
+
**kwargs,
|
| 156 |
+
)
|
third_party/MoGe/moge/model/dinov2/hub/utils.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import itertools
|
| 7 |
+
import math
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
import torch.nn as nn
|
| 11 |
+
import torch.nn.functional as F
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
_DINOV2_BASE_URL = "https://dl.fbaipublicfiles.com/dinov2"
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _make_dinov2_model_name(arch_name: str, patch_size: int, num_register_tokens: int = 0) -> str:
|
| 18 |
+
compact_arch_name = arch_name.replace("_", "")[:4]
|
| 19 |
+
registers_suffix = f"_reg{num_register_tokens}" if num_register_tokens else ""
|
| 20 |
+
return f"dinov2_{compact_arch_name}{patch_size}{registers_suffix}"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class CenterPadding(nn.Module):
|
| 24 |
+
def __init__(self, multiple):
|
| 25 |
+
super().__init__()
|
| 26 |
+
self.multiple = multiple
|
| 27 |
+
|
| 28 |
+
def _get_pad(self, size):
|
| 29 |
+
new_size = math.ceil(size / self.multiple) * self.multiple
|
| 30 |
+
pad_size = new_size - size
|
| 31 |
+
pad_size_left = pad_size // 2
|
| 32 |
+
pad_size_right = pad_size - pad_size_left
|
| 33 |
+
return pad_size_left, pad_size_right
|
| 34 |
+
|
| 35 |
+
@torch.inference_mode()
|
| 36 |
+
def forward(self, x):
|
| 37 |
+
pads = list(itertools.chain.from_iterable(self._get_pad(m) for m in x.shape[:1:-1]))
|
| 38 |
+
output = F.pad(x, pads)
|
| 39 |
+
return output
|
third_party/MoGe/moge/model/dinov2/layers/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
from .dino_head import DINOHead
|
| 7 |
+
from .mlp import Mlp
|
| 8 |
+
from .patch_embed import PatchEmbed
|
| 9 |
+
from .swiglu_ffn import SwiGLUFFN, SwiGLUFFNFused
|
| 10 |
+
from .block import NestedTensorBlock
|
| 11 |
+
from .attention import MemEffAttention
|
third_party/MoGe/moge/model/dinov2/layers/attention.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
# References:
|
| 7 |
+
# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
|
| 8 |
+
# https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py
|
| 9 |
+
|
| 10 |
+
import logging
|
| 11 |
+
import os
|
| 12 |
+
import warnings
|
| 13 |
+
|
| 14 |
+
import torch.nn.functional as F
|
| 15 |
+
from torch import Tensor
|
| 16 |
+
from torch import nn
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
logger = logging.getLogger("dinov2")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
XFORMERS_ENABLED = os.environ.get("XFORMERS_DISABLED") is None
|
| 23 |
+
try:
|
| 24 |
+
if XFORMERS_ENABLED:
|
| 25 |
+
from xformers.ops import memory_efficient_attention, unbind
|
| 26 |
+
|
| 27 |
+
XFORMERS_AVAILABLE = True
|
| 28 |
+
# warnings.warn("xFormers is available (Attention)")
|
| 29 |
+
else:
|
| 30 |
+
# warnings.warn("xFormers is disabled (Attention)")
|
| 31 |
+
raise ImportError
|
| 32 |
+
except ImportError:
|
| 33 |
+
XFORMERS_AVAILABLE = False
|
| 34 |
+
# warnings.warn("xFormers is not available (Attention)")
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class Attention(nn.Module):
|
| 38 |
+
def __init__(
|
| 39 |
+
self,
|
| 40 |
+
dim: int,
|
| 41 |
+
num_heads: int = 8,
|
| 42 |
+
qkv_bias: bool = False,
|
| 43 |
+
proj_bias: bool = True,
|
| 44 |
+
attn_drop: float = 0.0,
|
| 45 |
+
proj_drop: float = 0.0,
|
| 46 |
+
) -> None:
|
| 47 |
+
super().__init__()
|
| 48 |
+
self.num_heads = num_heads
|
| 49 |
+
head_dim = dim // num_heads
|
| 50 |
+
self.scale = head_dim**-0.5
|
| 51 |
+
|
| 52 |
+
self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
|
| 53 |
+
self.attn_drop = nn.Dropout(attn_drop)
|
| 54 |
+
self.proj = nn.Linear(dim, dim, bias=proj_bias)
|
| 55 |
+
self.proj_drop = nn.Dropout(proj_drop)
|
| 56 |
+
|
| 57 |
+
# # Deprecated implementation, extremely slow
|
| 58 |
+
# def forward(self, x: Tensor, attn_bias=None) -> Tensor:
|
| 59 |
+
# B, N, C = x.shape
|
| 60 |
+
# qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
|
| 61 |
+
# q, k, v = qkv[0] * self.scale, qkv[1], qkv[2]
|
| 62 |
+
# attn = q @ k.transpose(-2, -1)
|
| 63 |
+
# attn = attn.softmax(dim=-1)
|
| 64 |
+
# attn = self.attn_drop(attn)
|
| 65 |
+
# x = (attn @ v).transpose(1, 2).reshape(B, N, C)
|
| 66 |
+
# x = self.proj(x)
|
| 67 |
+
# x = self.proj_drop(x)
|
| 68 |
+
# return x
|
| 69 |
+
|
| 70 |
+
def forward(self, x: Tensor, attn_bias=None) -> Tensor:
|
| 71 |
+
B, N, C = x.shape
|
| 72 |
+
qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) # (3, B, H, N, C // H)
|
| 73 |
+
|
| 74 |
+
q, k, v = qkv.unbind(0) # (B, H, N, C // H)
|
| 75 |
+
|
| 76 |
+
x = F.scaled_dot_product_attention(q, k, v, attn_bias)
|
| 77 |
+
x = x.permute(0, 2, 1, 3).reshape(B, N, C)
|
| 78 |
+
|
| 79 |
+
x = self.proj(x)
|
| 80 |
+
x = self.proj_drop(x)
|
| 81 |
+
return x
|
| 82 |
+
|
| 83 |
+
class MemEffAttention(Attention):
|
| 84 |
+
def forward(self, x: Tensor, attn_bias=None) -> Tensor:
|
| 85 |
+
if not XFORMERS_AVAILABLE:
|
| 86 |
+
if attn_bias is not None:
|
| 87 |
+
raise AssertionError("xFormers is required for using nested tensors")
|
| 88 |
+
return super().forward(x)
|
| 89 |
+
|
| 90 |
+
B, N, C = x.shape
|
| 91 |
+
qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads)
|
| 92 |
+
|
| 93 |
+
q, k, v = unbind(qkv, 2)
|
| 94 |
+
|
| 95 |
+
x = memory_efficient_attention(q, k, v, attn_bias=attn_bias)
|
| 96 |
+
x = x.reshape([B, N, C])
|
| 97 |
+
|
| 98 |
+
x = self.proj(x)
|
| 99 |
+
x = self.proj_drop(x)
|
| 100 |
+
return x
|
third_party/MoGe/moge/model/dinov2/layers/block.py
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
# References:
|
| 7 |
+
# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
|
| 8 |
+
# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py
|
| 9 |
+
|
| 10 |
+
import logging
|
| 11 |
+
import os
|
| 12 |
+
from typing import Callable, List, Any, Tuple, Dict
|
| 13 |
+
import warnings
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
from torch import nn, Tensor
|
| 17 |
+
|
| 18 |
+
from .attention import Attention, MemEffAttention
|
| 19 |
+
from .drop_path import DropPath
|
| 20 |
+
from .layer_scale import LayerScale
|
| 21 |
+
from .mlp import Mlp
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
logger = logging.getLogger("dinov2")
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
XFORMERS_ENABLED = os.environ.get("XFORMERS_DISABLED") is None
|
| 28 |
+
try:
|
| 29 |
+
if XFORMERS_ENABLED:
|
| 30 |
+
from xformers.ops import fmha, scaled_index_add, index_select_cat
|
| 31 |
+
|
| 32 |
+
XFORMERS_AVAILABLE = True
|
| 33 |
+
# warnings.warn("xFormers is available (Block)")
|
| 34 |
+
else:
|
| 35 |
+
# warnings.warn("xFormers is disabled (Block)")
|
| 36 |
+
raise ImportError
|
| 37 |
+
except ImportError:
|
| 38 |
+
XFORMERS_AVAILABLE = False
|
| 39 |
+
# warnings.warn("xFormers is not available (Block)")
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class Block(nn.Module):
|
| 43 |
+
def __init__(
|
| 44 |
+
self,
|
| 45 |
+
dim: int,
|
| 46 |
+
num_heads: int,
|
| 47 |
+
mlp_ratio: float = 4.0,
|
| 48 |
+
qkv_bias: bool = False,
|
| 49 |
+
proj_bias: bool = True,
|
| 50 |
+
ffn_bias: bool = True,
|
| 51 |
+
drop: float = 0.0,
|
| 52 |
+
attn_drop: float = 0.0,
|
| 53 |
+
init_values=None,
|
| 54 |
+
drop_path: float = 0.0,
|
| 55 |
+
act_layer: Callable[..., nn.Module] = nn.GELU,
|
| 56 |
+
norm_layer: Callable[..., nn.Module] = nn.LayerNorm,
|
| 57 |
+
attn_class: Callable[..., nn.Module] = Attention,
|
| 58 |
+
ffn_layer: Callable[..., nn.Module] = Mlp,
|
| 59 |
+
) -> None:
|
| 60 |
+
super().__init__()
|
| 61 |
+
# print(f"biases: qkv: {qkv_bias}, proj: {proj_bias}, ffn: {ffn_bias}")
|
| 62 |
+
self.norm1 = norm_layer(dim)
|
| 63 |
+
self.attn = attn_class(
|
| 64 |
+
dim,
|
| 65 |
+
num_heads=num_heads,
|
| 66 |
+
qkv_bias=qkv_bias,
|
| 67 |
+
proj_bias=proj_bias,
|
| 68 |
+
attn_drop=attn_drop,
|
| 69 |
+
proj_drop=drop,
|
| 70 |
+
)
|
| 71 |
+
self.ls1 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
|
| 72 |
+
self.drop_path1 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
|
| 73 |
+
|
| 74 |
+
self.norm2 = norm_layer(dim)
|
| 75 |
+
mlp_hidden_dim = int(dim * mlp_ratio)
|
| 76 |
+
self.mlp = ffn_layer(
|
| 77 |
+
in_features=dim,
|
| 78 |
+
hidden_features=mlp_hidden_dim,
|
| 79 |
+
act_layer=act_layer,
|
| 80 |
+
drop=drop,
|
| 81 |
+
bias=ffn_bias,
|
| 82 |
+
)
|
| 83 |
+
self.ls2 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
|
| 84 |
+
self.drop_path2 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
|
| 85 |
+
|
| 86 |
+
self.sample_drop_ratio = drop_path
|
| 87 |
+
|
| 88 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 89 |
+
def attn_residual_func(x: Tensor) -> Tensor:
|
| 90 |
+
return self.ls1(self.attn(self.norm1(x)))
|
| 91 |
+
|
| 92 |
+
def ffn_residual_func(x: Tensor) -> Tensor:
|
| 93 |
+
return self.ls2(self.mlp(self.norm2(x)))
|
| 94 |
+
|
| 95 |
+
if self.training and self.sample_drop_ratio > 0.1:
|
| 96 |
+
# the overhead is compensated only for a drop path rate larger than 0.1
|
| 97 |
+
x = drop_add_residual_stochastic_depth(
|
| 98 |
+
x,
|
| 99 |
+
residual_func=attn_residual_func,
|
| 100 |
+
sample_drop_ratio=self.sample_drop_ratio,
|
| 101 |
+
)
|
| 102 |
+
x = drop_add_residual_stochastic_depth(
|
| 103 |
+
x,
|
| 104 |
+
residual_func=ffn_residual_func,
|
| 105 |
+
sample_drop_ratio=self.sample_drop_ratio,
|
| 106 |
+
)
|
| 107 |
+
elif self.training and self.sample_drop_ratio > 0.0:
|
| 108 |
+
x = x + self.drop_path1(attn_residual_func(x))
|
| 109 |
+
x = x + self.drop_path1(ffn_residual_func(x)) # FIXME: drop_path2
|
| 110 |
+
else:
|
| 111 |
+
x = x + attn_residual_func(x)
|
| 112 |
+
x = x + ffn_residual_func(x)
|
| 113 |
+
return x
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def drop_add_residual_stochastic_depth(
|
| 117 |
+
x: Tensor,
|
| 118 |
+
residual_func: Callable[[Tensor], Tensor],
|
| 119 |
+
sample_drop_ratio: float = 0.0,
|
| 120 |
+
) -> Tensor:
|
| 121 |
+
# 1) extract subset using permutation
|
| 122 |
+
b, n, d = x.shape
|
| 123 |
+
sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
|
| 124 |
+
brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
|
| 125 |
+
x_subset = x[brange]
|
| 126 |
+
|
| 127 |
+
# 2) apply residual_func to get residual
|
| 128 |
+
residual = residual_func(x_subset)
|
| 129 |
+
|
| 130 |
+
x_flat = x.flatten(1)
|
| 131 |
+
residual = residual.flatten(1)
|
| 132 |
+
|
| 133 |
+
residual_scale_factor = b / sample_subset_size
|
| 134 |
+
|
| 135 |
+
# 3) add the residual
|
| 136 |
+
x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor)
|
| 137 |
+
return x_plus_residual.view_as(x)
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def get_branges_scales(x, sample_drop_ratio=0.0):
|
| 141 |
+
b, n, d = x.shape
|
| 142 |
+
sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
|
| 143 |
+
brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
|
| 144 |
+
residual_scale_factor = b / sample_subset_size
|
| 145 |
+
return brange, residual_scale_factor
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def add_residual(x, brange, residual, residual_scale_factor, scaling_vector=None):
|
| 149 |
+
if scaling_vector is None:
|
| 150 |
+
x_flat = x.flatten(1)
|
| 151 |
+
residual = residual.flatten(1)
|
| 152 |
+
x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor)
|
| 153 |
+
else:
|
| 154 |
+
x_plus_residual = scaled_index_add(
|
| 155 |
+
x, brange, residual.to(dtype=x.dtype), scaling=scaling_vector, alpha=residual_scale_factor
|
| 156 |
+
)
|
| 157 |
+
return x_plus_residual
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
attn_bias_cache: Dict[Tuple, Any] = {}
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def get_attn_bias_and_cat(x_list, branges=None):
|
| 164 |
+
"""
|
| 165 |
+
this will perform the index select, cat the tensors, and provide the attn_bias from cache
|
| 166 |
+
"""
|
| 167 |
+
batch_sizes = [b.shape[0] for b in branges] if branges is not None else [x.shape[0] for x in x_list]
|
| 168 |
+
all_shapes = tuple((b, x.shape[1]) for b, x in zip(batch_sizes, x_list))
|
| 169 |
+
if all_shapes not in attn_bias_cache.keys():
|
| 170 |
+
seqlens = []
|
| 171 |
+
for b, x in zip(batch_sizes, x_list):
|
| 172 |
+
for _ in range(b):
|
| 173 |
+
seqlens.append(x.shape[1])
|
| 174 |
+
attn_bias = fmha.BlockDiagonalMask.from_seqlens(seqlens)
|
| 175 |
+
attn_bias._batch_sizes = batch_sizes
|
| 176 |
+
attn_bias_cache[all_shapes] = attn_bias
|
| 177 |
+
|
| 178 |
+
if branges is not None:
|
| 179 |
+
cat_tensors = index_select_cat([x.flatten(1) for x in x_list], branges).view(1, -1, x_list[0].shape[-1])
|
| 180 |
+
else:
|
| 181 |
+
tensors_bs1 = tuple(x.reshape([1, -1, *x.shape[2:]]) for x in x_list)
|
| 182 |
+
cat_tensors = torch.cat(tensors_bs1, dim=1)
|
| 183 |
+
|
| 184 |
+
return attn_bias_cache[all_shapes], cat_tensors
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def drop_add_residual_stochastic_depth_list(
|
| 188 |
+
x_list: List[Tensor],
|
| 189 |
+
residual_func: Callable[[Tensor, Any], Tensor],
|
| 190 |
+
sample_drop_ratio: float = 0.0,
|
| 191 |
+
scaling_vector=None,
|
| 192 |
+
) -> Tensor:
|
| 193 |
+
# 1) generate random set of indices for dropping samples in the batch
|
| 194 |
+
branges_scales = [get_branges_scales(x, sample_drop_ratio=sample_drop_ratio) for x in x_list]
|
| 195 |
+
branges = [s[0] for s in branges_scales]
|
| 196 |
+
residual_scale_factors = [s[1] for s in branges_scales]
|
| 197 |
+
|
| 198 |
+
# 2) get attention bias and index+concat the tensors
|
| 199 |
+
attn_bias, x_cat = get_attn_bias_and_cat(x_list, branges)
|
| 200 |
+
|
| 201 |
+
# 3) apply residual_func to get residual, and split the result
|
| 202 |
+
residual_list = attn_bias.split(residual_func(x_cat, attn_bias=attn_bias)) # type: ignore
|
| 203 |
+
|
| 204 |
+
outputs = []
|
| 205 |
+
for x, brange, residual, residual_scale_factor in zip(x_list, branges, residual_list, residual_scale_factors):
|
| 206 |
+
outputs.append(add_residual(x, brange, residual, residual_scale_factor, scaling_vector).view_as(x))
|
| 207 |
+
return outputs
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
class NestedTensorBlock(Block):
|
| 211 |
+
def forward_nested(self, x_list: List[Tensor]) -> List[Tensor]:
|
| 212 |
+
"""
|
| 213 |
+
x_list contains a list of tensors to nest together and run
|
| 214 |
+
"""
|
| 215 |
+
assert isinstance(self.attn, MemEffAttention)
|
| 216 |
+
|
| 217 |
+
if self.training and self.sample_drop_ratio > 0.0:
|
| 218 |
+
|
| 219 |
+
def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
|
| 220 |
+
return self.attn(self.norm1(x), attn_bias=attn_bias)
|
| 221 |
+
|
| 222 |
+
def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
|
| 223 |
+
return self.mlp(self.norm2(x))
|
| 224 |
+
|
| 225 |
+
x_list = drop_add_residual_stochastic_depth_list(
|
| 226 |
+
x_list,
|
| 227 |
+
residual_func=attn_residual_func,
|
| 228 |
+
sample_drop_ratio=self.sample_drop_ratio,
|
| 229 |
+
scaling_vector=self.ls1.gamma if isinstance(self.ls1, LayerScale) else None,
|
| 230 |
+
)
|
| 231 |
+
x_list = drop_add_residual_stochastic_depth_list(
|
| 232 |
+
x_list,
|
| 233 |
+
residual_func=ffn_residual_func,
|
| 234 |
+
sample_drop_ratio=self.sample_drop_ratio,
|
| 235 |
+
scaling_vector=self.ls2.gamma if isinstance(self.ls1, LayerScale) else None,
|
| 236 |
+
)
|
| 237 |
+
return x_list
|
| 238 |
+
else:
|
| 239 |
+
|
| 240 |
+
def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
|
| 241 |
+
return self.ls1(self.attn(self.norm1(x), attn_bias=attn_bias))
|
| 242 |
+
|
| 243 |
+
def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
|
| 244 |
+
return self.ls2(self.mlp(self.norm2(x)))
|
| 245 |
+
|
| 246 |
+
attn_bias, x = get_attn_bias_and_cat(x_list)
|
| 247 |
+
x = x + attn_residual_func(x, attn_bias=attn_bias)
|
| 248 |
+
x = x + ffn_residual_func(x)
|
| 249 |
+
return attn_bias.split(x)
|
| 250 |
+
|
| 251 |
+
def forward(self, x_or_x_list):
|
| 252 |
+
if isinstance(x_or_x_list, Tensor):
|
| 253 |
+
return super().forward(x_or_x_list)
|
| 254 |
+
elif isinstance(x_or_x_list, list):
|
| 255 |
+
if not XFORMERS_AVAILABLE:
|
| 256 |
+
raise AssertionError("xFormers is required for using nested tensors")
|
| 257 |
+
return self.forward_nested(x_or_x_list)
|
| 258 |
+
else:
|
| 259 |
+
raise AssertionError
|
third_party/MoGe/moge/model/dinov2/layers/dino_head.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
import torch.nn as nn
|
| 8 |
+
from torch.nn.init import trunc_normal_
|
| 9 |
+
from torch.nn.utils import weight_norm
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class DINOHead(nn.Module):
|
| 13 |
+
def __init__(
|
| 14 |
+
self,
|
| 15 |
+
in_dim,
|
| 16 |
+
out_dim,
|
| 17 |
+
use_bn=False,
|
| 18 |
+
nlayers=3,
|
| 19 |
+
hidden_dim=2048,
|
| 20 |
+
bottleneck_dim=256,
|
| 21 |
+
mlp_bias=True,
|
| 22 |
+
):
|
| 23 |
+
super().__init__()
|
| 24 |
+
nlayers = max(nlayers, 1)
|
| 25 |
+
self.mlp = _build_mlp(nlayers, in_dim, bottleneck_dim, hidden_dim=hidden_dim, use_bn=use_bn, bias=mlp_bias)
|
| 26 |
+
self.apply(self._init_weights)
|
| 27 |
+
self.last_layer = weight_norm(nn.Linear(bottleneck_dim, out_dim, bias=False))
|
| 28 |
+
self.last_layer.weight_g.data.fill_(1)
|
| 29 |
+
|
| 30 |
+
def _init_weights(self, m):
|
| 31 |
+
if isinstance(m, nn.Linear):
|
| 32 |
+
trunc_normal_(m.weight, std=0.02)
|
| 33 |
+
if isinstance(m, nn.Linear) and m.bias is not None:
|
| 34 |
+
nn.init.constant_(m.bias, 0)
|
| 35 |
+
|
| 36 |
+
def forward(self, x):
|
| 37 |
+
x = self.mlp(x)
|
| 38 |
+
eps = 1e-6 if x.dtype == torch.float16 else 1e-12
|
| 39 |
+
x = nn.functional.normalize(x, dim=-1, p=2, eps=eps)
|
| 40 |
+
x = self.last_layer(x)
|
| 41 |
+
return x
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _build_mlp(nlayers, in_dim, bottleneck_dim, hidden_dim=None, use_bn=False, bias=True):
|
| 45 |
+
if nlayers == 1:
|
| 46 |
+
return nn.Linear(in_dim, bottleneck_dim, bias=bias)
|
| 47 |
+
else:
|
| 48 |
+
layers = [nn.Linear(in_dim, hidden_dim, bias=bias)]
|
| 49 |
+
if use_bn:
|
| 50 |
+
layers.append(nn.BatchNorm1d(hidden_dim))
|
| 51 |
+
layers.append(nn.GELU())
|
| 52 |
+
for _ in range(nlayers - 2):
|
| 53 |
+
layers.append(nn.Linear(hidden_dim, hidden_dim, bias=bias))
|
| 54 |
+
if use_bn:
|
| 55 |
+
layers.append(nn.BatchNorm1d(hidden_dim))
|
| 56 |
+
layers.append(nn.GELU())
|
| 57 |
+
layers.append(nn.Linear(hidden_dim, bottleneck_dim, bias=bias))
|
| 58 |
+
return nn.Sequential(*layers)
|
third_party/MoGe/moge/model/dinov2/layers/drop_path.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
# References:
|
| 7 |
+
# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
|
| 8 |
+
# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/drop.py
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
from torch import nn
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def drop_path(x, drop_prob: float = 0.0, training: bool = False):
|
| 15 |
+
if drop_prob == 0.0 or not training:
|
| 16 |
+
return x
|
| 17 |
+
keep_prob = 1 - drop_prob
|
| 18 |
+
shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets
|
| 19 |
+
random_tensor = x.new_empty(shape).bernoulli_(keep_prob)
|
| 20 |
+
if keep_prob > 0.0:
|
| 21 |
+
random_tensor.div_(keep_prob)
|
| 22 |
+
output = x * random_tensor
|
| 23 |
+
return output
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class DropPath(nn.Module):
|
| 27 |
+
"""Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks)."""
|
| 28 |
+
|
| 29 |
+
def __init__(self, drop_prob=None):
|
| 30 |
+
super(DropPath, self).__init__()
|
| 31 |
+
self.drop_prob = drop_prob
|
| 32 |
+
|
| 33 |
+
def forward(self, x):
|
| 34 |
+
return drop_path(x, self.drop_prob, self.training)
|
third_party/MoGe/moge/model/dinov2/layers/layer_scale.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
# Modified from: https://github.com/huggingface/pytorch-image-models/blob/main/timm/models/vision_transformer.py#L103-L110
|
| 7 |
+
|
| 8 |
+
from typing import Union
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
from torch import Tensor
|
| 12 |
+
from torch import nn
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class LayerScale(nn.Module):
|
| 16 |
+
def __init__(
|
| 17 |
+
self,
|
| 18 |
+
dim: int,
|
| 19 |
+
init_values: Union[float, Tensor] = 1e-5,
|
| 20 |
+
inplace: bool = False,
|
| 21 |
+
) -> None:
|
| 22 |
+
super().__init__()
|
| 23 |
+
self.inplace = inplace
|
| 24 |
+
self.gamma = nn.Parameter(init_values * torch.ones(dim))
|
| 25 |
+
|
| 26 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 27 |
+
return x.mul_(self.gamma) if self.inplace else x * self.gamma
|
third_party/MoGe/moge/model/dinov2/layers/mlp.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
# References:
|
| 7 |
+
# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
|
| 8 |
+
# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/mlp.py
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
from typing import Callable, Optional
|
| 12 |
+
|
| 13 |
+
from torch import Tensor, nn
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class Mlp(nn.Module):
|
| 17 |
+
def __init__(
|
| 18 |
+
self,
|
| 19 |
+
in_features: int,
|
| 20 |
+
hidden_features: Optional[int] = None,
|
| 21 |
+
out_features: Optional[int] = None,
|
| 22 |
+
act_layer: Callable[..., nn.Module] = nn.GELU,
|
| 23 |
+
drop: float = 0.0,
|
| 24 |
+
bias: bool = True,
|
| 25 |
+
) -> None:
|
| 26 |
+
super().__init__()
|
| 27 |
+
out_features = out_features or in_features
|
| 28 |
+
hidden_features = hidden_features or in_features
|
| 29 |
+
self.fc1 = nn.Linear(in_features, hidden_features, bias=bias)
|
| 30 |
+
self.act = act_layer()
|
| 31 |
+
self.fc2 = nn.Linear(hidden_features, out_features, bias=bias)
|
| 32 |
+
self.drop = nn.Dropout(drop)
|
| 33 |
+
|
| 34 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 35 |
+
x = self.fc1(x)
|
| 36 |
+
x = self.act(x)
|
| 37 |
+
x = self.drop(x)
|
| 38 |
+
x = self.fc2(x)
|
| 39 |
+
x = self.drop(x)
|
| 40 |
+
return x
|
third_party/MoGe/moge/model/dinov2/layers/patch_embed.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
# References:
|
| 7 |
+
# https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
|
| 8 |
+
# https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py
|
| 9 |
+
|
| 10 |
+
from typing import Callable, Optional, Tuple, Union
|
| 11 |
+
|
| 12 |
+
from torch import Tensor
|
| 13 |
+
import torch.nn as nn
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def make_2tuple(x):
|
| 17 |
+
if isinstance(x, tuple):
|
| 18 |
+
assert len(x) == 2
|
| 19 |
+
return x
|
| 20 |
+
|
| 21 |
+
assert isinstance(x, int)
|
| 22 |
+
return (x, x)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class PatchEmbed(nn.Module):
|
| 26 |
+
"""
|
| 27 |
+
2D image to patch embedding: (B,C,H,W) -> (B,N,D)
|
| 28 |
+
|
| 29 |
+
Args:
|
| 30 |
+
img_size: Image size.
|
| 31 |
+
patch_size: Patch token size.
|
| 32 |
+
in_chans: Number of input image channels.
|
| 33 |
+
embed_dim: Number of linear projection output channels.
|
| 34 |
+
norm_layer: Normalization layer.
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
def __init__(
|
| 38 |
+
self,
|
| 39 |
+
img_size: Union[int, Tuple[int, int]] = 224,
|
| 40 |
+
patch_size: Union[int, Tuple[int, int]] = 16,
|
| 41 |
+
in_chans: int = 3,
|
| 42 |
+
embed_dim: int = 768,
|
| 43 |
+
norm_layer: Optional[Callable] = None,
|
| 44 |
+
flatten_embedding: bool = True,
|
| 45 |
+
) -> None:
|
| 46 |
+
super().__init__()
|
| 47 |
+
|
| 48 |
+
image_HW = make_2tuple(img_size)
|
| 49 |
+
patch_HW = make_2tuple(patch_size)
|
| 50 |
+
patch_grid_size = (
|
| 51 |
+
image_HW[0] // patch_HW[0],
|
| 52 |
+
image_HW[1] // patch_HW[1],
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
self.img_size = image_HW
|
| 56 |
+
self.patch_size = patch_HW
|
| 57 |
+
self.patches_resolution = patch_grid_size
|
| 58 |
+
self.num_patches = patch_grid_size[0] * patch_grid_size[1]
|
| 59 |
+
|
| 60 |
+
self.in_chans = in_chans
|
| 61 |
+
self.embed_dim = embed_dim
|
| 62 |
+
|
| 63 |
+
self.flatten_embedding = flatten_embedding
|
| 64 |
+
|
| 65 |
+
self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_HW, stride=patch_HW)
|
| 66 |
+
self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
|
| 67 |
+
|
| 68 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 69 |
+
_, _, H, W = x.shape
|
| 70 |
+
patch_H, patch_W = self.patch_size
|
| 71 |
+
|
| 72 |
+
assert H % patch_H == 0, f"Input image height {H} is not a multiple of patch height {patch_H}"
|
| 73 |
+
assert W % patch_W == 0, f"Input image width {W} is not a multiple of patch width: {patch_W}"
|
| 74 |
+
|
| 75 |
+
x = self.proj(x) # B C H W
|
| 76 |
+
H, W = x.size(2), x.size(3)
|
| 77 |
+
x = x.flatten(2).transpose(1, 2) # B HW C
|
| 78 |
+
x = self.norm(x)
|
| 79 |
+
if not self.flatten_embedding:
|
| 80 |
+
x = x.reshape(-1, H, W, self.embed_dim) # B H W C
|
| 81 |
+
return x
|
| 82 |
+
|
| 83 |
+
def flops(self) -> float:
|
| 84 |
+
Ho, Wo = self.patches_resolution
|
| 85 |
+
flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1])
|
| 86 |
+
if self.norm is not None:
|
| 87 |
+
flops += Ho * Wo * self.embed_dim
|
| 88 |
+
return flops
|
third_party/MoGe/moge/model/dinov2/layers/swiglu_ffn.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import os
|
| 7 |
+
from typing import Callable, Optional
|
| 8 |
+
import warnings
|
| 9 |
+
|
| 10 |
+
from torch import Tensor, nn
|
| 11 |
+
import torch.nn.functional as F
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class SwiGLUFFN(nn.Module):
|
| 15 |
+
def __init__(
|
| 16 |
+
self,
|
| 17 |
+
in_features: int,
|
| 18 |
+
hidden_features: Optional[int] = None,
|
| 19 |
+
out_features: Optional[int] = None,
|
| 20 |
+
act_layer: Callable[..., nn.Module] = None,
|
| 21 |
+
drop: float = 0.0,
|
| 22 |
+
bias: bool = True,
|
| 23 |
+
) -> None:
|
| 24 |
+
super().__init__()
|
| 25 |
+
out_features = out_features or in_features
|
| 26 |
+
hidden_features = hidden_features or in_features
|
| 27 |
+
self.w12 = nn.Linear(in_features, 2 * hidden_features, bias=bias)
|
| 28 |
+
self.w3 = nn.Linear(hidden_features, out_features, bias=bias)
|
| 29 |
+
|
| 30 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 31 |
+
x12 = self.w12(x)
|
| 32 |
+
x1, x2 = x12.chunk(2, dim=-1)
|
| 33 |
+
hidden = F.silu(x1) * x2
|
| 34 |
+
return self.w3(hidden)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
XFORMERS_ENABLED = os.environ.get("XFORMERS_DISABLED") is None
|
| 38 |
+
try:
|
| 39 |
+
if XFORMERS_ENABLED:
|
| 40 |
+
from xformers.ops import SwiGLU
|
| 41 |
+
|
| 42 |
+
XFORMERS_AVAILABLE = True
|
| 43 |
+
# warnings.warn("xFormers is available (SwiGLU)")
|
| 44 |
+
else:
|
| 45 |
+
# warnings.warn("xFormers is disabled (SwiGLU)")
|
| 46 |
+
raise ImportError
|
| 47 |
+
except ImportError:
|
| 48 |
+
SwiGLU = SwiGLUFFN
|
| 49 |
+
XFORMERS_AVAILABLE = False
|
| 50 |
+
|
| 51 |
+
# warnings.warn("xFormers is not available (SwiGLU)")
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class SwiGLUFFNFused(SwiGLU):
|
| 55 |
+
def __init__(
|
| 56 |
+
self,
|
| 57 |
+
in_features: int,
|
| 58 |
+
hidden_features: Optional[int] = None,
|
| 59 |
+
out_features: Optional[int] = None,
|
| 60 |
+
act_layer: Callable[..., nn.Module] = None,
|
| 61 |
+
drop: float = 0.0,
|
| 62 |
+
bias: bool = True,
|
| 63 |
+
) -> None:
|
| 64 |
+
out_features = out_features or in_features
|
| 65 |
+
hidden_features = hidden_features or in_features
|
| 66 |
+
hidden_features = (int(hidden_features * 2 / 3) + 7) // 8 * 8
|
| 67 |
+
super().__init__(
|
| 68 |
+
in_features=in_features,
|
| 69 |
+
hidden_features=hidden_features,
|
| 70 |
+
out_features=out_features,
|
| 71 |
+
bias=bias,
|
| 72 |
+
)
|
third_party/MoGe/moge/model/dinov2/models/__init__.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import logging
|
| 7 |
+
|
| 8 |
+
from . import vision_transformer as vits
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
logger = logging.getLogger("dinov2")
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def build_model(args, only_teacher=False, img_size=224):
|
| 15 |
+
args.arch = args.arch.removesuffix("_memeff")
|
| 16 |
+
if "vit" in args.arch:
|
| 17 |
+
vit_kwargs = dict(
|
| 18 |
+
img_size=img_size,
|
| 19 |
+
patch_size=args.patch_size,
|
| 20 |
+
init_values=args.layerscale,
|
| 21 |
+
ffn_layer=args.ffn_layer,
|
| 22 |
+
block_chunks=args.block_chunks,
|
| 23 |
+
qkv_bias=args.qkv_bias,
|
| 24 |
+
proj_bias=args.proj_bias,
|
| 25 |
+
ffn_bias=args.ffn_bias,
|
| 26 |
+
num_register_tokens=args.num_register_tokens,
|
| 27 |
+
interpolate_offset=args.interpolate_offset,
|
| 28 |
+
interpolate_antialias=args.interpolate_antialias,
|
| 29 |
+
)
|
| 30 |
+
teacher = vits.__dict__[args.arch](**vit_kwargs)
|
| 31 |
+
if only_teacher:
|
| 32 |
+
return teacher, teacher.embed_dim
|
| 33 |
+
student = vits.__dict__[args.arch](
|
| 34 |
+
**vit_kwargs,
|
| 35 |
+
drop_path_rate=args.drop_path_rate,
|
| 36 |
+
drop_path_uniform=args.drop_path_uniform,
|
| 37 |
+
)
|
| 38 |
+
embed_dim = student.embed_dim
|
| 39 |
+
return student, teacher, embed_dim
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def build_model_from_cfg(cfg, only_teacher=False):
|
| 43 |
+
return build_model(cfg.student, only_teacher=only_teacher, img_size=cfg.crops.global_crops_size)
|
third_party/MoGe/moge/model/dinov2/models/vision_transformer.py
ADDED
|
@@ -0,0 +1,407 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
# References:
|
| 7 |
+
# https://github.com/facebookresearch/dino/blob/main/vision_transformer.py
|
| 8 |
+
# https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py
|
| 9 |
+
|
| 10 |
+
from functools import partial
|
| 11 |
+
import math
|
| 12 |
+
import logging
|
| 13 |
+
from typing import Sequence, Tuple, Union, Callable, Optional, List
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn as nn
|
| 17 |
+
import torch.utils.checkpoint
|
| 18 |
+
from torch.nn.init import trunc_normal_
|
| 19 |
+
|
| 20 |
+
from ..layers import Mlp, PatchEmbed, SwiGLUFFNFused, MemEffAttention, NestedTensorBlock as Block
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
logger = logging.getLogger("dinov2")
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def named_apply(fn: Callable, module: nn.Module, name="", depth_first=True, include_root=False) -> nn.Module:
|
| 27 |
+
if not depth_first and include_root:
|
| 28 |
+
fn(module=module, name=name)
|
| 29 |
+
for child_name, child_module in module.named_children():
|
| 30 |
+
child_name = ".".join((name, child_name)) if name else child_name
|
| 31 |
+
named_apply(fn=fn, module=child_module, name=child_name, depth_first=depth_first, include_root=True)
|
| 32 |
+
if depth_first and include_root:
|
| 33 |
+
fn(module=module, name=name)
|
| 34 |
+
return module
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class BlockChunk(nn.ModuleList):
|
| 38 |
+
def forward(self, x):
|
| 39 |
+
for b in self:
|
| 40 |
+
x = b(x)
|
| 41 |
+
return x
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class DinoVisionTransformer(nn.Module):
|
| 45 |
+
def __init__(
|
| 46 |
+
self,
|
| 47 |
+
img_size=224,
|
| 48 |
+
patch_size=16,
|
| 49 |
+
in_chans=3,
|
| 50 |
+
embed_dim=768,
|
| 51 |
+
depth=12,
|
| 52 |
+
num_heads=12,
|
| 53 |
+
mlp_ratio=4.0,
|
| 54 |
+
qkv_bias=True,
|
| 55 |
+
ffn_bias=True,
|
| 56 |
+
proj_bias=True,
|
| 57 |
+
drop_path_rate=0.0,
|
| 58 |
+
drop_path_uniform=False,
|
| 59 |
+
init_values=None, # for layerscale: None or 0 => no layerscale
|
| 60 |
+
embed_layer=PatchEmbed,
|
| 61 |
+
act_layer=nn.GELU,
|
| 62 |
+
block_fn=Block,
|
| 63 |
+
ffn_layer="mlp",
|
| 64 |
+
block_chunks=1,
|
| 65 |
+
num_register_tokens=0,
|
| 66 |
+
interpolate_antialias=False,
|
| 67 |
+
interpolate_offset=0.1,
|
| 68 |
+
):
|
| 69 |
+
"""
|
| 70 |
+
Args:
|
| 71 |
+
img_size (int, tuple): input image size
|
| 72 |
+
patch_size (int, tuple): patch size
|
| 73 |
+
in_chans (int): number of input channels
|
| 74 |
+
embed_dim (int): embedding dimension
|
| 75 |
+
depth (int): depth of transformer
|
| 76 |
+
num_heads (int): number of attention heads
|
| 77 |
+
mlp_ratio (int): ratio of mlp hidden dim to embedding dim
|
| 78 |
+
qkv_bias (bool): enable bias for qkv if True
|
| 79 |
+
proj_bias (bool): enable bias for proj in attn if True
|
| 80 |
+
ffn_bias (bool): enable bias for ffn if True
|
| 81 |
+
drop_path_rate (float): stochastic depth rate
|
| 82 |
+
drop_path_uniform (bool): apply uniform drop rate across blocks
|
| 83 |
+
weight_init (str): weight init scheme
|
| 84 |
+
init_values (float): layer-scale init values
|
| 85 |
+
embed_layer (nn.Module): patch embedding layer
|
| 86 |
+
act_layer (nn.Module): MLP activation layer
|
| 87 |
+
block_fn (nn.Module): transformer block class
|
| 88 |
+
ffn_layer (str): "mlp", "swiglu", "swiglufused" or "identity"
|
| 89 |
+
block_chunks: (int) split block sequence into block_chunks units for FSDP wrap
|
| 90 |
+
num_register_tokens: (int) number of extra cls tokens (so-called "registers")
|
| 91 |
+
interpolate_antialias: (str) flag to apply anti-aliasing when interpolating positional embeddings
|
| 92 |
+
interpolate_offset: (float) work-around offset to apply when interpolating positional embeddings
|
| 93 |
+
"""
|
| 94 |
+
super().__init__()
|
| 95 |
+
norm_layer = partial(nn.LayerNorm, eps=1e-6)
|
| 96 |
+
|
| 97 |
+
self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models
|
| 98 |
+
self.num_tokens = 1
|
| 99 |
+
self.n_blocks = depth
|
| 100 |
+
self.num_heads = num_heads
|
| 101 |
+
self.patch_size = patch_size
|
| 102 |
+
self.num_register_tokens = num_register_tokens
|
| 103 |
+
self.interpolate_antialias = interpolate_antialias
|
| 104 |
+
self.interpolate_offset = interpolate_offset
|
| 105 |
+
|
| 106 |
+
self.patch_embed = embed_layer(img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
|
| 107 |
+
num_patches = self.patch_embed.num_patches
|
| 108 |
+
|
| 109 |
+
self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
|
| 110 |
+
self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim))
|
| 111 |
+
assert num_register_tokens >= 0
|
| 112 |
+
self.register_tokens = (
|
| 113 |
+
nn.Parameter(torch.zeros(1, num_register_tokens, embed_dim)) if num_register_tokens else None
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
if drop_path_uniform is True:
|
| 117 |
+
dpr = [drop_path_rate] * depth
|
| 118 |
+
else:
|
| 119 |
+
dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule
|
| 120 |
+
|
| 121 |
+
if ffn_layer == "mlp":
|
| 122 |
+
logger.info("using MLP layer as FFN")
|
| 123 |
+
ffn_layer = Mlp
|
| 124 |
+
elif ffn_layer == "swiglufused" or ffn_layer == "swiglu":
|
| 125 |
+
logger.info("using SwiGLU layer as FFN")
|
| 126 |
+
ffn_layer = SwiGLUFFNFused
|
| 127 |
+
elif ffn_layer == "identity":
|
| 128 |
+
logger.info("using Identity layer as FFN")
|
| 129 |
+
|
| 130 |
+
def f(*args, **kwargs):
|
| 131 |
+
return nn.Identity()
|
| 132 |
+
|
| 133 |
+
ffn_layer = f
|
| 134 |
+
else:
|
| 135 |
+
raise NotImplementedError
|
| 136 |
+
|
| 137 |
+
blocks_list = [
|
| 138 |
+
block_fn(
|
| 139 |
+
dim=embed_dim,
|
| 140 |
+
num_heads=num_heads,
|
| 141 |
+
mlp_ratio=mlp_ratio,
|
| 142 |
+
qkv_bias=qkv_bias,
|
| 143 |
+
proj_bias=proj_bias,
|
| 144 |
+
ffn_bias=ffn_bias,
|
| 145 |
+
drop_path=dpr[i],
|
| 146 |
+
norm_layer=norm_layer,
|
| 147 |
+
act_layer=act_layer,
|
| 148 |
+
ffn_layer=ffn_layer,
|
| 149 |
+
init_values=init_values,
|
| 150 |
+
)
|
| 151 |
+
for i in range(depth)
|
| 152 |
+
]
|
| 153 |
+
if block_chunks > 0:
|
| 154 |
+
self.chunked_blocks = True
|
| 155 |
+
chunked_blocks = []
|
| 156 |
+
chunksize = depth // block_chunks
|
| 157 |
+
for i in range(0, depth, chunksize):
|
| 158 |
+
# this is to keep the block index consistent if we chunk the block list
|
| 159 |
+
chunked_blocks.append([nn.Identity()] * i + blocks_list[i : i + chunksize])
|
| 160 |
+
self.blocks = nn.ModuleList([BlockChunk(p) for p in chunked_blocks])
|
| 161 |
+
else:
|
| 162 |
+
self.chunked_blocks = False
|
| 163 |
+
self.blocks = nn.ModuleList(blocks_list)
|
| 164 |
+
|
| 165 |
+
self.norm = norm_layer(embed_dim)
|
| 166 |
+
self.head = nn.Identity()
|
| 167 |
+
|
| 168 |
+
self.mask_token = nn.Parameter(torch.zeros(1, embed_dim))
|
| 169 |
+
|
| 170 |
+
self.init_weights()
|
| 171 |
+
|
| 172 |
+
@property
|
| 173 |
+
def onnx_compatible_mode(self):
|
| 174 |
+
return getattr(self, "_onnx_compatible_mode", False)
|
| 175 |
+
|
| 176 |
+
@onnx_compatible_mode.setter
|
| 177 |
+
def onnx_compatible_mode(self, value: bool):
|
| 178 |
+
self._onnx_compatible_mode = value
|
| 179 |
+
|
| 180 |
+
def init_weights(self):
|
| 181 |
+
trunc_normal_(self.pos_embed, std=0.02)
|
| 182 |
+
nn.init.normal_(self.cls_token, std=1e-6)
|
| 183 |
+
if self.register_tokens is not None:
|
| 184 |
+
nn.init.normal_(self.register_tokens, std=1e-6)
|
| 185 |
+
named_apply(init_weights_vit_timm, self)
|
| 186 |
+
|
| 187 |
+
def interpolate_pos_encoding(self, x, h, w):
|
| 188 |
+
previous_dtype = x.dtype
|
| 189 |
+
npatch = x.shape[1] - 1
|
| 190 |
+
batch_size = x.shape[0]
|
| 191 |
+
N = self.pos_embed.shape[1] - 1
|
| 192 |
+
if not self.onnx_compatible_mode and npatch == N and w == h:
|
| 193 |
+
return self.pos_embed
|
| 194 |
+
pos_embed = self.pos_embed.float()
|
| 195 |
+
class_pos_embed = pos_embed[:, 0, :]
|
| 196 |
+
patch_pos_embed = pos_embed[:, 1:, :]
|
| 197 |
+
dim = x.shape[-1]
|
| 198 |
+
h0, w0 = h // self.patch_size, w // self.patch_size
|
| 199 |
+
M = int(math.sqrt(N)) # Recover the number of patches in each dimension
|
| 200 |
+
assert N == M * M
|
| 201 |
+
kwargs = {}
|
| 202 |
+
if not self.onnx_compatible_mode and self.interpolate_offset > 0:
|
| 203 |
+
# Historical kludge: add a small number to avoid floating point error in the interpolation, see https://github.com/facebookresearch/dino/issues/8
|
| 204 |
+
# Note: still needed for backward-compatibility, the underlying operators are using both output size and scale factors
|
| 205 |
+
sx = float(w0 + self.interpolate_offset) / M
|
| 206 |
+
sy = float(h0 + self.interpolate_offset) / M
|
| 207 |
+
kwargs["scale_factor"] = (sy, sx)
|
| 208 |
+
else:
|
| 209 |
+
# Simply specify an output size instead of a scale factor
|
| 210 |
+
kwargs["size"] = (h0, w0)
|
| 211 |
+
|
| 212 |
+
patch_pos_embed = nn.functional.interpolate(
|
| 213 |
+
patch_pos_embed.reshape(1, M, M, dim).permute(0, 3, 1, 2),
|
| 214 |
+
mode="bicubic",
|
| 215 |
+
antialias=self.interpolate_antialias,
|
| 216 |
+
**kwargs,
|
| 217 |
+
)
|
| 218 |
+
|
| 219 |
+
assert (h0, w0) == patch_pos_embed.shape[-2:]
|
| 220 |
+
patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).flatten(1, 2)
|
| 221 |
+
return torch.cat((class_pos_embed[:, None, :].expand(patch_pos_embed.shape[0], -1, -1), patch_pos_embed), dim=1).to(previous_dtype)
|
| 222 |
+
|
| 223 |
+
def prepare_tokens_with_masks(self, x, masks=None):
|
| 224 |
+
B, nc, h, w = x.shape
|
| 225 |
+
x = self.patch_embed(x)
|
| 226 |
+
|
| 227 |
+
if masks is not None:
|
| 228 |
+
x = torch.where(masks.unsqueeze(-1), self.mask_token.to(x.dtype).unsqueeze(0), x)
|
| 229 |
+
|
| 230 |
+
x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1)
|
| 231 |
+
x = x + self.interpolate_pos_encoding(x, h, w)
|
| 232 |
+
|
| 233 |
+
if self.register_tokens is not None:
|
| 234 |
+
x = torch.cat(
|
| 235 |
+
(
|
| 236 |
+
x[:, :1],
|
| 237 |
+
self.register_tokens.expand(x.shape[0], -1, -1),
|
| 238 |
+
x[:, 1:],
|
| 239 |
+
),
|
| 240 |
+
dim=1,
|
| 241 |
+
)
|
| 242 |
+
|
| 243 |
+
return x
|
| 244 |
+
|
| 245 |
+
def forward_features_list(self, x_list, masks_list):
|
| 246 |
+
x = [self.prepare_tokens_with_masks(x, masks) for x, masks, ar in zip(x_list, masks_list)]
|
| 247 |
+
for blk in self.blocks:
|
| 248 |
+
x = blk(x)
|
| 249 |
+
|
| 250 |
+
all_x = x
|
| 251 |
+
output = []
|
| 252 |
+
for x, masks in zip(all_x, masks_list):
|
| 253 |
+
x_norm = self.norm(x)
|
| 254 |
+
output.append(
|
| 255 |
+
{
|
| 256 |
+
"x_norm_clstoken": x_norm[:, 0],
|
| 257 |
+
"x_norm_regtokens": x_norm[:, 1 : self.num_register_tokens + 1],
|
| 258 |
+
"x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :],
|
| 259 |
+
"x_prenorm": x,
|
| 260 |
+
"masks": masks,
|
| 261 |
+
}
|
| 262 |
+
)
|
| 263 |
+
return output
|
| 264 |
+
|
| 265 |
+
def forward_features(self, x, masks=None):
|
| 266 |
+
if isinstance(x, list):
|
| 267 |
+
return self.forward_features_list(x, masks)
|
| 268 |
+
|
| 269 |
+
x = self.prepare_tokens_with_masks(x, masks)
|
| 270 |
+
|
| 271 |
+
for blk in self.blocks:
|
| 272 |
+
x = blk(x)
|
| 273 |
+
|
| 274 |
+
x_norm = self.norm(x)
|
| 275 |
+
return {
|
| 276 |
+
"x_norm_clstoken": x_norm[:, 0],
|
| 277 |
+
"x_norm_regtokens": x_norm[:, 1 : self.num_register_tokens + 1],
|
| 278 |
+
"x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :],
|
| 279 |
+
"x_prenorm": x,
|
| 280 |
+
"masks": masks,
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
+
def _get_intermediate_layers_not_chunked(self, x, n=1):
|
| 284 |
+
x = self.prepare_tokens_with_masks(x)
|
| 285 |
+
# If n is an int, take the n last blocks. If it's a list, take them
|
| 286 |
+
output, total_block_len = [], len(self.blocks)
|
| 287 |
+
blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n
|
| 288 |
+
for i, blk in enumerate(self.blocks):
|
| 289 |
+
x = blk(x)
|
| 290 |
+
if i in blocks_to_take:
|
| 291 |
+
output.append(x)
|
| 292 |
+
assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found"
|
| 293 |
+
return output
|
| 294 |
+
|
| 295 |
+
def _get_intermediate_layers_chunked(self, x, n=1):
|
| 296 |
+
x = self.prepare_tokens_with_masks(x)
|
| 297 |
+
output, i, total_block_len = [], 0, len(self.blocks[-1])
|
| 298 |
+
# If n is an int, take the n last blocks. If it's a list, take them
|
| 299 |
+
blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n
|
| 300 |
+
for block_chunk in self.blocks:
|
| 301 |
+
for blk in block_chunk[i:]: # Passing the nn.Identity()
|
| 302 |
+
x = blk(x)
|
| 303 |
+
if i in blocks_to_take:
|
| 304 |
+
output.append(x)
|
| 305 |
+
i += 1
|
| 306 |
+
assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found"
|
| 307 |
+
return output
|
| 308 |
+
|
| 309 |
+
def get_intermediate_layers(
|
| 310 |
+
self,
|
| 311 |
+
x: torch.Tensor,
|
| 312 |
+
n: Union[int, Sequence] = 1, # Layers or n last layers to take
|
| 313 |
+
reshape: bool = False,
|
| 314 |
+
return_class_token: bool = False,
|
| 315 |
+
norm=True,
|
| 316 |
+
) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]:
|
| 317 |
+
if self.chunked_blocks:
|
| 318 |
+
outputs = self._get_intermediate_layers_chunked(x, n)
|
| 319 |
+
else:
|
| 320 |
+
outputs = self._get_intermediate_layers_not_chunked(x, n)
|
| 321 |
+
if norm:
|
| 322 |
+
outputs = [self.norm(out) for out in outputs]
|
| 323 |
+
class_tokens = [out[:, 0] for out in outputs]
|
| 324 |
+
outputs = [out[:, 1 + self.num_register_tokens :] for out in outputs]
|
| 325 |
+
if reshape:
|
| 326 |
+
B, _, w, h = x.shape
|
| 327 |
+
outputs = [
|
| 328 |
+
out.reshape(B, w // self.patch_size, h // self.patch_size, -1).permute(0, 3, 1, 2).contiguous()
|
| 329 |
+
for out in outputs
|
| 330 |
+
]
|
| 331 |
+
if return_class_token:
|
| 332 |
+
return tuple(zip(outputs, class_tokens))
|
| 333 |
+
return tuple(outputs)
|
| 334 |
+
|
| 335 |
+
def forward(self, *args, is_training=False, **kwargs):
|
| 336 |
+
ret = self.forward_features(*args, **kwargs)
|
| 337 |
+
if is_training:
|
| 338 |
+
return ret
|
| 339 |
+
else:
|
| 340 |
+
return self.head(ret["x_norm_clstoken"])
|
| 341 |
+
|
| 342 |
+
|
| 343 |
+
def init_weights_vit_timm(module: nn.Module, name: str = ""):
|
| 344 |
+
"""ViT weight initialization, original timm impl (for reproducibility)"""
|
| 345 |
+
if isinstance(module, nn.Linear):
|
| 346 |
+
trunc_normal_(module.weight, std=0.02)
|
| 347 |
+
if module.bias is not None:
|
| 348 |
+
nn.init.zeros_(module.bias)
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def vit_small(patch_size=16, num_register_tokens=0, **kwargs):
|
| 352 |
+
model = DinoVisionTransformer(
|
| 353 |
+
patch_size=patch_size,
|
| 354 |
+
embed_dim=384,
|
| 355 |
+
depth=12,
|
| 356 |
+
num_heads=6,
|
| 357 |
+
mlp_ratio=4,
|
| 358 |
+
block_fn=partial(Block, attn_class=MemEffAttention),
|
| 359 |
+
num_register_tokens=num_register_tokens,
|
| 360 |
+
**kwargs,
|
| 361 |
+
)
|
| 362 |
+
return model
|
| 363 |
+
|
| 364 |
+
|
| 365 |
+
def vit_base(patch_size=16, num_register_tokens=0, **kwargs):
|
| 366 |
+
model = DinoVisionTransformer(
|
| 367 |
+
patch_size=patch_size,
|
| 368 |
+
embed_dim=768,
|
| 369 |
+
depth=12,
|
| 370 |
+
num_heads=12,
|
| 371 |
+
mlp_ratio=4,
|
| 372 |
+
block_fn=partial(Block, attn_class=MemEffAttention),
|
| 373 |
+
num_register_tokens=num_register_tokens,
|
| 374 |
+
**kwargs,
|
| 375 |
+
)
|
| 376 |
+
return model
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
def vit_large(patch_size=16, num_register_tokens=0, **kwargs):
|
| 380 |
+
model = DinoVisionTransformer(
|
| 381 |
+
patch_size=patch_size,
|
| 382 |
+
embed_dim=1024,
|
| 383 |
+
depth=24,
|
| 384 |
+
num_heads=16,
|
| 385 |
+
mlp_ratio=4,
|
| 386 |
+
block_fn=partial(Block, attn_class=MemEffAttention),
|
| 387 |
+
num_register_tokens=num_register_tokens,
|
| 388 |
+
**kwargs,
|
| 389 |
+
)
|
| 390 |
+
return model
|
| 391 |
+
|
| 392 |
+
|
| 393 |
+
def vit_giant2(patch_size=16, num_register_tokens=0, **kwargs):
|
| 394 |
+
"""
|
| 395 |
+
Close to ViT-giant, with embed-dim 1536 and 24 heads => embed-dim per head 64
|
| 396 |
+
"""
|
| 397 |
+
model = DinoVisionTransformer(
|
| 398 |
+
patch_size=patch_size,
|
| 399 |
+
embed_dim=1536,
|
| 400 |
+
depth=40,
|
| 401 |
+
num_heads=24,
|
| 402 |
+
mlp_ratio=4,
|
| 403 |
+
block_fn=partial(Block, attn_class=MemEffAttention),
|
| 404 |
+
num_register_tokens=num_register_tokens,
|
| 405 |
+
**kwargs,
|
| 406 |
+
)
|
| 407 |
+
return model
|
third_party/MoGe/moge/model/dinov2/utils/__init__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
third_party/MoGe/moge/model/dinov2/utils/cluster.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
from enum import Enum
|
| 7 |
+
import os
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
from typing import Any, Dict, Optional
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class ClusterType(Enum):
|
| 13 |
+
AWS = "aws"
|
| 14 |
+
FAIR = "fair"
|
| 15 |
+
RSC = "rsc"
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _guess_cluster_type() -> ClusterType:
|
| 19 |
+
uname = os.uname()
|
| 20 |
+
if uname.sysname == "Linux":
|
| 21 |
+
if uname.release.endswith("-aws"):
|
| 22 |
+
# Linux kernel versions on AWS instances are of the form "5.4.0-1051-aws"
|
| 23 |
+
return ClusterType.AWS
|
| 24 |
+
elif uname.nodename.startswith("rsc"):
|
| 25 |
+
# Linux kernel versions on RSC instances are standard ones but hostnames start with "rsc"
|
| 26 |
+
return ClusterType.RSC
|
| 27 |
+
|
| 28 |
+
return ClusterType.FAIR
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def get_cluster_type(cluster_type: Optional[ClusterType] = None) -> Optional[ClusterType]:
|
| 32 |
+
if cluster_type is None:
|
| 33 |
+
return _guess_cluster_type()
|
| 34 |
+
|
| 35 |
+
return cluster_type
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def get_checkpoint_path(cluster_type: Optional[ClusterType] = None) -> Optional[Path]:
|
| 39 |
+
cluster_type = get_cluster_type(cluster_type)
|
| 40 |
+
if cluster_type is None:
|
| 41 |
+
return None
|
| 42 |
+
|
| 43 |
+
CHECKPOINT_DIRNAMES = {
|
| 44 |
+
ClusterType.AWS: "checkpoints",
|
| 45 |
+
ClusterType.FAIR: "checkpoint",
|
| 46 |
+
ClusterType.RSC: "checkpoint/dino",
|
| 47 |
+
}
|
| 48 |
+
return Path("/") / CHECKPOINT_DIRNAMES[cluster_type]
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def get_user_checkpoint_path(cluster_type: Optional[ClusterType] = None) -> Optional[Path]:
|
| 52 |
+
checkpoint_path = get_checkpoint_path(cluster_type)
|
| 53 |
+
if checkpoint_path is None:
|
| 54 |
+
return None
|
| 55 |
+
|
| 56 |
+
username = os.environ.get("USER")
|
| 57 |
+
assert username is not None
|
| 58 |
+
return checkpoint_path / username
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def get_slurm_partition(cluster_type: Optional[ClusterType] = None) -> Optional[str]:
|
| 62 |
+
cluster_type = get_cluster_type(cluster_type)
|
| 63 |
+
if cluster_type is None:
|
| 64 |
+
return None
|
| 65 |
+
|
| 66 |
+
SLURM_PARTITIONS = {
|
| 67 |
+
ClusterType.AWS: "learnlab",
|
| 68 |
+
ClusterType.FAIR: "learnlab",
|
| 69 |
+
ClusterType.RSC: "learn",
|
| 70 |
+
}
|
| 71 |
+
return SLURM_PARTITIONS[cluster_type]
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def get_slurm_executor_parameters(
|
| 75 |
+
nodes: int, num_gpus_per_node: int, cluster_type: Optional[ClusterType] = None, **kwargs
|
| 76 |
+
) -> Dict[str, Any]:
|
| 77 |
+
# create default parameters
|
| 78 |
+
params = {
|
| 79 |
+
"mem_gb": 0, # Requests all memory on a node, see https://slurm.schedmd.com/sbatch.html
|
| 80 |
+
"gpus_per_node": num_gpus_per_node,
|
| 81 |
+
"tasks_per_node": num_gpus_per_node, # one task per GPU
|
| 82 |
+
"cpus_per_task": 10,
|
| 83 |
+
"nodes": nodes,
|
| 84 |
+
"slurm_partition": get_slurm_partition(cluster_type),
|
| 85 |
+
}
|
| 86 |
+
# apply cluster-specific adjustments
|
| 87 |
+
cluster_type = get_cluster_type(cluster_type)
|
| 88 |
+
if cluster_type == ClusterType.AWS:
|
| 89 |
+
params["cpus_per_task"] = 12
|
| 90 |
+
del params["mem_gb"]
|
| 91 |
+
elif cluster_type == ClusterType.RSC:
|
| 92 |
+
params["cpus_per_task"] = 12
|
| 93 |
+
# set additional parameters / apply overrides
|
| 94 |
+
params.update(kwargs)
|
| 95 |
+
return params
|
third_party/MoGe/moge/model/dinov2/utils/config.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import math
|
| 7 |
+
import logging
|
| 8 |
+
import os
|
| 9 |
+
|
| 10 |
+
from omegaconf import OmegaConf
|
| 11 |
+
|
| 12 |
+
import dinov2.distributed as distributed
|
| 13 |
+
from dinov2.logging import setup_logging
|
| 14 |
+
from dinov2.utils import utils
|
| 15 |
+
from dinov2.configs import dinov2_default_config
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
logger = logging.getLogger("dinov2")
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def apply_scaling_rules_to_cfg(cfg): # to fix
|
| 22 |
+
if cfg.optim.scaling_rule == "sqrt_wrt_1024":
|
| 23 |
+
base_lr = cfg.optim.base_lr
|
| 24 |
+
cfg.optim.lr = base_lr
|
| 25 |
+
cfg.optim.lr *= math.sqrt(cfg.train.batch_size_per_gpu * distributed.get_global_size() / 1024.0)
|
| 26 |
+
logger.info(f"sqrt scaling learning rate; base: {base_lr}, new: {cfg.optim.lr}")
|
| 27 |
+
else:
|
| 28 |
+
raise NotImplementedError
|
| 29 |
+
return cfg
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def write_config(cfg, output_dir, name="config.yaml"):
|
| 33 |
+
logger.info(OmegaConf.to_yaml(cfg))
|
| 34 |
+
saved_cfg_path = os.path.join(output_dir, name)
|
| 35 |
+
with open(saved_cfg_path, "w") as f:
|
| 36 |
+
OmegaConf.save(config=cfg, f=f)
|
| 37 |
+
return saved_cfg_path
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def get_cfg_from_args(args):
|
| 41 |
+
args.output_dir = os.path.abspath(args.output_dir)
|
| 42 |
+
args.opts += [f"train.output_dir={args.output_dir}"]
|
| 43 |
+
default_cfg = OmegaConf.create(dinov2_default_config)
|
| 44 |
+
cfg = OmegaConf.load(args.config_file)
|
| 45 |
+
cfg = OmegaConf.merge(default_cfg, cfg, OmegaConf.from_cli(args.opts))
|
| 46 |
+
return cfg
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def default_setup(args):
|
| 50 |
+
distributed.enable(overwrite=True)
|
| 51 |
+
seed = getattr(args, "seed", 0)
|
| 52 |
+
rank = distributed.get_global_rank()
|
| 53 |
+
|
| 54 |
+
global logger
|
| 55 |
+
setup_logging(output=args.output_dir, level=logging.INFO)
|
| 56 |
+
logger = logging.getLogger("dinov2")
|
| 57 |
+
|
| 58 |
+
utils.fix_random_seeds(seed + rank)
|
| 59 |
+
logger.info("git:\n {}\n".format(utils.get_sha()))
|
| 60 |
+
logger.info("\n".join("%s: %s" % (k, str(v)) for k, v in sorted(dict(vars(args)).items())))
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def setup(args):
|
| 64 |
+
"""
|
| 65 |
+
Create configs and perform basic setups.
|
| 66 |
+
"""
|
| 67 |
+
cfg = get_cfg_from_args(args)
|
| 68 |
+
os.makedirs(args.output_dir, exist_ok=True)
|
| 69 |
+
default_setup(args)
|
| 70 |
+
apply_scaling_rules_to_cfg(cfg)
|
| 71 |
+
write_config(cfg, args.output_dir)
|
| 72 |
+
return cfg
|
third_party/MoGe/moge/model/dinov2/utils/dtype.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
from typing import Dict, Union
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
import torch
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
TypeSpec = Union[str, np.dtype, torch.dtype]
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
_NUMPY_TO_TORCH_DTYPE: Dict[np.dtype, torch.dtype] = {
|
| 17 |
+
np.dtype("bool"): torch.bool,
|
| 18 |
+
np.dtype("uint8"): torch.uint8,
|
| 19 |
+
np.dtype("int8"): torch.int8,
|
| 20 |
+
np.dtype("int16"): torch.int16,
|
| 21 |
+
np.dtype("int32"): torch.int32,
|
| 22 |
+
np.dtype("int64"): torch.int64,
|
| 23 |
+
np.dtype("float16"): torch.float16,
|
| 24 |
+
np.dtype("float32"): torch.float32,
|
| 25 |
+
np.dtype("float64"): torch.float64,
|
| 26 |
+
np.dtype("complex64"): torch.complex64,
|
| 27 |
+
np.dtype("complex128"): torch.complex128,
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def as_torch_dtype(dtype: TypeSpec) -> torch.dtype:
|
| 32 |
+
if isinstance(dtype, torch.dtype):
|
| 33 |
+
return dtype
|
| 34 |
+
if isinstance(dtype, str):
|
| 35 |
+
dtype = np.dtype(dtype)
|
| 36 |
+
assert isinstance(dtype, np.dtype), f"Expected an instance of nunpy dtype, got {type(dtype)}"
|
| 37 |
+
return _NUMPY_TO_TORCH_DTYPE[dtype]
|
third_party/MoGe/moge/model/dinov2/utils/param_groups.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
from collections import defaultdict
|
| 7 |
+
import logging
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
logger = logging.getLogger("dinov2")
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def get_vit_lr_decay_rate(name, lr_decay_rate=1.0, num_layers=12, force_is_backbone=False, chunked_blocks=False):
|
| 14 |
+
"""
|
| 15 |
+
Calculate lr decay rate for different ViT blocks.
|
| 16 |
+
Args:
|
| 17 |
+
name (string): parameter name.
|
| 18 |
+
lr_decay_rate (float): base lr decay rate.
|
| 19 |
+
num_layers (int): number of ViT blocks.
|
| 20 |
+
Returns:
|
| 21 |
+
lr decay rate for the given parameter.
|
| 22 |
+
"""
|
| 23 |
+
layer_id = num_layers + 1
|
| 24 |
+
if name.startswith("backbone") or force_is_backbone:
|
| 25 |
+
if (
|
| 26 |
+
".pos_embed" in name
|
| 27 |
+
or ".patch_embed" in name
|
| 28 |
+
or ".mask_token" in name
|
| 29 |
+
or ".cls_token" in name
|
| 30 |
+
or ".register_tokens" in name
|
| 31 |
+
):
|
| 32 |
+
layer_id = 0
|
| 33 |
+
elif force_is_backbone and (
|
| 34 |
+
"pos_embed" in name
|
| 35 |
+
or "patch_embed" in name
|
| 36 |
+
or "mask_token" in name
|
| 37 |
+
or "cls_token" in name
|
| 38 |
+
or "register_tokens" in name
|
| 39 |
+
):
|
| 40 |
+
layer_id = 0
|
| 41 |
+
elif ".blocks." in name and ".residual." not in name:
|
| 42 |
+
layer_id = int(name[name.find(".blocks.") :].split(".")[2]) + 1
|
| 43 |
+
elif chunked_blocks and "blocks." in name and "residual." not in name:
|
| 44 |
+
layer_id = int(name[name.find("blocks.") :].split(".")[2]) + 1
|
| 45 |
+
elif "blocks." in name and "residual." not in name:
|
| 46 |
+
layer_id = int(name[name.find("blocks.") :].split(".")[1]) + 1
|
| 47 |
+
|
| 48 |
+
return lr_decay_rate ** (num_layers + 1 - layer_id)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def get_params_groups_with_decay(model, lr_decay_rate=1.0, patch_embed_lr_mult=1.0):
|
| 52 |
+
chunked_blocks = False
|
| 53 |
+
if hasattr(model, "n_blocks"):
|
| 54 |
+
logger.info("chunked fsdp")
|
| 55 |
+
n_blocks = model.n_blocks
|
| 56 |
+
chunked_blocks = model.chunked_blocks
|
| 57 |
+
elif hasattr(model, "blocks"):
|
| 58 |
+
logger.info("first code branch")
|
| 59 |
+
n_blocks = len(model.blocks)
|
| 60 |
+
elif hasattr(model, "backbone"):
|
| 61 |
+
logger.info("second code branch")
|
| 62 |
+
n_blocks = len(model.backbone.blocks)
|
| 63 |
+
else:
|
| 64 |
+
logger.info("else code branch")
|
| 65 |
+
n_blocks = 0
|
| 66 |
+
all_param_groups = []
|
| 67 |
+
|
| 68 |
+
for name, param in model.named_parameters():
|
| 69 |
+
name = name.replace("_fsdp_wrapped_module.", "")
|
| 70 |
+
if not param.requires_grad:
|
| 71 |
+
continue
|
| 72 |
+
decay_rate = get_vit_lr_decay_rate(
|
| 73 |
+
name, lr_decay_rate, num_layers=n_blocks, force_is_backbone=n_blocks > 0, chunked_blocks=chunked_blocks
|
| 74 |
+
)
|
| 75 |
+
d = {"params": param, "is_last_layer": False, "lr_multiplier": decay_rate, "wd_multiplier": 1.0, "name": name}
|
| 76 |
+
|
| 77 |
+
if "last_layer" in name:
|
| 78 |
+
d.update({"is_last_layer": True})
|
| 79 |
+
|
| 80 |
+
if name.endswith(".bias") or "norm" in name or "gamma" in name:
|
| 81 |
+
d.update({"wd_multiplier": 0.0})
|
| 82 |
+
|
| 83 |
+
if "patch_embed" in name:
|
| 84 |
+
d.update({"lr_multiplier": d["lr_multiplier"] * patch_embed_lr_mult})
|
| 85 |
+
|
| 86 |
+
all_param_groups.append(d)
|
| 87 |
+
logger.info(f"""{name}: lr_multiplier: {d["lr_multiplier"]}, wd_multiplier: {d["wd_multiplier"]}""")
|
| 88 |
+
|
| 89 |
+
return all_param_groups
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def fuse_params_groups(all_params_groups, keys=("lr_multiplier", "wd_multiplier", "is_last_layer")):
|
| 93 |
+
fused_params_groups = defaultdict(lambda: {"params": []})
|
| 94 |
+
for d in all_params_groups:
|
| 95 |
+
identifier = ""
|
| 96 |
+
for k in keys:
|
| 97 |
+
identifier += k + str(d[k]) + "_"
|
| 98 |
+
|
| 99 |
+
for k in keys:
|
| 100 |
+
fused_params_groups[identifier][k] = d[k]
|
| 101 |
+
fused_params_groups[identifier]["params"].append(d["params"])
|
| 102 |
+
|
| 103 |
+
return fused_params_groups.values()
|
third_party/MoGe/moge/model/dinov2/utils/utils.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
| 2 |
+
#
|
| 3 |
+
# This source code is licensed under the Apache License, Version 2.0
|
| 4 |
+
# found in the LICENSE file in the root directory of this source tree.
|
| 5 |
+
|
| 6 |
+
import logging
|
| 7 |
+
import os
|
| 8 |
+
import random
|
| 9 |
+
import subprocess
|
| 10 |
+
from urllib.parse import urlparse
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
import torch
|
| 14 |
+
from torch import nn
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
logger = logging.getLogger("dinov2")
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def load_pretrained_weights(model, pretrained_weights, checkpoint_key):
|
| 21 |
+
if urlparse(pretrained_weights).scheme: # If it looks like an URL
|
| 22 |
+
state_dict = torch.hub.load_state_dict_from_url(pretrained_weights, map_location="cpu")
|
| 23 |
+
else:
|
| 24 |
+
state_dict = torch.load(pretrained_weights, map_location="cpu")
|
| 25 |
+
if checkpoint_key is not None and checkpoint_key in state_dict:
|
| 26 |
+
logger.info(f"Take key {checkpoint_key} in provided checkpoint dict")
|
| 27 |
+
state_dict = state_dict[checkpoint_key]
|
| 28 |
+
# remove `module.` prefix
|
| 29 |
+
state_dict = {k.replace("module.", ""): v for k, v in state_dict.items()}
|
| 30 |
+
# remove `backbone.` prefix induced by multicrop wrapper
|
| 31 |
+
state_dict = {k.replace("backbone.", ""): v for k, v in state_dict.items()}
|
| 32 |
+
msg = model.load_state_dict(state_dict, strict=False)
|
| 33 |
+
logger.info("Pretrained weights found at {} and loaded with msg: {}".format(pretrained_weights, msg))
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def fix_random_seeds(seed=31):
|
| 37 |
+
"""
|
| 38 |
+
Fix random seeds.
|
| 39 |
+
"""
|
| 40 |
+
torch.manual_seed(seed)
|
| 41 |
+
torch.cuda.manual_seed_all(seed)
|
| 42 |
+
np.random.seed(seed)
|
| 43 |
+
random.seed(seed)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def get_sha():
|
| 47 |
+
cwd = os.path.dirname(os.path.abspath(__file__))
|
| 48 |
+
|
| 49 |
+
def _run(command):
|
| 50 |
+
return subprocess.check_output(command, cwd=cwd).decode("ascii").strip()
|
| 51 |
+
|
| 52 |
+
sha = "N/A"
|
| 53 |
+
diff = "clean"
|
| 54 |
+
branch = "N/A"
|
| 55 |
+
try:
|
| 56 |
+
sha = _run(["git", "rev-parse", "HEAD"])
|
| 57 |
+
subprocess.check_output(["git", "diff"], cwd=cwd)
|
| 58 |
+
diff = _run(["git", "diff-index", "HEAD"])
|
| 59 |
+
diff = "has uncommitted changes" if diff else "clean"
|
| 60 |
+
branch = _run(["git", "rev-parse", "--abbrev-ref", "HEAD"])
|
| 61 |
+
except Exception:
|
| 62 |
+
pass
|
| 63 |
+
message = f"sha: {sha}, status: {diff}, branch: {branch}"
|
| 64 |
+
return message
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
class CosineScheduler(object):
|
| 68 |
+
def __init__(self, base_value, final_value, total_iters, warmup_iters=0, start_warmup_value=0, freeze_iters=0):
|
| 69 |
+
super().__init__()
|
| 70 |
+
self.final_value = final_value
|
| 71 |
+
self.total_iters = total_iters
|
| 72 |
+
|
| 73 |
+
freeze_schedule = np.zeros((freeze_iters))
|
| 74 |
+
|
| 75 |
+
warmup_schedule = np.linspace(start_warmup_value, base_value, warmup_iters)
|
| 76 |
+
|
| 77 |
+
iters = np.arange(total_iters - warmup_iters - freeze_iters)
|
| 78 |
+
schedule = final_value + 0.5 * (base_value - final_value) * (1 + np.cos(np.pi * iters / len(iters)))
|
| 79 |
+
self.schedule = np.concatenate((freeze_schedule, warmup_schedule, schedule))
|
| 80 |
+
|
| 81 |
+
assert len(self.schedule) == self.total_iters
|
| 82 |
+
|
| 83 |
+
def __getitem__(self, it):
|
| 84 |
+
if it >= self.total_iters:
|
| 85 |
+
return self.final_value
|
| 86 |
+
else:
|
| 87 |
+
return self.schedule[it]
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def has_batchnorms(model):
|
| 91 |
+
bn_types = (nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d, nn.SyncBatchNorm)
|
| 92 |
+
for name, module in model.named_modules():
|
| 93 |
+
if isinstance(module, bn_types):
|
| 94 |
+
return True
|
| 95 |
+
return False
|
third_party/MoGe/moge/model/modules.py
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import *
|
| 2 |
+
from numbers import Number
|
| 3 |
+
import importlib
|
| 4 |
+
import itertools
|
| 5 |
+
import functools
|
| 6 |
+
import sys
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
from torch import Tensor
|
| 10 |
+
import torch.nn as nn
|
| 11 |
+
import torch.nn.functional as F
|
| 12 |
+
|
| 13 |
+
from .dinov2.models.vision_transformer import DinoVisionTransformer
|
| 14 |
+
from .utils import wrap_dinov2_attention_with_sdpa, wrap_module_with_gradient_checkpointing, unwrap_module_with_gradient_checkpointing
|
| 15 |
+
from ..utils.geometry_torch import normalized_view_plane_uv
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class ResidualConvBlock(nn.Module):
|
| 19 |
+
def __init__(
|
| 20 |
+
self,
|
| 21 |
+
in_channels: int,
|
| 22 |
+
out_channels: int = None,
|
| 23 |
+
hidden_channels: int = None,
|
| 24 |
+
kernel_size: int = 3,
|
| 25 |
+
padding_mode: str = 'replicate',
|
| 26 |
+
activation: Literal['relu', 'leaky_relu', 'silu', 'elu'] = 'relu',
|
| 27 |
+
in_norm: Literal['group_norm', 'layer_norm', 'instance_norm', 'none'] = 'layer_norm',
|
| 28 |
+
hidden_norm: Literal['group_norm', 'layer_norm', 'instance_norm'] = 'group_norm',
|
| 29 |
+
):
|
| 30 |
+
super(ResidualConvBlock, self).__init__()
|
| 31 |
+
if out_channels is None:
|
| 32 |
+
out_channels = in_channels
|
| 33 |
+
if hidden_channels is None:
|
| 34 |
+
hidden_channels = in_channels
|
| 35 |
+
|
| 36 |
+
if activation =='relu':
|
| 37 |
+
activation_cls = nn.ReLU
|
| 38 |
+
elif activation == 'leaky_relu':
|
| 39 |
+
activation_cls = functools.partial(nn.LeakyReLU, negative_slope=0.2)
|
| 40 |
+
elif activation =='silu':
|
| 41 |
+
activation_cls = nn.SiLU
|
| 42 |
+
elif activation == 'elu':
|
| 43 |
+
activation_cls = nn.ELU
|
| 44 |
+
else:
|
| 45 |
+
raise ValueError(f'Unsupported activation function: {activation}')
|
| 46 |
+
|
| 47 |
+
self.layers = nn.Sequential(
|
| 48 |
+
nn.GroupNorm(in_channels // 32, in_channels) if in_norm == 'group_norm' else \
|
| 49 |
+
nn.GroupNorm(1, in_channels) if in_norm == 'layer_norm' else \
|
| 50 |
+
nn.InstanceNorm2d(in_channels) if in_norm == 'instance_norm' else \
|
| 51 |
+
nn.Identity(),
|
| 52 |
+
activation_cls(),
|
| 53 |
+
nn.Conv2d(in_channels, hidden_channels, kernel_size=kernel_size, padding=kernel_size // 2, padding_mode=padding_mode),
|
| 54 |
+
nn.GroupNorm(hidden_channels // 32, hidden_channels) if hidden_norm == 'group_norm' else \
|
| 55 |
+
nn.GroupNorm(1, hidden_channels) if hidden_norm == 'layer_norm' else \
|
| 56 |
+
nn.InstanceNorm2d(hidden_channels) if hidden_norm == 'instance_norm' else\
|
| 57 |
+
nn.Identity(),
|
| 58 |
+
activation_cls(),
|
| 59 |
+
nn.Conv2d(hidden_channels, out_channels, kernel_size=kernel_size, padding=kernel_size // 2, padding_mode=padding_mode)
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
self.skip_connection = nn.Conv2d(in_channels, out_channels, kernel_size=1, padding=0) if in_channels != out_channels else nn.Identity()
|
| 63 |
+
|
| 64 |
+
def forward(self, x):
|
| 65 |
+
skip = self.skip_connection(x)
|
| 66 |
+
x = self.layers(x)
|
| 67 |
+
x = x + skip
|
| 68 |
+
return x
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
class DINOv2Encoder(nn.Module):
|
| 72 |
+
"Wrapped DINOv2 encoder supporting gradient checkpointing. Input is RGB image in range [0, 1]."
|
| 73 |
+
backbone: DinoVisionTransformer
|
| 74 |
+
image_mean: torch.Tensor
|
| 75 |
+
image_std: torch.Tensor
|
| 76 |
+
dim_features: int
|
| 77 |
+
|
| 78 |
+
def __init__(self, backbone: str, intermediate_layers: Union[int, List[int]], dim_out: int, **deprecated_kwargs):
|
| 79 |
+
super(DINOv2Encoder, self).__init__()
|
| 80 |
+
|
| 81 |
+
self.intermediate_layers = intermediate_layers
|
| 82 |
+
|
| 83 |
+
# Load the backbone
|
| 84 |
+
self.hub_loader = getattr(importlib.import_module(".dinov2.hub.backbones", __package__), backbone)
|
| 85 |
+
self.backbone_name = backbone
|
| 86 |
+
self.backbone = self.hub_loader(pretrained=False)
|
| 87 |
+
|
| 88 |
+
self.dim_features = self.backbone.blocks[0].attn.qkv.in_features
|
| 89 |
+
self.num_features = intermediate_layers if isinstance(intermediate_layers, int) else len(intermediate_layers)
|
| 90 |
+
|
| 91 |
+
self.output_projections = nn.ModuleList([
|
| 92 |
+
nn.Conv2d(in_channels=self.dim_features, out_channels=dim_out, kernel_size=1, stride=1, padding=0,)
|
| 93 |
+
for _ in range(self.num_features)
|
| 94 |
+
])
|
| 95 |
+
|
| 96 |
+
self.register_buffer("image_mean", torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1))
|
| 97 |
+
self.register_buffer("image_std", torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1))
|
| 98 |
+
|
| 99 |
+
@property
|
| 100 |
+
def onnx_compatible_mode(self):
|
| 101 |
+
return getattr(self, "_onnx_compatible_mode", False)
|
| 102 |
+
|
| 103 |
+
@onnx_compatible_mode.setter
|
| 104 |
+
def onnx_compatible_mode(self, value: bool):
|
| 105 |
+
self._onnx_compatible_mode = value
|
| 106 |
+
self.backbone.onnx_compatible_mode = value
|
| 107 |
+
|
| 108 |
+
def init_weights(self):
|
| 109 |
+
pretrained_backbone_state_dict = self.hub_loader(pretrained=True).state_dict()
|
| 110 |
+
self.backbone.load_state_dict(pretrained_backbone_state_dict)
|
| 111 |
+
|
| 112 |
+
def enable_gradient_checkpointing(self):
|
| 113 |
+
for i in range(len(self.backbone.blocks)):
|
| 114 |
+
wrap_module_with_gradient_checkpointing(self.backbone.blocks[i])
|
| 115 |
+
|
| 116 |
+
def enable_pytorch_native_sdpa(self):
|
| 117 |
+
for i in range(len(self.backbone.blocks)):
|
| 118 |
+
wrap_dinov2_attention_with_sdpa(self.backbone.blocks[i].attn)
|
| 119 |
+
|
| 120 |
+
def forward(self, image: torch.Tensor, token_rows: Union[int, torch.LongTensor], token_cols: Union[int, torch.LongTensor], return_class_token: bool = False) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 121 |
+
image_14 = F.interpolate(image, (token_rows * 14, token_cols * 14), mode="bilinear", align_corners=False, antialias=not self.onnx_compatible_mode)
|
| 122 |
+
image_14 = (image_14 - self.image_mean) / self.image_std
|
| 123 |
+
|
| 124 |
+
# Get intermediate layers from the backbone
|
| 125 |
+
features = self.backbone.get_intermediate_layers(image_14, n=self.intermediate_layers, return_class_token=True)
|
| 126 |
+
|
| 127 |
+
# Project features to the desired dimensionality
|
| 128 |
+
x = torch.stack([
|
| 129 |
+
proj(feat.permute(0, 2, 1).unflatten(2, (token_rows, token_cols)).contiguous())
|
| 130 |
+
for proj, (feat, clstoken) in zip(self.output_projections, features)
|
| 131 |
+
], dim=1).sum(dim=1)
|
| 132 |
+
|
| 133 |
+
if return_class_token:
|
| 134 |
+
return x, features[-1][1]
|
| 135 |
+
else:
|
| 136 |
+
return x
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
class Resampler(nn.Sequential):
|
| 140 |
+
def __init__(self,
|
| 141 |
+
in_channels: int,
|
| 142 |
+
out_channels: int,
|
| 143 |
+
type_: Literal['pixel_shuffle', 'nearest', 'bilinear', 'conv_transpose', 'pixel_unshuffle', 'avg_pool', 'max_pool'],
|
| 144 |
+
scale_factor: int = 2,
|
| 145 |
+
):
|
| 146 |
+
if type_ == 'pixel_shuffle':
|
| 147 |
+
nn.Sequential.__init__(self,
|
| 148 |
+
nn.Conv2d(in_channels, out_channels * (scale_factor ** 2), kernel_size=3, stride=1, padding=1, padding_mode='replicate'),
|
| 149 |
+
nn.PixelShuffle(scale_factor),
|
| 150 |
+
nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, padding_mode='replicate')
|
| 151 |
+
)
|
| 152 |
+
for i in range(1, scale_factor ** 2):
|
| 153 |
+
self[0].weight.data[i::scale_factor ** 2] = self[0].weight.data[0::scale_factor ** 2]
|
| 154 |
+
self[0].bias.data[i::scale_factor ** 2] = self[0].bias.data[0::scale_factor ** 2]
|
| 155 |
+
elif type_ in ['nearest', 'bilinear']:
|
| 156 |
+
nn.Sequential.__init__(self,
|
| 157 |
+
nn.Upsample(scale_factor=scale_factor, mode=type_, align_corners=False if type_ == 'bilinear' else None),
|
| 158 |
+
nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1, padding_mode='replicate')
|
| 159 |
+
)
|
| 160 |
+
elif type_ == 'conv_transpose':
|
| 161 |
+
nn.Sequential.__init__(self,
|
| 162 |
+
nn.ConvTranspose2d(in_channels, out_channels, kernel_size=scale_factor, stride=scale_factor),
|
| 163 |
+
nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, padding_mode='replicate')
|
| 164 |
+
)
|
| 165 |
+
self[0].weight.data[:] = self[0].weight.data[:, :, :1, :1]
|
| 166 |
+
elif type_ == 'pixel_unshuffle':
|
| 167 |
+
nn.Sequential.__init__(self,
|
| 168 |
+
nn.PixelUnshuffle(scale_factor),
|
| 169 |
+
nn.Conv2d(in_channels * (scale_factor ** 2), out_channels, kernel_size=3, stride=1, padding=1, padding_mode='replicate')
|
| 170 |
+
)
|
| 171 |
+
elif type_ == 'avg_pool':
|
| 172 |
+
nn.Sequential.__init__(self,
|
| 173 |
+
nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1, padding_mode='replicate'),
|
| 174 |
+
nn.AvgPool2d(kernel_size=scale_factor, stride=scale_factor),
|
| 175 |
+
)
|
| 176 |
+
elif type_ == 'max_pool':
|
| 177 |
+
nn.Sequential.__init__(self,
|
| 178 |
+
nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1, padding_mode='replicate'),
|
| 179 |
+
nn.MaxPool2d(kernel_size=scale_factor, stride=scale_factor),
|
| 180 |
+
)
|
| 181 |
+
else:
|
| 182 |
+
raise ValueError(f'Unsupported resampler type: {type_}')
|
| 183 |
+
|
| 184 |
+
class MLP(nn.Sequential):
|
| 185 |
+
def __init__(self, dims: Sequence[int]):
|
| 186 |
+
nn.Sequential.__init__(self,
|
| 187 |
+
*itertools.chain(*[
|
| 188 |
+
(nn.Linear(dim_in, dim_out), nn.ReLU(inplace=True))
|
| 189 |
+
for dim_in, dim_out in zip(dims[:-2], dims[1:-1])
|
| 190 |
+
]),
|
| 191 |
+
nn.Linear(dims[-2], dims[-1]),
|
| 192 |
+
)
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
class ConvStack(nn.Module):
|
| 196 |
+
def __init__(self,
|
| 197 |
+
dim_in: List[Optional[int]],
|
| 198 |
+
dim_res_blocks: List[int],
|
| 199 |
+
dim_out: List[Optional[int]],
|
| 200 |
+
resamplers: Union[Literal['pixel_shuffle', 'nearest', 'bilinear', 'conv_transpose', 'pixel_unshuffle', 'avg_pool', 'max_pool'], List],
|
| 201 |
+
dim_times_res_block_hidden: int = 1,
|
| 202 |
+
num_res_blocks: int = 1,
|
| 203 |
+
res_block_in_norm: Literal['layer_norm', 'group_norm' , 'instance_norm', 'none'] = 'layer_norm',
|
| 204 |
+
res_block_hidden_norm: Literal['layer_norm', 'group_norm' , 'instance_norm', 'none'] = 'group_norm',
|
| 205 |
+
activation: Literal['relu', 'leaky_relu', 'silu', 'elu'] = 'relu',
|
| 206 |
+
):
|
| 207 |
+
super().__init__()
|
| 208 |
+
self.input_blocks = nn.ModuleList([
|
| 209 |
+
nn.Conv2d(dim_in_, dim_res_block_, kernel_size=1, stride=1, padding=0) if dim_in_ is not None else nn.Identity()
|
| 210 |
+
for dim_in_, dim_res_block_ in zip(dim_in if isinstance(dim_in, Sequence) else itertools.repeat(dim_in), dim_res_blocks)
|
| 211 |
+
])
|
| 212 |
+
self.resamplers = nn.ModuleList([
|
| 213 |
+
Resampler(dim_prev, dim_succ, scale_factor=2, type_=resampler)
|
| 214 |
+
for i, (dim_prev, dim_succ, resampler) in enumerate(zip(
|
| 215 |
+
dim_res_blocks[:-1],
|
| 216 |
+
dim_res_blocks[1:],
|
| 217 |
+
resamplers if isinstance(resamplers, Sequence) else itertools.repeat(resamplers)
|
| 218 |
+
))
|
| 219 |
+
])
|
| 220 |
+
self.res_blocks = nn.ModuleList([
|
| 221 |
+
nn.Sequential(
|
| 222 |
+
*(
|
| 223 |
+
ResidualConvBlock(
|
| 224 |
+
dim_res_block_, dim_res_block_, dim_times_res_block_hidden * dim_res_block_,
|
| 225 |
+
activation=activation, in_norm=res_block_in_norm, hidden_norm=res_block_hidden_norm
|
| 226 |
+
) for _ in range(num_res_blocks[i] if isinstance(num_res_blocks, list) else num_res_blocks)
|
| 227 |
+
)
|
| 228 |
+
) for i, dim_res_block_ in enumerate(dim_res_blocks)
|
| 229 |
+
])
|
| 230 |
+
self.output_blocks = nn.ModuleList([
|
| 231 |
+
nn.Conv2d(dim_res_block_, dim_out_, kernel_size=1, stride=1, padding=0) if dim_out_ is not None else nn.Identity()
|
| 232 |
+
for dim_out_, dim_res_block_ in zip(dim_out if isinstance(dim_out, Sequence) else itertools.repeat(dim_out), dim_res_blocks)
|
| 233 |
+
])
|
| 234 |
+
|
| 235 |
+
def enable_gradient_checkpointing(self):
|
| 236 |
+
for i in range(len(self.resamplers)):
|
| 237 |
+
self.resamplers[i] = wrap_module_with_gradient_checkpointing(self.resamplers[i])
|
| 238 |
+
for i in range(len(self.res_blocks)):
|
| 239 |
+
for j in range(len(self.res_blocks[i])):
|
| 240 |
+
self.res_blocks[i][j] = wrap_module_with_gradient_checkpointing(self.res_blocks[i][j])
|
| 241 |
+
|
| 242 |
+
def forward(self, in_features: List[torch.Tensor]):
|
| 243 |
+
out_features = []
|
| 244 |
+
for i in range(len(self.res_blocks)):
|
| 245 |
+
feature = self.input_blocks[i](in_features[i])
|
| 246 |
+
if i == 0:
|
| 247 |
+
x = feature
|
| 248 |
+
elif feature is not None:
|
| 249 |
+
x = x + feature
|
| 250 |
+
x = self.res_blocks[i](x)
|
| 251 |
+
out_features.append(self.output_blocks[i](x))
|
| 252 |
+
if i < len(self.res_blocks) - 1:
|
| 253 |
+
x = self.resamplers[i](x)
|
| 254 |
+
return out_features
|
third_party/MoGe/moge/model/utils.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import *
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
import torch.nn as nn
|
| 5 |
+
import torch.nn.functional as F
|
| 6 |
+
|
| 7 |
+
def wrap_module_with_gradient_checkpointing(module: nn.Module):
|
| 8 |
+
from torch.utils.checkpoint import checkpoint
|
| 9 |
+
class _CheckpointingWrapper(module.__class__):
|
| 10 |
+
_restore_cls = module.__class__
|
| 11 |
+
def forward(self, *args, **kwargs):
|
| 12 |
+
return checkpoint(super().forward, *args, use_reentrant=False, **kwargs)
|
| 13 |
+
|
| 14 |
+
module.__class__ = _CheckpointingWrapper
|
| 15 |
+
return module
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def unwrap_module_with_gradient_checkpointing(module: nn.Module):
|
| 19 |
+
module.__class__ = module.__class__._restore_cls
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def wrap_dinov2_attention_with_sdpa(module: nn.Module):
|
| 23 |
+
assert torch.__version__ >= '2.0', "SDPA requires PyTorch 2.0 or later"
|
| 24 |
+
class _AttentionWrapper(module.__class__):
|
| 25 |
+
def forward(self, x: torch.Tensor, attn_bias=None) -> torch.Tensor:
|
| 26 |
+
B, N, C = x.shape
|
| 27 |
+
qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) # (3, B, H, N, C // H)
|
| 28 |
+
|
| 29 |
+
q, k, v = torch.unbind(qkv, 0) # (B, H, N, C // H)
|
| 30 |
+
|
| 31 |
+
x = F.scaled_dot_product_attention(q, k, v, attn_bias)
|
| 32 |
+
x = x.permute(0, 2, 1, 3).reshape(B, N, C)
|
| 33 |
+
|
| 34 |
+
x = self.proj(x)
|
| 35 |
+
x = self.proj_drop(x)
|
| 36 |
+
return x
|
| 37 |
+
module.__class__ = _AttentionWrapper
|
| 38 |
+
return module
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def sync_ddp_hook(state, bucket: torch.distributed.GradBucket) -> torch.futures.Future[torch.Tensor]:
|
| 42 |
+
group_to_use = torch.distributed.group.WORLD
|
| 43 |
+
world_size = group_to_use.size()
|
| 44 |
+
grad = bucket.buffer()
|
| 45 |
+
grad.div_(world_size)
|
| 46 |
+
torch.distributed.all_reduce(grad, group=group_to_use)
|
| 47 |
+
fut = torch.futures.Future()
|
| 48 |
+
fut.set_result(grad)
|
| 49 |
+
return fut
|
third_party/MoGe/moge/model/v1.py
ADDED
|
@@ -0,0 +1,392 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import *
|
| 2 |
+
from numbers import Number
|
| 3 |
+
from functools import partial
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
import importlib
|
| 6 |
+
import warnings
|
| 7 |
+
import json
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
import torch.nn as nn
|
| 11 |
+
import torch.nn.functional as F
|
| 12 |
+
import torch.utils
|
| 13 |
+
import torch.utils.checkpoint
|
| 14 |
+
import torch.version
|
| 15 |
+
import utils3d
|
| 16 |
+
from huggingface_hub import hf_hub_download
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
from ..utils.geometry_torch import normalized_view_plane_uv, recover_focal_shift, gaussian_blur_2d, dilate_with_mask
|
| 20 |
+
from .utils import wrap_dinov2_attention_with_sdpa, wrap_module_with_gradient_checkpointing, unwrap_module_with_gradient_checkpointing
|
| 21 |
+
from ..utils.tools import timeit
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class ResidualConvBlock(nn.Module):
|
| 25 |
+
def __init__(self, in_channels: int, out_channels: int = None, hidden_channels: int = None, padding_mode: str = 'replicate', activation: Literal['relu', 'leaky_relu', 'silu', 'elu'] = 'relu', norm: Literal['group_norm', 'layer_norm'] = 'group_norm'):
|
| 26 |
+
super(ResidualConvBlock, self).__init__()
|
| 27 |
+
if out_channels is None:
|
| 28 |
+
out_channels = in_channels
|
| 29 |
+
if hidden_channels is None:
|
| 30 |
+
hidden_channels = in_channels
|
| 31 |
+
|
| 32 |
+
if activation =='relu':
|
| 33 |
+
activation_cls = lambda: nn.ReLU(inplace=True)
|
| 34 |
+
elif activation == 'leaky_relu':
|
| 35 |
+
activation_cls = lambda: nn.LeakyReLU(negative_slope=0.2, inplace=True)
|
| 36 |
+
elif activation =='silu':
|
| 37 |
+
activation_cls = lambda: nn.SiLU(inplace=True)
|
| 38 |
+
elif activation == 'elu':
|
| 39 |
+
activation_cls = lambda: nn.ELU(inplace=True)
|
| 40 |
+
else:
|
| 41 |
+
raise ValueError(f'Unsupported activation function: {activation}')
|
| 42 |
+
|
| 43 |
+
self.layers = nn.Sequential(
|
| 44 |
+
nn.GroupNorm(1, in_channels),
|
| 45 |
+
activation_cls(),
|
| 46 |
+
nn.Conv2d(in_channels, hidden_channels, kernel_size=3, padding=1, padding_mode=padding_mode),
|
| 47 |
+
nn.GroupNorm(hidden_channels // 32 if norm == 'group_norm' else 1, hidden_channels),
|
| 48 |
+
activation_cls(),
|
| 49 |
+
nn.Conv2d(hidden_channels, out_channels, kernel_size=3, padding=1, padding_mode=padding_mode)
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
self.skip_connection = nn.Conv2d(in_channels, out_channels, kernel_size=1, padding=0) if in_channels != out_channels else nn.Identity()
|
| 53 |
+
|
| 54 |
+
def forward(self, x):
|
| 55 |
+
skip = self.skip_connection(x)
|
| 56 |
+
x = self.layers(x)
|
| 57 |
+
x = x + skip
|
| 58 |
+
return x
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
class Head(nn.Module):
|
| 62 |
+
def __init__(
|
| 63 |
+
self,
|
| 64 |
+
num_features: int,
|
| 65 |
+
dim_in: int,
|
| 66 |
+
dim_out: List[int],
|
| 67 |
+
dim_proj: int = 512,
|
| 68 |
+
dim_upsample: List[int] = [256, 128, 128],
|
| 69 |
+
dim_times_res_block_hidden: int = 1,
|
| 70 |
+
num_res_blocks: int = 1,
|
| 71 |
+
res_block_norm: Literal['group_norm', 'layer_norm'] = 'group_norm',
|
| 72 |
+
last_res_blocks: int = 0,
|
| 73 |
+
last_conv_channels: int = 32,
|
| 74 |
+
last_conv_size: int = 1
|
| 75 |
+
):
|
| 76 |
+
super().__init__()
|
| 77 |
+
|
| 78 |
+
self.projects = nn.ModuleList([
|
| 79 |
+
nn.Conv2d(in_channels=dim_in, out_channels=dim_proj, kernel_size=1, stride=1, padding=0,) for _ in range(num_features)
|
| 80 |
+
])
|
| 81 |
+
|
| 82 |
+
self.upsample_blocks = nn.ModuleList([
|
| 83 |
+
nn.Sequential(
|
| 84 |
+
self._make_upsampler(in_ch + 2, out_ch),
|
| 85 |
+
*(ResidualConvBlock(out_ch, out_ch, dim_times_res_block_hidden * out_ch, activation="relu", norm=res_block_norm) for _ in range(num_res_blocks))
|
| 86 |
+
) for in_ch, out_ch in zip([dim_proj] + dim_upsample[:-1], dim_upsample)
|
| 87 |
+
])
|
| 88 |
+
|
| 89 |
+
self.output_block = nn.ModuleList([
|
| 90 |
+
self._make_output_block(
|
| 91 |
+
dim_upsample[-1] + 2, dim_out_, dim_times_res_block_hidden, last_res_blocks, last_conv_channels, last_conv_size, res_block_norm,
|
| 92 |
+
) for dim_out_ in dim_out
|
| 93 |
+
])
|
| 94 |
+
|
| 95 |
+
def _make_upsampler(self, in_channels: int, out_channels: int):
|
| 96 |
+
upsampler = nn.Sequential(
|
| 97 |
+
nn.ConvTranspose2d(in_channels, out_channels, kernel_size=2, stride=2),
|
| 98 |
+
nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, padding_mode='replicate')
|
| 99 |
+
)
|
| 100 |
+
upsampler[0].weight.data[:] = upsampler[0].weight.data[:, :, :1, :1]
|
| 101 |
+
return upsampler
|
| 102 |
+
|
| 103 |
+
def _make_output_block(self, dim_in: int, dim_out: int, dim_times_res_block_hidden: int, last_res_blocks: int, last_conv_channels: int, last_conv_size: int, res_block_norm: Literal['group_norm', 'layer_norm']):
|
| 104 |
+
return nn.Sequential(
|
| 105 |
+
nn.Conv2d(dim_in, last_conv_channels, kernel_size=3, stride=1, padding=1, padding_mode='replicate'),
|
| 106 |
+
*(ResidualConvBlock(last_conv_channels, last_conv_channels, dim_times_res_block_hidden * last_conv_channels, activation='relu', norm=res_block_norm) for _ in range(last_res_blocks)),
|
| 107 |
+
nn.ReLU(inplace=True),
|
| 108 |
+
nn.Conv2d(last_conv_channels, dim_out, kernel_size=last_conv_size, stride=1, padding=last_conv_size // 2, padding_mode='replicate'),
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
def forward(self, hidden_states: torch.Tensor, image: torch.Tensor):
|
| 112 |
+
img_h, img_w = image.shape[-2:]
|
| 113 |
+
patch_h, patch_w = img_h // 14, img_w // 14
|
| 114 |
+
|
| 115 |
+
# Process the hidden states
|
| 116 |
+
x = torch.stack([
|
| 117 |
+
proj(feat.permute(0, 2, 1).unflatten(2, (patch_h, patch_w)).contiguous())
|
| 118 |
+
for proj, (feat, clstoken) in zip(self.projects, hidden_states)
|
| 119 |
+
], dim=1).sum(dim=1)
|
| 120 |
+
|
| 121 |
+
# Upsample stage
|
| 122 |
+
# (patch_h, patch_w) -> (patch_h * 2, patch_w * 2) -> (patch_h * 4, patch_w * 4) -> (patch_h * 8, patch_w * 8)
|
| 123 |
+
for i, block in enumerate(self.upsample_blocks):
|
| 124 |
+
# UV coordinates is for awareness of image aspect ratio
|
| 125 |
+
uv = normalized_view_plane_uv(width=x.shape[-1], height=x.shape[-2], aspect_ratio=img_w / img_h, dtype=x.dtype, device=x.device)
|
| 126 |
+
uv = uv.permute(2, 0, 1).unsqueeze(0).expand(x.shape[0], -1, -1, -1)
|
| 127 |
+
x = torch.cat([x, uv], dim=1)
|
| 128 |
+
for layer in block:
|
| 129 |
+
x = torch.utils.checkpoint.checkpoint(layer, x, use_reentrant=False)
|
| 130 |
+
|
| 131 |
+
# (patch_h * 8, patch_w * 8) -> (img_h, img_w)
|
| 132 |
+
x = F.interpolate(x, (img_h, img_w), mode="bilinear", align_corners=False)
|
| 133 |
+
uv = normalized_view_plane_uv(width=x.shape[-1], height=x.shape[-2], aspect_ratio=img_w / img_h, dtype=x.dtype, device=x.device)
|
| 134 |
+
uv = uv.permute(2, 0, 1).unsqueeze(0).expand(x.shape[0], -1, -1, -1)
|
| 135 |
+
x = torch.cat([x, uv], dim=1)
|
| 136 |
+
|
| 137 |
+
if isinstance(self.output_block, nn.ModuleList):
|
| 138 |
+
output = [torch.utils.checkpoint.checkpoint(block, x, use_reentrant=False) for block in self.output_block]
|
| 139 |
+
else:
|
| 140 |
+
output = torch.utils.checkpoint.checkpoint(self.output_block, x, use_reentrant=False)
|
| 141 |
+
|
| 142 |
+
return output
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
class MoGeModel(nn.Module):
|
| 146 |
+
image_mean: torch.Tensor
|
| 147 |
+
image_std: torch.Tensor
|
| 148 |
+
|
| 149 |
+
def __init__(self,
|
| 150 |
+
encoder: str = 'dinov2_vitb14',
|
| 151 |
+
intermediate_layers: Union[int, List[int]] = 4,
|
| 152 |
+
dim_proj: int = 512,
|
| 153 |
+
dim_upsample: List[int] = [256, 128, 128],
|
| 154 |
+
dim_times_res_block_hidden: int = 1,
|
| 155 |
+
num_res_blocks: int = 1,
|
| 156 |
+
remap_output: Literal[False, True, 'linear', 'sinh', 'exp', 'sinh_exp'] = 'linear',
|
| 157 |
+
res_block_norm: Literal['group_norm', 'layer_norm'] = 'group_norm',
|
| 158 |
+
num_tokens_range: Tuple[Number, Number] = [1200, 2500],
|
| 159 |
+
last_res_blocks: int = 0,
|
| 160 |
+
last_conv_channels: int = 32,
|
| 161 |
+
last_conv_size: int = 1,
|
| 162 |
+
mask_threshold: float = 0.5,
|
| 163 |
+
**deprecated_kwargs
|
| 164 |
+
):
|
| 165 |
+
super(MoGeModel, self).__init__()
|
| 166 |
+
|
| 167 |
+
if deprecated_kwargs:
|
| 168 |
+
# Process legacy arguments
|
| 169 |
+
if 'trained_area_range' in deprecated_kwargs:
|
| 170 |
+
num_tokens_range = [deprecated_kwargs['trained_area_range'][0] // 14 ** 2, deprecated_kwargs['trained_area_range'][1] // 14 ** 2]
|
| 171 |
+
del deprecated_kwargs['trained_area_range']
|
| 172 |
+
warnings.warn(f"The following deprecated/invalid arguments are ignored: {deprecated_kwargs}")
|
| 173 |
+
|
| 174 |
+
self.encoder = encoder
|
| 175 |
+
self.remap_output = remap_output
|
| 176 |
+
self.intermediate_layers = intermediate_layers
|
| 177 |
+
self.num_tokens_range = num_tokens_range
|
| 178 |
+
self.mask_threshold = mask_threshold
|
| 179 |
+
|
| 180 |
+
# NOTE: We have copied the DINOv2 code in torchhub to this repository.
|
| 181 |
+
# Minimal modifications have been made: removing irrelevant code, unnecessary warnings and fixing importing issues.
|
| 182 |
+
hub_loader = getattr(importlib.import_module(".dinov2.hub.backbones", __package__), encoder)
|
| 183 |
+
self.backbone = hub_loader(pretrained=False)
|
| 184 |
+
dim_feature = self.backbone.blocks[0].attn.qkv.in_features
|
| 185 |
+
|
| 186 |
+
self.head = Head(
|
| 187 |
+
num_features=intermediate_layers if isinstance(intermediate_layers, int) else len(intermediate_layers),
|
| 188 |
+
dim_in=dim_feature,
|
| 189 |
+
dim_out=[3, 1],
|
| 190 |
+
dim_proj=dim_proj,
|
| 191 |
+
dim_upsample=dim_upsample,
|
| 192 |
+
dim_times_res_block_hidden=dim_times_res_block_hidden,
|
| 193 |
+
num_res_blocks=num_res_blocks,
|
| 194 |
+
res_block_norm=res_block_norm,
|
| 195 |
+
last_res_blocks=last_res_blocks,
|
| 196 |
+
last_conv_channels=last_conv_channels,
|
| 197 |
+
last_conv_size=last_conv_size
|
| 198 |
+
)
|
| 199 |
+
|
| 200 |
+
image_mean = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)
|
| 201 |
+
image_std = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)
|
| 202 |
+
|
| 203 |
+
self.register_buffer("image_mean", image_mean)
|
| 204 |
+
self.register_buffer("image_std", image_std)
|
| 205 |
+
|
| 206 |
+
@property
|
| 207 |
+
def device(self) -> torch.device:
|
| 208 |
+
return next(self.parameters()).device
|
| 209 |
+
|
| 210 |
+
@property
|
| 211 |
+
def dtype(self) -> torch.dtype:
|
| 212 |
+
return next(self.parameters()).dtype
|
| 213 |
+
|
| 214 |
+
@classmethod
|
| 215 |
+
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, Path, IO[bytes]], model_kwargs: Optional[Dict[str, Any]] = None, **hf_kwargs) -> 'MoGeModel':
|
| 216 |
+
"""
|
| 217 |
+
Load a model from a checkpoint file.
|
| 218 |
+
|
| 219 |
+
### Parameters:
|
| 220 |
+
- `pretrained_model_name_or_path`: path to the checkpoint file or repo id.
|
| 221 |
+
- `model_kwargs`: additional keyword arguments to override the parameters in the checkpoint.
|
| 222 |
+
- `hf_kwargs`: additional keyword arguments to pass to the `hf_hub_download` function. Ignored if `pretrained_model_name_or_path` is a local path.
|
| 223 |
+
|
| 224 |
+
### Returns:
|
| 225 |
+
- A new instance of `MoGe` with the parameters loaded from the checkpoint.
|
| 226 |
+
"""
|
| 227 |
+
if Path(pretrained_model_name_or_path).exists():
|
| 228 |
+
checkpoint = torch.load(pretrained_model_name_or_path, map_location='cpu', weights_only=True)
|
| 229 |
+
else:
|
| 230 |
+
cached_checkpoint_path = hf_hub_download(
|
| 231 |
+
repo_id=pretrained_model_name_or_path,
|
| 232 |
+
repo_type="model",
|
| 233 |
+
filename="model.pt",
|
| 234 |
+
**hf_kwargs
|
| 235 |
+
)
|
| 236 |
+
checkpoint = torch.load(cached_checkpoint_path, map_location='cpu', weights_only=True)
|
| 237 |
+
model_config = checkpoint['model_config']
|
| 238 |
+
if model_kwargs is not None:
|
| 239 |
+
model_config.update(model_kwargs)
|
| 240 |
+
model = cls(**model_config)
|
| 241 |
+
model.load_state_dict(checkpoint['model'])
|
| 242 |
+
return model
|
| 243 |
+
|
| 244 |
+
def init_weights(self):
|
| 245 |
+
"Load the backbone with pretrained dinov2 weights from torch hub"
|
| 246 |
+
state_dict = torch.hub.load('facebookresearch/dinov2', self.encoder, pretrained=True).state_dict()
|
| 247 |
+
self.backbone.load_state_dict(state_dict)
|
| 248 |
+
|
| 249 |
+
def enable_gradient_checkpointing(self):
|
| 250 |
+
for i in range(len(self.backbone.blocks)):
|
| 251 |
+
self.backbone.blocks[i] = wrap_module_with_gradient_checkpointing(self.backbone.blocks[i])
|
| 252 |
+
|
| 253 |
+
def _remap_points(self, points: torch.Tensor) -> torch.Tensor:
|
| 254 |
+
if self.remap_output == 'linear':
|
| 255 |
+
pass
|
| 256 |
+
elif self.remap_output =='sinh':
|
| 257 |
+
points = torch.sinh(points)
|
| 258 |
+
elif self.remap_output == 'exp':
|
| 259 |
+
xy, z = points.split([2, 1], dim=-1)
|
| 260 |
+
z = torch.exp(z)
|
| 261 |
+
points = torch.cat([xy * z, z], dim=-1)
|
| 262 |
+
elif self.remap_output =='sinh_exp':
|
| 263 |
+
xy, z = points.split([2, 1], dim=-1)
|
| 264 |
+
points = torch.cat([torch.sinh(xy), torch.exp(z)], dim=-1)
|
| 265 |
+
else:
|
| 266 |
+
raise ValueError(f"Invalid remap output type: {self.remap_output}")
|
| 267 |
+
return points
|
| 268 |
+
|
| 269 |
+
def forward(self, image: torch.Tensor, num_tokens: int) -> Dict[str, torch.Tensor]:
|
| 270 |
+
original_height, original_width = image.shape[-2:]
|
| 271 |
+
|
| 272 |
+
# Resize to expected resolution defined by num_tokens
|
| 273 |
+
resize_factor = ((num_tokens * 14 ** 2) / (original_height * original_width)) ** 0.5
|
| 274 |
+
resized_width, resized_height = int(original_width * resize_factor), int(original_height * resize_factor)
|
| 275 |
+
image = F.interpolate(image, (resized_height, resized_width), mode="bicubic", align_corners=False, antialias=True)
|
| 276 |
+
|
| 277 |
+
# Apply image transformation for DINOv2
|
| 278 |
+
image = (image - self.image_mean) / self.image_std
|
| 279 |
+
image_14 = F.interpolate(image, (resized_height // 14 * 14, resized_width // 14 * 14), mode="bilinear", align_corners=False, antialias=True)
|
| 280 |
+
|
| 281 |
+
# Get intermediate layers from the backbone
|
| 282 |
+
features = self.backbone.get_intermediate_layers(image_14, self.intermediate_layers, return_class_token=True)
|
| 283 |
+
|
| 284 |
+
# Predict points (and mask)
|
| 285 |
+
output = self.head(features, image)
|
| 286 |
+
points, mask = output
|
| 287 |
+
|
| 288 |
+
# Make sure fp32 precision for output
|
| 289 |
+
with torch.autocast(device_type=image.device.type, dtype=torch.float32):
|
| 290 |
+
# Resize to original resolution
|
| 291 |
+
points = F.interpolate(points, (original_height, original_width), mode='bilinear', align_corners=False, antialias=False)
|
| 292 |
+
mask = F.interpolate(mask, (original_height, original_width), mode='bilinear', align_corners=False, antialias=False)
|
| 293 |
+
|
| 294 |
+
# Post-process points and mask
|
| 295 |
+
points, mask = points.permute(0, 2, 3, 1), mask.squeeze(1)
|
| 296 |
+
points = self._remap_points(points) # slightly improves the performance in case of very large output values
|
| 297 |
+
|
| 298 |
+
return_dict = {'points': points, 'mask': mask}
|
| 299 |
+
return return_dict
|
| 300 |
+
|
| 301 |
+
@torch.inference_mode()
|
| 302 |
+
def infer(
|
| 303 |
+
self,
|
| 304 |
+
image: torch.Tensor,
|
| 305 |
+
fov_x: Union[Number, torch.Tensor] = None,
|
| 306 |
+
resolution_level: int = 9,
|
| 307 |
+
num_tokens: int = None,
|
| 308 |
+
apply_mask: bool = True,
|
| 309 |
+
force_projection: bool = True,
|
| 310 |
+
use_fp16: bool = True,
|
| 311 |
+
) -> Dict[str, torch.Tensor]:
|
| 312 |
+
"""
|
| 313 |
+
User-friendly inference function
|
| 314 |
+
|
| 315 |
+
### Parameters
|
| 316 |
+
- `image`: input image tensor of shape (B, 3, H, W) or (3, H, W)\
|
| 317 |
+
- `fov_x`: the horizontal camera FoV in degrees. If None, it will be inferred from the predicted point map. Default: None
|
| 318 |
+
- `resolution_level`: An integer [0-9] for the resolution level for inference.
|
| 319 |
+
The higher, the finer details will be captured, but slower. Defaults to 9. Note that it is irrelevant to the output size, which is always the same as the input size.
|
| 320 |
+
`resolution_level` actually controls `num_tokens`. See `num_tokens` for more details.
|
| 321 |
+
- `num_tokens`: number of tokens used for inference. A integer in the (suggested) range of `[1200, 2500]`.
|
| 322 |
+
`resolution_level` will be ignored if `num_tokens` is provided. Default: None
|
| 323 |
+
- `apply_mask`: if True, the output point map will be masked using the predicted mask. Default: True
|
| 324 |
+
- `force_projection`: if True, the output point map will be recomputed to match the projection constraint. Default: True
|
| 325 |
+
- `use_fp16`: if True, use mixed precision to speed up inference. Default: True
|
| 326 |
+
|
| 327 |
+
### Returns
|
| 328 |
+
|
| 329 |
+
A dictionary containing the following keys:
|
| 330 |
+
- `points`: output tensor of shape (B, H, W, 3) or (H, W, 3).
|
| 331 |
+
- `depth`: tensor of shape (B, H, W) or (H, W) containing the depth map.
|
| 332 |
+
- `intrinsics`: tensor of shape (B, 3, 3) or (3, 3) containing the camera intrinsics.
|
| 333 |
+
"""
|
| 334 |
+
if image.dim() == 3:
|
| 335 |
+
omit_batch_dim = True
|
| 336 |
+
image = image.unsqueeze(0)
|
| 337 |
+
else:
|
| 338 |
+
omit_batch_dim = False
|
| 339 |
+
image = image.to(dtype=self.dtype, device=self.device)
|
| 340 |
+
|
| 341 |
+
original_height, original_width = image.shape[-2:]
|
| 342 |
+
aspect_ratio = original_width / original_height
|
| 343 |
+
|
| 344 |
+
if num_tokens is None:
|
| 345 |
+
min_tokens, max_tokens = self.num_tokens_range
|
| 346 |
+
num_tokens = int(min_tokens + (resolution_level / 9) * (max_tokens - min_tokens))
|
| 347 |
+
|
| 348 |
+
with torch.autocast(device_type=self.device.type, dtype=torch.float16, enabled=use_fp16 and self.dtype != torch.float16):
|
| 349 |
+
output = self.forward(image, num_tokens)
|
| 350 |
+
points, mask = output['points'], output['mask']
|
| 351 |
+
|
| 352 |
+
# Always process the output in fp32 precision
|
| 353 |
+
with torch.autocast(device_type=self.device.type, dtype=torch.float32):
|
| 354 |
+
points, mask, fov_x = map(lambda x: x.float() if isinstance(x, torch.Tensor) else x, [points, mask, fov_x])
|
| 355 |
+
|
| 356 |
+
mask_binary = mask > self.mask_threshold
|
| 357 |
+
|
| 358 |
+
# Get camera-space point map. (Focal here is the focal length relative to half the image diagonal)
|
| 359 |
+
if fov_x is None:
|
| 360 |
+
focal, shift = recover_focal_shift(points, mask_binary)
|
| 361 |
+
else:
|
| 362 |
+
focal = aspect_ratio / (1 + aspect_ratio ** 2) ** 0.5 / torch.tan(torch.deg2rad(torch.as_tensor(fov_x, device=points.device, dtype=points.dtype) / 2))
|
| 363 |
+
if focal.ndim == 0:
|
| 364 |
+
focal = focal[None].expand(points.shape[0])
|
| 365 |
+
_, shift = recover_focal_shift(points, mask_binary, focal=focal)
|
| 366 |
+
fx = focal / 2 * (1 + aspect_ratio ** 2) ** 0.5 / aspect_ratio
|
| 367 |
+
fy = focal / 2 * (1 + aspect_ratio ** 2) ** 0.5
|
| 368 |
+
intrinsics = utils3d.pt.intrinsics_from_focal_center(fx, fy, torch.tensor(0.5, device=points.device, dtype=points.dtype), torch.tensor(0.5, device=points.device, dtype=points.dtype))
|
| 369 |
+
depth = points[..., 2] + shift[..., None, None]
|
| 370 |
+
|
| 371 |
+
# If projection constraint is forced, recompute the point map using the actual depth map
|
| 372 |
+
if force_projection:
|
| 373 |
+
points = utils3d.pt.depth_map_to_point_map(depth, intrinsics=intrinsics)
|
| 374 |
+
else:
|
| 375 |
+
points = points + torch.stack([torch.zeros_like(shift), torch.zeros_like(shift), shift], dim=-1)[..., None, None, :]
|
| 376 |
+
|
| 377 |
+
# Apply mask if needed
|
| 378 |
+
if apply_mask:
|
| 379 |
+
points = torch.where(mask_binary[..., None], points, torch.inf)
|
| 380 |
+
depth = torch.where(mask_binary, depth, torch.inf)
|
| 381 |
+
|
| 382 |
+
return_dict = {
|
| 383 |
+
'points': points,
|
| 384 |
+
'intrinsics': intrinsics,
|
| 385 |
+
'depth': depth,
|
| 386 |
+
'mask': mask_binary,
|
| 387 |
+
}
|
| 388 |
+
|
| 389 |
+
if omit_batch_dim:
|
| 390 |
+
return_dict = {k: v.squeeze(0) for k, v in return_dict.items()}
|
| 391 |
+
|
| 392 |
+
return return_dict
|
third_party/MoGe/moge/model/v2.py
ADDED
|
@@ -0,0 +1,303 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import *
|
| 2 |
+
from numbers import Number
|
| 3 |
+
from functools import partial
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
import warnings
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
import torch.nn as nn
|
| 9 |
+
import torch.nn.functional as F
|
| 10 |
+
import torch.utils
|
| 11 |
+
import torch.utils.checkpoint
|
| 12 |
+
import torch.amp
|
| 13 |
+
import torch.version
|
| 14 |
+
import utils3d
|
| 15 |
+
from huggingface_hub import hf_hub_download
|
| 16 |
+
|
| 17 |
+
from ..utils.geometry_torch import normalized_view_plane_uv, recover_focal_shift, angle_diff_vec3
|
| 18 |
+
from .utils import wrap_dinov2_attention_with_sdpa, wrap_module_with_gradient_checkpointing, unwrap_module_with_gradient_checkpointing
|
| 19 |
+
from .modules import DINOv2Encoder, MLP, ConvStack
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class MoGeModel(nn.Module):
|
| 23 |
+
encoder: DINOv2Encoder
|
| 24 |
+
neck: ConvStack
|
| 25 |
+
points_head: ConvStack
|
| 26 |
+
mask_head: ConvStack
|
| 27 |
+
scale_head: MLP
|
| 28 |
+
onnx_compatible_mode: bool
|
| 29 |
+
|
| 30 |
+
def __init__(self,
|
| 31 |
+
encoder: Dict[str, Any],
|
| 32 |
+
neck: Dict[str, Any],
|
| 33 |
+
points_head: Dict[str, Any] = None,
|
| 34 |
+
mask_head: Dict[str, Any] = None,
|
| 35 |
+
normal_head: Dict[str, Any] = None,
|
| 36 |
+
scale_head: Dict[str, Any] = None,
|
| 37 |
+
remap_output: Literal['linear', 'sinh', 'exp', 'sinh_exp'] = 'linear',
|
| 38 |
+
num_tokens_range: List[int] = [1200, 3600],
|
| 39 |
+
**deprecated_kwargs
|
| 40 |
+
):
|
| 41 |
+
super(MoGeModel, self).__init__()
|
| 42 |
+
if deprecated_kwargs:
|
| 43 |
+
warnings.warn(f"The following deprecated/invalid arguments are ignored: {deprecated_kwargs}")
|
| 44 |
+
|
| 45 |
+
self.remap_output = remap_output
|
| 46 |
+
self.num_tokens_range = num_tokens_range
|
| 47 |
+
|
| 48 |
+
self.encoder = DINOv2Encoder(**encoder)
|
| 49 |
+
self.neck = ConvStack(**neck)
|
| 50 |
+
if points_head is not None:
|
| 51 |
+
self.points_head = ConvStack(**points_head)
|
| 52 |
+
if mask_head is not None:
|
| 53 |
+
self.mask_head = ConvStack(**mask_head)
|
| 54 |
+
if normal_head is not None:
|
| 55 |
+
self.normal_head = ConvStack(**normal_head)
|
| 56 |
+
if scale_head is not None:
|
| 57 |
+
self.scale_head = MLP(**scale_head)
|
| 58 |
+
|
| 59 |
+
@property
|
| 60 |
+
def device(self) -> torch.device:
|
| 61 |
+
return next(self.parameters()).device
|
| 62 |
+
|
| 63 |
+
@property
|
| 64 |
+
def dtype(self) -> torch.dtype:
|
| 65 |
+
return next(self.parameters()).dtype
|
| 66 |
+
|
| 67 |
+
@property
|
| 68 |
+
def onnx_compatible_mode(self) -> bool:
|
| 69 |
+
return getattr(self, "_onnx_compatible_mode", False)
|
| 70 |
+
|
| 71 |
+
@onnx_compatible_mode.setter
|
| 72 |
+
def onnx_compatible_mode(self, value: bool):
|
| 73 |
+
self._onnx_compatible_mode = value
|
| 74 |
+
self.encoder.onnx_compatible_mode = value
|
| 75 |
+
|
| 76 |
+
@classmethod
|
| 77 |
+
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, Path, IO[bytes]], model_kwargs: Optional[Dict[str, Any]] = None, **hf_kwargs) -> 'MoGeModel':
|
| 78 |
+
"""
|
| 79 |
+
Load a model from a checkpoint file.
|
| 80 |
+
|
| 81 |
+
### Parameters:
|
| 82 |
+
- `pretrained_model_name_or_path`: path to the checkpoint file or repo id.
|
| 83 |
+
- `compiled`
|
| 84 |
+
- `model_kwargs`: additional keyword arguments to override the parameters in the checkpoint.
|
| 85 |
+
- `hf_kwargs`: additional keyword arguments to pass to the `hf_hub_download` function. Ignored if `pretrained_model_name_or_path` is a local path.
|
| 86 |
+
|
| 87 |
+
### Returns:
|
| 88 |
+
- A new instance of `MoGe` with the parameters loaded from the checkpoint.
|
| 89 |
+
"""
|
| 90 |
+
if Path(pretrained_model_name_or_path).exists():
|
| 91 |
+
checkpoint_path = pretrained_model_name_or_path
|
| 92 |
+
else:
|
| 93 |
+
checkpoint_path = hf_hub_download(
|
| 94 |
+
repo_id=pretrained_model_name_or_path,
|
| 95 |
+
repo_type="model",
|
| 96 |
+
filename="model.pt",
|
| 97 |
+
**hf_kwargs
|
| 98 |
+
)
|
| 99 |
+
checkpoint = torch.load(checkpoint_path, map_location='cpu', weights_only=True)
|
| 100 |
+
|
| 101 |
+
model_config = checkpoint['model_config']
|
| 102 |
+
if model_kwargs is not None:
|
| 103 |
+
model_config.update(model_kwargs)
|
| 104 |
+
model = cls(**model_config)
|
| 105 |
+
model.load_state_dict(checkpoint['model'], strict=False)
|
| 106 |
+
|
| 107 |
+
return model
|
| 108 |
+
|
| 109 |
+
def init_weights(self):
|
| 110 |
+
self.encoder.init_weights()
|
| 111 |
+
|
| 112 |
+
def enable_gradient_checkpointing(self):
|
| 113 |
+
self.encoder.enable_gradient_checkpointing()
|
| 114 |
+
self.neck.enable_gradient_checkpointing()
|
| 115 |
+
for head in ['points_head', 'normal_head', 'mask_head']:
|
| 116 |
+
if hasattr(self, head):
|
| 117 |
+
getattr(self, head).enable_gradient_checkpointing()
|
| 118 |
+
|
| 119 |
+
def enable_pytorch_native_sdpa(self):
|
| 120 |
+
self.encoder.enable_pytorch_native_sdpa()
|
| 121 |
+
|
| 122 |
+
def _remap_points(self, points: torch.Tensor) -> torch.Tensor:
|
| 123 |
+
if self.remap_output == 'linear':
|
| 124 |
+
pass
|
| 125 |
+
elif self.remap_output =='sinh':
|
| 126 |
+
points = torch.sinh(points)
|
| 127 |
+
elif self.remap_output == 'exp':
|
| 128 |
+
xy, z = points.split([2, 1], dim=-1)
|
| 129 |
+
z = torch.exp(z)
|
| 130 |
+
points = torch.cat([xy * z, z], dim=-1)
|
| 131 |
+
elif self.remap_output =='sinh_exp':
|
| 132 |
+
xy, z = points.split([2, 1], dim=-1)
|
| 133 |
+
points = torch.cat([torch.sinh(xy), torch.exp(z)], dim=-1)
|
| 134 |
+
else:
|
| 135 |
+
raise ValueError(f"Invalid remap output type: {self.remap_output}")
|
| 136 |
+
return points
|
| 137 |
+
|
| 138 |
+
def forward(self, image: torch.Tensor, num_tokens: Union[int, torch.LongTensor]) -> Dict[str, torch.Tensor]:
|
| 139 |
+
batch_size, _, img_h, img_w = image.shape
|
| 140 |
+
device, dtype = image.device, image.dtype
|
| 141 |
+
|
| 142 |
+
aspect_ratio = img_w / img_h
|
| 143 |
+
base_h, base_w = (num_tokens / aspect_ratio) ** 0.5, (num_tokens * aspect_ratio) ** 0.5
|
| 144 |
+
if isinstance(base_h, torch.Tensor):
|
| 145 |
+
base_h, base_w = base_h.round().long(), base_w.round().long()
|
| 146 |
+
else:
|
| 147 |
+
base_h, base_w = round(base_h), round(base_w)
|
| 148 |
+
|
| 149 |
+
# Backbones encoding
|
| 150 |
+
features, cls_token = self.encoder(image, base_h, base_w, return_class_token=True)
|
| 151 |
+
features = [features, None, None, None, None]
|
| 152 |
+
|
| 153 |
+
# Concat UVs for aspect ratio input
|
| 154 |
+
for level in range(5):
|
| 155 |
+
uv = normalized_view_plane_uv(width=base_w * 2 ** level, height=base_h * 2 ** level, aspect_ratio=aspect_ratio, dtype=dtype, device=device)
|
| 156 |
+
uv = uv.permute(2, 0, 1).unsqueeze(0).expand(batch_size, -1, -1, -1)
|
| 157 |
+
if features[level] is None:
|
| 158 |
+
features[level] = uv
|
| 159 |
+
else:
|
| 160 |
+
features[level] = torch.concat([features[level], uv], dim=1)
|
| 161 |
+
|
| 162 |
+
# Shared neck
|
| 163 |
+
features = self.neck(features)
|
| 164 |
+
|
| 165 |
+
# Heads decoding
|
| 166 |
+
points, normal, mask = (getattr(self, head)(features)[-1] if hasattr(self, head) else None for head in ['points_head', 'normal_head', 'mask_head'])
|
| 167 |
+
metric_scale = self.scale_head(cls_token) if hasattr(self, 'scale_head') else None
|
| 168 |
+
|
| 169 |
+
# Resize
|
| 170 |
+
points, normal, mask = (F.interpolate(v, (img_h, img_w), mode='bilinear', align_corners=False, antialias=False) if v is not None else None for v in [points, normal, mask])
|
| 171 |
+
|
| 172 |
+
# Remap output
|
| 173 |
+
if points is not None:
|
| 174 |
+
points = points.permute(0, 2, 3, 1)
|
| 175 |
+
points = self._remap_points(points) # slightly improves the performance in case of very large output values
|
| 176 |
+
if normal is not None:
|
| 177 |
+
normal = normal.permute(0, 2, 3, 1)
|
| 178 |
+
normal = F.normalize(normal, dim=-1)
|
| 179 |
+
if mask is not None:
|
| 180 |
+
mask = mask.squeeze(1).sigmoid()
|
| 181 |
+
if metric_scale is not None:
|
| 182 |
+
metric_scale = metric_scale.squeeze(1).exp()
|
| 183 |
+
|
| 184 |
+
return_dict = {
|
| 185 |
+
'points': points,
|
| 186 |
+
'normal': normal,
|
| 187 |
+
'mask': mask,
|
| 188 |
+
'metric_scale': metric_scale
|
| 189 |
+
}
|
| 190 |
+
return_dict = {k: v for k, v in return_dict.items() if v is not None}
|
| 191 |
+
|
| 192 |
+
return return_dict
|
| 193 |
+
|
| 194 |
+
@torch.inference_mode()
|
| 195 |
+
def infer(
|
| 196 |
+
self,
|
| 197 |
+
image: torch.Tensor,
|
| 198 |
+
num_tokens: int = None,
|
| 199 |
+
resolution_level: int = 9,
|
| 200 |
+
force_projection: bool = True,
|
| 201 |
+
apply_mask: bool = True,
|
| 202 |
+
fov_x: Optional[Union[Number, torch.Tensor]] = None,
|
| 203 |
+
use_fp16: bool = True,
|
| 204 |
+
) -> Dict[str, torch.Tensor]:
|
| 205 |
+
"""
|
| 206 |
+
User-friendly inference function
|
| 207 |
+
|
| 208 |
+
### Parameters
|
| 209 |
+
- `image`: input image tensor of shape (B, 3, H, W) or (3, H, W)
|
| 210 |
+
- `num_tokens`: the number of base ViT tokens to use for inference, `'least'` or `'most'` or an integer. Suggested range: 1200 ~ 2500.
|
| 211 |
+
More tokens will result in significantly higher accuracy and finer details, but slower inference time. Default: `'most'`.
|
| 212 |
+
- `force_projection`: if True, the output point map will be computed using the actual depth map. Default: True
|
| 213 |
+
- `apply_mask`: if True, the output point map will be masked using the predicted mask. Default: True
|
| 214 |
+
- `fov_x`: the horizontal camera FoV in degrees. If None, it will be inferred from the predicted point map. Default: None
|
| 215 |
+
- `use_fp16`: if True, use mixed precision to speed up inference. Default: True
|
| 216 |
+
|
| 217 |
+
### Returns
|
| 218 |
+
|
| 219 |
+
A dictionary containing the following keys:
|
| 220 |
+
- `points`: output tensor of shape (B, H, W, 3) or (H, W, 3).
|
| 221 |
+
- `depth`: tensor of shape (B, H, W) or (H, W) containing the depth map.
|
| 222 |
+
- `intrinsics`: tensor of shape (B, 3, 3) or (3, 3) containing the camera intrinsics.
|
| 223 |
+
"""
|
| 224 |
+
if image.dim() == 3:
|
| 225 |
+
omit_batch_dim = True
|
| 226 |
+
image = image.unsqueeze(0)
|
| 227 |
+
else:
|
| 228 |
+
omit_batch_dim = False
|
| 229 |
+
image = image.to(dtype=self.dtype, device=self.device)
|
| 230 |
+
|
| 231 |
+
original_height, original_width = image.shape[-2:]
|
| 232 |
+
area = original_height * original_width
|
| 233 |
+
aspect_ratio = original_width / original_height
|
| 234 |
+
|
| 235 |
+
# Determine the number of base tokens to use
|
| 236 |
+
if num_tokens is None:
|
| 237 |
+
min_tokens, max_tokens = self.num_tokens_range
|
| 238 |
+
num_tokens = int(min_tokens + (resolution_level / 9) * (max_tokens - min_tokens))
|
| 239 |
+
|
| 240 |
+
# Forward pass
|
| 241 |
+
with torch.autocast(device_type=self.device.type, dtype=torch.float16, enabled=use_fp16 and self.dtype != torch.float16):
|
| 242 |
+
output = self.forward(image, num_tokens=num_tokens)
|
| 243 |
+
points, normal, mask, metric_scale = (output.get(k, None) for k in ['points', 'normal', 'mask', 'metric_scale'])
|
| 244 |
+
|
| 245 |
+
# Always process the output in fp32 precision
|
| 246 |
+
points, normal, mask, metric_scale, fov_x = map(lambda x: x.float() if isinstance(x, torch.Tensor) else x, [points, normal, mask, metric_scale, fov_x])
|
| 247 |
+
with torch.autocast(device_type=self.device.type, dtype=torch.float32):
|
| 248 |
+
if mask is not None:
|
| 249 |
+
mask_binary = mask > 0.5
|
| 250 |
+
else:
|
| 251 |
+
mask_binary = None
|
| 252 |
+
|
| 253 |
+
if points is not None:
|
| 254 |
+
# Convert affine point map to camera-space. Recover depth and intrinsics from point map.
|
| 255 |
+
# NOTE: Focal here is the focal length relative to half the image diagonal
|
| 256 |
+
if fov_x is None:
|
| 257 |
+
# Recover focal and shift from predicted point map
|
| 258 |
+
focal, shift = recover_focal_shift(points, mask_binary)
|
| 259 |
+
else:
|
| 260 |
+
# Focal is known, recover shift only
|
| 261 |
+
focal = aspect_ratio / (1 + aspect_ratio ** 2) ** 0.5 / torch.tan(torch.deg2rad(torch.as_tensor(fov_x, device=points.device, dtype=points.dtype) / 2))
|
| 262 |
+
if focal.ndim == 0:
|
| 263 |
+
focal = focal[None].expand(points.shape[0])
|
| 264 |
+
_, shift = recover_focal_shift(points, mask_binary, focal=focal)
|
| 265 |
+
fx, fy = focal / 2 * (1 + aspect_ratio ** 2) ** 0.5 / aspect_ratio, focal / 2 * (1 + aspect_ratio ** 2) ** 0.5
|
| 266 |
+
intrinsics = utils3d.pt.intrinsics_from_focal_center(fx, fy, torch.tensor(0.5, device=points.device, dtype=points.dtype), torch.tensor(0.5, device=points.device, dtype=points.dtype))
|
| 267 |
+
points[..., 2] += shift[..., None, None]
|
| 268 |
+
if mask_binary is not None:
|
| 269 |
+
mask_binary &= points[..., 2] > 0 # in case depth is contains negative values (which should never happen in practice)
|
| 270 |
+
depth = points[..., 2].clone()
|
| 271 |
+
else:
|
| 272 |
+
depth, intrinsics = None, None
|
| 273 |
+
|
| 274 |
+
# If projection constraint is forced, recompute the point map using the actual depth map & intrinsics
|
| 275 |
+
if force_projection and depth is not None:
|
| 276 |
+
points = utils3d.pt.depth_map_to_point_map(depth, intrinsics=intrinsics)
|
| 277 |
+
|
| 278 |
+
# Apply metric scale
|
| 279 |
+
if metric_scale is not None:
|
| 280 |
+
if points is not None:
|
| 281 |
+
points *= metric_scale[:, None, None, None]
|
| 282 |
+
if depth is not None:
|
| 283 |
+
depth *= metric_scale[:, None, None]
|
| 284 |
+
|
| 285 |
+
# Apply mask
|
| 286 |
+
if apply_mask and mask_binary is not None:
|
| 287 |
+
points = torch.where(mask_binary[..., None], points, torch.inf) if points is not None else None
|
| 288 |
+
depth = torch.where(mask_binary, depth, torch.inf) if depth is not None else None
|
| 289 |
+
normal = torch.where(mask_binary[..., None], normal, torch.zeros_like(normal)) if normal is not None else None
|
| 290 |
+
|
| 291 |
+
return_dict = {
|
| 292 |
+
'points': points,
|
| 293 |
+
'intrinsics': intrinsics,
|
| 294 |
+
'depth': depth,
|
| 295 |
+
'mask': mask_binary,
|
| 296 |
+
'normal': normal
|
| 297 |
+
}
|
| 298 |
+
return_dict = {k: v for k, v in return_dict.items() if v is not None}
|
| 299 |
+
|
| 300 |
+
if omit_batch_dim:
|
| 301 |
+
return_dict = {k: v.squeeze(0) for k, v in return_dict.items()}
|
| 302 |
+
|
| 303 |
+
return return_dict
|
third_party/MoGe/moge/utils/__init__.py
ADDED
|
File without changes
|
third_party/MoGe/moge/utils/alignment.py
ADDED
|
@@ -0,0 +1,416 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import *
|
| 2 |
+
import math
|
| 3 |
+
from collections import namedtuple
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
import torch
|
| 7 |
+
import torch.nn as nn
|
| 8 |
+
import torch.nn.functional as F
|
| 9 |
+
import torch.types
|
| 10 |
+
import utils3d
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def scatter_min(size: int, dim: int, index: torch.LongTensor, src: torch.Tensor) -> torch.return_types.min:
|
| 14 |
+
"Scatter the minimum value along the given dimension of `input` into `src` at the indices specified in `index`."
|
| 15 |
+
shape = src.shape[:dim] + (size,) + src.shape[dim + 1:]
|
| 16 |
+
minimum = torch.full(shape, float('inf'), dtype=src.dtype, device=src.device).scatter_reduce(dim=dim, index=index, src=src, reduce='amin', include_self=False)
|
| 17 |
+
minimum_where = torch.where(src == torch.gather(minimum, dim=dim, index=index))
|
| 18 |
+
indices = torch.full(shape, -1, dtype=torch.long, device=src.device)
|
| 19 |
+
indices[(*minimum_where[:dim], index[minimum_where], *minimum_where[dim + 1:])] = minimum_where[dim]
|
| 20 |
+
return torch.return_types.min((minimum, indices))
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def split_batch_fwd(fn: Callable, chunk_size: int, *args, **kwargs):
|
| 24 |
+
batch_size = next(x for x in (*args, *kwargs.values()) if isinstance(x, torch.Tensor)).shape[0]
|
| 25 |
+
n_chunks = batch_size // chunk_size + (batch_size % chunk_size > 0)
|
| 26 |
+
splited_args = tuple(arg.split(chunk_size, dim=0) if isinstance(arg, torch.Tensor) else [arg] * n_chunks for arg in args)
|
| 27 |
+
splited_kwargs = {k: [v.split(chunk_size, dim=0) if isinstance(v, torch.Tensor) else [v] * n_chunks] for k, v in kwargs.items()}
|
| 28 |
+
results = []
|
| 29 |
+
for i in range(n_chunks):
|
| 30 |
+
chunk_args = tuple(arg[i] for arg in splited_args)
|
| 31 |
+
chunk_kwargs = {k: v[i] for k, v in splited_kwargs.items()}
|
| 32 |
+
results.append(fn(*chunk_args, **chunk_kwargs))
|
| 33 |
+
|
| 34 |
+
if isinstance(results[0], tuple):
|
| 35 |
+
return tuple(torch.cat(r, dim=0) for r in zip(*results))
|
| 36 |
+
else:
|
| 37 |
+
return torch.cat(results, dim=0)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _pad_inf(x_: torch.Tensor):
|
| 41 |
+
return torch.cat([torch.full_like(x_[..., :1], -torch.inf), x_, torch.full_like(x_[..., :1], torch.inf)], dim=-1)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _pad_cumsum(cumsum: torch.Tensor):
|
| 45 |
+
return torch.cat([torch.zeros_like(cumsum[..., :1]), cumsum, cumsum[..., -1:]], dim=-1)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _compute_residual(a: torch.Tensor, xyw: torch.Tensor, trunc: float):
|
| 49 |
+
return a.mul(xyw[..., 0]).sub_(xyw[..., 1]).abs_().mul_(xyw[..., 2]).clamp_max_(trunc).sum(dim=-1)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def align(x: torch.Tensor, y: torch.Tensor, w: torch.Tensor, trunc: Optional[Union[float, torch.Tensor]] = None, eps: float = 1e-7) -> Tuple[torch.Tensor, torch.Tensor, torch.LongTensor]:
|
| 53 |
+
"""
|
| 54 |
+
If trunc is None, solve `min sum_i w_i * |a * x_i - y_i|`, otherwise solve `min sum_i min(trunc, w_i * |a * x_i - y_i|)`.
|
| 55 |
+
|
| 56 |
+
w_i must be >= 0.
|
| 57 |
+
|
| 58 |
+
### Parameters:
|
| 59 |
+
- `x`: tensor of shape (..., n)
|
| 60 |
+
- `y`: tensor of shape (..., n)
|
| 61 |
+
- `w`: tensor of shape (..., n)
|
| 62 |
+
- `trunc`: optional, float or tensor of shape (..., n) or None
|
| 63 |
+
|
| 64 |
+
### Returns:
|
| 65 |
+
- `a`: tensor of shape (...), differentiable
|
| 66 |
+
- `loss`: tensor of shape (...), value of loss function at `a`, detached
|
| 67 |
+
- `index`: tensor of shape (...), where a = y[idx] / x[idx]
|
| 68 |
+
"""
|
| 69 |
+
if trunc is None:
|
| 70 |
+
x, y, w = torch.broadcast_tensors(x, y, w)
|
| 71 |
+
sign = torch.sign(x)
|
| 72 |
+
x, y = x * sign, y * sign
|
| 73 |
+
y_div_x = y / x.clamp_min(eps)
|
| 74 |
+
y_div_x, argsort = y_div_x.sort(dim=-1)
|
| 75 |
+
|
| 76 |
+
wx = torch.gather(x * w, dim=-1, index=argsort)
|
| 77 |
+
derivatives = 2 * wx.cumsum(dim=-1) - wx.sum(dim=-1, keepdim=True)
|
| 78 |
+
search = torch.searchsorted(derivatives, torch.zeros_like(derivatives[..., :1]), side='left').clamp_max(derivatives.shape[-1] - 1)
|
| 79 |
+
|
| 80 |
+
a = y_div_x.gather(dim=-1, index=search).squeeze(-1)
|
| 81 |
+
index = argsort.gather(dim=-1, index=search).squeeze(-1)
|
| 82 |
+
loss = (w * (a[..., None] * x - y).abs()).sum(dim=-1)
|
| 83 |
+
|
| 84 |
+
else:
|
| 85 |
+
# Reshape to (batch_size, n) for simplicity
|
| 86 |
+
x, y, w = torch.broadcast_tensors(x, y, w)
|
| 87 |
+
batch_shape = x.shape[:-1]
|
| 88 |
+
batch_size = math.prod(batch_shape)
|
| 89 |
+
x, y, w = x.reshape(-1, x.shape[-1]), y.reshape(-1, y.shape[-1]), w.reshape(-1, w.shape[-1])
|
| 90 |
+
|
| 91 |
+
sign = torch.sign(x)
|
| 92 |
+
x, y = x * sign, y * sign
|
| 93 |
+
wx, wy = w * x, w * y
|
| 94 |
+
xyw = torch.stack([x, y, w], dim=-1) # Stacked for convenient gathering
|
| 95 |
+
|
| 96 |
+
y_div_x = A = y / x.clamp_min(eps)
|
| 97 |
+
B = (wy - trunc) / wx.clamp_min(eps)
|
| 98 |
+
C = (wy + trunc) / wx.clamp_min(eps)
|
| 99 |
+
with torch.no_grad():
|
| 100 |
+
# Caculate prefix sum by orders of A, B, C
|
| 101 |
+
A, A_argsort = A.sort(dim=-1)
|
| 102 |
+
Q_A = torch.cumsum(torch.gather(wx, dim=-1, index=A_argsort), dim=-1)
|
| 103 |
+
A, Q_A = _pad_inf(A), _pad_cumsum(Q_A) # Pad [-inf, A1, ..., An, inf] and [0, Q1, ..., Qn, Qn] to handle edge cases.
|
| 104 |
+
|
| 105 |
+
B, B_argsort = B.sort(dim=-1)
|
| 106 |
+
Q_B = torch.cumsum(torch.gather(wx, dim=-1, index=B_argsort), dim=-1)
|
| 107 |
+
B, Q_B = _pad_inf(B), _pad_cumsum(Q_B)
|
| 108 |
+
|
| 109 |
+
C, C_argsort = C.sort(dim=-1)
|
| 110 |
+
Q_C = torch.cumsum(torch.gather(wx, dim=-1, index=C_argsort), dim=-1)
|
| 111 |
+
C, Q_C = _pad_inf(C), _pad_cumsum(Q_C)
|
| 112 |
+
|
| 113 |
+
# Caculate left and right derivative of A
|
| 114 |
+
j_A = torch.searchsorted(A, y_div_x, side='left').sub_(1)
|
| 115 |
+
j_B = torch.searchsorted(B, y_div_x, side='left').sub_(1)
|
| 116 |
+
j_C = torch.searchsorted(C, y_div_x, side='left').sub_(1)
|
| 117 |
+
left_derivative = 2 * torch.gather(Q_A, dim=-1, index=j_A) - torch.gather(Q_B, dim=-1, index=j_B) - torch.gather(Q_C, dim=-1, index=j_C)
|
| 118 |
+
j_A = torch.searchsorted(A, y_div_x, side='right').sub_(1)
|
| 119 |
+
j_B = torch.searchsorted(B, y_div_x, side='right').sub_(1)
|
| 120 |
+
j_C = torch.searchsorted(C, y_div_x, side='right').sub_(1)
|
| 121 |
+
right_derivative = 2 * torch.gather(Q_A, dim=-1, index=j_A) - torch.gather(Q_B, dim=-1, index=j_B) - torch.gather(Q_C, dim=-1, index=j_C)
|
| 122 |
+
|
| 123 |
+
# Find extrema
|
| 124 |
+
is_extrema = (left_derivative < 0) & (right_derivative >= 0)
|
| 125 |
+
is_extrema[..., 0] |= ~is_extrema.any(dim=-1) # In case all derivatives are zero, take the first one as extrema.
|
| 126 |
+
where_extrema_batch, where_extrema_index = torch.where(is_extrema)
|
| 127 |
+
|
| 128 |
+
# Calculate objective value at extrema
|
| 129 |
+
extrema_a = y_div_x[where_extrema_batch, where_extrema_index] # (num_extrema,)
|
| 130 |
+
MAX_ELEMENTS = 4096 ** 2 # Split into small batches to avoid OOM in case there are too many extrema.(~1G)
|
| 131 |
+
SPLIT_SIZE = MAX_ELEMENTS // x.shape[-1]
|
| 132 |
+
extrema_value = torch.cat([
|
| 133 |
+
_compute_residual(extrema_a_split[:, None], xyw[extrema_i_split, :, :], trunc)
|
| 134 |
+
for extrema_a_split, extrema_i_split in zip(extrema_a.split(SPLIT_SIZE), where_extrema_batch.split(SPLIT_SIZE))
|
| 135 |
+
]) # (num_extrema,)
|
| 136 |
+
|
| 137 |
+
# Find minima among corresponding extrema
|
| 138 |
+
minima, indices = scatter_min(size=batch_size, dim=0, index=where_extrema_batch, src=extrema_value) # (batch_size,)
|
| 139 |
+
index = where_extrema_index[indices]
|
| 140 |
+
|
| 141 |
+
a = torch.gather(y, dim=-1, index=index[..., None]) / torch.gather(x, dim=-1, index=index[..., None]).clamp_min(eps)
|
| 142 |
+
a = a.reshape(batch_shape)
|
| 143 |
+
loss = minima.reshape(batch_shape)
|
| 144 |
+
index = index.reshape(batch_shape)
|
| 145 |
+
|
| 146 |
+
return a, loss, index
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def align_depth_scale(depth_src: torch.Tensor, depth_tgt: torch.Tensor, weight: Optional[torch.Tensor], trunc: Optional[Union[float, torch.Tensor]] = None):
|
| 150 |
+
"""
|
| 151 |
+
Align `depth_src` to `depth_tgt` with given constant weights.
|
| 152 |
+
|
| 153 |
+
### Parameters:
|
| 154 |
+
- `depth_src: torch.Tensor` of shape (..., N)
|
| 155 |
+
- `depth_tgt: torch.Tensor` of shape (..., N)
|
| 156 |
+
|
| 157 |
+
"""
|
| 158 |
+
scale, _, _ = align(depth_src, depth_tgt, weight, trunc)
|
| 159 |
+
|
| 160 |
+
return scale
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def align_depth_affine(depth_src: torch.Tensor, depth_tgt: torch.Tensor, weight: Optional[torch.Tensor], trunc: Optional[Union[float, torch.Tensor]] = None):
|
| 164 |
+
"""
|
| 165 |
+
Align `depth_src` to `depth_tgt` with given constant weights.
|
| 166 |
+
|
| 167 |
+
### Parameters:
|
| 168 |
+
- `depth_src: torch.Tensor` of shape (..., N)
|
| 169 |
+
- `depth_tgt: torch.Tensor` of shape (..., N)
|
| 170 |
+
- `weight: torch.Tensor` of shape (..., N)
|
| 171 |
+
- `trunc: float` or tensor of shape (..., N) or None
|
| 172 |
+
|
| 173 |
+
### Returns:
|
| 174 |
+
- `scale: torch.Tensor` of shape (...).
|
| 175 |
+
- `shift: torch.Tensor` of shape (...).
|
| 176 |
+
"""
|
| 177 |
+
dtype, device = depth_src.dtype, depth_src.device
|
| 178 |
+
|
| 179 |
+
# Flatten batch dimensions for simplicity
|
| 180 |
+
batch_shape, n = depth_src.shape[:-1], depth_src.shape[-1]
|
| 181 |
+
batch_size = math.prod(batch_shape)
|
| 182 |
+
depth_src, depth_tgt, weight = depth_src.reshape(batch_size, n), depth_tgt.reshape(batch_size, n), weight.reshape(batch_size, n)
|
| 183 |
+
|
| 184 |
+
# Here, we take anchors only for non-zero weights.
|
| 185 |
+
# Although the results will be still correct even anchor points have zero weight,
|
| 186 |
+
# it is wasting computation and may cause instability in some cases, e.g. too many extrema.
|
| 187 |
+
anchors_where_batch, anchors_where_n = torch.where(weight > 0)
|
| 188 |
+
|
| 189 |
+
# Stop gradient when solving optimal anchors
|
| 190 |
+
with torch.no_grad():
|
| 191 |
+
depth_src_anchor = depth_src[anchors_where_batch, anchors_where_n] # (anchors)
|
| 192 |
+
depth_tgt_anchor = depth_tgt[anchors_where_batch, anchors_where_n] # (anchors)
|
| 193 |
+
|
| 194 |
+
depth_src_anchored = depth_src[anchors_where_batch, :] - depth_src_anchor[..., None] # (anchors, n)
|
| 195 |
+
depth_tgt_anchored = depth_tgt[anchors_where_batch, :] - depth_tgt_anchor[..., None] # (anchors, n)
|
| 196 |
+
weight_anchored = weight[anchors_where_batch, :] # (anchors, n)
|
| 197 |
+
|
| 198 |
+
scale, loss, index = align(depth_src_anchored, depth_tgt_anchored, weight_anchored, trunc) # (anchors)
|
| 199 |
+
|
| 200 |
+
loss, index_anchor = scatter_min(size=batch_size, dim=0, index=anchors_where_batch, src=loss) # (batch_size,)
|
| 201 |
+
|
| 202 |
+
# Reproduce by indexing for shorter compute graph
|
| 203 |
+
index_1 = anchors_where_n[index_anchor] # (batch_size,)
|
| 204 |
+
index_2 = index[index_anchor] # (batch_size,)
|
| 205 |
+
|
| 206 |
+
tgt_1, src_1 = torch.gather(depth_tgt, dim=1, index=index_1[..., None]).squeeze(-1), torch.gather(depth_src, dim=1, index=index_1[..., None]).squeeze(-1)
|
| 207 |
+
tgt_2, src_2 = torch.gather(depth_tgt, dim=1, index=index_2[..., None]).squeeze(-1), torch.gather(depth_src, dim=1, index=index_2[..., None]).squeeze(-1)
|
| 208 |
+
|
| 209 |
+
scale = (tgt_2 - tgt_1) / torch.where(src_2 != src_1, src_2 - src_1, 1e-7)
|
| 210 |
+
shift = tgt_1 - scale * src_1
|
| 211 |
+
|
| 212 |
+
scale, shift = scale.reshape(batch_shape), shift.reshape(batch_shape)
|
| 213 |
+
|
| 214 |
+
return scale, shift
|
| 215 |
+
|
| 216 |
+
def align_depth_affine_irls(depth_src: torch.Tensor, depth_tgt: torch.Tensor, weight: Optional[torch.Tensor], max_iter: int = 100, eps: float = 1e-12):
|
| 217 |
+
"""
|
| 218 |
+
Align `depth_src` to `depth_tgt` with given constant weights using IRLS.
|
| 219 |
+
"""
|
| 220 |
+
dtype, device = depth_src.dtype, depth_src.device
|
| 221 |
+
|
| 222 |
+
w = weight
|
| 223 |
+
x = torch.stack([depth_src, torch.ones_like(depth_src)], dim=-1)
|
| 224 |
+
y = depth_tgt
|
| 225 |
+
|
| 226 |
+
for i in range(max_iter):
|
| 227 |
+
beta = (x.transpose(-1, -2) @ (w * y)) @ (x.transpose(-1, -2) @ (w[..., None] * x)).inverse().transpose(-2, -1)
|
| 228 |
+
w = 1 / (y - (x @ beta[..., None])[..., 0]).abs().clamp_min(eps)
|
| 229 |
+
|
| 230 |
+
return beta[..., 0], beta[..., 1]
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def align_points_scale(points_src: torch.Tensor, points_tgt: torch.Tensor, weight: Optional[torch.Tensor], trunc: Optional[Union[float, torch.Tensor]] = None):
|
| 234 |
+
"""
|
| 235 |
+
### Parameters:
|
| 236 |
+
- `points_src: torch.Tensor` of shape (..., N, 3)
|
| 237 |
+
- `points_tgt: torch.Tensor` of shape (..., N, 3)
|
| 238 |
+
- `weight: torch.Tensor` of shape (..., N)
|
| 239 |
+
|
| 240 |
+
### Returns:
|
| 241 |
+
- `a: torch.Tensor` of shape (...). Only positive solutions are garunteed. You should filter out negative scales before using it.
|
| 242 |
+
- `b: torch.Tensor` of shape (...)
|
| 243 |
+
"""
|
| 244 |
+
dtype, device = points_src.dtype, points_src.device
|
| 245 |
+
|
| 246 |
+
scale, _, _ = align(points_src.flatten(-2), points_tgt.flatten(-2), weight[..., None].expand_as(points_src).flatten(-2), trunc)
|
| 247 |
+
|
| 248 |
+
return scale
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def align_points_scale_z_shift(points_src: torch.Tensor, points_tgt: torch.Tensor, weight: Optional[torch.Tensor], trunc: Optional[Union[float, torch.Tensor]] = None):
|
| 252 |
+
"""
|
| 253 |
+
Align `points_src` to `points_tgt` with respect to a shared xyz scale and z shift.
|
| 254 |
+
It is similar to `align_affine` but scale and shift are applied to different dimensions.
|
| 255 |
+
|
| 256 |
+
### Parameters:
|
| 257 |
+
- `points_src: torch.Tensor` of shape (..., N, 3)
|
| 258 |
+
- `points_tgt: torch.Tensor` of shape (..., N, 3)
|
| 259 |
+
- `weights: torch.Tensor` of shape (..., N)
|
| 260 |
+
|
| 261 |
+
### Returns:
|
| 262 |
+
- `scale: torch.Tensor` of shape (...).
|
| 263 |
+
- `shift: torch.Tensor` of shape (..., 3). x and y shifts are zeros.
|
| 264 |
+
"""
|
| 265 |
+
dtype, device = points_src.dtype, points_src.device
|
| 266 |
+
|
| 267 |
+
# Flatten batch dimensions for simplicity
|
| 268 |
+
batch_shape, n = points_src.shape[:-2], points_src.shape[-2]
|
| 269 |
+
batch_size = math.prod(batch_shape)
|
| 270 |
+
points_src, points_tgt, weight = points_src.reshape(batch_size, n, 3), points_tgt.reshape(batch_size, n, 3), weight.reshape(batch_size, n)
|
| 271 |
+
|
| 272 |
+
# Take anchors
|
| 273 |
+
anchor_where_batch, anchor_where_n = torch.where(weight > 0)
|
| 274 |
+
with torch.no_grad():
|
| 275 |
+
zeros = torch.zeros(anchor_where_batch.shape[0], device=device, dtype=dtype)
|
| 276 |
+
points_src_anchor = torch.stack([zeros, zeros, points_src[anchor_where_batch, anchor_where_n, 2]], dim=-1) # (anchors, 3)
|
| 277 |
+
points_tgt_anchor = torch.stack([zeros, zeros, points_tgt[anchor_where_batch, anchor_where_n, 2]], dim=-1) # (anchors, 3)
|
| 278 |
+
|
| 279 |
+
points_src_anchored = points_src[anchor_where_batch, :, :] - points_src_anchor[..., None, :] # (anchors, n, 3)
|
| 280 |
+
points_tgt_anchored = points_tgt[anchor_where_batch, :, :] - points_tgt_anchor[..., None, :] # (anchors, n, 3)
|
| 281 |
+
weight_anchored = weight[anchor_where_batch, :, None].expand(-1, -1, 3) # (anchors, n, 3)
|
| 282 |
+
|
| 283 |
+
# Solve optimal scale and shift for each anchor
|
| 284 |
+
MAX_ELEMENTS = 2 ** 20
|
| 285 |
+
scale, loss, index = split_batch_fwd(align, MAX_ELEMENTS // n, points_src_anchored.flatten(-2), points_tgt_anchored.flatten(-2), weight_anchored.flatten(-2), trunc) # (anchors,)
|
| 286 |
+
|
| 287 |
+
loss, index_anchor = scatter_min(size=batch_size, dim=0, index=anchor_where_batch, src=loss) # (batch_size,)
|
| 288 |
+
|
| 289 |
+
# Reproduce by indexing for shorter compute graph
|
| 290 |
+
index_2 = index[index_anchor] # (batch_size,) [0, 3n)
|
| 291 |
+
index_1 = anchor_where_n[index_anchor] * 3 + index_2 % 3 # (batch_size,) [0, 3n)
|
| 292 |
+
|
| 293 |
+
zeros = torch.zeros((batch_size, n), device=device, dtype=dtype)
|
| 294 |
+
points_tgt_00z, points_src_00z = torch.stack([zeros, zeros, points_tgt[..., 2]], dim=-1), torch.stack([zeros, zeros, points_src[..., 2]], dim=-1)
|
| 295 |
+
tgt_1, src_1 = torch.gather(points_tgt_00z.flatten(-2), dim=1, index=index_1[..., None]).squeeze(-1), torch.gather(points_src_00z.flatten(-2), dim=1, index=index_1[..., None]).squeeze(-1)
|
| 296 |
+
tgt_2, src_2 = torch.gather(points_tgt.flatten(-2), dim=1, index=index_2[..., None]).squeeze(-1), torch.gather(points_src.flatten(-2), dim=1, index=index_2[..., None]).squeeze(-1)
|
| 297 |
+
|
| 298 |
+
scale = (tgt_2 - tgt_1) / torch.where(src_2 != src_1, src_2 - src_1, 1.0)
|
| 299 |
+
shift = torch.gather(points_tgt_00z, dim=1, index=(index_1 // 3)[..., None, None].expand(-1, -1, 3)).squeeze(-2) - scale[..., None] * torch.gather(points_src_00z, dim=1, index=(index_1 // 3)[..., None, None].expand(-1, -1, 3)).squeeze(-2)
|
| 300 |
+
scale, shift = scale.reshape(batch_shape), shift.reshape(*batch_shape, 3)
|
| 301 |
+
|
| 302 |
+
return scale, shift
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
def align_points_scale_xyz_shift(points_src: torch.Tensor, points_tgt: torch.Tensor, weight: Optional[torch.Tensor], trunc: Optional[Union[float, torch.Tensor]] = None, max_iters: int = 30, eps: float = 1e-6):
|
| 306 |
+
"""
|
| 307 |
+
Align `points_src` to `points_tgt` with respect to a shared xyz scale and z shift.
|
| 308 |
+
It is similar to `align_affine` but scale and shift are applied to different dimensions.
|
| 309 |
+
|
| 310 |
+
### Parameters:
|
| 311 |
+
- `points_src: torch.Tensor` of shape (..., N, 3)
|
| 312 |
+
- `points_tgt: torch.Tensor` of shape (..., N, 3)
|
| 313 |
+
- `weights: torch.Tensor` of shape (..., N)
|
| 314 |
+
|
| 315 |
+
### Returns:
|
| 316 |
+
- `scale: torch.Tensor` of shape (...).
|
| 317 |
+
- `shift: torch.Tensor` of shape (..., 3)
|
| 318 |
+
"""
|
| 319 |
+
dtype, device = points_src.dtype, points_src.device
|
| 320 |
+
|
| 321 |
+
# Flatten batch dimensions for simplicity
|
| 322 |
+
batch_shape, n = points_src.shape[:-2], points_src.shape[-2]
|
| 323 |
+
batch_size = math.prod(batch_shape)
|
| 324 |
+
points_src, points_tgt, weight = points_src.reshape(batch_size, n, 3), points_tgt.reshape(batch_size, n, 3), weight.reshape(batch_size, n)
|
| 325 |
+
|
| 326 |
+
# Take anchors
|
| 327 |
+
anchor_where_batch, anchor_where_n = torch.where(weight > 0)
|
| 328 |
+
|
| 329 |
+
with torch.no_grad():
|
| 330 |
+
points_src_anchor = points_src[anchor_where_batch, anchor_where_n] # (anchors, 3)
|
| 331 |
+
points_tgt_anchor = points_tgt[anchor_where_batch, anchor_where_n] # (anchors, 3)
|
| 332 |
+
|
| 333 |
+
points_src_anchored = points_src[anchor_where_batch, :, :] - points_src_anchor[..., None, :] # (anchors, n, 3)
|
| 334 |
+
points_tgt_anchored = points_tgt[anchor_where_batch, :, :] - points_tgt_anchor[..., None, :] # (anchors, n, 3)
|
| 335 |
+
weight_anchored = weight[anchor_where_batch, :, None].expand(-1, -1, 3) # (anchors, n, 3)
|
| 336 |
+
|
| 337 |
+
# Solve optimal scale and shift for each anchor
|
| 338 |
+
MAX_ELEMENTS = 2 ** 20
|
| 339 |
+
scale, loss, index = split_batch_fwd(align, MAX_ELEMENTS // 2, points_src_anchored.flatten(-2), points_tgt_anchored.flatten(-2), weight_anchored.flatten(-2), trunc) # (anchors,)
|
| 340 |
+
|
| 341 |
+
# Get optimal scale and shift for each batch element
|
| 342 |
+
loss, index_anchor = scatter_min(size=batch_size, dim=0, index=anchor_where_batch, src=loss) # (batch_size,)
|
| 343 |
+
|
| 344 |
+
index_2 = index[index_anchor] # (batch_size,) [0, 3n)
|
| 345 |
+
index_1 = anchor_where_n[index_anchor] * 3 + index_2 % 3 # (batch_size,) [0, 3n)
|
| 346 |
+
|
| 347 |
+
src_1, tgt_1 = torch.gather(points_src.flatten(-2), dim=1, index=index_1[..., None]).squeeze(-1), torch.gather(points_tgt.flatten(-2), dim=1, index=index_1[..., None]).squeeze(-1)
|
| 348 |
+
src_2, tgt_2 = torch.gather(points_src.flatten(-2), dim=1, index=index_2[..., None]).squeeze(-1), torch.gather(points_tgt.flatten(-2), dim=1, index=index_2[..., None]).squeeze(-1)
|
| 349 |
+
|
| 350 |
+
scale = (tgt_2 - tgt_1) / torch.where(src_2 != src_1, src_2 - src_1, 1.0)
|
| 351 |
+
shift = torch.gather(points_tgt, dim=1, index=(index_1 // 3)[..., None, None].expand(-1, -1, 3)).squeeze(-2) - scale[..., None] * torch.gather(points_src, dim=1, index=(index_1 // 3)[..., None, None].expand(-1, -1, 3)).squeeze(-2)
|
| 352 |
+
|
| 353 |
+
scale, shift = scale.reshape(batch_shape), shift.reshape(*batch_shape, 3)
|
| 354 |
+
|
| 355 |
+
return scale, shift
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
def align_points_z_shift(points_src: torch.Tensor, points_tgt: torch.Tensor, weight: Optional[torch.Tensor], trunc: Optional[Union[float, torch.Tensor]] = None, max_iters: int = 30, eps: float = 1e-6):
|
| 359 |
+
"""
|
| 360 |
+
Align `points_src` to `points_tgt` with respect to a Z-axis shift.
|
| 361 |
+
|
| 362 |
+
### Parameters:
|
| 363 |
+
- `points_src: torch.Tensor` of shape (..., N, 3)
|
| 364 |
+
- `points_tgt: torch.Tensor` of shape (..., N, 3)
|
| 365 |
+
- `weights: torch.Tensor` of shape (..., N)
|
| 366 |
+
|
| 367 |
+
### Returns:
|
| 368 |
+
- `scale: torch.Tensor` of shape (...).
|
| 369 |
+
- `shift: torch.Tensor` of shape (..., 3)
|
| 370 |
+
"""
|
| 371 |
+
dtype, device = points_src.dtype, points_src.device
|
| 372 |
+
|
| 373 |
+
shift, _, _ = align(torch.ones_like(points_src[..., 2]), points_tgt[..., 2] - points_src[..., 2], weight, trunc)
|
| 374 |
+
shift = torch.stack([torch.zeros_like(shift), torch.zeros_like(shift), shift], dim=-1)
|
| 375 |
+
|
| 376 |
+
return shift
|
| 377 |
+
|
| 378 |
+
|
| 379 |
+
def align_points_xyz_shift(points_src: torch.Tensor, points_tgt: torch.Tensor, weight: Optional[torch.Tensor], trunc: Optional[Union[float, torch.Tensor]] = None, max_iters: int = 30, eps: float = 1e-6):
|
| 380 |
+
"""
|
| 381 |
+
Align `points_src` to `points_tgt` with respect to a Z-axis shift.
|
| 382 |
+
|
| 383 |
+
### Parameters:
|
| 384 |
+
- `points_src: torch.Tensor` of shape (..., N, 3)
|
| 385 |
+
- `points_tgt: torch.Tensor` of shape (..., N, 3)
|
| 386 |
+
- `weights: torch.Tensor` of shape (..., N)
|
| 387 |
+
|
| 388 |
+
### Returns:
|
| 389 |
+
- `scale: torch.Tensor` of shape (...).
|
| 390 |
+
- `shift: torch.Tensor` of shape (..., 3)
|
| 391 |
+
"""
|
| 392 |
+
dtype, device = points_src.dtype, points_src.device
|
| 393 |
+
|
| 394 |
+
shift, _, _ = align(torch.ones_like(points_src).swapaxes(-2, -1), (points_tgt - points_src).swapaxes(-2, -1), weight[..., None, :], trunc)
|
| 395 |
+
|
| 396 |
+
return shift
|
| 397 |
+
|
| 398 |
+
|
| 399 |
+
def align_affine_lstsq(x: torch.Tensor, y: torch.Tensor, w: torch.Tensor = None) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 400 |
+
"""
|
| 401 |
+
Solve `min sum_i w_i * (a * x_i + b - y_i ) ^ 2`, where `a` and `b` are scalars, with respect to `a` and `b` using least squares.
|
| 402 |
+
|
| 403 |
+
### Parameters:
|
| 404 |
+
- `x: torch.Tensor` of shape (..., N)
|
| 405 |
+
- `y: torch.Tensor` of shape (..., N)
|
| 406 |
+
- `w: torch.Tensor` of shape (..., N)
|
| 407 |
+
|
| 408 |
+
### Returns:
|
| 409 |
+
- `a: torch.Tensor` of shape (...,)
|
| 410 |
+
- `b: torch.Tensor` of shape (...,)
|
| 411 |
+
"""
|
| 412 |
+
w_sqrt = torch.ones_like(x) if w is None else w.sqrt()
|
| 413 |
+
A = torch.stack([w_sqrt * x, torch.ones_like(x)], dim=-1)
|
| 414 |
+
B = (w_sqrt * y)[..., None]
|
| 415 |
+
a, b = torch.linalg.lstsq(A, B)[0].squeeze(-1).unbind(-1)
|
| 416 |
+
return a, b
|
third_party/MoGe/moge/utils/data_augmentation.py
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import json
|
| 3 |
+
import time
|
| 4 |
+
import random
|
| 5 |
+
from typing import *
|
| 6 |
+
import itertools
|
| 7 |
+
from numbers import Number
|
| 8 |
+
import io
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
import cv2
|
| 12 |
+
from PIL import Image
|
| 13 |
+
import torch
|
| 14 |
+
import torchvision.transforms.v2.functional as TF
|
| 15 |
+
import utils3d
|
| 16 |
+
from scipy.signal import fftconvolve
|
| 17 |
+
|
| 18 |
+
from ..utils.geometry_numpy import harmonic_mean_numpy, norm3d, depth_occlusion_edge_numpy
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def sample_perspective(
|
| 22 |
+
src_intrinsics: np.ndarray,
|
| 23 |
+
tgt_aspect: float,
|
| 24 |
+
center_augmentation: float,
|
| 25 |
+
fov_range_absolute: Tuple[float, float],
|
| 26 |
+
fov_range_relative: Tuple[float, float],
|
| 27 |
+
rng: np.random.Generator = None
|
| 28 |
+
) -> Tuple[np.ndarray, np.ndarray]:
|
| 29 |
+
raw_horizontal, raw_vertical = abs(1.0 / src_intrinsics[0, 0]), abs(1.0 / src_intrinsics[1, 1])
|
| 30 |
+
raw_fov_x, raw_fov_y = utils3d.np.intrinsics_to_fov(src_intrinsics)
|
| 31 |
+
|
| 32 |
+
# 1. set target fov
|
| 33 |
+
fov_range_absolute_min, fov_range_absolute_max = fov_range_absolute
|
| 34 |
+
fov_range_relative_min, fov_range_relative_max = fov_range_relative
|
| 35 |
+
tgt_fov_x_min = min(fov_range_relative_min * raw_fov_x, utils3d.focal_to_fov(utils3d.fov_to_focal(fov_range_relative_min * raw_fov_y) / tgt_aspect))
|
| 36 |
+
tgt_fov_x_max = min(fov_range_relative_max * raw_fov_x, utils3d.focal_to_fov(utils3d.fov_to_focal(fov_range_relative_max * raw_fov_y) / tgt_aspect))
|
| 37 |
+
tgt_fov_x_min, tgt_fov_max = max(np.deg2rad(fov_range_absolute_min), tgt_fov_x_min), min(np.deg2rad(fov_range_absolute_max), tgt_fov_x_max)
|
| 38 |
+
tgt_fov_x = rng.uniform(min(tgt_fov_x_min, tgt_fov_x_max), tgt_fov_x_max)
|
| 39 |
+
tgt_fov_y = utils3d.focal_to_fov(utils3d.np.fov_to_focal(tgt_fov_x) * tgt_aspect)
|
| 40 |
+
|
| 41 |
+
# 2. set target image center (principal point) and the corresponding z-direction in raw camera space
|
| 42 |
+
center_dtheta = center_augmentation * rng.uniform(-0.5, 0.5) * (raw_fov_x - tgt_fov_x)
|
| 43 |
+
center_dphi = center_augmentation * rng.uniform(-0.5, 0.5) * (raw_fov_y - tgt_fov_y)
|
| 44 |
+
cu, cv = 0.5 + 0.5 * np.tan(center_dtheta) / np.tan(raw_fov_x / 2), 0.5 + 0.5 * np.tan(center_dphi) / np.tan(raw_fov_y / 2)
|
| 45 |
+
direction = utils3d.np.unproject_cv(np.array([[cu, cv]], dtype=np.float32), np.array([1.0], dtype=np.float32), intrinsics=src_intrinsics)[0]
|
| 46 |
+
|
| 47 |
+
# 3. obtain the rotation matrix for homography warping (new_ext = R * old_ext)
|
| 48 |
+
R = utils3d.np.rotation_matrix_from_vectors(direction, np.array([0, 0, 1], dtype=np.float32))
|
| 49 |
+
|
| 50 |
+
# 4. shrink the target view to fit into the warped image
|
| 51 |
+
corners = np.array([[0, 0], [0, 1], [1, 1], [1, 0]], dtype=np.float32)
|
| 52 |
+
corners = np.concatenate([corners, np.ones((4, 1), dtype=np.float32)], axis=1) @ (np.linalg.inv(src_intrinsics).T @ R.T) # corners in viewport's camera plane
|
| 53 |
+
corners = corners[:, :2] / corners[:, 2:3]
|
| 54 |
+
tgt_horizontal, tgt_vertical = np.tan(tgt_fov_x / 2) * 2, np.tan(tgt_fov_y / 2) * 2
|
| 55 |
+
warp_horizontal, warp_vertical = float('inf'), float('inf')
|
| 56 |
+
for i in range(4):
|
| 57 |
+
intersection, _ = utils3d.np.ray_intersection(
|
| 58 |
+
np.array([0., 0.]), np.array([[tgt_aspect, 1.0], [tgt_aspect, -1.0]]),
|
| 59 |
+
corners[i - 1], corners[i] - corners[i - 1],
|
| 60 |
+
)
|
| 61 |
+
warp_horizontal, warp_vertical = min(warp_horizontal, 2 * np.abs(intersection[:, 0]).min()), min(warp_vertical, 2 * np.abs(intersection[:, 1]).min())
|
| 62 |
+
tgt_horizontal, tgt_vertical = min(tgt_horizontal, warp_horizontal), min(tgt_vertical, warp_vertical)
|
| 63 |
+
|
| 64 |
+
# 5. obtain the target intrinsics
|
| 65 |
+
fx, fy = 1 / tgt_horizontal, 1 / tgt_vertical
|
| 66 |
+
tgt_intrinsics = utils3d.np.intrinsics_from_focal_center(fx, fy, 0.5, 0.5).astype(np.float32)
|
| 67 |
+
|
| 68 |
+
return tgt_intrinsics, R
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def warp_perspective(
|
| 72 |
+
src_map: np.ndarray = None,
|
| 73 |
+
transform: np.ndarray = None,
|
| 74 |
+
tgt_size: Tuple[int, int] = None,
|
| 75 |
+
interpolation: Literal['nearest', 'bilinear', 'lanczos'] = 'nearest',
|
| 76 |
+
sparse_mask: np.ndarray = None,
|
| 77 |
+
):
|
| 78 |
+
"""Perspective warping with careful resampling.
|
| 79 |
+
- For `lanczos`, use PIL to resize first to reduce aliasing.
|
| 80 |
+
- For `nearest` with sparse input, use mask-aware nearest resize to avoid losing points.
|
| 81 |
+
- For `bilinear` or `nearest` with dense input, directly use cv2.remap.
|
| 82 |
+
|
| 83 |
+
- `transform` is the matrix that transforms homogeneous pixel coordinates of source image to those of target image, i.e., `p_tgt = transform @ p_src`.
|
| 84 |
+
"""
|
| 85 |
+
|
| 86 |
+
tgt_height, tgt_width = tgt_size
|
| 87 |
+
src_height, src_width = src_map.shape[:2]
|
| 88 |
+
|
| 89 |
+
# source to target transform
|
| 90 |
+
transform_pixel = np.array([[tgt_width, 0, -0.5], [0, tgt_height, -0.5], [0, 0, 1]], dtype=np.float32) @ transform @ np.array([[1 / src_width, 0, 0.5 / src_width], [0, 1 / src_height, 0.5 / src_height], [0, 0, 1]], dtype=np.float32)
|
| 91 |
+
# Get scale factor at the target center
|
| 92 |
+
w = np.dot(np.linalg.inv(transform_pixel)[2, :], np.array([tgt_width / 2, tgt_height / 2, 1], dtype=np.float32))
|
| 93 |
+
scale_x, scale_y = w * np.linalg.norm(transform_pixel[:2, :2], axis=0)
|
| 94 |
+
|
| 95 |
+
if interpolation == 'lanczos' and (scale_x < 0.8 or scale_y < 0.8):
|
| 96 |
+
# If lanczos & downsampling, use PIL to resize first to reduce aliasing
|
| 97 |
+
src_height, src_width = max(round(src_height * scale_y * 1.25), 16), max(round(src_width * scale_x * 1.25), 16)
|
| 98 |
+
src_map = np.array(Image.fromarray(src_map).resize((src_width, src_height), Image.Resampling.LANCZOS))
|
| 99 |
+
elif interpolation == 'nearest' and sparse_mask is not None and (scale_x < 1 or scale_y < 1):
|
| 100 |
+
# If nearest and sparse, use mask-aware nearest resize first to avoid losing points
|
| 101 |
+
src_height, src_width = max(round(src_height * scale_y), 16), max(round(src_width * scale_x), 16)
|
| 102 |
+
src_map, _ = utils3d.np.masked_nearest_resize(src_map, mask=sparse_mask, size=(src_height, src_width))
|
| 103 |
+
|
| 104 |
+
# Recompute the pixel-space transform after resizing
|
| 105 |
+
transform_pixel = np.array([[tgt_width, 0, -0.5], [0, tgt_height, -0.5], [0, 0, 1]], dtype=np.float32) @ transform @ np.array([[1 / src_width, 0, 0.5 / src_width], [0, 1 / src_height, 0.5 / src_height], [0, 0, 1]], dtype=np.float32)
|
| 106 |
+
|
| 107 |
+
# Remap
|
| 108 |
+
cv2_interpolation = {'nearest': cv2.INTER_NEAREST, 'bilinear': cv2.INTER_LINEAR, 'lanczos': cv2.INTER_LANCZOS4}[interpolation]
|
| 109 |
+
tgt_map = cv2.warpPerspective(src_map, transform_pixel, (tgt_width, tgt_height), flags=cv2_interpolation)
|
| 110 |
+
|
| 111 |
+
return tgt_map
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def image_color_augmentation(image: np.ndarray, augmentations: List[Dict[str, Any]], rng: np.random.Generator = None, depth: np.ndarray = None):
|
| 115 |
+
height, width = image.shape[:2]
|
| 116 |
+
if rng is None:
|
| 117 |
+
rng = np.random.default_rng()
|
| 118 |
+
if 'jittering' in augmentations:
|
| 119 |
+
image = torch.from_numpy(image).permute(2, 0, 1)
|
| 120 |
+
image = TF.adjust_brightness(image, rng.uniform(0.9, 1.1))
|
| 121 |
+
image = TF.adjust_contrast(image, rng.uniform(0.9, 1.1))
|
| 122 |
+
image = TF.adjust_saturation(image, rng.uniform(0.9, 1.1))
|
| 123 |
+
image = TF.adjust_hue(image, rng.uniform(-0.05, 0.05))
|
| 124 |
+
image = TF.adjust_gamma(image, rng.uniform(0.9, 1.1))
|
| 125 |
+
image = image.permute(1, 2, 0).numpy()
|
| 126 |
+
if 'dof' in augmentations:
|
| 127 |
+
assert depth is not None, 'Depth map is required for DOF augmentation'
|
| 128 |
+
if rng.uniform() < 0.5:
|
| 129 |
+
dof_strength = rng.integers(12)
|
| 130 |
+
disp = 1 / depth
|
| 131 |
+
finite_mask = np.isfinite(depth)
|
| 132 |
+
disp_min, disp_max = disp[finite_mask].min(), disp[finite_mask].max()
|
| 133 |
+
disp = cv2.inpaint(np.nan_to_num(disp, nan=1), np.isnan(disp).astype(np.uint8), 3, cv2.INPAINT_TELEA).clip(0, disp_max)
|
| 134 |
+
dof_focus = rng.uniform(disp_min, disp_max)
|
| 135 |
+
image = depth_of_field(image, disp, dof_focus, dof_strength)
|
| 136 |
+
if 'shot_noise' in augmentations:
|
| 137 |
+
if rng.uniform() < 0.5:
|
| 138 |
+
k = np.exp(rng.uniform(np.log(100), np.log(10000))) / 255
|
| 139 |
+
image = (rng.poisson(image * k) / k).clip(0, 255).astype(np.uint8)
|
| 140 |
+
if 'blurring' in augmentations:
|
| 141 |
+
if rng.uniform() < 0.5:
|
| 142 |
+
ratio = rng.uniform(0.25, 1)
|
| 143 |
+
image = cv2.resize(cv2.resize(image, (int(width * ratio), int(height * ratio)), interpolation=cv2.INTER_AREA), (width, height), interpolation=rng.choice([cv2.INTER_LINEAR_EXACT, cv2.INTER_CUBIC, cv2.INTER_LANCZOS4]))
|
| 144 |
+
if 'jpeg_loss' in augmentations:
|
| 145 |
+
if rng.uniform() < 0.5:
|
| 146 |
+
image = cv2.imdecode(cv2.imencode('.jpg', image, [cv2.IMWRITE_JPEG_QUALITY, rng.integers(20, 100)])[1], cv2.IMREAD_COLOR)
|
| 147 |
+
|
| 148 |
+
return image
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def disk_kernel(radius: int) -> np.ndarray:
|
| 153 |
+
"""
|
| 154 |
+
Generate disk kernel with given radius.
|
| 155 |
+
|
| 156 |
+
Args:
|
| 157 |
+
radius (int): Radius of the disk (in pixels).
|
| 158 |
+
|
| 159 |
+
Returns:
|
| 160 |
+
np.ndarray: (2*radius+1, 2*radius+1) normalized convolution kernel.
|
| 161 |
+
"""
|
| 162 |
+
# Create coordinate grid centered at (0,0)
|
| 163 |
+
L = np.arange(-radius, radius + 1)
|
| 164 |
+
X, Y = np.meshgrid(L, L)
|
| 165 |
+
# Generate disk: region inside circle with radius R is 1
|
| 166 |
+
kernel = ((X**2 + Y**2) <= radius**2).astype(np.float32)
|
| 167 |
+
# Normalize the kernel
|
| 168 |
+
kernel /= np.sum(kernel)
|
| 169 |
+
return kernel
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def disk_blur(image: np.ndarray, radius: int) -> np.ndarray:
|
| 173 |
+
"""
|
| 174 |
+
Apply disk blur to an image using FFT convolution.
|
| 175 |
+
|
| 176 |
+
Args:
|
| 177 |
+
image (np.ndarray): Input image, can be grayscale or color.
|
| 178 |
+
radius (int): Blur radius (in pixels).
|
| 179 |
+
|
| 180 |
+
Returns:
|
| 181 |
+
np.ndarray: Blurred image.
|
| 182 |
+
"""
|
| 183 |
+
if radius == 0:
|
| 184 |
+
return image
|
| 185 |
+
kernel = disk_kernel(radius)
|
| 186 |
+
if image.ndim == 2:
|
| 187 |
+
blurred = fftconvolve(image, kernel, mode='same')
|
| 188 |
+
elif image.ndim == 3:
|
| 189 |
+
channels = []
|
| 190 |
+
for i in range(image.shape[2]):
|
| 191 |
+
blurred_channel = fftconvolve(image[..., i], kernel, mode='same')
|
| 192 |
+
channels.append(blurred_channel)
|
| 193 |
+
blurred = np.stack(channels, axis=-1)
|
| 194 |
+
else:
|
| 195 |
+
raise ValueError("Image must be 2D or 3D.")
|
| 196 |
+
return blurred
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def depth_of_field(
|
| 200 |
+
img: np.ndarray,
|
| 201 |
+
disp: np.ndarray,
|
| 202 |
+
focus_disp : float,
|
| 203 |
+
max_blur_radius : int = 10,
|
| 204 |
+
) -> np.ndarray:
|
| 205 |
+
"""
|
| 206 |
+
Apply depth of field effect to an image.
|
| 207 |
+
|
| 208 |
+
Args:
|
| 209 |
+
img (numpy.ndarray): (H, W, 3) input image.
|
| 210 |
+
depth (numpy.ndarray): (H, W) depth map of the scene.
|
| 211 |
+
focus_depth (float): Focus depth of the lens.
|
| 212 |
+
strength (float): Strength of the depth of field effect.
|
| 213 |
+
max_blur_radius (int): Maximum blur radius (in pixels).
|
| 214 |
+
|
| 215 |
+
Returns:
|
| 216 |
+
numpy.ndarray: (H, W, 3) output image with depth of field effect applied.
|
| 217 |
+
"""
|
| 218 |
+
# Precalculate dialated depth map for each blur radius
|
| 219 |
+
max_disp = np.max(disp)
|
| 220 |
+
disp = disp / max_disp
|
| 221 |
+
focus_disp = focus_disp / max_disp
|
| 222 |
+
dilated_disp = []
|
| 223 |
+
for radius in range(max_blur_radius + 1):
|
| 224 |
+
dilated_disp.append(cv2.dilate(disp, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (2 * radius + 1, 2 * radius + 1)), iterations=1))
|
| 225 |
+
|
| 226 |
+
# Determine the blur radius for each pixel based on the depth map
|
| 227 |
+
blur_radii = np.clip(np.abs(disp - focus_disp) * max_blur_radius, 0, max_blur_radius).astype(np.int32)
|
| 228 |
+
for radius in range(max_blur_radius + 1):
|
| 229 |
+
dialted_blur_radii = np.clip(np.abs(dilated_disp[radius] - focus_disp) * max_blur_radius, 0, max_blur_radius).astype(np.int32)
|
| 230 |
+
mask = (dialted_blur_radii >= radius) & (dialted_blur_radii >= blur_radii) & (dilated_disp[radius] > disp)
|
| 231 |
+
blur_radii[mask] = dialted_blur_radii[mask]
|
| 232 |
+
blur_radii = np.clip(blur_radii, 0, max_blur_radius)
|
| 233 |
+
blur_radii = cv2.blur(blur_radii, (5, 5))
|
| 234 |
+
|
| 235 |
+
# Precalculate the blured image for each blur radius
|
| 236 |
+
unique_radii = np.unique(blur_radii)
|
| 237 |
+
precomputed = {}
|
| 238 |
+
for radius in range(max_blur_radius + 1):
|
| 239 |
+
if radius not in unique_radii:
|
| 240 |
+
continue
|
| 241 |
+
precomputed[radius] = disk_blur(img, radius)
|
| 242 |
+
|
| 243 |
+
# Composit the blured image for each pixel
|
| 244 |
+
output = np.zeros_like(img)
|
| 245 |
+
for r in unique_radii:
|
| 246 |
+
mask = blur_radii == r
|
| 247 |
+
output[mask] = precomputed[r][mask]
|
| 248 |
+
|
| 249 |
+
return output
|
| 250 |
+
|