Spaces:
Sleeping
Sleeping
Giulio Federico commited on
Commit ·
2039c18
1
Parent(s): 565d54b
Upload of the code without weights
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitignore +1 -0
- app.py +63 -0
- configs/baseline.yaml +125 -0
- configs/baseline_1_to_infinity.yaml +125 -0
- configs/baseline_for_ablation.yaml +122 -0
- configs/baseline_for_test.yaml +126 -0
- configs/baseline_geometric_loss.yaml +148 -0
- configs/baseline_global.yaml +125 -0
- configs/baseline_triplane.yaml +126 -0
- configs/baseline_triplane2.yaml +126 -0
- configs/baseline_triplane3.yaml +126 -0
- configs/baseline_triplane_with_3d_conv.yaml +126 -0
- configs/baseline_with_distillation.yaml +124 -0
- configs/baseline_with_projection.yaml +126 -0
- configs/loomnet_global2.log +2 -0
- configs/loomnet_triplane.yaml.save +1 -0
- configs/nerf.yaml +26 -0
- configs/neus.yaml +27 -0
- inference.py +141 -0
- loomnet/data/__init__.py +0 -0
- loomnet/data/__pycache__/__init__.cpython-311.pyc +0 -0
- loomnet/data/__pycache__/multiview.cpython-311.pyc +0 -0
- loomnet/data/__pycache__/multiview_infinity.cpython-311.pyc +0 -0
- loomnet/data/multiview.py +375 -0
- loomnet/data/multiview_infinity.py +410 -0
- loomnet/models/__init__.py +0 -0
- loomnet/models/__pycache__/__init__.cpython-311.pyc +0 -0
- loomnet/models/__pycache__/mv_model.cpython-311.pyc +0 -0
- loomnet/models/__pycache__/mv_model_1_to_infinity.cpython-311.pyc +0 -0
- loomnet/models/__pycache__/mv_model_infinity.cpython-311.pyc +0 -0
- loomnet/models/__pycache__/mv_model_triplane.cpython-311.pyc +0 -0
- loomnet/models/__pycache__/mv_model_triplane_with_3d_conv.cpython-311.pyc +0 -0
- loomnet/models/__pycache__/mv_model_v7.cpython-311.pyc +0 -0
- loomnet/models/__pycache__/mv_model_v8.cpython-311.pyc +0 -0
- loomnet/models/__pycache__/mv_model_with_distillation.cpython-311.pyc +0 -0
- loomnet/models/blocks/__init__.py +0 -0
- loomnet/models/blocks/__pycache__/__init__.cpython-311.pyc +0 -0
- loomnet/models/blocks/__pycache__/generic_blocks.cpython-311.pyc +0 -0
- loomnet/models/blocks/__pycache__/pointwise_spatial_transformer.cpython-311.pyc +0 -0
- loomnet/models/blocks/__pycache__/resnet.cpython-311.pyc +0 -0
- loomnet/models/blocks/__pycache__/resnet_fpn.cpython-311.pyc +0 -0
- loomnet/models/blocks/__pycache__/transformer_t.cpython-311.pyc +0 -0
- loomnet/models/blocks/attention.py +739 -0
- loomnet/models/blocks/attention_processor.py +871 -0
- loomnet/models/blocks/generic_blocks.py +295 -0
- loomnet/models/blocks/pointwise_spatial_transformer.py +179 -0
- loomnet/models/blocks/resnet.py +357 -0
- loomnet/models/blocks/resnet_fpn.py +372 -0
- loomnet/models/blocks/transformer.py +221 -0
- loomnet/models/blocks/transformer_t.py +268 -0
.gitignore
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
*.ckpt
|
app.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import gradio as gr
|
| 2 |
+
import spaces
|
| 3 |
+
import torch
|
| 4 |
+
import hydra
|
| 5 |
+
import lightning as L
|
| 6 |
+
from einops import rearrange
|
| 7 |
+
from PIL import Image
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
# I tuoi import
|
| 11 |
+
from loomnet.utils import load_config
|
| 12 |
+
from inference import prepare_inputs
|
| 13 |
+
|
| 14 |
+
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
| 15 |
+
OUTPUT_DIR = Path("outputs")
|
| 16 |
+
OUTPUT_DIR.mkdir(exist_ok=True)
|
| 17 |
+
|
| 18 |
+
# Il modello viene caricato nella RAM dello spazio cloud all'avvio
|
| 19 |
+
cfg = load_config("configs/baseline.yaml", cli_args=[])
|
| 20 |
+
model = hydra.utils.instantiate(cfg.system)
|
| 21 |
+
model.load_weights(cfg.resume)
|
| 22 |
+
model = model.to(DEVICE).eval()
|
| 23 |
+
|
| 24 |
+
@spaces.GPU
|
| 25 |
+
def run_demo(image_path, elevation, seed):
|
| 26 |
+
if image_path is None:
|
| 27 |
+
return None, None
|
| 28 |
+
|
| 29 |
+
L.seed_everything(int(seed), workers=True)
|
| 30 |
+
data = prepare_inputs(image_path, elevation, "ele30")
|
| 31 |
+
for k, v in data.items():
|
| 32 |
+
data[k] = v.unsqueeze(0).to(DEVICE)
|
| 33 |
+
if k not in ["k_near_indices"]: data[k] = data[k].float()
|
| 34 |
+
|
| 35 |
+
with torch.no_grad():
|
| 36 |
+
images_pred = model._generate_images(data)
|
| 37 |
+
|
| 38 |
+
gif_path = str(OUTPUT_DIR / "out.gif")
|
| 39 |
+
jpg_path = str(OUTPUT_DIR / "out.jpg")
|
| 40 |
+
|
| 41 |
+
image_list = [Image.fromarray(img) for img in images_pred[0]]
|
| 42 |
+
image_list[0].save(gif_path, save_all=True, append_images=image_list[1:], duration=100, loop=0)
|
| 43 |
+
|
| 44 |
+
full_image = rearrange(images_pred, "b m h w c -> (b h) (m w) c")
|
| 45 |
+
Image.fromarray(full_image).save(jpg_path)
|
| 46 |
+
|
| 47 |
+
return jpg_path, gif_path
|
| 48 |
+
|
| 49 |
+
with gr.Blocks() as demo:
|
| 50 |
+
with gr.Row():
|
| 51 |
+
with gr.Column():
|
| 52 |
+
img_in = gr.Image(type="filepath", label="Immagine Input")
|
| 53 |
+
elevation_in = gr.Slider(-90, 90, value=12.0, label="Elevation")
|
| 54 |
+
seed_in = gr.Number(value=0, label="Seed")
|
| 55 |
+
btn = gr.Button("Genera")
|
| 56 |
+
with gr.Column():
|
| 57 |
+
out_jpg = gr.Image(label="Viste Generate")
|
| 58 |
+
out_gif = gr.Image(label="GIF")
|
| 59 |
+
|
| 60 |
+
btn.click(fn=run_demo, inputs=[img_in, elevation_in, seed_in], outputs=[out_jpg, out_gif])
|
| 61 |
+
|
| 62 |
+
if __name__ == "__main__":
|
| 63 |
+
demo.launch()
|
configs/baseline.yaml
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: loomnet_global
|
| 2 |
+
tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
|
| 3 |
+
version: loomnet_v2-stable-zero123-n16f16-pabs
|
| 4 |
+
description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
|
| 5 |
+
output_dir: "outputs"
|
| 6 |
+
|
| 7 |
+
#NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
|
| 8 |
+
resume: epoch=47-step=37544.ckpt
|
| 9 |
+
|
| 10 |
+
extras:
|
| 11 |
+
# for dataset
|
| 12 |
+
# num_views: 17 # 1 reference view + 16 sampling views
|
| 13 |
+
# k_near_views: 16 # our baseline uses 16 nearest views
|
| 14 |
+
# sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
|
| 15 |
+
num_views: 17 # 1 reference view + 16 sampling views
|
| 16 |
+
k_near_views: 1 # our baseline uses 16 nearest views
|
| 17 |
+
sample_views_mode: "random"
|
| 18 |
+
img_wh: [256, 256]
|
| 19 |
+
bg_color: white
|
| 20 |
+
load_only_weights:
|
| 21 |
+
|
| 22 |
+
data:
|
| 23 |
+
_target_: loomnet.data.multiview.MultiViewDataModule
|
| 24 |
+
train_dataset:
|
| 25 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 26 |
+
type_of_dataset: "training"
|
| 27 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 28 |
+
img_wh: ${extras.img_wh}
|
| 29 |
+
bg_color: ${extras.bg_color}
|
| 30 |
+
num_views: ${extras.num_views}
|
| 31 |
+
k_near_views: ${extras.k_near_views}
|
| 32 |
+
ref_view_mode: "random" #64
|
| 33 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 34 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
|
| 35 |
+
use_abs_elevation: true # for stable zero123
|
| 36 |
+
train_batch_size: 1 #15
|
| 37 |
+
val_dataset:
|
| 38 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 39 |
+
type_of_dataset: "validation"
|
| 40 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 41 |
+
img_wh: ${extras.img_wh}
|
| 42 |
+
bg_color: ${extras.bg_color}
|
| 43 |
+
num_views: ${extras.num_views}
|
| 44 |
+
k_near_views: ${extras.k_near_views}
|
| 45 |
+
ref_view_mode: "random" #64
|
| 46 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 47 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
|
| 48 |
+
use_abs_elevation: true # for stable zero123
|
| 49 |
+
max_samples_per_category: 1
|
| 50 |
+
val_batch_size: 1 # [NB]: during validation we will use double val_batch_size due to cfg
|
| 51 |
+
test_dataset:
|
| 52 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 53 |
+
type_of_dataset: "testing"
|
| 54 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
|
| 55 |
+
img_wh: ${extras.img_wh}
|
| 56 |
+
bg_color: ${extras.bg_color}
|
| 57 |
+
num_views: ${extras.num_views}
|
| 58 |
+
k_near_views: ${extras.k_near_views}
|
| 59 |
+
ref_view_mode: 64
|
| 60 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 61 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
|
| 62 |
+
use_abs_elevation: true
|
| 63 |
+
test_batch_size: 1
|
| 64 |
+
num_workers: 16 #64
|
| 65 |
+
pin_memory: true
|
| 66 |
+
|
| 67 |
+
system:
|
| 68 |
+
_target_: loomnet.systems.i2mv_system_v8.I2MVSystem
|
| 69 |
+
base_model_id: "ashawkey/stable-zero123-diffusers"
|
| 70 |
+
mv_model:
|
| 71 |
+
_target_: loomnet.models.mv_model_v8.MVModel
|
| 72 |
+
_partial_: true
|
| 73 |
+
cp_block_model:
|
| 74 |
+
_target_: loomnet.models.extras.feature_aggregator_v8.FeatureAggregator
|
| 75 |
+
_partial_: true
|
| 76 |
+
num_views: ${extras.num_views}
|
| 77 |
+
n_samples_per_ray: 64 #number of samples M per ray
|
| 78 |
+
num_weaving_layers_per_decoder_block: [3, 4, 6, 6] #how many times the weaving stages is repeated for each decoder layer
|
| 79 |
+
num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
|
| 80 |
+
plane_channel_dim: ${system.mv_model.plane_channel_dim}
|
| 81 |
+
plane_resolution: ${system.mv_model.plane_resolution}
|
| 82 |
+
splat_plane_resolution: 64 #the resolution of each splatted plane
|
| 83 |
+
latent_rendering_stages: [False, False, False, True] #in what stage to use latent rendering
|
| 84 |
+
do_local_bilinear_splatting: True
|
| 85 |
+
do_global_rendering: False
|
| 86 |
+
use_positional_encoding: True
|
| 87 |
+
|
| 88 |
+
insert_stages: ["mid", "up"]
|
| 89 |
+
insert_up_layers: [0, 1, 2]
|
| 90 |
+
use_residual: false
|
| 91 |
+
plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
|
| 92 |
+
plane_channel_dim: 512 #the number of channels of each latent plane
|
| 93 |
+
lr: 1e-5
|
| 94 |
+
cfg: 0.1
|
| 95 |
+
tv_weight: 5e-4 #loss weight for triplane
|
| 96 |
+
report_to: none #TODO --> wandn
|
| 97 |
+
|
| 98 |
+
trainer:
|
| 99 |
+
_target_: lightning.pytorch.trainer.Trainer
|
| 100 |
+
default_root_dir: ${output_dir}
|
| 101 |
+
max_steps: 370001
|
| 102 |
+
check_val_every_n_epoch: 1
|
| 103 |
+
log_every_n_steps: 5
|
| 104 |
+
num_sanity_val_steps: 0
|
| 105 |
+
enable_progress_bar: true
|
| 106 |
+
strategy: ddp_find_unused_parameters_true
|
| 107 |
+
accelerator: gpu
|
| 108 |
+
precision: 16-mixed # mixed precision for extra speed-up
|
| 109 |
+
accumulate_grad_batches: 8
|
| 110 |
+
gradient_clip_val: 1.0
|
| 111 |
+
|
| 112 |
+
callbacks:
|
| 113 |
+
model_checkpoint:
|
| 114 |
+
_target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 115 |
+
save_top_k: 1
|
| 116 |
+
#every_n_train_steps: 10000
|
| 117 |
+
#every_n_epochs: 12
|
| 118 |
+
dirpath: "${output_dir}/${version}/checkpoints"
|
| 119 |
+
|
| 120 |
+
# logger:
|
| 121 |
+
# wandb:
|
| 122 |
+
# _target_: lightning.pytorch.loggers.wandb.WandbLogger
|
| 123 |
+
# project: "${name}"
|
| 124 |
+
# save_dir: ${output_dir}
|
| 125 |
+
# name: "${version}"
|
configs/baseline_1_to_infinity.yaml
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: loomnet_1_to_infinity_baseline_1
|
| 2 |
+
tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
|
| 3 |
+
version: loomnet_v2-stable-zero123-n16f16-pabs
|
| 4 |
+
description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
|
| 5 |
+
output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
|
| 6 |
+
|
| 7 |
+
#NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
|
| 8 |
+
resume: /raid/homes/giulio.federico/LoomNet/output_dir/loomnet_1_to_infinity_baseline_1/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=49-step=67001.ckpt
|
| 9 |
+
|
| 10 |
+
extras:
|
| 11 |
+
# for dataset
|
| 12 |
+
# num_views: 17 # 1 reference view + 16 sampling views
|
| 13 |
+
# k_near_views: 16 # our baseline uses 16 nearest views
|
| 14 |
+
# sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
|
| 15 |
+
num_views: 2 # 1 reference view + 32 sampling views
|
| 16 |
+
k_near_views: 1 # our baseline uses 16 nearest views
|
| 17 |
+
sample_views_mode: "random"
|
| 18 |
+
img_wh: [256, 256]
|
| 19 |
+
bg_color: white
|
| 20 |
+
load_only_weights:
|
| 21 |
+
|
| 22 |
+
data:
|
| 23 |
+
_target_: loomnet.data.multiview.MultiViewDataModule
|
| 24 |
+
train_dataset:
|
| 25 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 26 |
+
type_of_dataset: "training"
|
| 27 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 28 |
+
img_wh: ${extras.img_wh}
|
| 29 |
+
bg_color: ${extras.bg_color}
|
| 30 |
+
num_views: ${extras.num_views}
|
| 31 |
+
k_near_views: ${extras.k_near_views}
|
| 32 |
+
ref_view_mode: "random" #64
|
| 33 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 34 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
|
| 35 |
+
use_abs_elevation: true # for stable zero123
|
| 36 |
+
train_batch_size: 2 #15
|
| 37 |
+
val_dataset:
|
| 38 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 39 |
+
type_of_dataset: "validation"
|
| 40 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 41 |
+
img_wh: ${extras.img_wh}
|
| 42 |
+
bg_color: ${extras.bg_color}
|
| 43 |
+
num_views: ${extras.num_views}
|
| 44 |
+
k_near_views: ${extras.k_near_views}
|
| 45 |
+
ref_view_mode: "random" #64
|
| 46 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 47 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
|
| 48 |
+
use_abs_elevation: true # for stable zero123
|
| 49 |
+
max_samples_per_category: 1
|
| 50 |
+
val_batch_size: 2 # [NB]: during validation we will use double val_batch_size due to cfg
|
| 51 |
+
test_dataset:
|
| 52 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 53 |
+
type_of_dataset: "testing"
|
| 54 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
|
| 55 |
+
img_wh: ${extras.img_wh}
|
| 56 |
+
bg_color: ${extras.bg_color}
|
| 57 |
+
num_views: ${extras.num_views}
|
| 58 |
+
k_near_views: ${extras.k_near_views}
|
| 59 |
+
ref_view_mode: 64
|
| 60 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 61 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
|
| 62 |
+
use_abs_elevation: true
|
| 63 |
+
test_batch_size: 1
|
| 64 |
+
num_workers: 16 #64
|
| 65 |
+
pin_memory: true
|
| 66 |
+
|
| 67 |
+
system:
|
| 68 |
+
_target_: loomnet.systems.i2mv_system_1_to_infinity.I2MVSystem
|
| 69 |
+
base_model_id: "ashawkey/stable-zero123-diffusers"
|
| 70 |
+
mv_model:
|
| 71 |
+
_target_: loomnet.models.mv_model_1_to_infinity.MVModel
|
| 72 |
+
_partial_: true
|
| 73 |
+
cp_block_model:
|
| 74 |
+
_target_: loomnet.models.extras.feature_aggregator_1_to_infinity.FeatureAggregator
|
| 75 |
+
_partial_: true
|
| 76 |
+
num_views: ${extras.num_views}
|
| 77 |
+
n_samples_per_ray: 16 #number of samples M per ray
|
| 78 |
+
num_weaving_layers_per_decoder_block: [3, 4, 6, 8] #how many times the weaving stages is repeated for each decoder layer
|
| 79 |
+
num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
|
| 80 |
+
plane_channel_dim: ${system.mv_model.plane_channel_dim}
|
| 81 |
+
plane_resolution: ${system.mv_model.plane_resolution}
|
| 82 |
+
splat_plane_resolution: 64 #the resolution of each splatted plane
|
| 83 |
+
latent_rendering_stages: [False, False, False, True] #in what stage to use latent rendering
|
| 84 |
+
do_local_bilinear_splatting: True
|
| 85 |
+
use_positional_encoding: True
|
| 86 |
+
num_views_latent_space: 16 #num views for building the shared latent space
|
| 87 |
+
|
| 88 |
+
insert_stages: ["mid", "up"]
|
| 89 |
+
insert_up_layers: [0, 1, 2, 3]
|
| 90 |
+
use_residual: false
|
| 91 |
+
plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
|
| 92 |
+
plane_channel_dim: 512 #the number of channels of each latent plane
|
| 93 |
+
lr: 1e-5
|
| 94 |
+
cfg: 0.1
|
| 95 |
+
tv_weight: 5e-4 #loss weight for triplane
|
| 96 |
+
report_to: none #TODO --> wandn
|
| 97 |
+
|
| 98 |
+
trainer:
|
| 99 |
+
_target_: lightning.pytorch.trainer.Trainer
|
| 100 |
+
default_root_dir: ${output_dir}
|
| 101 |
+
max_steps: 300000
|
| 102 |
+
check_val_every_n_epoch: 1
|
| 103 |
+
log_every_n_steps: 5
|
| 104 |
+
num_sanity_val_steps: 0
|
| 105 |
+
enable_progress_bar: true
|
| 106 |
+
strategy: ddp_find_unused_parameters_true
|
| 107 |
+
accelerator: gpu
|
| 108 |
+
precision: 16-mixed # mixed precision for extra speed-up
|
| 109 |
+
accumulate_grad_batches: 4
|
| 110 |
+
gradient_clip_val: 1.0
|
| 111 |
+
|
| 112 |
+
callbacks:
|
| 113 |
+
model_checkpoint:
|
| 114 |
+
_target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 115 |
+
save_top_k: 1
|
| 116 |
+
#every_n_train_steps: 10000
|
| 117 |
+
#every_n_epochs: 12
|
| 118 |
+
dirpath: "${output_dir}/${version}/checkpoints"
|
| 119 |
+
|
| 120 |
+
# logger:
|
| 121 |
+
# wandb:
|
| 122 |
+
# _target_: lightning.pytorch.loggers.wandb.WandbLogger
|
| 123 |
+
# project: "${name}"
|
| 124 |
+
# save_dir: ${output_dir}
|
| 125 |
+
# name: "${version}"
|
configs/baseline_for_ablation.yaml
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: loomnet_v8_baseline_6
|
| 2 |
+
tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
|
| 3 |
+
version: loomnet_v2-stable-zero123-n16f16-pabs
|
| 4 |
+
description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
|
| 5 |
+
output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
|
| 6 |
+
|
| 7 |
+
resume: /raid/homes/giulio.federico/LoomNet/output_dir/loomnet_v8_baseline_6/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=10-step=21186.ckpt
|
| 8 |
+
|
| 9 |
+
extras:
|
| 10 |
+
# for dataset
|
| 11 |
+
# num_views: 17 # 1 reference view + 16 sampling views
|
| 12 |
+
# k_near_views: 16 # our baseline uses 16 nearest views
|
| 13 |
+
# sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
|
| 14 |
+
num_views: 17 # 1 reference view + 16 sampling views
|
| 15 |
+
k_near_views: 1 # our baseline uses 16 nearest views
|
| 16 |
+
sample_views_mode: "random"
|
| 17 |
+
img_wh: [256, 256]
|
| 18 |
+
bg_color: white
|
| 19 |
+
|
| 20 |
+
data:
|
| 21 |
+
_target_: loomnet.data.multiview.MultiViewDataModule
|
| 22 |
+
train_dataset:
|
| 23 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 24 |
+
type_of_dataset: "training"
|
| 25 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 26 |
+
img_wh: ${extras.img_wh}
|
| 27 |
+
bg_color: ${extras.bg_color}
|
| 28 |
+
num_views: ${extras.num_views}
|
| 29 |
+
k_near_views: ${extras.k_near_views}
|
| 30 |
+
ref_view_mode: "random" #64
|
| 31 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 32 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
|
| 33 |
+
use_abs_elevation: true # for stable zero123
|
| 34 |
+
train_batch_size: 2 #15
|
| 35 |
+
val_dataset:
|
| 36 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 37 |
+
type_of_dataset: "validation"
|
| 38 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 39 |
+
img_wh: ${extras.img_wh}
|
| 40 |
+
bg_color: ${extras.bg_color}
|
| 41 |
+
num_views: ${extras.num_views}
|
| 42 |
+
k_near_views: ${extras.k_near_views}
|
| 43 |
+
ref_view_mode: "random" #64
|
| 44 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 45 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
|
| 46 |
+
use_abs_elevation: true # for stable zero123
|
| 47 |
+
max_samples_per_category: 1
|
| 48 |
+
val_batch_size: 2 # [NB]: during validation we will use double val_batch_size due to cfg
|
| 49 |
+
test_dataset:
|
| 50 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 51 |
+
type_of_dataset: "testing"
|
| 52 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
|
| 53 |
+
img_wh: ${extras.img_wh}
|
| 54 |
+
bg_color: ${extras.bg_color}
|
| 55 |
+
num_views: ${extras.num_views}
|
| 56 |
+
k_near_views: ${extras.k_near_views}
|
| 57 |
+
ref_view_mode: 64
|
| 58 |
+
sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] #[60,67,74,81,88,95,4,9,14,19,24,31,38,45,52,59] #${extras.sample_views_mode}
|
| 59 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
|
| 60 |
+
use_abs_elevation: true
|
| 61 |
+
test_batch_size: 1
|
| 62 |
+
num_workers: 16 #64
|
| 63 |
+
pin_memory: true
|
| 64 |
+
|
| 65 |
+
system:
|
| 66 |
+
_target_: loomnet.systems.i2mv_system_v8.I2MVSystem
|
| 67 |
+
base_model_id: "ashawkey/stable-zero123-diffusers"
|
| 68 |
+
mv_model:
|
| 69 |
+
_target_: loomnet.models.mv_model_v8.MVModel
|
| 70 |
+
_partial_: true
|
| 71 |
+
cp_block_model:
|
| 72 |
+
_target_: loomnet.models.extras.feature_aggregator_v8.FeatureAggregator
|
| 73 |
+
_partial_: true
|
| 74 |
+
num_views: ${extras.num_views}
|
| 75 |
+
n_samples_per_ray: 16 #number of samples M per ray
|
| 76 |
+
num_weaving_layers_per_decoder_block: [3, 4, 6, 8] #how many times the weaving stages is repeated for each decoder layer
|
| 77 |
+
num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
|
| 78 |
+
plane_channel_dim: ${system.mv_model.plane_channel_dim}
|
| 79 |
+
plane_resolution: ${system.mv_model.plane_resolution}
|
| 80 |
+
splat_plane_resolution: 64 #the resolution of each splatted plane
|
| 81 |
+
latent_rendering_stages: [False, False, False, True] #in what stage to use latent rendering
|
| 82 |
+
do_local_bilinear_splatting: True
|
| 83 |
+
use_positional_encoding: False
|
| 84 |
+
|
| 85 |
+
insert_stages: ["mid", "up"]
|
| 86 |
+
insert_up_layers: [0, 1, 2, 3]
|
| 87 |
+
use_residual: false
|
| 88 |
+
plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
|
| 89 |
+
plane_channel_dim: 512 #the number of channels of each latent plane
|
| 90 |
+
lr: 1e-5
|
| 91 |
+
cfg: 0.1
|
| 92 |
+
tv_weight: 5e-4 #loss weight for triplane
|
| 93 |
+
report_to: none #TODO --> wandn
|
| 94 |
+
|
| 95 |
+
trainer:
|
| 96 |
+
_target_: lightning.pytorch.trainer.Trainer
|
| 97 |
+
default_root_dir: ${output_dir}
|
| 98 |
+
max_steps: 37001
|
| 99 |
+
check_val_every_n_epoch: 1
|
| 100 |
+
log_every_n_steps: 5
|
| 101 |
+
num_sanity_val_steps: 0
|
| 102 |
+
enable_progress_bar: true
|
| 103 |
+
strategy: ddp_find_unused_parameters_true
|
| 104 |
+
accelerator: gpu
|
| 105 |
+
precision: 16-mixed # mixed precision for extra speed-up
|
| 106 |
+
accumulate_grad_batches: 4
|
| 107 |
+
gradient_clip_val: 1.0
|
| 108 |
+
|
| 109 |
+
callbacks:
|
| 110 |
+
model_checkpoint:
|
| 111 |
+
_target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 112 |
+
save_top_k: 1
|
| 113 |
+
#every_n_train_steps: 10000
|
| 114 |
+
#every_n_epochs: 12
|
| 115 |
+
dirpath: "${output_dir}/${version}/checkpoints"
|
| 116 |
+
|
| 117 |
+
# logger:
|
| 118 |
+
# wandb:
|
| 119 |
+
# _target_: lightning.pytorch.loggers.wandb.WandbLogger
|
| 120 |
+
# project: "${name}"
|
| 121 |
+
# save_dir: ${output_dir}
|
| 122 |
+
# name: "${version}"
|
configs/baseline_for_test.yaml
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: loomnet_v8_da-eliminare_per_paper
|
| 2 |
+
tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
|
| 3 |
+
version: loomnet_v2-stable-zero123-n16f16-pabs
|
| 4 |
+
description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
|
| 5 |
+
output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
|
| 6 |
+
|
| 7 |
+
resume: /raid/homes/giulio.federico/LoomNet/output_dir/loomnet_v9/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=47-step=37544.ckpt
|
| 8 |
+
|
| 9 |
+
extras:
|
| 10 |
+
# for dataset
|
| 11 |
+
# num_views: 17 # 1 reference view + 16 sampling views
|
| 12 |
+
# k_near_views: 16 # our baseline uses 16 nearest views
|
| 13 |
+
# sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
|
| 14 |
+
num_views: 17 # 1 reference view + 16 sampling views
|
| 15 |
+
k_near_views: 1 # our baseline uses 16 nearest views
|
| 16 |
+
sample_views_mode: [4, 9, 14, 19, 24, 31, 38, 45, 52, 59, 60, 67, 74, 81, 88, 95] #[4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94]
|
| 17 |
+
img_wh: [256, 256]
|
| 18 |
+
bg_color: white
|
| 19 |
+
|
| 20 |
+
data:
|
| 21 |
+
_target_: loomnet.data.multiview.MultiViewDataModule
|
| 22 |
+
train_dataset:
|
| 23 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 24 |
+
type_of_dataset: "training"
|
| 25 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 26 |
+
img_wh: ${extras.img_wh}
|
| 27 |
+
bg_color: ${extras.bg_color}
|
| 28 |
+
num_views: ${extras.num_views}
|
| 29 |
+
k_near_views: ${extras.k_near_views}
|
| 30 |
+
ref_view_mode: "random" #64
|
| 31 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 32 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
|
| 33 |
+
use_abs_elevation: true # for stable zero123
|
| 34 |
+
train_batch_size: 2 #15
|
| 35 |
+
val_dataset:
|
| 36 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 37 |
+
type_of_dataset: "validation"
|
| 38 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 39 |
+
img_wh: ${extras.img_wh}
|
| 40 |
+
bg_color: ${extras.bg_color}
|
| 41 |
+
num_views: ${extras.num_views}
|
| 42 |
+
k_near_views: ${extras.k_near_views}
|
| 43 |
+
ref_view_mode: "random" #64
|
| 44 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 45 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
|
| 46 |
+
use_abs_elevation: true # for stable zero123
|
| 47 |
+
max_samples_per_category: 1
|
| 48 |
+
val_batch_size: 2 # [NB]: during validation we will use double val_batch_size due to cfg
|
| 49 |
+
test_dataset:
|
| 50 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 51 |
+
type_of_dataset: "testing"
|
| 52 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
|
| 53 |
+
img_wh: ${extras.img_wh}
|
| 54 |
+
bg_color: ${extras.bg_color}
|
| 55 |
+
num_views: ${extras.num_views}
|
| 56 |
+
k_near_views: ${extras.k_near_views}
|
| 57 |
+
ref_view_mode: 64
|
| 58 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 59 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
|
| 60 |
+
use_abs_elevation: true
|
| 61 |
+
test_batch_size: 1
|
| 62 |
+
num_workers: 16 #64
|
| 63 |
+
pin_memory: true
|
| 64 |
+
|
| 65 |
+
system:
|
| 66 |
+
_target_: loomnet.systems.i2mv_system_v8.I2MVSystem
|
| 67 |
+
base_model_id: "ashawkey/stable-zero123-diffusers"
|
| 68 |
+
use_geometric_loss: False
|
| 69 |
+
mv_model:
|
| 70 |
+
_target_: loomnet.models.mv_model_v8.MVModel
|
| 71 |
+
_partial_: true
|
| 72 |
+
cp_block_model:
|
| 73 |
+
# _target_: loomnet.models.extras.feature_aggregator.FeatureAggregator
|
| 74 |
+
# _partial_: true
|
| 75 |
+
# depth: 2
|
| 76 |
+
# use_resnet: true
|
| 77 |
+
# use_fhw_attn: false
|
| 78 |
+
# pos_encs: ["abs"] # ["abs", "ref"]
|
| 79 |
+
_target_: loomnet.models.extras.feature_aggregator_v8.FeatureAggregator
|
| 80 |
+
_partial_: true
|
| 81 |
+
num_views: ${extras.num_views}
|
| 82 |
+
# n_heads_transformers: [16, 16, 16] #quante teste
|
| 83 |
+
# d_heads_transformers: [64, 64, 64] #la dimensione di ogni testa
|
| 84 |
+
n_heads_transformers: [16, 16, 16] #quante teste
|
| 85 |
+
d_heads_transformers: [32, 32, 32] #la dimensione di ogni testa
|
| 86 |
+
depths_transformers: [2, None, 2]
|
| 87 |
+
ambassadors_feature_grid_resolution: {4: 2, 8: 4, 16: 8, 32: 16}
|
| 88 |
+
number_nearby_hubs_per_projected_point: {4: 4, 8: 4, 16: 4, 32: 4}
|
| 89 |
+
num_latent_points: {4: 32, 8: 64, 16: 128, 32: 256}
|
| 90 |
+
insert_stages: ["mid", "up"]
|
| 91 |
+
insert_up_layers: [0, 1, 2, 3]
|
| 92 |
+
use_residual: false
|
| 93 |
+
render_options:
|
| 94 |
+
n_samples: 8
|
| 95 |
+
lr: 1e-5
|
| 96 |
+
cfg: 0.1
|
| 97 |
+
report_to: none #TODO --> wandn
|
| 98 |
+
|
| 99 |
+
trainer:
|
| 100 |
+
_target_: lightning.pytorch.trainer.Trainer
|
| 101 |
+
default_root_dir: ${output_dir}
|
| 102 |
+
max_steps: 360001
|
| 103 |
+
check_val_every_n_epoch: 1
|
| 104 |
+
log_every_n_steps: 5
|
| 105 |
+
num_sanity_val_steps: 0
|
| 106 |
+
enable_progress_bar: true
|
| 107 |
+
strategy: ddp_find_unused_parameters_true
|
| 108 |
+
accelerator: gpu
|
| 109 |
+
precision: 16-mixed # mixed precision for extra speed-up
|
| 110 |
+
accumulate_grad_batches: 4
|
| 111 |
+
gradient_clip_val: 1.0
|
| 112 |
+
|
| 113 |
+
callbacks:
|
| 114 |
+
model_checkpoint:
|
| 115 |
+
_target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 116 |
+
save_top_k: 1
|
| 117 |
+
#every_n_train_steps: 10000
|
| 118 |
+
#every_n_epochs: 12
|
| 119 |
+
dirpath: "${output_dir}/${version}/checkpoints"
|
| 120 |
+
|
| 121 |
+
# logger:
|
| 122 |
+
# wandb:
|
| 123 |
+
# _target_: lightning.pytorch.loggers.wandb.WandbLogger
|
| 124 |
+
# project: "${name}"
|
| 125 |
+
# save_dir: ${output_dir}
|
| 126 |
+
# name: "${version}"
|
configs/baseline_geometric_loss.yaml
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: loomnet_with_geometric_loss_point_matching_baseline_1
|
| 2 |
+
tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
|
| 3 |
+
version: loomnet_v2-stable-zero123-n16f16-pabs
|
| 4 |
+
description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
|
| 5 |
+
output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
|
| 6 |
+
|
| 7 |
+
resume: /raid/homes/giulio.federico/LoomNet/output_dir/loomnet_with_geometric_loss_point_matching_baseline_1/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=1-step=7702.ckpt
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
extras:
|
| 11 |
+
# for dataset
|
| 12 |
+
# num_views: 17 # 1 reference view + 16 sampling views
|
| 13 |
+
# k_near_views: 16 # our baseline uses 16 nearest views
|
| 14 |
+
# sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
|
| 15 |
+
num_views: 17 # 1 reference view + 16 sampling views
|
| 16 |
+
k_near_views: 4 # our baseline uses 16 nearest views
|
| 17 |
+
sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94]
|
| 18 |
+
img_wh: [256, 256]
|
| 19 |
+
bg_color: white
|
| 20 |
+
load_only_weights: #/raid/homes/giulio.federico/LoomNet/output_dir/loomnet_v9/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=47-step=37544.ckpt
|
| 21 |
+
|
| 22 |
+
token_based_geometric_loss:
|
| 23 |
+
max_geometric_loss_weight: 0.1
|
| 24 |
+
min_geometric_loss_weight: 0.005
|
| 25 |
+
geometric_loss_scheduler: "cosine"
|
| 26 |
+
target_vggt_layers: [4,11,17,23]
|
| 27 |
+
|
| 28 |
+
register_token_based_geometric_loss:
|
| 29 |
+
max_geometric_loss_weight: 0.1
|
| 30 |
+
min_geometric_loss_weight: 0.005
|
| 31 |
+
geometric_loss_scheduler: "cosine"
|
| 32 |
+
target_vggt_layers: [4,11,17,23]
|
| 33 |
+
|
| 34 |
+
point_matching_geometric_loss:
|
| 35 |
+
k_near_views: ${extras.k_near_views} #for each view to check we will use extras.k_near_views-1 near views
|
| 36 |
+
num_points: 32
|
| 37 |
+
num_views_to_check: 3 #how many random views to use to check consistency
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
data:
|
| 41 |
+
_target_: loomnet.data.multiview.MultiViewDataModule
|
| 42 |
+
train_dataset:
|
| 43 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 44 |
+
type_of_dataset: "training"
|
| 45 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 46 |
+
img_wh: ${extras.img_wh}
|
| 47 |
+
bg_color: ${extras.bg_color}
|
| 48 |
+
num_views: ${extras.num_views}
|
| 49 |
+
k_near_views: ${extras.k_near_views}
|
| 50 |
+
ref_view_mode: "random" #64
|
| 51 |
+
sample_views_mode: "random"
|
| 52 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
|
| 53 |
+
use_abs_elevation: true # for stable zero123
|
| 54 |
+
train_batch_size: 1 #USA SEMPRE BATCH SIZE 1
|
| 55 |
+
val_dataset:
|
| 56 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 57 |
+
type_of_dataset: "validation"
|
| 58 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
|
| 59 |
+
img_wh: ${extras.img_wh}
|
| 60 |
+
bg_color: ${extras.bg_color}
|
| 61 |
+
num_views: ${extras.num_views}
|
| 62 |
+
k_near_views: ${extras.k_near_views}
|
| 63 |
+
ref_view_mode: "random" #64
|
| 64 |
+
sample_views_mode: "random"
|
| 65 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
|
| 66 |
+
use_abs_elevation: true # for stable zero123
|
| 67 |
+
max_samples_per_category: 1
|
| 68 |
+
val_batch_size: 2 # [NB]: during validation we will use double val_batch_size due to cfg
|
| 69 |
+
test_dataset:
|
| 70 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 71 |
+
type_of_dataset: "testing"
|
| 72 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
|
| 73 |
+
img_wh: ${extras.img_wh}
|
| 74 |
+
bg_color: ${extras.bg_color}
|
| 75 |
+
num_views: ${extras.num_views}
|
| 76 |
+
k_near_views: ${extras.k_near_views}
|
| 77 |
+
ref_view_mode: 64
|
| 78 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 79 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
|
| 80 |
+
use_abs_elevation: true
|
| 81 |
+
test_batch_size: 1
|
| 82 |
+
num_workers: 16 #64
|
| 83 |
+
pin_memory: true
|
| 84 |
+
|
| 85 |
+
system:
|
| 86 |
+
_target_: loomnet.systems.i2mv_system_for_geometric_loss.I2MVSystem
|
| 87 |
+
base_model_id: "ashawkey/stable-zero123-diffusers"
|
| 88 |
+
mv_model:
|
| 89 |
+
_target_: loomnet.models.mv_model_v8.MVModel
|
| 90 |
+
_partial_: true
|
| 91 |
+
cp_block_model:
|
| 92 |
+
_target_: loomnet.models.extras.feature_aggregator_v8.FeatureAggregator
|
| 93 |
+
_partial_: true
|
| 94 |
+
num_views: ${extras.num_views}
|
| 95 |
+
n_samples_per_ray: 16 #number of samples M per ray
|
| 96 |
+
num_weaving_layers_per_decoder_block: [3, 4, 6, 8] #how many times the weaving stages is repeated for each decoder layer
|
| 97 |
+
num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
|
| 98 |
+
plane_channel_dim: ${system.mv_model.plane_channel_dim}
|
| 99 |
+
plane_resolution: ${system.mv_model.plane_resolution}
|
| 100 |
+
splat_plane_resolution: 64 #the resolution of each splatted plane
|
| 101 |
+
latent_rendering_stages: [False, False, False, True] #in what stage to use latent rendering
|
| 102 |
+
do_local_bilinear_splatting: True
|
| 103 |
+
use_positional_encoding: True
|
| 104 |
+
|
| 105 |
+
insert_stages: ["mid", "up"]
|
| 106 |
+
insert_up_layers: [0, 1, 2, 3]
|
| 107 |
+
use_residual: false
|
| 108 |
+
plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
|
| 109 |
+
plane_channel_dim: 512 #the number of channels of each latent plane
|
| 110 |
+
|
| 111 |
+
#for geometric (vggt) loss
|
| 112 |
+
type_of_geometric_loss: 'point_matching_geometric_loss' #['token_based_geometric_loss', 'register_token_based_geometric_loss']
|
| 113 |
+
use_vggt_gradient_checkpoint: True #If there are too many views or nearby views, it is better to use it to avoid going out of memory (even if it will be slower)
|
| 114 |
+
extras: ${extras}
|
| 115 |
+
|
| 116 |
+
lr: 1e-5
|
| 117 |
+
cfg: 0.1
|
| 118 |
+
tv_weight: 5e-4 #loss weight for triplane
|
| 119 |
+
report_to: none #TODO --> wandn
|
| 120 |
+
|
| 121 |
+
trainer:
|
| 122 |
+
_target_: lightning.pytorch.trainer.Trainer
|
| 123 |
+
default_root_dir: ${output_dir}
|
| 124 |
+
max_steps: 15000
|
| 125 |
+
check_val_every_n_epoch: 1
|
| 126 |
+
log_every_n_steps: 5
|
| 127 |
+
num_sanity_val_steps: 0
|
| 128 |
+
enable_progress_bar: true
|
| 129 |
+
strategy: ddp_find_unused_parameters_true
|
| 130 |
+
accelerator: gpu
|
| 131 |
+
precision: 16-mixed # mixed precision for extra speed-up
|
| 132 |
+
accumulate_grad_batches: 3
|
| 133 |
+
gradient_clip_val: 1.0
|
| 134 |
+
|
| 135 |
+
callbacks:
|
| 136 |
+
model_checkpoint:
|
| 137 |
+
_target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 138 |
+
save_top_k: 1
|
| 139 |
+
#every_n_train_steps: 10000
|
| 140 |
+
#every_n_epochs: 12
|
| 141 |
+
dirpath: "${output_dir}/${version}/checkpoints"
|
| 142 |
+
|
| 143 |
+
# logger:
|
| 144 |
+
# wandb:
|
| 145 |
+
# _target_: lightning.pytorch.loggers.wandb.WandbLogger
|
| 146 |
+
# project: "${name}"
|
| 147 |
+
# save_dir: ${output_dir}
|
| 148 |
+
# name: "${version}"
|
configs/baseline_global.yaml
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: loomnet_global
|
| 2 |
+
tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
|
| 3 |
+
version: loomnet_v2-stable-zero123-n16f16-pabs
|
| 4 |
+
description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
|
| 5 |
+
output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
|
| 6 |
+
|
| 7 |
+
#NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
|
| 8 |
+
resume: #/raid/homes/giulio.federico/LoomNet/output_dir/loomnet_v9/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=47-step=37544.ckpt
|
| 9 |
+
|
| 10 |
+
extras:
|
| 11 |
+
# for dataset
|
| 12 |
+
# num_views: 17 # 1 reference view + 16 sampling views
|
| 13 |
+
# k_near_views: 16 # our baseline uses 16 nearest views
|
| 14 |
+
# sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
|
| 15 |
+
num_views: 17 # 1 reference view + 16 sampling views
|
| 16 |
+
k_near_views: 1 # our baseline uses 16 nearest views
|
| 17 |
+
sample_views_mode: "random"
|
| 18 |
+
img_wh: [256, 256]
|
| 19 |
+
bg_color: white
|
| 20 |
+
load_only_weights:
|
| 21 |
+
|
| 22 |
+
data:
|
| 23 |
+
_target_: loomnet.data.multiview.MultiViewDataModule
|
| 24 |
+
train_dataset:
|
| 25 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 26 |
+
type_of_dataset: "training"
|
| 27 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 28 |
+
img_wh: ${extras.img_wh}
|
| 29 |
+
bg_color: ${extras.bg_color}
|
| 30 |
+
num_views: ${extras.num_views}
|
| 31 |
+
k_near_views: ${extras.k_near_views}
|
| 32 |
+
ref_view_mode: "random" #64
|
| 33 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 34 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
|
| 35 |
+
use_abs_elevation: true # for stable zero123
|
| 36 |
+
train_batch_size: 1 #15
|
| 37 |
+
val_dataset:
|
| 38 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 39 |
+
type_of_dataset: "validation"
|
| 40 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 41 |
+
img_wh: ${extras.img_wh}
|
| 42 |
+
bg_color: ${extras.bg_color}
|
| 43 |
+
num_views: ${extras.num_views}
|
| 44 |
+
k_near_views: ${extras.k_near_views}
|
| 45 |
+
ref_view_mode: "random" #64
|
| 46 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 47 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
|
| 48 |
+
use_abs_elevation: true # for stable zero123
|
| 49 |
+
max_samples_per_category: 1
|
| 50 |
+
val_batch_size: 1 # [NB]: during validation we will use double val_batch_size due to cfg
|
| 51 |
+
test_dataset:
|
| 52 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 53 |
+
type_of_dataset: "testing"
|
| 54 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
|
| 55 |
+
img_wh: ${extras.img_wh}
|
| 56 |
+
bg_color: ${extras.bg_color}
|
| 57 |
+
num_views: ${extras.num_views}
|
| 58 |
+
k_near_views: ${extras.k_near_views}
|
| 59 |
+
ref_view_mode: 64
|
| 60 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 61 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
|
| 62 |
+
use_abs_elevation: true
|
| 63 |
+
test_batch_size: 1
|
| 64 |
+
num_workers: 16 #64
|
| 65 |
+
pin_memory: true
|
| 66 |
+
|
| 67 |
+
system:
|
| 68 |
+
_target_: loomnet.systems.i2mv_system_global.I2MVSystem
|
| 69 |
+
base_model_id: "ashawkey/stable-zero123-diffusers"
|
| 70 |
+
mv_model:
|
| 71 |
+
_target_: loomnet.models.mv_model_v8.MVModel
|
| 72 |
+
_partial_: true
|
| 73 |
+
cp_block_model:
|
| 74 |
+
_target_: loomnet.models.extras.feature_aggregator_global.FeatureAggregator
|
| 75 |
+
_partial_: true
|
| 76 |
+
num_views: ${extras.num_views}
|
| 77 |
+
n_samples_per_ray: 64 #number of samples M per ray
|
| 78 |
+
num_weaving_layers_per_decoder_block: [3, 4, 6, 12] #how many times the weaving stages is repeated for each decoder layer
|
| 79 |
+
num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
|
| 80 |
+
plane_channel_dim: ${system.mv_model.plane_channel_dim}
|
| 81 |
+
plane_resolution: ${system.mv_model.plane_resolution}
|
| 82 |
+
splat_plane_resolution: 128 #the resolution of each splatted plane
|
| 83 |
+
triplane_upsample_channel_dim: 32
|
| 84 |
+
latent_rendering_stages: [False, False, False, True] #in what stage to use latent rendering
|
| 85 |
+
do_local_bilinear_splatting: True
|
| 86 |
+
use_positional_encoding: True
|
| 87 |
+
|
| 88 |
+
insert_stages: ["up"]
|
| 89 |
+
insert_up_layers: [3]
|
| 90 |
+
use_residual: false
|
| 91 |
+
plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
|
| 92 |
+
plane_channel_dim: 512 #the number of channels of each latent plane
|
| 93 |
+
lr: 1e-5
|
| 94 |
+
cfg: 0.1
|
| 95 |
+
tv_weight: 5e-4 #loss weight for triplane
|
| 96 |
+
report_to: none #TODO --> wandn
|
| 97 |
+
|
| 98 |
+
trainer:
|
| 99 |
+
_target_: lightning.pytorch.trainer.Trainer
|
| 100 |
+
default_root_dir: ${output_dir}
|
| 101 |
+
max_steps: 37001
|
| 102 |
+
check_val_every_n_epoch: 1
|
| 103 |
+
log_every_n_steps: 5
|
| 104 |
+
num_sanity_val_steps: 0
|
| 105 |
+
enable_progress_bar: true
|
| 106 |
+
strategy: ddp_find_unused_parameters_true
|
| 107 |
+
accelerator: gpu
|
| 108 |
+
precision: 16-mixed # mixed precision for extra speed-up
|
| 109 |
+
accumulate_grad_batches: 4
|
| 110 |
+
gradient_clip_val: 1.0
|
| 111 |
+
|
| 112 |
+
callbacks:
|
| 113 |
+
model_checkpoint:
|
| 114 |
+
_target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 115 |
+
save_top_k: 1
|
| 116 |
+
#every_n_train_steps: 10000
|
| 117 |
+
#every_n_epochs: 12
|
| 118 |
+
dirpath: "${output_dir}/${version}/checkpoints"
|
| 119 |
+
|
| 120 |
+
# logger:
|
| 121 |
+
# wandb:
|
| 122 |
+
# _target_: lightning.pytorch.loggers.wandb.WandbLogger
|
| 123 |
+
# project: "${name}"
|
| 124 |
+
# save_dir: ${output_dir}
|
| 125 |
+
# name: "${version}"
|
configs/baseline_triplane.yaml
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: loomnet_triplane_4
|
| 2 |
+
tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
|
| 3 |
+
version: loomnet_v2-stable-zero123-n16f16-pabs
|
| 4 |
+
description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
|
| 5 |
+
output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
|
| 6 |
+
|
| 7 |
+
#NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
|
| 8 |
+
resume:
|
| 9 |
+
|
| 10 |
+
extras:
|
| 11 |
+
# for dataset
|
| 12 |
+
# num_views: 17 # 1 reference view + 16 sampling views
|
| 13 |
+
# k_near_views: 16 # our baseline uses 16 nearest views
|
| 14 |
+
# sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
|
| 15 |
+
num_views: 2 # 1 reference view + 16 sampling views
|
| 16 |
+
k_near_views: 1 # our baseline uses 16 nearest views
|
| 17 |
+
sample_views_mode: "random"
|
| 18 |
+
img_wh: [256, 256]
|
| 19 |
+
bg_color: white
|
| 20 |
+
load_only_weights:
|
| 21 |
+
|
| 22 |
+
data:
|
| 23 |
+
_target_: loomnet.data.multiview.MultiViewDataModule
|
| 24 |
+
train_dataset:
|
| 25 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 26 |
+
type_of_dataset: "training"
|
| 27 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 28 |
+
img_wh: ${extras.img_wh}
|
| 29 |
+
bg_color: ${extras.bg_color}
|
| 30 |
+
num_views: ${extras.num_views}
|
| 31 |
+
k_near_views: ${extras.k_near_views}
|
| 32 |
+
ref_view_mode: "random" #64
|
| 33 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 34 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
|
| 35 |
+
use_abs_elevation: true # for stable zero123
|
| 36 |
+
train_batch_size: 6 #15
|
| 37 |
+
val_dataset:
|
| 38 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 39 |
+
type_of_dataset: "validation"
|
| 40 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 41 |
+
img_wh: ${extras.img_wh}
|
| 42 |
+
bg_color: ${extras.bg_color}
|
| 43 |
+
num_views: ${extras.num_views}
|
| 44 |
+
k_near_views: ${extras.k_near_views}
|
| 45 |
+
ref_view_mode: "random" #64
|
| 46 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 47 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
|
| 48 |
+
use_abs_elevation: true # for stable zero123
|
| 49 |
+
max_samples_per_category: 1
|
| 50 |
+
val_batch_size: 1 # [NB]: during validation we will use double val_batch_size due to cfg
|
| 51 |
+
test_dataset:
|
| 52 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 53 |
+
type_of_dataset: "testing"
|
| 54 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
|
| 55 |
+
img_wh: ${extras.img_wh}
|
| 56 |
+
bg_color: ${extras.bg_color}
|
| 57 |
+
num_views: ${extras.num_views}
|
| 58 |
+
k_near_views: ${extras.k_near_views}
|
| 59 |
+
ref_view_mode: 64
|
| 60 |
+
sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94]
|
| 61 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
|
| 62 |
+
use_abs_elevation: true
|
| 63 |
+
test_batch_size: 1
|
| 64 |
+
num_workers: 16 #64
|
| 65 |
+
pin_memory: true
|
| 66 |
+
|
| 67 |
+
system:
|
| 68 |
+
_target_: loomnet.systems.i2mv_system_v8.I2MVSystem
|
| 69 |
+
base_model_id: "ashawkey/stable-zero123-diffusers"
|
| 70 |
+
mv_model:
|
| 71 |
+
_target_: loomnet.models.mv_model_triplane.MVModel
|
| 72 |
+
_partial_: true
|
| 73 |
+
cp_block_model:
|
| 74 |
+
_target_: loomnet.models.extras.feature_aggregator_triplane.FeatureAggregator
|
| 75 |
+
_partial_: true
|
| 76 |
+
num_views: ${extras.num_views}
|
| 77 |
+
n_samples_per_ray: 48 #number of samples M per ray
|
| 78 |
+
num_weaving_layers_per_decoder_block: [4, 4, 5, 6] #how many times the weaving stages is repeated for each decoder layer
|
| 79 |
+
num_weaving_heads_per_decoder_block: [8, 8, 8, 16]
|
| 80 |
+
plane_channel_dim: ${system.mv_model.plane_channel_dim}
|
| 81 |
+
plane_resolution: ${system.mv_model.plane_resolution}
|
| 82 |
+
#plane_channel_dim_after_upsample: 240
|
| 83 |
+
splat_plane_resolutions: [64, 64, 64, 64] #the resolution of each splatted plane
|
| 84 |
+
latent_rendering_stages: [True, True, True, True] #in what stage to use latent rendering
|
| 85 |
+
do_local_bilinear_splatting: True
|
| 86 |
+
do_global_rendering: True
|
| 87 |
+
use_positional_encoding: True
|
| 88 |
+
|
| 89 |
+
insert_stages: ["mid", "up"]
|
| 90 |
+
insert_up_layers: [0, 1, 2, 3] #[0, 1, 2, 3]
|
| 91 |
+
use_residual: true
|
| 92 |
+
plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
|
| 93 |
+
plane_channel_dim: 512 #the number of channels of each latent plane
|
| 94 |
+
lr: 1e-5
|
| 95 |
+
cfg: 0.1
|
| 96 |
+
tv_weight: 5e-3 #loss weight for triplane
|
| 97 |
+
report_to: none #TODO --> wandn
|
| 98 |
+
|
| 99 |
+
trainer:
|
| 100 |
+
_target_: lightning.pytorch.trainer.Trainer
|
| 101 |
+
default_root_dir: ${output_dir}
|
| 102 |
+
max_steps: 946209
|
| 103 |
+
check_val_every_n_epoch: 1
|
| 104 |
+
log_every_n_steps: 50
|
| 105 |
+
num_sanity_val_steps: 0
|
| 106 |
+
enable_progress_bar: true
|
| 107 |
+
strategy: ddp_find_unused_parameters_true
|
| 108 |
+
accelerator: gpu
|
| 109 |
+
precision: 16-mixed # mixed precision for extra speed-up
|
| 110 |
+
accumulate_grad_batches: 16
|
| 111 |
+
gradient_clip_val: 1.0
|
| 112 |
+
|
| 113 |
+
callbacks:
|
| 114 |
+
model_checkpoint:
|
| 115 |
+
_target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 116 |
+
save_top_k: 1
|
| 117 |
+
#every_n_train_steps: 10000
|
| 118 |
+
#every_n_epochs: 12
|
| 119 |
+
dirpath: "${output_dir}/${version}/checkpoints"
|
| 120 |
+
|
| 121 |
+
# logger:
|
| 122 |
+
# wandb:
|
| 123 |
+
# _target_: lightning.pytorch.loggers.wandb.WandbLogger
|
| 124 |
+
# project: "${name}"
|
| 125 |
+
# save_dir: ${output_dir}
|
| 126 |
+
# name: "${version}"
|
configs/baseline_triplane2.yaml
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: loomnet_triplane_4
|
| 2 |
+
tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
|
| 3 |
+
version: loomnet_v2-stable-zero123-n16f16-pabs
|
| 4 |
+
description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
|
| 5 |
+
output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
|
| 6 |
+
|
| 7 |
+
#NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
|
| 8 |
+
resume: #/4/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=7-step=7704.ckpt
|
| 9 |
+
|
| 10 |
+
extras:
|
| 11 |
+
# for dataset
|
| 12 |
+
# num_views: 17 # 1 reference view + 16 sampling views
|
| 13 |
+
# k_near_views: 16 # our baseline uses 16 nearest views
|
| 14 |
+
# sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
|
| 15 |
+
num_views: 17 # 1 reference view + 16 sampling views
|
| 16 |
+
k_near_views: 1 # our baseline uses 16 nearest views
|
| 17 |
+
sample_views_mode: "random"
|
| 18 |
+
img_wh: [256, 256]
|
| 19 |
+
bg_color: white
|
| 20 |
+
load_only_weights:
|
| 21 |
+
|
| 22 |
+
data:
|
| 23 |
+
_target_: loomnet.data.multiview.MultiViewDataModule
|
| 24 |
+
train_dataset:
|
| 25 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 26 |
+
type_of_dataset: "training"
|
| 27 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 28 |
+
img_wh: ${extras.img_wh}
|
| 29 |
+
bg_color: ${extras.bg_color}
|
| 30 |
+
num_views: ${extras.num_views}
|
| 31 |
+
k_near_views: ${extras.k_near_views}
|
| 32 |
+
ref_view_mode: "random" #64
|
| 33 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 34 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
|
| 35 |
+
use_abs_elevation: true # for stable zero123
|
| 36 |
+
train_batch_size: 1 #15
|
| 37 |
+
val_dataset:
|
| 38 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 39 |
+
type_of_dataset: "validation"
|
| 40 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 41 |
+
img_wh: ${extras.img_wh}
|
| 42 |
+
bg_color: ${extras.bg_color}
|
| 43 |
+
num_views: ${extras.num_views}
|
| 44 |
+
k_near_views: ${extras.k_near_views}
|
| 45 |
+
ref_view_mode: "random" #64
|
| 46 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 47 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
|
| 48 |
+
use_abs_elevation: true # for stable zero123
|
| 49 |
+
max_samples_per_category: 1
|
| 50 |
+
val_batch_size: 1 # [NB]: during validation we will use double val_batch_size due to cfg
|
| 51 |
+
test_dataset:
|
| 52 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 53 |
+
type_of_dataset: "testing"
|
| 54 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
|
| 55 |
+
img_wh: ${extras.img_wh}
|
| 56 |
+
bg_color: ${extras.bg_color}
|
| 57 |
+
num_views: ${extras.num_views}
|
| 58 |
+
k_near_views: ${extras.k_near_views}
|
| 59 |
+
ref_view_mode: 64
|
| 60 |
+
sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94]
|
| 61 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
|
| 62 |
+
use_abs_elevation: true
|
| 63 |
+
test_batch_size: 1
|
| 64 |
+
num_workers: 16 #64
|
| 65 |
+
pin_memory: true
|
| 66 |
+
|
| 67 |
+
system:
|
| 68 |
+
_target_: loomnet.systems.i2mv_system_v8.I2MVSystem
|
| 69 |
+
base_model_id: "ashawkey/stable-zero123-diffusers"
|
| 70 |
+
mv_model:
|
| 71 |
+
_target_: loomnet.models.mv_model_triplane.MVModel
|
| 72 |
+
_partial_: true
|
| 73 |
+
cp_block_model:
|
| 74 |
+
_target_: loomnet.models.extras.feature_aggregator_triplane.FeatureAggregator
|
| 75 |
+
_partial_: true
|
| 76 |
+
num_views: ${extras.num_views}
|
| 77 |
+
n_samples_per_ray: 32 #number of samples M per ray
|
| 78 |
+
num_weaving_layers_per_decoder_block: [4, 4, 4, 4] #how many times the weaving stages is repeated for each decoder layer
|
| 79 |
+
num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
|
| 80 |
+
plane_channel_dim: ${system.mv_model.plane_channel_dim}
|
| 81 |
+
plane_resolution: ${system.mv_model.plane_resolution}
|
| 82 |
+
#plane_channel_dim_after_upsample: 240
|
| 83 |
+
splat_plane_resolutions: [64, 64, 64, 64] #the resolution of each splatted plane
|
| 84 |
+
latent_rendering_stages: [True, True, True, True] #in what stage to use latent rendering
|
| 85 |
+
do_local_bilinear_splatting: True
|
| 86 |
+
do_global_rendering: True
|
| 87 |
+
use_positional_encoding: True
|
| 88 |
+
|
| 89 |
+
insert_stages: ["mid", "up"]
|
| 90 |
+
insert_up_layers: [0, 1, 2, 3] #[0, 1, 2, 3]
|
| 91 |
+
use_residual: false
|
| 92 |
+
plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
|
| 93 |
+
plane_channel_dim: 512 #the number of channels of each latent plane
|
| 94 |
+
lr: 1e-5
|
| 95 |
+
cfg: 0.1
|
| 96 |
+
tv_weight: 5e-4 #loss weight for triplane
|
| 97 |
+
report_to: none #TODO --> wandn
|
| 98 |
+
|
| 99 |
+
trainer:
|
| 100 |
+
_target_: lightning.pytorch.trainer.Trainer
|
| 101 |
+
default_root_dir: ${output_dir}
|
| 102 |
+
max_steps: 946209
|
| 103 |
+
check_val_every_n_epoch: 1
|
| 104 |
+
log_every_n_steps: 50
|
| 105 |
+
num_sanity_val_steps: 0
|
| 106 |
+
enable_progress_bar: true
|
| 107 |
+
strategy: ddp_find_unused_parameters_true
|
| 108 |
+
accelerator: gpu
|
| 109 |
+
precision: 16-mixed # mixed precision for extra speed-up
|
| 110 |
+
accumulate_grad_batches: 16
|
| 111 |
+
gradient_clip_val: 1.0
|
| 112 |
+
|
| 113 |
+
callbacks:
|
| 114 |
+
model_checkpoint:
|
| 115 |
+
_target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 116 |
+
save_top_k: 1
|
| 117 |
+
#every_n_train_steps: 10000
|
| 118 |
+
#every_n_epochs: 12
|
| 119 |
+
dirpath: "${output_dir}/${version}/checkpoints"
|
| 120 |
+
|
| 121 |
+
# logger:
|
| 122 |
+
# wandb:
|
| 123 |
+
# _target_: lightning.pytorch.loggers.wandb.WandbLogger
|
| 124 |
+
# project: "${name}"
|
| 125 |
+
# save_dir: ${output_dir}
|
| 126 |
+
# name: "${version}"
|
configs/baseline_triplane3.yaml
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: loomnet_triplane_4
|
| 2 |
+
tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
|
| 3 |
+
version: loomnet_v2-stable-zero123-n16f16-pabs
|
| 4 |
+
description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
|
| 5 |
+
output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
|
| 6 |
+
|
| 7 |
+
#NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
|
| 8 |
+
resume: /raid/homes/giulio.federico/LoomNet/output_dir/loomnet_triplane_4/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=69-step=67410.ckpt
|
| 9 |
+
|
| 10 |
+
extras:
|
| 11 |
+
# for dataset
|
| 12 |
+
# num_views: 17 # 1 reference view + 16 sampling views
|
| 13 |
+
# k_near_views: 16 # our baseline uses 16 nearest views
|
| 14 |
+
# sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
|
| 15 |
+
num_views: 17 # 1 reference view + 16 sampling views
|
| 16 |
+
k_near_views: 1 # our baseline uses 16 nearest views
|
| 17 |
+
sample_views_mode: "random"
|
| 18 |
+
img_wh: [256, 256]
|
| 19 |
+
bg_color: white
|
| 20 |
+
load_only_weights:
|
| 21 |
+
|
| 22 |
+
data:
|
| 23 |
+
_target_: loomnet.data.multiview.MultiViewDataModule
|
| 24 |
+
train_dataset:
|
| 25 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 26 |
+
type_of_dataset: "training"
|
| 27 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 28 |
+
img_wh: ${extras.img_wh}
|
| 29 |
+
bg_color: ${extras.bg_color}
|
| 30 |
+
num_views: ${extras.num_views}
|
| 31 |
+
k_near_views: ${extras.k_near_views}
|
| 32 |
+
ref_view_mode: "random" #64
|
| 33 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 34 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
|
| 35 |
+
use_abs_elevation: true # for stable zero123
|
| 36 |
+
train_batch_size: 1 #15
|
| 37 |
+
val_dataset:
|
| 38 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 39 |
+
type_of_dataset: "validation"
|
| 40 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 41 |
+
img_wh: ${extras.img_wh}
|
| 42 |
+
bg_color: ${extras.bg_color}
|
| 43 |
+
num_views: ${extras.num_views}
|
| 44 |
+
k_near_views: ${extras.k_near_views}
|
| 45 |
+
ref_view_mode: "random" #64
|
| 46 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 47 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
|
| 48 |
+
use_abs_elevation: true # for stable zero123
|
| 49 |
+
max_samples_per_category: 1
|
| 50 |
+
val_batch_size: 1 # [NB]: during validation we will use double val_batch_size due to cfg
|
| 51 |
+
test_dataset:
|
| 52 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 53 |
+
type_of_dataset: "testing"
|
| 54 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
|
| 55 |
+
img_wh: ${extras.img_wh}
|
| 56 |
+
bg_color: ${extras.bg_color}
|
| 57 |
+
num_views: ${extras.num_views}
|
| 58 |
+
k_near_views: ${extras.k_near_views}
|
| 59 |
+
ref_view_mode: 64
|
| 60 |
+
sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94]
|
| 61 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
|
| 62 |
+
use_abs_elevation: true
|
| 63 |
+
test_batch_size: 1
|
| 64 |
+
num_workers: 16 #64
|
| 65 |
+
pin_memory: true
|
| 66 |
+
|
| 67 |
+
system:
|
| 68 |
+
_target_: loomnet.systems.i2mv_system_v8.I2MVSystem
|
| 69 |
+
base_model_id: "ashawkey/stable-zero123-diffusers"
|
| 70 |
+
mv_model:
|
| 71 |
+
_target_: loomnet.models.mv_model_triplane.MVModel
|
| 72 |
+
_partial_: true
|
| 73 |
+
cp_block_model:
|
| 74 |
+
_target_: loomnet.models.extras.feature_aggregator_triplane.FeatureAggregator
|
| 75 |
+
_partial_: true
|
| 76 |
+
num_views: ${extras.num_views}
|
| 77 |
+
n_samples_per_ray: 32 #number of samples M per ray
|
| 78 |
+
num_weaving_layers_per_decoder_block: [4, 4, 4, 4] #how many times the weaving stages is repeated for each decoder layer
|
| 79 |
+
num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
|
| 80 |
+
plane_channel_dim: ${system.mv_model.plane_channel_dim}
|
| 81 |
+
plane_resolution: ${system.mv_model.plane_resolution}
|
| 82 |
+
#plane_channel_dim_after_upsample: 240
|
| 83 |
+
splat_plane_resolutions: [64, 64, 64, 64] #the resolution of each splatted plane
|
| 84 |
+
latent_rendering_stages: [True, True, True, True] #in what stage to use latent rendering
|
| 85 |
+
do_local_bilinear_splatting: True
|
| 86 |
+
do_global_rendering: True
|
| 87 |
+
use_positional_encoding: True
|
| 88 |
+
|
| 89 |
+
insert_stages: ["mid", "up"]
|
| 90 |
+
insert_up_layers: [0, 1, 2, 3] #[0, 1, 2, 3]
|
| 91 |
+
use_residual: false
|
| 92 |
+
plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
|
| 93 |
+
plane_channel_dim: 512 #the number of channels of each latent plane
|
| 94 |
+
lr: 1e-5
|
| 95 |
+
cfg: 0.1
|
| 96 |
+
tv_weight: 5e-4 #loss weight for triplane
|
| 97 |
+
report_to: none #TODO --> wandn
|
| 98 |
+
|
| 99 |
+
trainer:
|
| 100 |
+
_target_: lightning.pytorch.trainer.Trainer
|
| 101 |
+
default_root_dir: ${output_dir}
|
| 102 |
+
max_steps: 946209
|
| 103 |
+
check_val_every_n_epoch: 1
|
| 104 |
+
log_every_n_steps: 50
|
| 105 |
+
num_sanity_val_steps: 0
|
| 106 |
+
enable_progress_bar: true
|
| 107 |
+
strategy: ddp_find_unused_parameters_true
|
| 108 |
+
accelerator: gpu
|
| 109 |
+
precision: 16-mixed # mixed precision for extra speed-up
|
| 110 |
+
accumulate_grad_batches: 16
|
| 111 |
+
gradient_clip_val: 1.0
|
| 112 |
+
|
| 113 |
+
callbacks:
|
| 114 |
+
model_checkpoint:
|
| 115 |
+
_target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 116 |
+
save_top_k: 1
|
| 117 |
+
#every_n_train_steps: 10000
|
| 118 |
+
#every_n_epochs: 12
|
| 119 |
+
dirpath: "${output_dir}/${version}/checkpoints"
|
| 120 |
+
|
| 121 |
+
# logger:
|
| 122 |
+
# wandb:
|
| 123 |
+
# _target_: lightning.pytorch.loggers.wandb.WandbLogger
|
| 124 |
+
# project: "${name}"
|
| 125 |
+
# save_dir: ${output_dir}
|
| 126 |
+
# name: "${version}"
|
configs/baseline_triplane_with_3d_conv.yaml
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: loomnet_triplane_4
|
| 2 |
+
tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
|
| 3 |
+
version: loomnet_v2-stable-zero123-n16f16-pabs
|
| 4 |
+
description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
|
| 5 |
+
output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
|
| 6 |
+
|
| 7 |
+
#NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
|
| 8 |
+
resume:
|
| 9 |
+
|
| 10 |
+
extras:
|
| 11 |
+
# for dataset
|
| 12 |
+
# num_views: 17 # 1 reference view + 16 sampling views
|
| 13 |
+
# k_near_views: 16 # our baseline uses 16 nearest views
|
| 14 |
+
# sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
|
| 15 |
+
num_views: 17 # 1 reference view + 16 sampling views
|
| 16 |
+
k_near_views: 1 # our baseline uses 16 nearest views
|
| 17 |
+
sample_views_mode: "random"
|
| 18 |
+
img_wh: [256, 256]
|
| 19 |
+
bg_color: white
|
| 20 |
+
load_only_weights:
|
| 21 |
+
|
| 22 |
+
data:
|
| 23 |
+
_target_: loomnet.data.multiview.MultiViewDataModule
|
| 24 |
+
train_dataset:
|
| 25 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 26 |
+
type_of_dataset: "training"
|
| 27 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 28 |
+
img_wh: ${extras.img_wh}
|
| 29 |
+
bg_color: ${extras.bg_color}
|
| 30 |
+
num_views: ${extras.num_views}
|
| 31 |
+
k_near_views: ${extras.k_near_views}
|
| 32 |
+
ref_view_mode: "random" #64
|
| 33 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 34 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
|
| 35 |
+
use_abs_elevation: true # for stable zero123
|
| 36 |
+
train_batch_size: 1 #15
|
| 37 |
+
val_dataset:
|
| 38 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 39 |
+
type_of_dataset: "validation"
|
| 40 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 41 |
+
img_wh: ${extras.img_wh}
|
| 42 |
+
bg_color: ${extras.bg_color}
|
| 43 |
+
num_views: ${extras.num_views}
|
| 44 |
+
k_near_views: ${extras.k_near_views}
|
| 45 |
+
ref_view_mode: "random" #64
|
| 46 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 47 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
|
| 48 |
+
use_abs_elevation: true # for stable zero123
|
| 49 |
+
max_samples_per_category: 1
|
| 50 |
+
val_batch_size: 1 # [NB]: during validation we will use double val_batch_size due to cfg
|
| 51 |
+
test_dataset:
|
| 52 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 53 |
+
type_of_dataset: "testing"
|
| 54 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
|
| 55 |
+
img_wh: ${extras.img_wh}
|
| 56 |
+
bg_color: ${extras.bg_color}
|
| 57 |
+
num_views: ${extras.num_views}
|
| 58 |
+
k_near_views: ${extras.k_near_views}
|
| 59 |
+
ref_view_mode: 64
|
| 60 |
+
sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94]
|
| 61 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
|
| 62 |
+
use_abs_elevation: true
|
| 63 |
+
test_batch_size: 1
|
| 64 |
+
num_workers: 16 #64
|
| 65 |
+
pin_memory: true
|
| 66 |
+
|
| 67 |
+
system:
|
| 68 |
+
_target_: loomnet.systems.i2mv_system_v8.I2MVSystem
|
| 69 |
+
base_model_id: "ashawkey/stable-zero123-diffusers"
|
| 70 |
+
mv_model:
|
| 71 |
+
_target_: loomnet.models.mv_model_triplane_with_3d_conv.MVModel
|
| 72 |
+
_partial_: true
|
| 73 |
+
cp_block_model:
|
| 74 |
+
_target_: loomnet.models.extras.feature_aggregator_triplane_with_3d_conv.FeatureAggregator
|
| 75 |
+
_partial_: true
|
| 76 |
+
num_views: ${extras.num_views}
|
| 77 |
+
n_samples_per_ray: 48 #number of samples M per ray
|
| 78 |
+
num_weaving_layers_per_decoder_block: [4, 4, 5, 6] #how many times the weaving stages is repeated for each decoder layer
|
| 79 |
+
num_weaving_heads_per_decoder_block: [8, 8, 8, 16]
|
| 80 |
+
plane_channel_dim: ${system.mv_model.plane_channel_dim}
|
| 81 |
+
plane_resolution: ${system.mv_model.plane_resolution}
|
| 82 |
+
#plane_channel_dim_after_upsample: 240
|
| 83 |
+
splat_plane_resolutions: [64, 64, 64, 64] #the resolution of each splatted plane
|
| 84 |
+
latent_rendering_stages: [True, True, True, True] #in what stage to use latent rendering
|
| 85 |
+
do_local_bilinear_splatting: True
|
| 86 |
+
do_global_rendering: True
|
| 87 |
+
use_positional_encoding: True
|
| 88 |
+
|
| 89 |
+
insert_stages: ["mid", "up"]
|
| 90 |
+
insert_up_layers: [0, 1, 2, 3] #[0, 1, 2, 3]
|
| 91 |
+
use_residual: true
|
| 92 |
+
plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
|
| 93 |
+
plane_channel_dim: 512 #the number of channels of each latent plane
|
| 94 |
+
lr: 1e-5
|
| 95 |
+
cfg: 0.1
|
| 96 |
+
tv_weight: 5e-3 #loss weight for triplane
|
| 97 |
+
report_to: none #TODO --> wandn
|
| 98 |
+
|
| 99 |
+
trainer:
|
| 100 |
+
_target_: lightning.pytorch.trainer.Trainer
|
| 101 |
+
default_root_dir: ${output_dir}
|
| 102 |
+
max_steps: 946209
|
| 103 |
+
check_val_every_n_epoch: 1
|
| 104 |
+
log_every_n_steps: 50
|
| 105 |
+
num_sanity_val_steps: 0
|
| 106 |
+
enable_progress_bar: true
|
| 107 |
+
strategy: ddp_find_unused_parameters_true
|
| 108 |
+
accelerator: gpu
|
| 109 |
+
precision: 16-mixed # mixed precision for extra speed-up
|
| 110 |
+
accumulate_grad_batches: 16
|
| 111 |
+
gradient_clip_val: 1.0
|
| 112 |
+
|
| 113 |
+
callbacks:
|
| 114 |
+
model_checkpoint:
|
| 115 |
+
_target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 116 |
+
save_top_k: 1
|
| 117 |
+
#every_n_train_steps: 10000
|
| 118 |
+
#every_n_epochs: 12
|
| 119 |
+
dirpath: "${output_dir}/${version}/checkpoints"
|
| 120 |
+
|
| 121 |
+
# logger:
|
| 122 |
+
# wandb:
|
| 123 |
+
# _target_: lightning.pytorch.loggers.wandb.WandbLogger
|
| 124 |
+
# project: "${name}"
|
| 125 |
+
# save_dir: ${output_dir}
|
| 126 |
+
# name: "${version}"
|
configs/baseline_with_distillation.yaml
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: loomnet_with_distillation_3
|
| 2 |
+
tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
|
| 3 |
+
version: loomnet_v2-stable-zero123-n16f16-pabs
|
| 4 |
+
description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
|
| 5 |
+
output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
|
| 6 |
+
|
| 7 |
+
#NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
|
| 8 |
+
resume: /raid/homes/giulio.federico/LoomNet/output_dir/loomnet_v9/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=47-step=37544.ckpt
|
| 9 |
+
|
| 10 |
+
extras:
|
| 11 |
+
# for dataset
|
| 12 |
+
# num_views: 17 # 1 reference view + 16 sampling views
|
| 13 |
+
# k_near_views: 16 # our baseline uses 16 nearest views
|
| 14 |
+
# sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
|
| 15 |
+
num_views: 17 # 1 reference view + 16 sampling views
|
| 16 |
+
k_near_views: 1 # our baseline uses 16 nearest views
|
| 17 |
+
sample_views_mode: "random"
|
| 18 |
+
img_wh: [256, 256]
|
| 19 |
+
bg_color: white
|
| 20 |
+
load_only_weights:
|
| 21 |
+
|
| 22 |
+
data:
|
| 23 |
+
_target_: loomnet.data.multiview.MultiViewDataModule
|
| 24 |
+
train_dataset:
|
| 25 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 26 |
+
type_of_dataset: "training"
|
| 27 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 28 |
+
img_wh: ${extras.img_wh}
|
| 29 |
+
bg_color: ${extras.bg_color}
|
| 30 |
+
num_views: ${extras.num_views}
|
| 31 |
+
k_near_views: ${extras.k_near_views}
|
| 32 |
+
ref_view_mode: "random" #64
|
| 33 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 34 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
|
| 35 |
+
use_abs_elevation: true # for stable zero123
|
| 36 |
+
train_batch_size: 2 #15
|
| 37 |
+
val_dataset:
|
| 38 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 39 |
+
type_of_dataset: "validation"
|
| 40 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 41 |
+
img_wh: ${extras.img_wh}
|
| 42 |
+
bg_color: ${extras.bg_color}
|
| 43 |
+
num_views: ${extras.num_views}
|
| 44 |
+
k_near_views: ${extras.k_near_views}
|
| 45 |
+
ref_view_mode: "random" #64
|
| 46 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 47 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
|
| 48 |
+
use_abs_elevation: true # for stable zero123
|
| 49 |
+
max_samples_per_category: 1
|
| 50 |
+
val_batch_size: 2 # [NB]: during validation we will use double val_batch_size due to cfg
|
| 51 |
+
test_dataset:
|
| 52 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 53 |
+
type_of_dataset: "testing"
|
| 54 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
|
| 55 |
+
img_wh: ${extras.img_wh}
|
| 56 |
+
bg_color: ${extras.bg_color}
|
| 57 |
+
num_views: ${extras.num_views}
|
| 58 |
+
k_near_views: ${extras.k_near_views}
|
| 59 |
+
ref_view_mode: 64
|
| 60 |
+
sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94]
|
| 61 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
|
| 62 |
+
use_abs_elevation: true
|
| 63 |
+
test_batch_size: 1
|
| 64 |
+
num_workers: 16 #64
|
| 65 |
+
pin_memory: true
|
| 66 |
+
|
| 67 |
+
system:
|
| 68 |
+
_target_: loomnet.systems.i2mv_system_with_distillation.I2MVSystem
|
| 69 |
+
base_model_id: "ashawkey/stable-zero123-diffusers"
|
| 70 |
+
mv_model:
|
| 71 |
+
_target_: loomnet.models.mv_model_with_distillation.MVModel
|
| 72 |
+
_partial_: true
|
| 73 |
+
cp_block_model:
|
| 74 |
+
_target_: loomnet.models.extras.feature_aggregator_with_distillation.FeatureAggregator
|
| 75 |
+
_partial_: true
|
| 76 |
+
num_views: ${extras.num_views}
|
| 77 |
+
n_samples_per_ray: 16 #number of samples M per ray
|
| 78 |
+
num_weaving_layers_per_decoder_block: [3, 4, 6, 8] #how many times the weaving stages is repeated for each decoder layer
|
| 79 |
+
num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
|
| 80 |
+
plane_channel_dim: ${system.mv_model.plane_channel_dim}
|
| 81 |
+
plane_resolution: ${system.mv_model.plane_resolution}
|
| 82 |
+
splat_plane_resolution: 64 #the resolution of each splatted plane
|
| 83 |
+
latent_rendering_stages: [False, False, False, True] #in what stage to use latent rendering
|
| 84 |
+
do_local_bilinear_splatting: True
|
| 85 |
+
use_positional_encoding: True
|
| 86 |
+
|
| 87 |
+
insert_stages: ["mid", "up"]
|
| 88 |
+
insert_up_layers: [0, 1, 2, 3]
|
| 89 |
+
use_residual: false
|
| 90 |
+
plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
|
| 91 |
+
plane_channel_dim: 512 #the number of channels of each latent plane
|
| 92 |
+
lr: 1e-5
|
| 93 |
+
cfg: 0.1
|
| 94 |
+
tv_weight: 5e-4 #loss weight for triplane
|
| 95 |
+
report_to: none #TODO --> wandn
|
| 96 |
+
|
| 97 |
+
trainer:
|
| 98 |
+
_target_: lightning.pytorch.trainer.Trainer
|
| 99 |
+
default_root_dir: ${output_dir}
|
| 100 |
+
max_steps: 52545
|
| 101 |
+
check_val_every_n_epoch: 1
|
| 102 |
+
log_every_n_steps: 5
|
| 103 |
+
num_sanity_val_steps: 0
|
| 104 |
+
enable_progress_bar: true
|
| 105 |
+
strategy: ddp_find_unused_parameters_true
|
| 106 |
+
accelerator: gpu
|
| 107 |
+
precision: 16-mixed # mixed precision for extra speed-up
|
| 108 |
+
accumulate_grad_batches: 4
|
| 109 |
+
gradient_clip_val: 1.0
|
| 110 |
+
|
| 111 |
+
callbacks:
|
| 112 |
+
model_checkpoint:
|
| 113 |
+
_target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 114 |
+
save_top_k: 1
|
| 115 |
+
#every_n_train_steps: 10000
|
| 116 |
+
#every_n_epochs: 12
|
| 117 |
+
dirpath: "${output_dir}/${version}/checkpoints"
|
| 118 |
+
|
| 119 |
+
# logger:
|
| 120 |
+
# wandb:
|
| 121 |
+
# _target_: lightning.pytorch.loggers.wandb.WandbLogger
|
| 122 |
+
# project: "${name}"
|
| 123 |
+
# save_dir: ${output_dir}
|
| 124 |
+
# name: "${version}"
|
configs/baseline_with_projection.yaml
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: loomnet
|
| 2 |
+
tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
|
| 3 |
+
version: loomnet_v2-stable-zero123-n16f16-pabs
|
| 4 |
+
description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
|
| 5 |
+
output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
|
| 6 |
+
|
| 7 |
+
#NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
|
| 8 |
+
resume: /raid/homes/giulio.federico/LoomNet/output_dir/loomnet/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=60-step=39162.ckpt
|
| 9 |
+
|
| 10 |
+
extras:
|
| 11 |
+
# for dataset
|
| 12 |
+
# num_views: 17 # 1 reference view + 16 sampling views
|
| 13 |
+
# k_near_views: 16 # our baseline uses 16 nearest views
|
| 14 |
+
# sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
|
| 15 |
+
num_views: 2 # 1 reference view + 16 sampling views
|
| 16 |
+
k_near_views: 1 # our baseline uses 16 nearest views
|
| 17 |
+
sample_views_mode: "random"
|
| 18 |
+
img_wh: [256, 256]
|
| 19 |
+
bg_color: white
|
| 20 |
+
load_only_weights:
|
| 21 |
+
|
| 22 |
+
data:
|
| 23 |
+
_target_: loomnet.data.multiview.MultiViewDataModule
|
| 24 |
+
train_dataset:
|
| 25 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 26 |
+
type_of_dataset: "training"
|
| 27 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 28 |
+
img_wh: ${extras.img_wh}
|
| 29 |
+
bg_color: ${extras.bg_color}
|
| 30 |
+
num_views: ${extras.num_views}
|
| 31 |
+
k_near_views: ${extras.k_near_views}
|
| 32 |
+
ref_view_mode: "random" #64
|
| 33 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 34 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
|
| 35 |
+
use_abs_elevation: true # for stable zero123
|
| 36 |
+
train_batch_size: 1 #15
|
| 37 |
+
val_dataset:
|
| 38 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 39 |
+
type_of_dataset: "validation"
|
| 40 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
|
| 41 |
+
img_wh: ${extras.img_wh}
|
| 42 |
+
bg_color: ${extras.bg_color}
|
| 43 |
+
num_views: ${extras.num_views}
|
| 44 |
+
k_near_views: ${extras.k_near_views}
|
| 45 |
+
ref_view_mode: "random" #64
|
| 46 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 47 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
|
| 48 |
+
use_abs_elevation: true # for stable zero123
|
| 49 |
+
max_samples_per_category: 1
|
| 50 |
+
val_batch_size: 2 # [NB]: during validation we will use double val_batch_size due to cfg
|
| 51 |
+
test_dataset:
|
| 52 |
+
_target_: loomnet.data.multiview.MultiViewDataset
|
| 53 |
+
type_of_dataset: "testing"
|
| 54 |
+
root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
|
| 55 |
+
img_wh: ${extras.img_wh}
|
| 56 |
+
bg_color: ${extras.bg_color}
|
| 57 |
+
num_views: ${extras.num_views}
|
| 58 |
+
k_near_views: ${extras.k_near_views}
|
| 59 |
+
ref_view_mode: 64
|
| 60 |
+
sample_views_mode: ${extras.sample_views_mode}
|
| 61 |
+
caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
|
| 62 |
+
use_abs_elevation: true
|
| 63 |
+
test_batch_size: 1
|
| 64 |
+
num_workers: 16 #64
|
| 65 |
+
pin_memory: true
|
| 66 |
+
|
| 67 |
+
system:
|
| 68 |
+
_target_: loomnet.systems.i2mv_system_with_projection.I2MVSystem
|
| 69 |
+
base_model_id: "ashawkey/stable-zero123-diffusers"
|
| 70 |
+
mv_model:
|
| 71 |
+
_target_: loomnet.models.mv_model_v8.MVModel
|
| 72 |
+
_partial_: true
|
| 73 |
+
cp_block_model:
|
| 74 |
+
_target_: loomnet.models.extras.feature_aggregator_with_projection.FeatureAggregator
|
| 75 |
+
_partial_: true
|
| 76 |
+
num_views: ${extras.num_views}
|
| 77 |
+
n_samples_per_ray: 64 #number of samples M per ray
|
| 78 |
+
num_weaving_layers_per_decoder_block: [4, 4, 6, 8] #how many times the weaving stages is repeated for each decoder layer
|
| 79 |
+
num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
|
| 80 |
+
plane_channel_dim: ${system.mv_model.plane_channel_dim}
|
| 81 |
+
plane_resolution: ${system.mv_model.plane_resolution}
|
| 82 |
+
splat_plane_resolution: 128 #the resolution of each splatted plane
|
| 83 |
+
triplane_upsample_channel_dim: 32
|
| 84 |
+
latent_rendering_stages: [False, False, False, True] #in what stage to use latent rendering
|
| 85 |
+
do_local_bilinear_splatting: True
|
| 86 |
+
do_global_rendering: True
|
| 87 |
+
use_positional_encoding: True
|
| 88 |
+
|
| 89 |
+
insert_stages: ["up"]
|
| 90 |
+
insert_up_layers: [3]
|
| 91 |
+
use_residual: false
|
| 92 |
+
plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
|
| 93 |
+
plane_channel_dim: 512 #the number of channels of each latent plane
|
| 94 |
+
lr: 1e-5
|
| 95 |
+
cfg: 0.1
|
| 96 |
+
tv_weight: 5e-4 #loss weight for triplane
|
| 97 |
+
report_to: none #TODO --> wandn
|
| 98 |
+
|
| 99 |
+
trainer:
|
| 100 |
+
_target_: lightning.pytorch.trainer.Trainer
|
| 101 |
+
default_root_dir: ${output_dir}
|
| 102 |
+
max_steps: 370001
|
| 103 |
+
check_val_every_n_epoch: 1
|
| 104 |
+
log_every_n_steps: 5
|
| 105 |
+
num_sanity_val_steps: 0
|
| 106 |
+
enable_progress_bar: true
|
| 107 |
+
strategy: ddp_find_unused_parameters_true
|
| 108 |
+
accelerator: gpu
|
| 109 |
+
precision: 16-mixed # mixed precision for extra speed-up
|
| 110 |
+
accumulate_grad_batches: 4
|
| 111 |
+
gradient_clip_val: 1.0
|
| 112 |
+
|
| 113 |
+
callbacks:
|
| 114 |
+
model_checkpoint:
|
| 115 |
+
_target_: lightning.pytorch.callbacks.ModelCheckpoint
|
| 116 |
+
save_top_k: 1
|
| 117 |
+
#every_n_train_steps: 10000
|
| 118 |
+
#every_n_epochs: 12
|
| 119 |
+
dirpath: "${output_dir}/${version}/checkpoints"
|
| 120 |
+
|
| 121 |
+
# logger:
|
| 122 |
+
# wandb:
|
| 123 |
+
# _target_: lightning.pytorch.loggers.wandb.WandbLogger
|
| 124 |
+
# project: "${name}"
|
| 125 |
+
# save_dir: ${output_dir}
|
| 126 |
+
# name: "${version}"
|
configs/loomnet_global2.log
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
nohup: ignoring input
|
| 2 |
+
nohup: failed to run command 'torchrun': No such file or directory
|
configs/loomnet_triplane.yaml.save
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
^x
|
configs/nerf.yaml
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
model:
|
| 2 |
+
base_lr: 1.0e-2
|
| 3 |
+
target: renderer.renderer.RendererTrainer
|
| 4 |
+
params:
|
| 5 |
+
total_steps: 2000
|
| 6 |
+
warm_up_steps: 100
|
| 7 |
+
train_batch_num: 40960
|
| 8 |
+
test_batch_num: 40960
|
| 9 |
+
renderer: ngp
|
| 10 |
+
cube_bound: 0.6
|
| 11 |
+
use_mask: true
|
| 12 |
+
lambda_rgb_loss: 0.5
|
| 13 |
+
lambda_mask_loss: 10.0
|
| 14 |
+
meta_camera_path: "/raid/homes/giulio.federico/LoomNet/src/meta/fixed_30_meta.pkl"
|
| 15 |
+
|
| 16 |
+
data:
|
| 17 |
+
target: renderer.dummy_dataset.DummyDataset
|
| 18 |
+
params: {}
|
| 19 |
+
|
| 20 |
+
callbacks:
|
| 21 |
+
save_interval: 5000
|
| 22 |
+
|
| 23 |
+
trainer:
|
| 24 |
+
val_check_interval: 500
|
| 25 |
+
max_steps: 2000
|
| 26 |
+
|
configs/neus.yaml
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
model:
|
| 2 |
+
base_lr: 5.0e-4
|
| 3 |
+
target: renderer.renderer.RendererTrainer
|
| 4 |
+
params:
|
| 5 |
+
total_steps: 2000
|
| 6 |
+
warm_up_steps: 100
|
| 7 |
+
train_batch_num: 3584
|
| 8 |
+
train_batch_fg_num: 512
|
| 9 |
+
test_batch_num: 4096
|
| 10 |
+
use_mask: true
|
| 11 |
+
lambda_rgb_loss: 0.5
|
| 12 |
+
lambda_mask_loss: 1.0
|
| 13 |
+
lambda_eikonal_loss: 0.1
|
| 14 |
+
use_warm_up: true
|
| 15 |
+
meta_camera_path: /raid/homes/giulio.federico/LoomNet/competitors/SyncDreamer/meta_info/camera-16.pkl
|
| 16 |
+
|
| 17 |
+
data:
|
| 18 |
+
target: renderer.dummy_dataset.DummyDataset
|
| 19 |
+
params: {}
|
| 20 |
+
|
| 21 |
+
callbacks:
|
| 22 |
+
save_interval: 500
|
| 23 |
+
|
| 24 |
+
trainer:
|
| 25 |
+
val_check_interval: 500
|
| 26 |
+
max_steps: 2000
|
| 27 |
+
|
inference.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
import json
|
| 3 |
+
import math
|
| 4 |
+
import os
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from typing import List
|
| 7 |
+
|
| 8 |
+
import hydra
|
| 9 |
+
import lightning as L
|
| 10 |
+
import numpy as np
|
| 11 |
+
import torch
|
| 12 |
+
from einops import rearrange
|
| 13 |
+
from lightning import LightningModule
|
| 14 |
+
from PIL import Image
|
| 15 |
+
|
| 16 |
+
from loomnet.utils import load_config
|
| 17 |
+
from loomnet.utils.media import get_bg_color, load_image
|
| 18 |
+
from loomnet.utils.pose import get_k_near_views
|
| 19 |
+
|
| 20 |
+
WIDTH, HEIGHT = 256, 256
|
| 21 |
+
K_NEAR_VIEWS = 16
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def prepare_inputs(input_img: str, input_elevation: float, sample_views_mode: str):
|
| 25 |
+
bg_color = get_bg_color("white")
|
| 26 |
+
input_img = load_image(
|
| 27 |
+
input_img, (WIDTH, HEIGHT), bg_color, return_type="pt"
|
| 28 |
+
).permute(2, 0, 1)
|
| 29 |
+
|
| 30 |
+
meta_fp = f"meta_info/transforms_{sample_views_mode}.json"
|
| 31 |
+
with open(meta_fp, "r") as f:
|
| 32 |
+
meta = json.load(f)
|
| 33 |
+
|
| 34 |
+
# Camera intrinsics
|
| 35 |
+
fov = meta["camera_angle_x"]
|
| 36 |
+
focal_length = 1 / (2 * np.tan(0.5 * fov))
|
| 37 |
+
intrinsics_4x4 = torch.tensor(
|
| 38 |
+
[
|
| 39 |
+
[focal_length, 0, 0.5, 0],
|
| 40 |
+
[0, focal_length, 0.5, 0],
|
| 41 |
+
[0, 0, 1, 0],
|
| 42 |
+
[0, 0, 0, 1],
|
| 43 |
+
]
|
| 44 |
+
)
|
| 45 |
+
|
| 46 |
+
# Camera extrinsics
|
| 47 |
+
num_views = len(meta["frames"])
|
| 48 |
+
elevations, azimuths, c2w_matrixs = [], [], []
|
| 49 |
+
for frame in meta["frames"]:
|
| 50 |
+
elevations.append(frame["elevation"])
|
| 51 |
+
azimuths.append(frame["azimuth"])
|
| 52 |
+
c2w_matrixs.append(frame["transform_matrix"])
|
| 53 |
+
elevations = torch.tensor(elevations) # (N,)
|
| 54 |
+
azimuths = torch.tensor(azimuths) # (N,)
|
| 55 |
+
c2w_matrixs = torch.tensor(c2w_matrixs) # (N, 4, 4)
|
| 56 |
+
c2w_matrixs[:, :, 1:3] *= -1 # blender to opencv
|
| 57 |
+
|
| 58 |
+
# concat intrinsics and extrinsics
|
| 59 |
+
intrinsics_4x4 = (
|
| 60 |
+
intrinsics_4x4.unsqueeze(0).repeat(num_views, 1, 1).reshape(num_views, 16)
|
| 61 |
+
)
|
| 62 |
+
_c2w_matrixs = c2w_matrixs.reshape(num_views, 16)
|
| 63 |
+
camera_params = torch.cat([intrinsics_4x4, _c2w_matrixs], dim=1)
|
| 64 |
+
|
| 65 |
+
# find nearest views
|
| 66 |
+
k_near_indices = get_k_near_views(elevations, azimuths, K_NEAR_VIEWS, num_views)
|
| 67 |
+
|
| 68 |
+
# normalize elevations and azimuths
|
| 69 |
+
input_elevation = torch.tensor([input_elevation / 180 * math.pi])
|
| 70 |
+
d_elevations = (elevations - input_elevation).reshape(-1, 1)
|
| 71 |
+
d_azimuths = azimuths.reshape(-1, 1) % (2 * math.pi)
|
| 72 |
+
|
| 73 |
+
#distances = torch.zeros_like(d_elevations)
|
| 74 |
+
distances = torch.tensor(input_elevation).repeat(num_views).reshape(-1, 1)
|
| 75 |
+
|
| 76 |
+
return {
|
| 77 |
+
"image_0": input_img,
|
| 78 |
+
"elevations": d_elevations,
|
| 79 |
+
"azimuths": d_azimuths,
|
| 80 |
+
"distances": distances,
|
| 81 |
+
"c2w_matrixs": c2w_matrixs,
|
| 82 |
+
"cameras": camera_params,
|
| 83 |
+
"k_near_indices": k_near_indices,
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def main():
|
| 88 |
+
parser = argparse.ArgumentParser()
|
| 89 |
+
parser.add_argument("--config", type=str, default="configs/baseline.yaml")
|
| 90 |
+
parser.add_argument("--input_img", type=str, default="/raid/homes/giulio.federico/LoomNet/da_eliminare/render_0065.png")
|
| 91 |
+
parser.add_argument("--output_dir", type=str, default="/raid/homes/giulio.federico/LoomNet/output_dir")
|
| 92 |
+
parser.add_argument("--elevation", type=float, default=0.0)
|
| 93 |
+
parser.add_argument(
|
| 94 |
+
"--sample_views_mode", type=str, choices=["ele30", "ele40"], default="ele30"
|
| 95 |
+
)
|
| 96 |
+
parser.add_argument("--seed", type=int, default=42)
|
| 97 |
+
parser.add_argument("--device", type=str, default="cuda")
|
| 98 |
+
args, extras = parser.parse_known_args()
|
| 99 |
+
|
| 100 |
+
output_dir = Path(args.output_dir)
|
| 101 |
+
output_dir.mkdir(exist_ok=True, parents=True)
|
| 102 |
+
|
| 103 |
+
L.seed_everything(args.seed, workers=True)
|
| 104 |
+
cfg = load_config(args.config, cli_args=extras)
|
| 105 |
+
|
| 106 |
+
# prepare model
|
| 107 |
+
model: LightningModule = hydra.utils.instantiate(cfg.system)
|
| 108 |
+
model.load_weights(cfg.resume)
|
| 109 |
+
model = model.to(args.device).eval()
|
| 110 |
+
print(f"Loaded model from {cfg.resume}")
|
| 111 |
+
|
| 112 |
+
# prepare data
|
| 113 |
+
data = prepare_inputs(args.input_img, args.elevation, args.sample_views_mode)
|
| 114 |
+
for k, v in data.items():
|
| 115 |
+
data[k] = v.unsqueeze(0).to(args.device)
|
| 116 |
+
if k not in ["k_near_indices"]:
|
| 117 |
+
data[k] = data[k].float()
|
| 118 |
+
|
| 119 |
+
# generate
|
| 120 |
+
with torch.no_grad():
|
| 121 |
+
images_pred = model._generate_images(data)
|
| 122 |
+
|
| 123 |
+
# save
|
| 124 |
+
image_base_name = os.path.basename(args.input_img).split(".")[0]
|
| 125 |
+
image_list = []
|
| 126 |
+
for image in images_pred[0]:
|
| 127 |
+
image_list.append(Image.fromarray(image))
|
| 128 |
+
image_list[0].save(
|
| 129 |
+
output_dir / f"{image_base_name}.gif",
|
| 130 |
+
save_all=True,
|
| 131 |
+
append_images=image_list[1:],
|
| 132 |
+
duration=100,
|
| 133 |
+
loop=0,
|
| 134 |
+
)
|
| 135 |
+
|
| 136 |
+
full_image = rearrange(images_pred, "b m h w c -> (b h) (m w) c")
|
| 137 |
+
Image.fromarray(full_image).save(output_dir / f"{image_base_name}.jpg")
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
if __name__ == "__main__":
|
| 141 |
+
main()
|
loomnet/data/__init__.py
ADDED
|
File without changes
|
loomnet/data/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (177 Bytes). View file
|
|
|
loomnet/data/__pycache__/multiview.cpython-311.pyc
ADDED
|
Binary file (19.7 kB). View file
|
|
|
loomnet/data/__pycache__/multiview_infinity.cpython-311.pyc
ADDED
|
Binary file (20.5 kB). View file
|
|
|
loomnet/data/multiview.py
ADDED
|
@@ -0,0 +1,375 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import math
|
| 3 |
+
import os
|
| 4 |
+
from os.path import join as osp
|
| 5 |
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
| 6 |
+
import random
|
| 7 |
+
import numpy as np
|
| 8 |
+
import torch
|
| 9 |
+
from lightning import LightningDataModule
|
| 10 |
+
from omegaconf import DictConfig, ListConfig
|
| 11 |
+
from PIL import Image
|
| 12 |
+
from torch.utils.data import DataLoader, Dataset
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class MultiViewDataset(Dataset):
|
| 16 |
+
def __init__(
|
| 17 |
+
self,
|
| 18 |
+
type_of_dataset: str,
|
| 19 |
+
root_dir: str,
|
| 20 |
+
num_views: int,
|
| 21 |
+
bg_color: str,
|
| 22 |
+
img_wh: Tuple[int, int],
|
| 23 |
+
caption_path: str,
|
| 24 |
+
ref_view_mode: Union[str, List] = "random",
|
| 25 |
+
sample_views_mode: Union[str, List] = "random",
|
| 26 |
+
k_near_views: Optional[int] = None,
|
| 27 |
+
max_samples_per_category: Optional[int] = None,
|
| 28 |
+
use_abs_elevation: bool = False,
|
| 29 |
+
seed = 0
|
| 30 |
+
):
|
| 31 |
+
#data = []
|
| 32 |
+
|
| 33 |
+
# # Loop per ogni cartella di categoria
|
| 34 |
+
# for category in os.listdir(root_dir):
|
| 35 |
+
# category_path = os.path.join(root_dir, category)
|
| 36 |
+
|
| 37 |
+
# if os.path.isdir(category_path):
|
| 38 |
+
|
| 39 |
+
# uids = os.listdir(category_path)
|
| 40 |
+
# #se validazione, prendiamo per semplicità solo due campioni
|
| 41 |
+
# if max_samples_per_category is not None:
|
| 42 |
+
# random.seed(seed)
|
| 43 |
+
# uids = random.sample(uids, min(max_samples_per_category, len(uids)))
|
| 44 |
+
# for uid in uids:
|
| 45 |
+
|
| 46 |
+
# uid_path = os.path.join(root_dir, category, uid)
|
| 47 |
+
# obj_data = {"obj_dir": uid_path, "caption": ""}
|
| 48 |
+
|
| 49 |
+
# data.append(obj_data)
|
| 50 |
+
|
| 51 |
+
data = []
|
| 52 |
+
# Loop per ogni cartella di categoria
|
| 53 |
+
for category in os.listdir(root_dir):
|
| 54 |
+
category_path = os.path.join(root_dir, category)
|
| 55 |
+
|
| 56 |
+
if os.path.isdir(category_path):
|
| 57 |
+
|
| 58 |
+
uids = [f for f in os.listdir(category_path) if not os.path.isfile(os.path.join(category_path, f))] #os.listdir(category_path)
|
| 59 |
+
|
| 60 |
+
if len(uids)!=0:
|
| 61 |
+
for uid in uids:
|
| 62 |
+
|
| 63 |
+
uid_path = os.path.join(root_dir, category, uid)
|
| 64 |
+
obj_data = {"obj_dir": uid_path, "caption": ""}
|
| 65 |
+
|
| 66 |
+
data.append(obj_data)
|
| 67 |
+
else:
|
| 68 |
+
uid_path = os.path.join(root_dir, category)
|
| 69 |
+
obj_data = {"obj_dir": uid_path, "caption": ""}
|
| 70 |
+
|
| 71 |
+
data.append(obj_data)
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
#da eliminare (per velocità prendo solo i primi 8)
|
| 75 |
+
if type_of_dataset=="validation":
|
| 76 |
+
random.seed(3) # esempio: random.seed(42)
|
| 77 |
+
indices = random.sample(range(len(data)), 16) # prendo 96 indici casuali unici da 0 a 999
|
| 78 |
+
data = [data[i] for i in indices]
|
| 79 |
+
|
| 80 |
+
#data = data[:50] #TODO ELIMINA
|
| 81 |
+
|
| 82 |
+
self.data = data
|
| 83 |
+
self.num_views = num_views
|
| 84 |
+
self.bg_color = bg_color
|
| 85 |
+
self.img_wh = img_wh
|
| 86 |
+
self.ref_view_mode = ref_view_mode
|
| 87 |
+
self.sample_views_mode = sample_views_mode
|
| 88 |
+
self.k_near_views = k_near_views if k_near_views else num_views
|
| 89 |
+
self.use_abs_elevation = use_abs_elevation
|
| 90 |
+
|
| 91 |
+
def get_bg_color(self, bg_color):
|
| 92 |
+
if bg_color == "white":
|
| 93 |
+
bg_color = np.array([1.0, 1.0, 1.0], dtype=np.float32)
|
| 94 |
+
elif bg_color == "black":
|
| 95 |
+
bg_color = np.array([0.0, 0.0, 0.0], dtype=np.float32)
|
| 96 |
+
elif bg_color == "gray":
|
| 97 |
+
bg_color = np.array([0.5, 0.5, 0.5], dtype=np.float32)
|
| 98 |
+
elif bg_color == "random":
|
| 99 |
+
bg_color = np.random.rand(3)
|
| 100 |
+
elif bg_color == "random_gray":
|
| 101 |
+
bg_color = np.random.rand(1) * 0.5 + 0.3
|
| 102 |
+
bg_color = np.array([bg_color] * 3, dtype=np.float32)
|
| 103 |
+
elif isinstance(bg_color, float):
|
| 104 |
+
bg_color = np.array([bg_color] * 3, dtype=np.float32)
|
| 105 |
+
else:
|
| 106 |
+
raise NotImplementedError
|
| 107 |
+
return bg_color
|
| 108 |
+
|
| 109 |
+
def load_image(self, img_path, bg_color, rescale=True, return_type="np"):
|
| 110 |
+
# not using cv2 as may load in uint16 format
|
| 111 |
+
# img = cv2.imread(img_path, cv2.IMREAD_UNCHANGED) # [0, 255]
|
| 112 |
+
# img = cv2.resize(img, self.img_wh, interpolation=cv2.INTER_CUBIC)
|
| 113 |
+
# pil always returns uint8
|
| 114 |
+
img = np.array(Image.open(img_path).resize(self.img_wh))
|
| 115 |
+
img = img.astype(np.float32) / 255.0 # [0, 1]
|
| 116 |
+
|
| 117 |
+
if img.shape[-1] == 4:
|
| 118 |
+
alpha = img[..., 3:4]
|
| 119 |
+
img = img[..., :3] * alpha + bg_color * (1 - alpha)
|
| 120 |
+
|
| 121 |
+
if rescale:
|
| 122 |
+
img = img * 2.0 - 1.0 # to -1 ~ 1
|
| 123 |
+
|
| 124 |
+
if return_type == "np":
|
| 125 |
+
pass
|
| 126 |
+
elif return_type == "pt":
|
| 127 |
+
img = torch.from_numpy(img)
|
| 128 |
+
else:
|
| 129 |
+
raise NotImplementedError
|
| 130 |
+
|
| 131 |
+
return img
|
| 132 |
+
|
| 133 |
+
def get_k_near_views(self, elevations, azimuths, k_near_views, num_views):
|
| 134 |
+
views = torch.cat((elevations.unsqueeze(1), azimuths.unsqueeze(1)), dim=1)
|
| 135 |
+
distances = torch.cdist(views, views)
|
| 136 |
+
torch.fill_(distances.diagonal(), 0.0)
|
| 137 |
+
k_nearest_indices = torch.topk(distances, k_near_views, largest=False).indices
|
| 138 |
+
|
| 139 |
+
return k_nearest_indices
|
| 140 |
+
|
| 141 |
+
def __len__(self):
|
| 142 |
+
return len(self.data)
|
| 143 |
+
|
| 144 |
+
def check_view_validaty(self, path_img):
|
| 145 |
+
img = np.array(Image.open(os.path.join(path_img)))
|
| 146 |
+
|
| 147 |
+
#check if transparent
|
| 148 |
+
alpha = img[..., 3]
|
| 149 |
+
is_fully_transparent = np.all(alpha == 0)
|
| 150 |
+
|
| 151 |
+
return not(is_fully_transparent)
|
| 152 |
+
|
| 153 |
+
def __getitem__(self, index):
|
| 154 |
+
#{'obj_dir': path to folder uid, 'caption': ""}
|
| 155 |
+
sample = self.data[index]
|
| 156 |
+
obj_dir = sample["obj_dir"]
|
| 157 |
+
with open(osp(obj_dir, "meta.json"), "r") as f:
|
| 158 |
+
meta = json.load(f)
|
| 159 |
+
|
| 160 |
+
# no use
|
| 161 |
+
caption = sample["caption"]
|
| 162 |
+
|
| 163 |
+
# camera intrinsics
|
| 164 |
+
fov = meta["camera_angle_x"]
|
| 165 |
+
focal_length = 0.5 * 1 / np.tan(0.5 * fov) # FIXME: hard-code
|
| 166 |
+
intrinsics = np.array(
|
| 167 |
+
[
|
| 168 |
+
[focal_length, 0, 0.5],
|
| 169 |
+
[0, focal_length, 0.5],
|
| 170 |
+
[0, 0, 1],
|
| 171 |
+
],
|
| 172 |
+
dtype=np.float32,
|
| 173 |
+
)
|
| 174 |
+
intrinsics = torch.from_numpy(intrinsics)
|
| 175 |
+
intrinsics_4x4 = torch.zeros(4, 4)
|
| 176 |
+
intrinsics_4x4[:3, :3] = intrinsics
|
| 177 |
+
intrinsics_4x4[3, 3] = 1.0
|
| 178 |
+
|
| 179 |
+
# sample or select ref view ids
|
| 180 |
+
num_views_all = len(meta["locations"])
|
| 181 |
+
if self.ref_view_mode == "random":
|
| 182 |
+
check = False
|
| 183 |
+
c = 0
|
| 184 |
+
while not(check):
|
| 185 |
+
ref_view_id = np.random.randint(num_views_all)
|
| 186 |
+
#chech if ref view is not transparent
|
| 187 |
+
check = self.check_view_validaty(os.path.join(obj_dir, meta["locations"][ref_view_id]["frames"][0]["name"]))
|
| 188 |
+
c=c+1
|
| 189 |
+
if c == self.num_views: #DA ELIMINARE
|
| 190 |
+
print(f"The file at {obj_dir} directory do not contain any valid image....")
|
| 191 |
+
|
| 192 |
+
elif isinstance(self.ref_view_mode, int):
|
| 193 |
+
ref_view_id = self.ref_view_mode
|
| 194 |
+
else:
|
| 195 |
+
raise NotImplementedError
|
| 196 |
+
|
| 197 |
+
# sample or select view ids in a mode
|
| 198 |
+
num_views_all = len(meta["locations"])
|
| 199 |
+
if self.sample_views_mode == "random":
|
| 200 |
+
view_ids = np.random.choice(
|
| 201 |
+
num_views_all, self.num_views - 1, replace=False
|
| 202 |
+
)
|
| 203 |
+
elif isinstance(self.sample_views_mode, list) or isinstance(
|
| 204 |
+
self.sample_views_mode, ListConfig
|
| 205 |
+
):
|
| 206 |
+
assert len(self.sample_views_mode) == self.num_views - 1, "Invalid view_ids"
|
| 207 |
+
view_ids = self.sample_views_mode
|
| 208 |
+
else:
|
| 209 |
+
raise NotImplementedError
|
| 210 |
+
view_ids = np.insert(view_ids, 0, ref_view_id)
|
| 211 |
+
locations = [meta["locations"][i] for i in view_ids]
|
| 212 |
+
|
| 213 |
+
# load images, elevations, azimuths, c2w_matrixs
|
| 214 |
+
bg_color = self.get_bg_color(self.bg_color)
|
| 215 |
+
img_paths, img_tensors, elevations, azimuths, c2w_matrixs = [], [], [], [], []
|
| 216 |
+
for loc in locations:
|
| 217 |
+
img_path = osp(obj_dir, loc["frames"][0]["name"])
|
| 218 |
+
img = self.load_image(img_path, bg_color, return_type="pt").permute(2, 0, 1)
|
| 219 |
+
img_tensors.append(img)
|
| 220 |
+
img_paths.append(img_path)
|
| 221 |
+
elevations.append(loc["elevation"])
|
| 222 |
+
azimuths.append(loc["azimuth"])
|
| 223 |
+
c2w_matrixs.append(loc["transform_matrix"])
|
| 224 |
+
|
| 225 |
+
# concat and stack
|
| 226 |
+
img_tensors = torch.stack(img_tensors, dim=0).float() # (Nv, 3, H, W)
|
| 227 |
+
elevations = torch.tensor(elevations).float() # (Nv,)
|
| 228 |
+
azimuths = torch.tensor(azimuths).float() # (Nv,)
|
| 229 |
+
c2w_matrixs = torch.tensor(c2w_matrixs).float() # (Nv, 4, 4)
|
| 230 |
+
|
| 231 |
+
# blender to opencv
|
| 232 |
+
c2w_matrixs[:, :, 1:3] *= -1
|
| 233 |
+
|
| 234 |
+
intrinsics_matrixs = intrinsics.unsqueeze(0).repeat(
|
| 235 |
+
self.num_views, 1, 1
|
| 236 |
+
) # (Nv, 3, 3)
|
| 237 |
+
intrinsics_matrixs_4x4 = intrinsics_4x4.unsqueeze(0).repeat(
|
| 238 |
+
self.num_views, 1, 1
|
| 239 |
+
) # (Nv, 4, 4)
|
| 240 |
+
|
| 241 |
+
# flatten intrinsics_matrixs_4x4 and c2w_matrixs to (Nv, 16), and concat them
|
| 242 |
+
intrinsics_matrixs_4x4 = intrinsics_matrixs_4x4.reshape(self.num_views, 16)
|
| 243 |
+
c2w_matrixs = c2w_matrixs.reshape(self.num_views, 16)
|
| 244 |
+
camera_params = torch.cat((intrinsics_matrixs_4x4, c2w_matrixs), dim=1)
|
| 245 |
+
c2w_matrixs = c2w_matrixs.reshape(self.num_views, 4, 4)
|
| 246 |
+
|
| 247 |
+
# find nearest views
|
| 248 |
+
if self.k_near_views > self.num_views:
|
| 249 |
+
raise ValueError("k_near_views should be no larger than num_views")
|
| 250 |
+
else:
|
| 251 |
+
k_near_indices = self.get_k_near_views(
|
| 252 |
+
elevations[1:], azimuths[1:], self.k_near_views, self.num_views - 1
|
| 253 |
+
) # NOTE: we use the 1st view as the reference view
|
| 254 |
+
d_elevations = (elevations - elevations[0:1]).reshape(-1, 1)
|
| 255 |
+
d_azimuths = (azimuths - azimuths[0:1]).reshape(-1, 1) % (2 * math.pi)
|
| 256 |
+
|
| 257 |
+
if self.use_abs_elevation: # stable zero123 use abs_elevation as distance input
|
| 258 |
+
distances = elevations[0:1].repeat(self.num_views).reshape(-1, 1)
|
| 259 |
+
else:
|
| 260 |
+
distances = torch.zeros_like(d_elevations)
|
| 261 |
+
|
| 262 |
+
return {
|
| 263 |
+
"view_ids": torch.tensor(view_ids),
|
| 264 |
+
"images": img_tensors[1:],
|
| 265 |
+
"image_0": img_tensors[0],
|
| 266 |
+
"elevations": d_elevations[1:],
|
| 267 |
+
"azimuths": d_azimuths[1:],
|
| 268 |
+
"distances": distances[1:],
|
| 269 |
+
"c2w_matrixs": c2w_matrixs[1:],
|
| 270 |
+
"intrinsics_matrixs": intrinsics_matrixs[1:],
|
| 271 |
+
"cameras": camera_params[1:],
|
| 272 |
+
"caption": caption,
|
| 273 |
+
"k_near_indices": k_near_indices,
|
| 274 |
+
"obj_dir": obj_dir.split('/')[-1]
|
| 275 |
+
}
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
class MultiViewDataModule(LightningDataModule):
|
| 279 |
+
def __init__(
|
| 280 |
+
self,
|
| 281 |
+
train_dataset: Dataset[Any],
|
| 282 |
+
val_dataset: Dataset[Any],
|
| 283 |
+
test_dataset: Dataset[Any],
|
| 284 |
+
train_batch_size: int = 1,
|
| 285 |
+
val_batch_size: int = 1,
|
| 286 |
+
test_batch_size: int = 1,
|
| 287 |
+
num_workers: Optional[int] = None,
|
| 288 |
+
pin_memory: bool = True,
|
| 289 |
+
):
|
| 290 |
+
super().__init__()
|
| 291 |
+
|
| 292 |
+
# this line allows to access init params with 'self.hparams' attribute
|
| 293 |
+
# also ensures init params will be stored in ckpt
|
| 294 |
+
self.save_hyperparameters(logger=False)
|
| 295 |
+
|
| 296 |
+
self.data_train = train_dataset
|
| 297 |
+
self.data_val = val_dataset
|
| 298 |
+
self.data_test = test_dataset
|
| 299 |
+
|
| 300 |
+
self.num_workers = num_workers if num_workers else train_batch_size * 2
|
| 301 |
+
|
| 302 |
+
def prepare_data(self) -> None:
|
| 303 |
+
# TODO: check if data is available
|
| 304 |
+
pass
|
| 305 |
+
|
| 306 |
+
def _dataloader(
|
| 307 |
+
self, dataset: Dataset, batch_size: int, shuffle: bool
|
| 308 |
+
) -> DataLoader[Any]:
|
| 309 |
+
return DataLoader(
|
| 310 |
+
dataset,
|
| 311 |
+
batch_size=batch_size,
|
| 312 |
+
shuffle=shuffle,
|
| 313 |
+
num_workers=self.num_workers,
|
| 314 |
+
pin_memory=self.hparams.pin_memory,
|
| 315 |
+
)
|
| 316 |
+
|
| 317 |
+
def train_dataloader(self) -> DataLoader[Any]:
|
| 318 |
+
return DataLoader(
|
| 319 |
+
self.data_train,
|
| 320 |
+
batch_size=self.hparams.train_batch_size,
|
| 321 |
+
shuffle=True,
|
| 322 |
+
num_workers=self.num_workers,
|
| 323 |
+
pin_memory=self.hparams.pin_memory,
|
| 324 |
+
)
|
| 325 |
+
|
| 326 |
+
def val_dataloader(self) -> DataLoader[Any]:
|
| 327 |
+
# support multiple validation datasets
|
| 328 |
+
if isinstance(self.data_val, ListConfig):
|
| 329 |
+
return [
|
| 330 |
+
self._dataloader(dataset, self.hparams.val_batch_size, False)
|
| 331 |
+
for dataset in self.data_val
|
| 332 |
+
]
|
| 333 |
+
elif isinstance(self.data_val, DictConfig):
|
| 334 |
+
return [
|
| 335 |
+
self._dataloader(dataset, self.hparams.val_batch_size, False)
|
| 336 |
+
for _, dataset in self.data_val.items()
|
| 337 |
+
]
|
| 338 |
+
else:
|
| 339 |
+
return self._dataloader(self.data_val, self.hparams.val_batch_size, False)
|
| 340 |
+
|
| 341 |
+
def test_dataloader(self) -> DataLoader[Any]:
|
| 342 |
+
# support multiple test datasets
|
| 343 |
+
if isinstance(self.data_test, ListConfig):
|
| 344 |
+
return [
|
| 345 |
+
self._dataloader(dataset, self.hparams.test_batch_size, False)
|
| 346 |
+
for dataset in self.data_test
|
| 347 |
+
]
|
| 348 |
+
elif isinstance(self.data_test, DictConfig):
|
| 349 |
+
return [
|
| 350 |
+
self._dataloader(dataset, self.hparams.test_batch_size, False)
|
| 351 |
+
for _, dataset in self.data_test.items()
|
| 352 |
+
]
|
| 353 |
+
else:
|
| 354 |
+
return self._dataloader(self.data_test, self.hparams.test_batch_size, False)
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
if __name__ == "__main__":
|
| 358 |
+
from torchvision.utils import save_image
|
| 359 |
+
|
| 360 |
+
dataset = MultiViewDataset(
|
| 361 |
+
root_dir="/mnt/pfs/data/render_lvis_hzh",
|
| 362 |
+
num_views=17,
|
| 363 |
+
k_near_views=4,
|
| 364 |
+
bg_color="white",
|
| 365 |
+
img_wh=(256, 256),
|
| 366 |
+
ref_view_mode=2,
|
| 367 |
+
sample_views_mode="random",
|
| 368 |
+
caption_path="data/render_lvis_hzh/caption_train.txt",
|
| 369 |
+
use_abs_elevation=True,
|
| 370 |
+
)
|
| 371 |
+
|
| 372 |
+
sample = dataset[0]
|
| 373 |
+
print(sample["view_ids"])
|
| 374 |
+
save_image((sample["image_0"] + 1.0) / 2, "temp_0.jpg", nrow=1, value_range=(0, 1))
|
| 375 |
+
save_image((sample["images"] + 1.0) / 2, "temp.jpg", nrow=8, value_range=(0, 1))
|
loomnet/data/multiview_infinity.py
ADDED
|
@@ -0,0 +1,410 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import math
|
| 3 |
+
import os
|
| 4 |
+
from os.path import join as osp
|
| 5 |
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
| 6 |
+
import random
|
| 7 |
+
import numpy as np
|
| 8 |
+
import torch
|
| 9 |
+
from lightning import LightningDataModule
|
| 10 |
+
from omegaconf import DictConfig, ListConfig
|
| 11 |
+
from PIL import Image
|
| 12 |
+
from torch.utils.data import DataLoader, Dataset
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class MultiViewDataset(Dataset):
|
| 16 |
+
def __init__(
|
| 17 |
+
self,
|
| 18 |
+
type_of_dataset: str,
|
| 19 |
+
root_dir: str,
|
| 20 |
+
num_views: int,
|
| 21 |
+
num_derived_views: int,
|
| 22 |
+
bg_color: str,
|
| 23 |
+
img_wh: Tuple[int, int],
|
| 24 |
+
caption_path: str,
|
| 25 |
+
ref_view_mode: Union[str, List] = "random",
|
| 26 |
+
sample_views_mode: Union[str, List] = "random",
|
| 27 |
+
sample_derived_views_mode: Union[str, List] = "random",
|
| 28 |
+
k_near_views: Optional[int] = None,
|
| 29 |
+
max_samples_per_category: Optional[int] = None,
|
| 30 |
+
use_abs_elevation: bool = False,
|
| 31 |
+
seed = 0
|
| 32 |
+
):
|
| 33 |
+
#data = []
|
| 34 |
+
|
| 35 |
+
# # Loop per ogni cartella di categoria
|
| 36 |
+
# for category in os.listdir(root_dir):
|
| 37 |
+
# category_path = os.path.join(root_dir, category)
|
| 38 |
+
|
| 39 |
+
# if os.path.isdir(category_path):
|
| 40 |
+
|
| 41 |
+
# uids = os.listdir(category_path)
|
| 42 |
+
# #se validazione, prendiamo per semplicità solo due campioni
|
| 43 |
+
# if max_samples_per_category is not None:
|
| 44 |
+
# random.seed(seed)
|
| 45 |
+
# uids = random.sample(uids, min(max_samples_per_category, len(uids)))
|
| 46 |
+
# for uid in uids:
|
| 47 |
+
|
| 48 |
+
# uid_path = os.path.join(root_dir, category, uid)
|
| 49 |
+
# obj_data = {"obj_dir": uid_path, "caption": ""}
|
| 50 |
+
|
| 51 |
+
# data.append(obj_data)
|
| 52 |
+
|
| 53 |
+
data = []
|
| 54 |
+
# Loop per ogni cartella di categoria
|
| 55 |
+
for category in os.listdir(root_dir):
|
| 56 |
+
category_path = os.path.join(root_dir, category)
|
| 57 |
+
|
| 58 |
+
if os.path.isdir(category_path):
|
| 59 |
+
|
| 60 |
+
uids = [f for f in os.listdir(category_path) if not os.path.isfile(os.path.join(category_path, f))] #os.listdir(category_path)
|
| 61 |
+
|
| 62 |
+
if len(uids)!=0:
|
| 63 |
+
for uid in uids:
|
| 64 |
+
|
| 65 |
+
uid_path = os.path.join(root_dir, category, uid)
|
| 66 |
+
obj_data = {"obj_dir": uid_path, "caption": ""}
|
| 67 |
+
|
| 68 |
+
data.append(obj_data)
|
| 69 |
+
else:
|
| 70 |
+
uid_path = os.path.join(root_dir, category)
|
| 71 |
+
obj_data = {"obj_dir": uid_path, "caption": ""}
|
| 72 |
+
|
| 73 |
+
data.append(obj_data)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
#da eliminare (per velocità prendo solo i primi 8)
|
| 77 |
+
if type_of_dataset=="validation":
|
| 78 |
+
data = data[:144]
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
self.data = data
|
| 82 |
+
self.num_views = num_views
|
| 83 |
+
self.num_derived_views = num_derived_views
|
| 84 |
+
self.bg_color = bg_color
|
| 85 |
+
self.img_wh = img_wh
|
| 86 |
+
self.ref_view_mode = ref_view_mode
|
| 87 |
+
self.sample_views_mode = sample_views_mode
|
| 88 |
+
self.sample_derived_views_mode = sample_derived_views_mode
|
| 89 |
+
self.k_near_views = k_near_views if k_near_views else num_views
|
| 90 |
+
self.use_abs_elevation = use_abs_elevation
|
| 91 |
+
|
| 92 |
+
def get_bg_color(self, bg_color):
|
| 93 |
+
if bg_color == "white":
|
| 94 |
+
bg_color = np.array([1.0, 1.0, 1.0], dtype=np.float32)
|
| 95 |
+
elif bg_color == "black":
|
| 96 |
+
bg_color = np.array([0.0, 0.0, 0.0], dtype=np.float32)
|
| 97 |
+
elif bg_color == "gray":
|
| 98 |
+
bg_color = np.array([0.5, 0.5, 0.5], dtype=np.float32)
|
| 99 |
+
elif bg_color == "random":
|
| 100 |
+
bg_color = np.random.rand(3)
|
| 101 |
+
elif bg_color == "random_gray":
|
| 102 |
+
bg_color = np.random.rand(1) * 0.5 + 0.3
|
| 103 |
+
bg_color = np.array([bg_color] * 3, dtype=np.float32)
|
| 104 |
+
elif isinstance(bg_color, float):
|
| 105 |
+
bg_color = np.array([bg_color] * 3, dtype=np.float32)
|
| 106 |
+
else:
|
| 107 |
+
raise NotImplementedError
|
| 108 |
+
return bg_color
|
| 109 |
+
|
| 110 |
+
def load_image(self, img_path, bg_color, rescale=True, return_type="np"):
|
| 111 |
+
# not using cv2 as may load in uint16 format
|
| 112 |
+
# img = cv2.imread(img_path, cv2.IMREAD_UNCHANGED) # [0, 255]
|
| 113 |
+
# img = cv2.resize(img, self.img_wh, interpolation=cv2.INTER_CUBIC)
|
| 114 |
+
# pil always returns uint8
|
| 115 |
+
img = np.array(Image.open(img_path).resize(self.img_wh))
|
| 116 |
+
img = img.astype(np.float32) / 255.0 # [0, 1]
|
| 117 |
+
|
| 118 |
+
if img.shape[-1] == 4:
|
| 119 |
+
alpha = img[..., 3:4]
|
| 120 |
+
img = img[..., :3] * alpha + bg_color * (1 - alpha)
|
| 121 |
+
|
| 122 |
+
if rescale:
|
| 123 |
+
img = img * 2.0 - 1.0 # to -1 ~ 1
|
| 124 |
+
|
| 125 |
+
if return_type == "np":
|
| 126 |
+
pass
|
| 127 |
+
elif return_type == "pt":
|
| 128 |
+
img = torch.from_numpy(img)
|
| 129 |
+
else:
|
| 130 |
+
raise NotImplementedError
|
| 131 |
+
|
| 132 |
+
return img
|
| 133 |
+
|
| 134 |
+
def get_k_near_anchors(self, elevations, azimuths, k_near_views, num_views, exclude_ids=None):
|
| 135 |
+
views = torch.cat((elevations.unsqueeze(1), azimuths.unsqueeze(1)), dim=1)
|
| 136 |
+
distances = torch.cdist(views, views)
|
| 137 |
+
|
| 138 |
+
# Imposta la diagonale a inf per non contare sé stessi
|
| 139 |
+
distances.fill_diagonal_(float('inf'))
|
| 140 |
+
|
| 141 |
+
if exclude_ids is not None:
|
| 142 |
+
# Esclude gli ID impostando la distanza a inf
|
| 143 |
+
distances[:, exclude_ids] = float('inf')
|
| 144 |
+
|
| 145 |
+
#escludi le ancore
|
| 146 |
+
distances = distances[num_views:]
|
| 147 |
+
|
| 148 |
+
k_nearest_indices = torch.topk(distances, k_near_views, largest=False).indices
|
| 149 |
+
|
| 150 |
+
return k_nearest_indices
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def __len__(self):
|
| 154 |
+
return len(self.data)
|
| 155 |
+
|
| 156 |
+
def check_view_validaty(self, path_img):
|
| 157 |
+
img = np.array(Image.open(os.path.join(path_img)))
|
| 158 |
+
|
| 159 |
+
#check if transparent
|
| 160 |
+
alpha = img[..., 3]
|
| 161 |
+
is_fully_transparent = np.all(alpha == 0)
|
| 162 |
+
|
| 163 |
+
return not(is_fully_transparent)
|
| 164 |
+
|
| 165 |
+
def __getitem__(self, index):
|
| 166 |
+
#{'obj_dir': path to folder uid, 'caption': ""}
|
| 167 |
+
sample = self.data[index]
|
| 168 |
+
obj_dir = sample["obj_dir"]
|
| 169 |
+
with open(osp(obj_dir, "meta.json"), "r") as f:
|
| 170 |
+
meta = json.load(f)
|
| 171 |
+
|
| 172 |
+
# no use
|
| 173 |
+
caption = sample["caption"]
|
| 174 |
+
|
| 175 |
+
# camera intrinsics
|
| 176 |
+
fov = meta["camera_angle_x"]
|
| 177 |
+
focal_length = 0.5 * 1 / np.tan(0.5 * fov) # FIXME: hard-code
|
| 178 |
+
intrinsics = np.array(
|
| 179 |
+
[
|
| 180 |
+
[focal_length, 0, 0.5],
|
| 181 |
+
[0, focal_length, 0.5],
|
| 182 |
+
[0, 0, 1],
|
| 183 |
+
],
|
| 184 |
+
dtype=np.float32,
|
| 185 |
+
)
|
| 186 |
+
intrinsics = torch.from_numpy(intrinsics)
|
| 187 |
+
intrinsics_4x4 = torch.zeros(4, 4)
|
| 188 |
+
intrinsics_4x4[:3, :3] = intrinsics
|
| 189 |
+
intrinsics_4x4[3, 3] = 1.0
|
| 190 |
+
|
| 191 |
+
# sample or select ref view ids
|
| 192 |
+
num_views_all = len(meta["locations"])
|
| 193 |
+
if self.ref_view_mode == "random":
|
| 194 |
+
check = False
|
| 195 |
+
c = 0
|
| 196 |
+
while not(check):
|
| 197 |
+
ref_view_id = np.random.randint(num_views_all)
|
| 198 |
+
#chech if ref view is not transparent
|
| 199 |
+
check = self.check_view_validaty(os.path.join(obj_dir, meta["locations"][ref_view_id]["frames"][0]["name"]))
|
| 200 |
+
c=c+1
|
| 201 |
+
if c == self.num_views: #DA ELIMINARE
|
| 202 |
+
print(f"The file at {obj_dir} directory do not contain any valid image....")
|
| 203 |
+
|
| 204 |
+
elif isinstance(self.ref_view_mode, int):
|
| 205 |
+
ref_view_id = self.ref_view_mode
|
| 206 |
+
else:
|
| 207 |
+
raise NotImplementedError
|
| 208 |
+
|
| 209 |
+
# sample or select anchor view ids
|
| 210 |
+
num_views_all = len(meta["locations"])
|
| 211 |
+
if self.sample_views_mode == "random":
|
| 212 |
+
view_ids = np.random.choice(
|
| 213 |
+
num_views_all, self.num_views - 1, replace=False
|
| 214 |
+
)
|
| 215 |
+
elif isinstance(self.sample_views_mode, list) or isinstance(
|
| 216 |
+
self.sample_views_mode, ListConfig
|
| 217 |
+
):
|
| 218 |
+
assert len(self.sample_views_mode) == self.num_views - 1, "Invalid view_ids"
|
| 219 |
+
view_ids = self.sample_views_mode
|
| 220 |
+
else:
|
| 221 |
+
raise NotImplementedError
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
# sample or select derived view ids
|
| 225 |
+
if self.sample_derived_views_mode == "random":
|
| 226 |
+
derived_view_ids = np.random.choice(
|
| 227 |
+
num_views_all, self.num_derived_views, replace=False
|
| 228 |
+
)
|
| 229 |
+
elif isinstance(self.sample_derived_views_mode, list) or isinstance(
|
| 230 |
+
self.sample_derived_views_mode, ListConfig
|
| 231 |
+
):
|
| 232 |
+
derived_view_ids = self.sample_derived_views_mode
|
| 233 |
+
else:
|
| 234 |
+
raise NotImplementedError
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
#insert reference view in the first position
|
| 239 |
+
view_ids = np.insert(view_ids, 0, ref_view_id)
|
| 240 |
+
#insert derived views in the last positions
|
| 241 |
+
view_ids = np.insert(view_ids, self.num_views, derived_view_ids)
|
| 242 |
+
locations = [meta["locations"][i] for i in view_ids]
|
| 243 |
+
|
| 244 |
+
# load images, elevations, azimuths, c2w_matrixs
|
| 245 |
+
bg_color = self.get_bg_color(self.bg_color)
|
| 246 |
+
img_paths, img_tensors, elevations, azimuths, c2w_matrixs = [], [], [], [], []
|
| 247 |
+
for loc in locations:
|
| 248 |
+
img_path = osp(obj_dir, loc["frames"][0]["name"])
|
| 249 |
+
img = self.load_image(img_path, bg_color, return_type="pt").permute(2, 0, 1)
|
| 250 |
+
img_tensors.append(img)
|
| 251 |
+
img_paths.append(img_path)
|
| 252 |
+
elevations.append(loc["elevation"])
|
| 253 |
+
azimuths.append(loc["azimuth"])
|
| 254 |
+
c2w_matrixs.append(loc["transform_matrix"])
|
| 255 |
+
|
| 256 |
+
# concat and stack
|
| 257 |
+
img_tensors = torch.stack(img_tensors, dim=0).float() # (Nv, 3, H, W)
|
| 258 |
+
elevations = torch.tensor(elevations).float() # (Nv,)
|
| 259 |
+
azimuths = torch.tensor(azimuths).float() # (Nv,)
|
| 260 |
+
c2w_matrixs = torch.tensor(c2w_matrixs).float() # (Nv, 4, 4)
|
| 261 |
+
|
| 262 |
+
# blender to opencv
|
| 263 |
+
c2w_matrixs[:, :, 1:3] *= -1
|
| 264 |
+
|
| 265 |
+
intrinsics_matrixs = intrinsics.unsqueeze(0).repeat(
|
| 266 |
+
self.num_views + self.num_derived_views, 1, 1
|
| 267 |
+
) # (Nv, 3, 3)
|
| 268 |
+
intrinsics_matrixs_4x4 = intrinsics_4x4.unsqueeze(0).repeat(
|
| 269 |
+
self.num_views + self.num_derived_views, 1, 1
|
| 270 |
+
) # (Nv, 4, 4)
|
| 271 |
+
|
| 272 |
+
# flatten intrinsics_matrixs_4x4 and c2w_matrixs to (Nv, 16), and concat them
|
| 273 |
+
intrinsics_matrixs_4x4 = intrinsics_matrixs_4x4.reshape(self.num_views + self.num_derived_views, 16)
|
| 274 |
+
c2w_matrixs = c2w_matrixs.reshape(self.num_views + self.num_derived_views, 16)
|
| 275 |
+
camera_params = torch.cat((intrinsics_matrixs_4x4, c2w_matrixs), dim=1)
|
| 276 |
+
c2w_matrixs = c2w_matrixs.reshape(self.num_views + self.num_derived_views, 4, 4)
|
| 277 |
+
|
| 278 |
+
# find nearest views
|
| 279 |
+
if self.k_near_views > self.num_views:
|
| 280 |
+
raise ValueError("k_near_views should be no larger than num_views")
|
| 281 |
+
else:
|
| 282 |
+
k_near_indices = self.get_k_near_anchors(
|
| 283 |
+
elevations[1:], azimuths[1:], self.k_near_views, self.num_views - 1, exclude_ids=np.arange(self.num_views-1, self.num_views-1 + self.num_derived_views)
|
| 284 |
+
) # NOTE: we use the 1st view as the reference view
|
| 285 |
+
#k_near_indices = -1
|
| 286 |
+
|
| 287 |
+
d_elevations = (elevations - elevations[0:1]).reshape(-1, 1)
|
| 288 |
+
d_azimuths = (azimuths - azimuths[0:1]).reshape(-1, 1) % (2 * math.pi)
|
| 289 |
+
|
| 290 |
+
if self.use_abs_elevation: # stable zero123 use abs_elevation as distance input
|
| 291 |
+
distances = elevations[0:1].repeat(self.num_views + self.num_derived_views).reshape(-1, 1)
|
| 292 |
+
else:
|
| 293 |
+
distances = torch.zeros_like(d_elevations)
|
| 294 |
+
|
| 295 |
+
return {
|
| 296 |
+
"view_ids": torch.tensor(view_ids),
|
| 297 |
+
"images": img_tensors[1:],
|
| 298 |
+
"image_0": img_tensors[0],
|
| 299 |
+
"elevations": d_elevations[1:],
|
| 300 |
+
"azimuths": d_azimuths[1:],
|
| 301 |
+
"distances": distances[1:],
|
| 302 |
+
"c2w_matrixs": c2w_matrixs[1:],
|
| 303 |
+
"intrinsics_matrixs": intrinsics_matrixs[1:],
|
| 304 |
+
"cameras": camera_params[1:],
|
| 305 |
+
"caption": caption,
|
| 306 |
+
"k_near_indices": k_near_indices,
|
| 307 |
+
"obj_dir": obj_dir.split('/')[-1],
|
| 308 |
+
"num_anchor_images": self.num_views-1,
|
| 309 |
+
"num_derived_images": self.num_derived_views
|
| 310 |
+
}
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
class MultiViewDataModule(LightningDataModule):
|
| 314 |
+
def __init__(
|
| 315 |
+
self,
|
| 316 |
+
train_dataset: Dataset[Any],
|
| 317 |
+
val_dataset: Dataset[Any],
|
| 318 |
+
test_dataset: Dataset[Any],
|
| 319 |
+
train_batch_size: int = 1,
|
| 320 |
+
val_batch_size: int = 1,
|
| 321 |
+
test_batch_size: int = 1,
|
| 322 |
+
num_workers: Optional[int] = None,
|
| 323 |
+
pin_memory: bool = True,
|
| 324 |
+
):
|
| 325 |
+
super().__init__()
|
| 326 |
+
|
| 327 |
+
# this line allows to access init params with 'self.hparams' attribute
|
| 328 |
+
# also ensures init params will be stored in ckpt
|
| 329 |
+
self.save_hyperparameters(logger=False)
|
| 330 |
+
|
| 331 |
+
self.data_train = train_dataset
|
| 332 |
+
self.data_val = val_dataset
|
| 333 |
+
self.data_test = test_dataset
|
| 334 |
+
|
| 335 |
+
self.num_workers = num_workers if num_workers else train_batch_size * 2
|
| 336 |
+
|
| 337 |
+
def prepare_data(self) -> None:
|
| 338 |
+
# TODO: check if data is available
|
| 339 |
+
pass
|
| 340 |
+
|
| 341 |
+
def _dataloader(
|
| 342 |
+
self, dataset: Dataset, batch_size: int, shuffle: bool
|
| 343 |
+
) -> DataLoader[Any]:
|
| 344 |
+
return DataLoader(
|
| 345 |
+
dataset,
|
| 346 |
+
batch_size=batch_size,
|
| 347 |
+
shuffle=shuffle,
|
| 348 |
+
num_workers=self.num_workers,
|
| 349 |
+
pin_memory=self.hparams.pin_memory,
|
| 350 |
+
)
|
| 351 |
+
|
| 352 |
+
def train_dataloader(self) -> DataLoader[Any]:
|
| 353 |
+
return DataLoader(
|
| 354 |
+
self.data_train,
|
| 355 |
+
batch_size=self.hparams.train_batch_size,
|
| 356 |
+
shuffle=True,
|
| 357 |
+
num_workers=self.num_workers,
|
| 358 |
+
pin_memory=self.hparams.pin_memory,
|
| 359 |
+
)
|
| 360 |
+
|
| 361 |
+
def val_dataloader(self) -> DataLoader[Any]:
|
| 362 |
+
# support multiple validation datasets
|
| 363 |
+
if isinstance(self.data_val, ListConfig):
|
| 364 |
+
return [
|
| 365 |
+
self._dataloader(dataset, self.hparams.val_batch_size, False)
|
| 366 |
+
for dataset in self.data_val
|
| 367 |
+
]
|
| 368 |
+
elif isinstance(self.data_val, DictConfig):
|
| 369 |
+
return [
|
| 370 |
+
self._dataloader(dataset, self.hparams.val_batch_size, False)
|
| 371 |
+
for _, dataset in self.data_val.items()
|
| 372 |
+
]
|
| 373 |
+
else:
|
| 374 |
+
return self._dataloader(self.data_val, self.hparams.val_batch_size, False)
|
| 375 |
+
|
| 376 |
+
def test_dataloader(self) -> DataLoader[Any]:
|
| 377 |
+
# support multiple test datasets
|
| 378 |
+
if isinstance(self.data_test, ListConfig):
|
| 379 |
+
return [
|
| 380 |
+
self._dataloader(dataset, self.hparams.test_batch_size, False)
|
| 381 |
+
for dataset in self.data_test
|
| 382 |
+
]
|
| 383 |
+
elif isinstance(self.data_test, DictConfig):
|
| 384 |
+
return [
|
| 385 |
+
self._dataloader(dataset, self.hparams.test_batch_size, False)
|
| 386 |
+
for _, dataset in self.data_test.items()
|
| 387 |
+
]
|
| 388 |
+
else:
|
| 389 |
+
return self._dataloader(self.data_test, self.hparams.test_batch_size, False)
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
if __name__ == "__main__":
|
| 393 |
+
from torchvision.utils import save_image
|
| 394 |
+
|
| 395 |
+
dataset = MultiViewDataset(
|
| 396 |
+
root_dir="/mnt/pfs/data/render_lvis_hzh",
|
| 397 |
+
num_views=17,
|
| 398 |
+
k_near_views=4,
|
| 399 |
+
bg_color="white",
|
| 400 |
+
img_wh=(256, 256),
|
| 401 |
+
ref_view_mode=2,
|
| 402 |
+
sample_views_mode="random",
|
| 403 |
+
caption_path="data/render_lvis_hzh/caption_train.txt",
|
| 404 |
+
use_abs_elevation=True,
|
| 405 |
+
)
|
| 406 |
+
|
| 407 |
+
sample = dataset[0]
|
| 408 |
+
print(sample["view_ids"])
|
| 409 |
+
save_image((sample["image_0"] + 1.0) / 2, "temp_0.jpg", nrow=1, value_range=(0, 1))
|
| 410 |
+
save_image((sample["images"] + 1.0) / 2, "temp.jpg", nrow=8, value_range=(0, 1))
|
loomnet/models/__init__.py
ADDED
|
File without changes
|
loomnet/models/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (179 Bytes). View file
|
|
|
loomnet/models/__pycache__/mv_model.cpython-311.pyc
ADDED
|
Binary file (12 kB). View file
|
|
|
loomnet/models/__pycache__/mv_model_1_to_infinity.cpython-311.pyc
ADDED
|
Binary file (11.5 kB). View file
|
|
|
loomnet/models/__pycache__/mv_model_infinity.cpython-311.pyc
ADDED
|
Binary file (12.5 kB). View file
|
|
|
loomnet/models/__pycache__/mv_model_triplane.cpython-311.pyc
ADDED
|
Binary file (15.5 kB). View file
|
|
|
loomnet/models/__pycache__/mv_model_triplane_with_3d_conv.cpython-311.pyc
ADDED
|
Binary file (15.6 kB). View file
|
|
|
loomnet/models/__pycache__/mv_model_v7.cpython-311.pyc
ADDED
|
Binary file (11.1 kB). View file
|
|
|
loomnet/models/__pycache__/mv_model_v8.cpython-311.pyc
ADDED
|
Binary file (11.6 kB). View file
|
|
|
loomnet/models/__pycache__/mv_model_with_distillation.cpython-311.pyc
ADDED
|
Binary file (11.8 kB). View file
|
|
|
loomnet/models/blocks/__init__.py
ADDED
|
File without changes
|
loomnet/models/blocks/__pycache__/__init__.cpython-311.pyc
ADDED
|
Binary file (182 Bytes). View file
|
|
|
loomnet/models/blocks/__pycache__/generic_blocks.cpython-311.pyc
ADDED
|
Binary file (16.5 kB). View file
|
|
|
loomnet/models/blocks/__pycache__/pointwise_spatial_transformer.cpython-311.pyc
ADDED
|
Binary file (6.39 kB). View file
|
|
|
loomnet/models/blocks/__pycache__/resnet.cpython-311.pyc
ADDED
|
Binary file (18.5 kB). View file
|
|
|
loomnet/models/blocks/__pycache__/resnet_fpn.cpython-311.pyc
ADDED
|
Binary file (16.6 kB). View file
|
|
|
loomnet/models/blocks/__pycache__/transformer_t.cpython-311.pyc
ADDED
|
Binary file (14.9 kB). View file
|
|
|
loomnet/models/blocks/attention.py
ADDED
|
@@ -0,0 +1,739 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Adapted from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention.py
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
from typing import Callable, Optional
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
import torch.nn.functional as F
|
| 8 |
+
from diffusers.configuration_utils import ConfigMixin, register_to_config
|
| 9 |
+
from diffusers.models.attention import AdaLayerNorm, FeedForward
|
| 10 |
+
from diffusers.models.modeling_utils import ModelMixin
|
| 11 |
+
from diffusers.utils import BaseOutput
|
| 12 |
+
from diffusers.utils.import_utils import is_xformers_available
|
| 13 |
+
from einops import rearrange, repeat
|
| 14 |
+
from torch import nn
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@dataclass
|
| 18 |
+
class Transformer3DModelOutput(BaseOutput):
|
| 19 |
+
sample: torch.FloatTensor
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
if is_xformers_available():
|
| 23 |
+
import xformers
|
| 24 |
+
import xformers.ops
|
| 25 |
+
else:
|
| 26 |
+
xformers = None
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class CrossAttention(nn.Module):
|
| 30 |
+
r"""
|
| 31 |
+
A cross attention layer.
|
| 32 |
+
|
| 33 |
+
Parameters:
|
| 34 |
+
query_dim (`int`): The number of channels in the query.
|
| 35 |
+
cross_attention_dim (`int`, *optional*):
|
| 36 |
+
The number of channels in the encoder_hidden_states. If not given, defaults to `query_dim`.
|
| 37 |
+
heads (`int`, *optional*, defaults to 8): The number of heads to use for multi-head attention.
|
| 38 |
+
dim_head (`int`, *optional*, defaults to 64): The number of channels in each head.
|
| 39 |
+
dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
|
| 40 |
+
bias (`bool`, *optional*, defaults to False):
|
| 41 |
+
Set to `True` for the query, key, and value linear layers to contain a bias parameter.
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
def __init__(
|
| 45 |
+
self,
|
| 46 |
+
query_dim: int,
|
| 47 |
+
cross_attention_dim: Optional[int] = None,
|
| 48 |
+
heads: int = 8,
|
| 49 |
+
dim_head: int = 64,
|
| 50 |
+
dropout: float = 0.0,
|
| 51 |
+
bias=False,
|
| 52 |
+
upcast_attention: bool = False,
|
| 53 |
+
upcast_softmax: bool = False,
|
| 54 |
+
added_kv_proj_dim: Optional[int] = None,
|
| 55 |
+
norm_num_groups: Optional[int] = None,
|
| 56 |
+
):
|
| 57 |
+
super().__init__()
|
| 58 |
+
inner_dim = dim_head * heads
|
| 59 |
+
cross_attention_dim = (
|
| 60 |
+
cross_attention_dim if cross_attention_dim is not None else query_dim
|
| 61 |
+
)
|
| 62 |
+
self.upcast_attention = upcast_attention
|
| 63 |
+
self.upcast_softmax = upcast_softmax
|
| 64 |
+
|
| 65 |
+
self.scale = dim_head**-0.5
|
| 66 |
+
|
| 67 |
+
self.heads = heads
|
| 68 |
+
# for slice_size > 0 the attention score computation
|
| 69 |
+
# is split across the batch axis to save memory
|
| 70 |
+
# You can set slice_size with `set_attention_slice`
|
| 71 |
+
self.sliceable_head_dim = heads
|
| 72 |
+
self._slice_size = None
|
| 73 |
+
self._use_memory_efficient_attention_xformers = False
|
| 74 |
+
self.added_kv_proj_dim = added_kv_proj_dim
|
| 75 |
+
|
| 76 |
+
if norm_num_groups is not None:
|
| 77 |
+
self.group_norm = nn.GroupNorm(
|
| 78 |
+
num_channels=inner_dim,
|
| 79 |
+
num_groups=norm_num_groups,
|
| 80 |
+
eps=1e-5,
|
| 81 |
+
affine=True,
|
| 82 |
+
)
|
| 83 |
+
else:
|
| 84 |
+
self.group_norm = None
|
| 85 |
+
|
| 86 |
+
self.to_q = nn.Linear(query_dim, inner_dim, bias=bias)
|
| 87 |
+
self.to_k = nn.Linear(cross_attention_dim, inner_dim, bias=bias)
|
| 88 |
+
self.to_v = nn.Linear(cross_attention_dim, inner_dim, bias=bias)
|
| 89 |
+
if self.added_kv_proj_dim is not None:
|
| 90 |
+
self.add_k_proj = nn.Linear(added_kv_proj_dim, cross_attention_dim)
|
| 91 |
+
self.add_v_proj = nn.Linear(added_kv_proj_dim, cross_attention_dim)
|
| 92 |
+
|
| 93 |
+
self.to_out = nn.ModuleList([])
|
| 94 |
+
self.to_out.append(nn.Linear(inner_dim, query_dim))
|
| 95 |
+
self.to_out.append(nn.Dropout(dropout))
|
| 96 |
+
|
| 97 |
+
def reshape_heads_to_batch_dim(self, tensor):
|
| 98 |
+
batch_size, seq_len, dim = tensor.shape
|
| 99 |
+
head_size = self.heads
|
| 100 |
+
tensor = tensor.reshape(batch_size, seq_len, head_size, dim // head_size)
|
| 101 |
+
tensor = tensor.permute(0, 2, 1, 3).reshape(
|
| 102 |
+
batch_size * head_size, seq_len, dim // head_size
|
| 103 |
+
)
|
| 104 |
+
return tensor
|
| 105 |
+
|
| 106 |
+
def reshape_batch_dim_to_heads(self, tensor):
|
| 107 |
+
batch_size, seq_len, dim = tensor.shape
|
| 108 |
+
head_size = self.heads
|
| 109 |
+
tensor = tensor.reshape(batch_size // head_size, head_size, seq_len, dim)
|
| 110 |
+
tensor = tensor.permute(0, 2, 1, 3).reshape(
|
| 111 |
+
batch_size // head_size, seq_len, dim * head_size
|
| 112 |
+
)
|
| 113 |
+
return tensor
|
| 114 |
+
|
| 115 |
+
def set_attention_slice(self, slice_size):
|
| 116 |
+
if slice_size is not None and slice_size > self.sliceable_head_dim:
|
| 117 |
+
raise ValueError(
|
| 118 |
+
f"slice_size {slice_size} has to be smaller or equal to {self.sliceable_head_dim}."
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
self._slice_size = slice_size
|
| 122 |
+
|
| 123 |
+
def forward(
|
| 124 |
+
self, hidden_states, encoder_hidden_states=None, attention_mask=None, meta=None
|
| 125 |
+
):
|
| 126 |
+
batch_size, sequence_length, _ = hidden_states.shape
|
| 127 |
+
encoder_hidden_states = encoder_hidden_states
|
| 128 |
+
|
| 129 |
+
if self.group_norm is not None:
|
| 130 |
+
hidden_states = self.group_norm(hidden_states.transpose(1, 2)).transpose(
|
| 131 |
+
1, 2
|
| 132 |
+
)
|
| 133 |
+
|
| 134 |
+
query = self.to_q(hidden_states)
|
| 135 |
+
dim = query.shape[-1]
|
| 136 |
+
query = self.reshape_heads_to_batch_dim(query)
|
| 137 |
+
|
| 138 |
+
if self.added_kv_proj_dim is not None:
|
| 139 |
+
key = self.to_k(hidden_states)
|
| 140 |
+
value = self.to_v(hidden_states)
|
| 141 |
+
encoder_hidden_states_key_proj = self.add_k_proj(encoder_hidden_states)
|
| 142 |
+
encoder_hidden_states_value_proj = self.add_v_proj(encoder_hidden_states)
|
| 143 |
+
|
| 144 |
+
key = self.reshape_heads_to_batch_dim(key)
|
| 145 |
+
value = self.reshape_heads_to_batch_dim(value)
|
| 146 |
+
encoder_hidden_states_key_proj = self.reshape_heads_to_batch_dim(
|
| 147 |
+
encoder_hidden_states_key_proj
|
| 148 |
+
)
|
| 149 |
+
encoder_hidden_states_value_proj = self.reshape_heads_to_batch_dim(
|
| 150 |
+
encoder_hidden_states_value_proj
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
key = torch.concat([encoder_hidden_states_key_proj, key], dim=1)
|
| 154 |
+
value = torch.concat([encoder_hidden_states_value_proj, value], dim=1)
|
| 155 |
+
else:
|
| 156 |
+
encoder_hidden_states = (
|
| 157 |
+
encoder_hidden_states
|
| 158 |
+
if encoder_hidden_states is not None
|
| 159 |
+
else hidden_states
|
| 160 |
+
)
|
| 161 |
+
key = self.to_k(encoder_hidden_states)
|
| 162 |
+
value = self.to_v(encoder_hidden_states)
|
| 163 |
+
|
| 164 |
+
key = self.reshape_heads_to_batch_dim(key)
|
| 165 |
+
value = self.reshape_heads_to_batch_dim(value)
|
| 166 |
+
|
| 167 |
+
if attention_mask is not None:
|
| 168 |
+
if attention_mask.shape[-1] != query.shape[1]:
|
| 169 |
+
target_length = query.shape[1]
|
| 170 |
+
attention_mask = F.pad(attention_mask, (0, target_length), value=0.0)
|
| 171 |
+
attention_mask = attention_mask.repeat_interleave(self.heads, dim=0)
|
| 172 |
+
|
| 173 |
+
# attention, what we cannot get enough of
|
| 174 |
+
if self._use_memory_efficient_attention_xformers:
|
| 175 |
+
hidden_states = self._memory_efficient_attention_xformers(
|
| 176 |
+
query, key, value, attention_mask
|
| 177 |
+
)
|
| 178 |
+
# Some versions of xformers return output in fp32, cast it back to the dtype of the input
|
| 179 |
+
hidden_states = hidden_states.to(query.dtype)
|
| 180 |
+
else:
|
| 181 |
+
if self._slice_size is None or query.shape[0] // self._slice_size == 1:
|
| 182 |
+
hidden_states = self._attention(query, key, value, attention_mask)
|
| 183 |
+
else:
|
| 184 |
+
hidden_states = self._sliced_attention(
|
| 185 |
+
query, key, value, sequence_length, dim, attention_mask
|
| 186 |
+
)
|
| 187 |
+
|
| 188 |
+
# linear proj
|
| 189 |
+
hidden_states = self.to_out[0](hidden_states)
|
| 190 |
+
|
| 191 |
+
# dropout
|
| 192 |
+
hidden_states = self.to_out[1](hidden_states)
|
| 193 |
+
return hidden_states
|
| 194 |
+
|
| 195 |
+
def _attention(self, query, key, value, attention_mask=None):
|
| 196 |
+
if self.upcast_attention:
|
| 197 |
+
query = query.float()
|
| 198 |
+
key = key.float()
|
| 199 |
+
|
| 200 |
+
attention_scores = torch.baddbmm(
|
| 201 |
+
torch.empty(
|
| 202 |
+
query.shape[0],
|
| 203 |
+
query.shape[1],
|
| 204 |
+
key.shape[1],
|
| 205 |
+
dtype=query.dtype,
|
| 206 |
+
device=query.device,
|
| 207 |
+
),
|
| 208 |
+
query,
|
| 209 |
+
key.transpose(-1, -2),
|
| 210 |
+
beta=0,
|
| 211 |
+
alpha=self.scale,
|
| 212 |
+
)
|
| 213 |
+
|
| 214 |
+
if attention_mask is not None:
|
| 215 |
+
attention_scores = attention_scores + attention_mask
|
| 216 |
+
|
| 217 |
+
if self.upcast_softmax:
|
| 218 |
+
attention_scores = attention_scores.float()
|
| 219 |
+
|
| 220 |
+
attention_probs = attention_scores.softmax(dim=-1)
|
| 221 |
+
|
| 222 |
+
# cast back to the original dtype
|
| 223 |
+
attention_probs = attention_probs.to(value.dtype)
|
| 224 |
+
|
| 225 |
+
# compute attention output
|
| 226 |
+
hidden_states = torch.bmm(attention_probs, value)
|
| 227 |
+
|
| 228 |
+
# reshape hidden_states
|
| 229 |
+
hidden_states = self.reshape_batch_dim_to_heads(hidden_states)
|
| 230 |
+
return hidden_states
|
| 231 |
+
|
| 232 |
+
def _sliced_attention(
|
| 233 |
+
self, query, key, value, sequence_length, dim, attention_mask
|
| 234 |
+
):
|
| 235 |
+
batch_size_attention = query.shape[0]
|
| 236 |
+
hidden_states = torch.zeros(
|
| 237 |
+
(batch_size_attention, sequence_length, dim // self.heads),
|
| 238 |
+
device=query.device,
|
| 239 |
+
dtype=query.dtype,
|
| 240 |
+
)
|
| 241 |
+
slice_size = (
|
| 242 |
+
self._slice_size if self._slice_size is not None else hidden_states.shape[0]
|
| 243 |
+
)
|
| 244 |
+
for i in range(hidden_states.shape[0] // slice_size):
|
| 245 |
+
start_idx = i * slice_size
|
| 246 |
+
end_idx = (i + 1) * slice_size
|
| 247 |
+
|
| 248 |
+
query_slice = query[start_idx:end_idx]
|
| 249 |
+
key_slice = key[start_idx:end_idx]
|
| 250 |
+
|
| 251 |
+
if self.upcast_attention:
|
| 252 |
+
query_slice = query_slice.float()
|
| 253 |
+
key_slice = key_slice.float()
|
| 254 |
+
|
| 255 |
+
attn_slice = torch.baddbmm(
|
| 256 |
+
torch.empty(
|
| 257 |
+
slice_size,
|
| 258 |
+
query.shape[1],
|
| 259 |
+
key.shape[1],
|
| 260 |
+
dtype=query_slice.dtype,
|
| 261 |
+
device=query.device,
|
| 262 |
+
),
|
| 263 |
+
query_slice,
|
| 264 |
+
key_slice.transpose(-1, -2),
|
| 265 |
+
beta=0,
|
| 266 |
+
alpha=self.scale,
|
| 267 |
+
)
|
| 268 |
+
|
| 269 |
+
if attention_mask is not None:
|
| 270 |
+
attn_slice = attn_slice + attention_mask[start_idx:end_idx]
|
| 271 |
+
|
| 272 |
+
if self.upcast_softmax:
|
| 273 |
+
attn_slice = attn_slice.float()
|
| 274 |
+
|
| 275 |
+
attn_slice = attn_slice.softmax(dim=-1)
|
| 276 |
+
|
| 277 |
+
# cast back to the original dtype
|
| 278 |
+
attn_slice = attn_slice.to(value.dtype)
|
| 279 |
+
attn_slice = torch.bmm(attn_slice, value[start_idx:end_idx])
|
| 280 |
+
|
| 281 |
+
hidden_states[start_idx:end_idx] = attn_slice
|
| 282 |
+
|
| 283 |
+
# reshape hidden_states
|
| 284 |
+
hidden_states = self.reshape_batch_dim_to_heads(hidden_states)
|
| 285 |
+
return hidden_states
|
| 286 |
+
|
| 287 |
+
def _memory_efficient_attention_xformers(self, query, key, value, attention_mask):
|
| 288 |
+
# TODO attention_mask
|
| 289 |
+
query = query.contiguous()
|
| 290 |
+
key = key.contiguous()
|
| 291 |
+
value = value.contiguous()
|
| 292 |
+
hidden_states = xformers.ops.memory_efficient_attention(
|
| 293 |
+
query, key, value, attn_bias=attention_mask
|
| 294 |
+
)
|
| 295 |
+
hidden_states = self.reshape_batch_dim_to_heads(hidden_states)
|
| 296 |
+
return hidden_states
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
class Transformer3DModel(ModelMixin, ConfigMixin):
|
| 300 |
+
@register_to_config
|
| 301 |
+
def __init__(
|
| 302 |
+
self,
|
| 303 |
+
num_attention_heads: int = 16,
|
| 304 |
+
attention_head_dim: int = 88,
|
| 305 |
+
in_channels: Optional[int] = None,
|
| 306 |
+
num_layers: int = 1,
|
| 307 |
+
dropout: float = 0.0,
|
| 308 |
+
norm_num_groups: int = 32,
|
| 309 |
+
cross_attention_dim: Optional[int] = None,
|
| 310 |
+
attention_bias: bool = False,
|
| 311 |
+
activation_fn: str = "geglu",
|
| 312 |
+
num_embeds_ada_norm: Optional[int] = None,
|
| 313 |
+
use_linear_projection: bool = False,
|
| 314 |
+
only_cross_attention: bool = False,
|
| 315 |
+
upcast_attention: bool = False,
|
| 316 |
+
):
|
| 317 |
+
super().__init__()
|
| 318 |
+
self.use_linear_projection = use_linear_projection
|
| 319 |
+
self.num_attention_heads = num_attention_heads
|
| 320 |
+
self.attention_head_dim = attention_head_dim
|
| 321 |
+
inner_dim = num_attention_heads * attention_head_dim
|
| 322 |
+
|
| 323 |
+
# Define input layers
|
| 324 |
+
self.in_channels = in_channels
|
| 325 |
+
|
| 326 |
+
self.norm = torch.nn.GroupNorm(
|
| 327 |
+
num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True
|
| 328 |
+
)
|
| 329 |
+
if use_linear_projection:
|
| 330 |
+
self.proj_in = nn.Linear(in_channels, inner_dim)
|
| 331 |
+
else:
|
| 332 |
+
self.proj_in = nn.Conv2d(
|
| 333 |
+
in_channels, inner_dim, kernel_size=1, stride=1, padding=0
|
| 334 |
+
)
|
| 335 |
+
|
| 336 |
+
# Define transformers blocks
|
| 337 |
+
self.transformer_blocks = nn.ModuleList(
|
| 338 |
+
[
|
| 339 |
+
BasicTransformerBlock(
|
| 340 |
+
inner_dim,
|
| 341 |
+
num_attention_heads,
|
| 342 |
+
attention_head_dim,
|
| 343 |
+
dropout=dropout,
|
| 344 |
+
cross_attention_dim=cross_attention_dim,
|
| 345 |
+
activation_fn=activation_fn,
|
| 346 |
+
num_embeds_ada_norm=num_embeds_ada_norm,
|
| 347 |
+
attention_bias=attention_bias,
|
| 348 |
+
only_cross_attention=only_cross_attention,
|
| 349 |
+
upcast_attention=upcast_attention,
|
| 350 |
+
)
|
| 351 |
+
for d in range(num_layers)
|
| 352 |
+
]
|
| 353 |
+
)
|
| 354 |
+
|
| 355 |
+
# Define output layers
|
| 356 |
+
if use_linear_projection:
|
| 357 |
+
self.proj_out = nn.Linear(in_channels, inner_dim)
|
| 358 |
+
else:
|
| 359 |
+
self.proj_out = nn.Conv2d(
|
| 360 |
+
inner_dim, in_channels, kernel_size=1, stride=1, padding=0
|
| 361 |
+
)
|
| 362 |
+
|
| 363 |
+
def forward(
|
| 364 |
+
self,
|
| 365 |
+
hidden_states: torch.Tensor,
|
| 366 |
+
encoder_hidden_states: Optional[torch.Tensor] = None,
|
| 367 |
+
timestep: Optional[torch.LongTensor] = None,
|
| 368 |
+
return_dict: bool = True,
|
| 369 |
+
meta=None,
|
| 370 |
+
):
|
| 371 |
+
# Input
|
| 372 |
+
assert (
|
| 373 |
+
hidden_states.dim() == 5
|
| 374 |
+
), f"Expected hidden_states to have ndim=5, but got ndim={hidden_states.dim()}."
|
| 375 |
+
video_length = hidden_states.shape[2]
|
| 376 |
+
hidden_states = rearrange(hidden_states, "b c f h w -> (b f) c h w")
|
| 377 |
+
batch, _, height, weight = hidden_states.shape
|
| 378 |
+
residual = hidden_states
|
| 379 |
+
|
| 380 |
+
hidden_states = self.norm(hidden_states)
|
| 381 |
+
if not self.use_linear_projection:
|
| 382 |
+
hidden_states = self.proj_in(hidden_states)
|
| 383 |
+
inner_dim = hidden_states.shape[1]
|
| 384 |
+
hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(
|
| 385 |
+
batch, height * weight, inner_dim
|
| 386 |
+
)
|
| 387 |
+
else:
|
| 388 |
+
inner_dim = hidden_states.shape[1]
|
| 389 |
+
hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(
|
| 390 |
+
batch, height * weight, inner_dim
|
| 391 |
+
)
|
| 392 |
+
hidden_states = self.proj_in(hidden_states)
|
| 393 |
+
|
| 394 |
+
# Blocks
|
| 395 |
+
for block in self.transformer_blocks:
|
| 396 |
+
hidden_states = block(
|
| 397 |
+
hidden_states,
|
| 398 |
+
encoder_hidden_states=encoder_hidden_states,
|
| 399 |
+
timestep=timestep,
|
| 400 |
+
meta=meta,
|
| 401 |
+
)
|
| 402 |
+
|
| 403 |
+
# Output
|
| 404 |
+
if not self.use_linear_projection:
|
| 405 |
+
hidden_states = (
|
| 406 |
+
hidden_states.reshape(batch, height, weight, inner_dim)
|
| 407 |
+
.permute(0, 3, 1, 2)
|
| 408 |
+
.contiguous()
|
| 409 |
+
)
|
| 410 |
+
hidden_states = self.proj_out(hidden_states)
|
| 411 |
+
else:
|
| 412 |
+
hidden_states = self.proj_out(hidden_states)
|
| 413 |
+
hidden_states = (
|
| 414 |
+
hidden_states.reshape(batch, height, weight, inner_dim)
|
| 415 |
+
.permute(0, 3, 1, 2)
|
| 416 |
+
.contiguous()
|
| 417 |
+
)
|
| 418 |
+
|
| 419 |
+
output = hidden_states + residual
|
| 420 |
+
|
| 421 |
+
output = rearrange(output, "(b f) c h w -> b c f h w", f=video_length)
|
| 422 |
+
if not return_dict:
|
| 423 |
+
return (output,)
|
| 424 |
+
|
| 425 |
+
return Transformer3DModelOutput(sample=output)
|
| 426 |
+
|
| 427 |
+
|
| 428 |
+
class BasicTransformerBlock(nn.Module):
|
| 429 |
+
def __init__(
|
| 430 |
+
self,
|
| 431 |
+
dim: int,
|
| 432 |
+
num_attention_heads: int,
|
| 433 |
+
attention_head_dim: int,
|
| 434 |
+
dropout=0.0,
|
| 435 |
+
cross_attention_dim: Optional[int] = None,
|
| 436 |
+
activation_fn: str = "geglu",
|
| 437 |
+
num_embeds_ada_norm: Optional[int] = None,
|
| 438 |
+
attention_bias: bool = False,
|
| 439 |
+
only_cross_attention: bool = False,
|
| 440 |
+
upcast_attention: bool = False,
|
| 441 |
+
):
|
| 442 |
+
super().__init__()
|
| 443 |
+
self.only_cross_attention = only_cross_attention
|
| 444 |
+
self.use_ada_layer_norm = num_embeds_ada_norm is not None
|
| 445 |
+
# SC-Attn
|
| 446 |
+
self.attn1 = KNearsetAttention(
|
| 447 |
+
query_dim=dim,
|
| 448 |
+
heads=num_attention_heads,
|
| 449 |
+
dim_head=attention_head_dim,
|
| 450 |
+
dropout=dropout,
|
| 451 |
+
bias=attention_bias,
|
| 452 |
+
upcast_attention=upcast_attention,
|
| 453 |
+
)
|
| 454 |
+
# self.attn1= CrossAttention(
|
| 455 |
+
# query_dim=dim,
|
| 456 |
+
# heads=num_attention_heads,
|
| 457 |
+
# dim_head=attention_head_dim,
|
| 458 |
+
# dropout=dropout,
|
| 459 |
+
# bias=attention_bias,
|
| 460 |
+
# upcast_attention=upcast_attention,
|
| 461 |
+
# )
|
| 462 |
+
self.norm1 = (
|
| 463 |
+
AdaLayerNorm(dim, num_embeds_ada_norm)
|
| 464 |
+
if self.use_ada_layer_norm
|
| 465 |
+
else nn.LayerNorm(dim)
|
| 466 |
+
)
|
| 467 |
+
|
| 468 |
+
# Cross-Attn
|
| 469 |
+
if cross_attention_dim is not None:
|
| 470 |
+
self.attn2 = CrossAttention(
|
| 471 |
+
query_dim=dim,
|
| 472 |
+
cross_attention_dim=cross_attention_dim,
|
| 473 |
+
heads=num_attention_heads,
|
| 474 |
+
dim_head=attention_head_dim,
|
| 475 |
+
dropout=dropout,
|
| 476 |
+
bias=attention_bias,
|
| 477 |
+
upcast_attention=upcast_attention,
|
| 478 |
+
)
|
| 479 |
+
else:
|
| 480 |
+
self.attn2 = None
|
| 481 |
+
|
| 482 |
+
if cross_attention_dim is not None:
|
| 483 |
+
self.norm2 = (
|
| 484 |
+
AdaLayerNorm(dim, num_embeds_ada_norm)
|
| 485 |
+
if self.use_ada_layer_norm
|
| 486 |
+
else nn.LayerNorm(dim)
|
| 487 |
+
)
|
| 488 |
+
else:
|
| 489 |
+
self.norm2 = None
|
| 490 |
+
|
| 491 |
+
# Feed-forward
|
| 492 |
+
self.ff = FeedForward(dim, dropout=dropout, activation_fn=activation_fn)
|
| 493 |
+
self.norm3 = nn.LayerNorm(dim)
|
| 494 |
+
|
| 495 |
+
def set_use_memory_efficient_attention_xformers(
|
| 496 |
+
self,
|
| 497 |
+
use_memory_efficient_attention_xformers: bool,
|
| 498 |
+
attention_op: Optional[Callable] = None,
|
| 499 |
+
):
|
| 500 |
+
if not is_xformers_available():
|
| 501 |
+
print("Here is how to install it")
|
| 502 |
+
raise ModuleNotFoundError(
|
| 503 |
+
"Refer to https://github.com/facebookresearch/xformers for more information on how to install"
|
| 504 |
+
" xformers",
|
| 505 |
+
name="xformers",
|
| 506 |
+
)
|
| 507 |
+
elif not torch.cuda.is_available():
|
| 508 |
+
raise ValueError(
|
| 509 |
+
"torch.cuda.is_available() should be True but is False. xformers' memory efficient attention is only"
|
| 510 |
+
" available for GPU "
|
| 511 |
+
)
|
| 512 |
+
else:
|
| 513 |
+
try:
|
| 514 |
+
# Make sure we can run the memory efficient attention
|
| 515 |
+
_ = xformers.ops.memory_efficient_attention(
|
| 516 |
+
torch.randn((1, 2, 40), device="cuda"),
|
| 517 |
+
torch.randn((1, 2, 40), device="cuda"),
|
| 518 |
+
torch.randn((1, 2, 40), device="cuda"),
|
| 519 |
+
)
|
| 520 |
+
except Exception as e:
|
| 521 |
+
raise e
|
| 522 |
+
self.attn1._use_memory_efficient_attention_xformers = (
|
| 523 |
+
use_memory_efficient_attention_xformers
|
| 524 |
+
)
|
| 525 |
+
if self.attn2 is not None:
|
| 526 |
+
self.attn2._use_memory_efficient_attention_xformers = (
|
| 527 |
+
use_memory_efficient_attention_xformers
|
| 528 |
+
)
|
| 529 |
+
# self.attn_temp._use_memory_efficient_attention_xformers = use_memory_efficient_attention_xformers
|
| 530 |
+
|
| 531 |
+
def forward(
|
| 532 |
+
self,
|
| 533 |
+
hidden_states,
|
| 534 |
+
encoder_hidden_states=None,
|
| 535 |
+
timestep=None,
|
| 536 |
+
attention_mask=None,
|
| 537 |
+
meta=None,
|
| 538 |
+
):
|
| 539 |
+
# SparseCausal-Attention
|
| 540 |
+
norm_hidden_states = (
|
| 541 |
+
self.norm1(hidden_states, timestep)
|
| 542 |
+
if self.use_ada_layer_norm
|
| 543 |
+
else self.norm1(hidden_states)
|
| 544 |
+
)
|
| 545 |
+
|
| 546 |
+
if self.only_cross_attention:
|
| 547 |
+
hidden_states = (
|
| 548 |
+
self.attn1(
|
| 549 |
+
norm_hidden_states,
|
| 550 |
+
encoder_hidden_states,
|
| 551 |
+
attention_mask=attention_mask,
|
| 552 |
+
)
|
| 553 |
+
+ hidden_states
|
| 554 |
+
)
|
| 555 |
+
else:
|
| 556 |
+
hidden_states = (
|
| 557 |
+
self.attn1(
|
| 558 |
+
norm_hidden_states,
|
| 559 |
+
attention_mask=attention_mask,
|
| 560 |
+
meta=meta,
|
| 561 |
+
)
|
| 562 |
+
+ hidden_states
|
| 563 |
+
)
|
| 564 |
+
if self.attn2 is not None:
|
| 565 |
+
# Cross-Attention
|
| 566 |
+
norm_hidden_states = (
|
| 567 |
+
self.norm2(hidden_states, timestep)
|
| 568 |
+
if self.use_ada_layer_norm
|
| 569 |
+
else self.norm2(hidden_states)
|
| 570 |
+
)
|
| 571 |
+
hidden_states = (
|
| 572 |
+
self.attn2(
|
| 573 |
+
norm_hidden_states,
|
| 574 |
+
encoder_hidden_states=encoder_hidden_states,
|
| 575 |
+
attention_mask=attention_mask,
|
| 576 |
+
)
|
| 577 |
+
+ hidden_states
|
| 578 |
+
)
|
| 579 |
+
|
| 580 |
+
# Feed-forward
|
| 581 |
+
hidden_states = self.ff(self.norm3(hidden_states)) + hidden_states
|
| 582 |
+
return hidden_states
|
| 583 |
+
|
| 584 |
+
|
| 585 |
+
class SparseCausalAttention(CrossAttention):
|
| 586 |
+
def forward(
|
| 587 |
+
self,
|
| 588 |
+
hidden_states,
|
| 589 |
+
encoder_hidden_states=None,
|
| 590 |
+
attention_mask=None,
|
| 591 |
+
video_length=None,
|
| 592 |
+
):
|
| 593 |
+
batch_size, sequence_length, _ = hidden_states.shape
|
| 594 |
+
|
| 595 |
+
encoder_hidden_states = encoder_hidden_states
|
| 596 |
+
|
| 597 |
+
if self.group_norm is not None:
|
| 598 |
+
hidden_states = self.group_norm(hidden_states.transpose(1, 2)).transpose(
|
| 599 |
+
1, 2
|
| 600 |
+
)
|
| 601 |
+
|
| 602 |
+
query = self.to_q(hidden_states)
|
| 603 |
+
dim = query.shape[-1]
|
| 604 |
+
query = self.reshape_heads_to_batch_dim(query)
|
| 605 |
+
|
| 606 |
+
if self.added_kv_proj_dim is not None:
|
| 607 |
+
raise NotImplementedError
|
| 608 |
+
|
| 609 |
+
encoder_hidden_states = (
|
| 610 |
+
encoder_hidden_states
|
| 611 |
+
if encoder_hidden_states is not None
|
| 612 |
+
else hidden_states
|
| 613 |
+
)
|
| 614 |
+
key = self.to_k(encoder_hidden_states)
|
| 615 |
+
value = self.to_v(encoder_hidden_states)
|
| 616 |
+
|
| 617 |
+
former_frame_index = torch.arange(video_length) - 1
|
| 618 |
+
former_frame_index[0] = 0
|
| 619 |
+
|
| 620 |
+
key = rearrange(key, "(b f) d c -> b f d c", f=video_length)
|
| 621 |
+
key = torch.cat([key[:, [0] * video_length], key[:, former_frame_index]], dim=2)
|
| 622 |
+
key = rearrange(key, "b f d c -> (b f) d c")
|
| 623 |
+
|
| 624 |
+
value = rearrange(value, "(b f) d c -> b f d c", f=video_length)
|
| 625 |
+
value = torch.cat(
|
| 626 |
+
[value[:, [0] * video_length], value[:, former_frame_index]], dim=2
|
| 627 |
+
)
|
| 628 |
+
value = rearrange(value, "b f d c -> (b f) d c")
|
| 629 |
+
|
| 630 |
+
key = self.reshape_heads_to_batch_dim(key)
|
| 631 |
+
value = self.reshape_heads_to_batch_dim(value)
|
| 632 |
+
|
| 633 |
+
if attention_mask is not None:
|
| 634 |
+
if attention_mask.shape[-1] != query.shape[1]:
|
| 635 |
+
target_length = query.shape[1]
|
| 636 |
+
attention_mask = F.pad(attention_mask, (0, target_length), value=0.0)
|
| 637 |
+
attention_mask = attention_mask.repeat_interleave(self.heads, dim=0)
|
| 638 |
+
|
| 639 |
+
# attention, what we cannot get enough of
|
| 640 |
+
if self._use_memory_efficient_attention_xformers:
|
| 641 |
+
hidden_states = self._memory_efficient_attention_xformers(
|
| 642 |
+
query, key, value, attention_mask
|
| 643 |
+
)
|
| 644 |
+
# Some versions of xformers return output in fp32, cast it back to the dtype of the input
|
| 645 |
+
hidden_states = hidden_states.to(query.dtype)
|
| 646 |
+
else:
|
| 647 |
+
if self._slice_size is None or query.shape[0] // self._slice_size == 1:
|
| 648 |
+
hidden_states = self._attention(query, key, value, attention_mask)
|
| 649 |
+
else:
|
| 650 |
+
hidden_states = self._sliced_attention(
|
| 651 |
+
query, key, value, sequence_length, dim, attention_mask
|
| 652 |
+
)
|
| 653 |
+
|
| 654 |
+
# linear proj
|
| 655 |
+
hidden_states = self.to_out[0](hidden_states)
|
| 656 |
+
|
| 657 |
+
# dropout
|
| 658 |
+
hidden_states = self.to_out[1](hidden_states)
|
| 659 |
+
return hidden_states
|
| 660 |
+
|
| 661 |
+
|
| 662 |
+
class KNearsetAttention(CrossAttention):
|
| 663 |
+
def forward(
|
| 664 |
+
self, hidden_states, encoder_hidden_states=None, attention_mask=None, meta=None
|
| 665 |
+
):
|
| 666 |
+
video_length = meta["video_length"]
|
| 667 |
+
k_near_indices = meta["k_near_indices"] # B X F X K
|
| 668 |
+
batch_size, sequence_length, _ = hidden_states.shape
|
| 669 |
+
|
| 670 |
+
encoder_hidden_states = encoder_hidden_states
|
| 671 |
+
|
| 672 |
+
if self.group_norm is not None:
|
| 673 |
+
hidden_states = self.group_norm(hidden_states.transpose(1, 2)).transpose(
|
| 674 |
+
1, 2
|
| 675 |
+
)
|
| 676 |
+
|
| 677 |
+
query = self.to_q(hidden_states)
|
| 678 |
+
dim = query.shape[-1]
|
| 679 |
+
query = self.reshape_heads_to_batch_dim(query)
|
| 680 |
+
|
| 681 |
+
if self.added_kv_proj_dim is not None:
|
| 682 |
+
raise NotImplementedError
|
| 683 |
+
|
| 684 |
+
encoder_hidden_states = (
|
| 685 |
+
encoder_hidden_states
|
| 686 |
+
if encoder_hidden_states is not None
|
| 687 |
+
else hidden_states
|
| 688 |
+
)
|
| 689 |
+
key = self.to_k(encoder_hidden_states)
|
| 690 |
+
value = self.to_v(encoder_hidden_states)
|
| 691 |
+
|
| 692 |
+
key = repeat(key, "(b f) d c -> b f k d c", f=video_length, k=video_length)
|
| 693 |
+
|
| 694 |
+
b, f_views, k_views = k_near_indices.shape
|
| 695 |
+
b_ids = (
|
| 696 |
+
torch.arange(b)[:, None, None].expand(-1, f_views, k_views).to(key.device)
|
| 697 |
+
)
|
| 698 |
+
f_ids = (
|
| 699 |
+
torch.arange(f_views)[None, :, None].expand(b, -1, k_views).to(key.device)
|
| 700 |
+
)
|
| 701 |
+
|
| 702 |
+
k_nearest_key = key[b_ids, f_ids, k_near_indices] # b x f x k x d x c
|
| 703 |
+
|
| 704 |
+
key = rearrange(key, "b f k d c -> (b f) (k d) c")
|
| 705 |
+
|
| 706 |
+
value = repeat(value, "(b f) d c -> b f k d c", f=video_length, k=video_length)
|
| 707 |
+
k_nearest_value = value[b_ids, f_ids, k_near_indices] # b x f x k x d x c
|
| 708 |
+
value = rearrange(value, "b f k d c -> (b f) (k d) c")
|
| 709 |
+
|
| 710 |
+
key = self.reshape_heads_to_batch_dim(key)
|
| 711 |
+
value = self.reshape_heads_to_batch_dim(value)
|
| 712 |
+
|
| 713 |
+
if attention_mask is not None:
|
| 714 |
+
if attention_mask.shape[-1] != query.shape[1]:
|
| 715 |
+
target_length = query.shape[1]
|
| 716 |
+
attention_mask = F.pad(attention_mask, (0, target_length), value=0.0)
|
| 717 |
+
attention_mask = attention_mask.repeat_interleave(self.heads, dim=0)
|
| 718 |
+
|
| 719 |
+
# attention, what we cannot get enough of
|
| 720 |
+
if self._use_memory_efficient_attention_xformers:
|
| 721 |
+
hidden_states = self._memory_efficient_attention_xformers(
|
| 722 |
+
query, key, value, attention_mask
|
| 723 |
+
)
|
| 724 |
+
# Some versions of xformers return output in fp32, cast it back to the dtype of the input
|
| 725 |
+
hidden_states = hidden_states.to(query.dtype)
|
| 726 |
+
else:
|
| 727 |
+
if self._slice_size is None or query.shape[0] // self._slice_size == 1:
|
| 728 |
+
hidden_states = self._attention(query, key, value, attention_mask)
|
| 729 |
+
else:
|
| 730 |
+
hidden_states = self._sliced_attention(
|
| 731 |
+
query, key, value, sequence_length, dim, attention_mask
|
| 732 |
+
)
|
| 733 |
+
|
| 734 |
+
# linear proj
|
| 735 |
+
hidden_states = self.to_out[0](hidden_states)
|
| 736 |
+
|
| 737 |
+
# dropout
|
| 738 |
+
hidden_states = self.to_out[1](hidden_states)
|
| 739 |
+
return hidden_states
|
loomnet/models/blocks/attention_processor.py
ADDED
|
@@ -0,0 +1,871 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
import torch.nn.functional as F
|
| 5 |
+
from einops import rearrange, repeat
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class AttnProcessor(nn.Module):
|
| 9 |
+
r"""
|
| 10 |
+
Default processor for performing attention-related computations.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
def __init__(
|
| 14 |
+
self,
|
| 15 |
+
hidden_size=None,
|
| 16 |
+
cross_attention_dim=None,
|
| 17 |
+
):
|
| 18 |
+
super().__init__()
|
| 19 |
+
|
| 20 |
+
def __call__(
|
| 21 |
+
self,
|
| 22 |
+
attn,
|
| 23 |
+
hidden_states,
|
| 24 |
+
encoder_hidden_states=None,
|
| 25 |
+
attention_mask=None,
|
| 26 |
+
temb=None,
|
| 27 |
+
):
|
| 28 |
+
residual = hidden_states
|
| 29 |
+
|
| 30 |
+
if attn.spatial_norm is not None:
|
| 31 |
+
hidden_states = attn.spatial_norm(hidden_states, temb)
|
| 32 |
+
|
| 33 |
+
input_ndim = hidden_states.ndim
|
| 34 |
+
|
| 35 |
+
if input_ndim == 4:
|
| 36 |
+
batch_size, channel, height, width = hidden_states.shape
|
| 37 |
+
hidden_states = hidden_states.view(
|
| 38 |
+
batch_size, channel, height * width
|
| 39 |
+
).transpose(1, 2)
|
| 40 |
+
|
| 41 |
+
batch_size, sequence_length, _ = (
|
| 42 |
+
hidden_states.shape
|
| 43 |
+
if encoder_hidden_states is None
|
| 44 |
+
else encoder_hidden_states.shape
|
| 45 |
+
)
|
| 46 |
+
attention_mask = attn.prepare_attention_mask(
|
| 47 |
+
attention_mask, sequence_length, batch_size
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
if attn.group_norm is not None:
|
| 51 |
+
hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(
|
| 52 |
+
1, 2
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
query = attn.to_q(hidden_states)
|
| 56 |
+
|
| 57 |
+
if encoder_hidden_states is None:
|
| 58 |
+
encoder_hidden_states = hidden_states
|
| 59 |
+
elif attn.norm_cross:
|
| 60 |
+
encoder_hidden_states = attn.norm_encoder_hidden_states(
|
| 61 |
+
encoder_hidden_states
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
key = attn.to_k(encoder_hidden_states)
|
| 65 |
+
value = attn.to_v(encoder_hidden_states)
|
| 66 |
+
|
| 67 |
+
query = attn.head_to_batch_dim(query)
|
| 68 |
+
key = attn.head_to_batch_dim(key)
|
| 69 |
+
value = attn.head_to_batch_dim(value)
|
| 70 |
+
|
| 71 |
+
attention_probs = attn.get_attention_scores(query, key, attention_mask)
|
| 72 |
+
hidden_states = torch.bmm(attention_probs, value)
|
| 73 |
+
hidden_states = attn.batch_to_head_dim(hidden_states)
|
| 74 |
+
|
| 75 |
+
# linear proj
|
| 76 |
+
hidden_states = attn.to_out[0](hidden_states)
|
| 77 |
+
# dropout
|
| 78 |
+
hidden_states = attn.to_out[1](hidden_states)
|
| 79 |
+
|
| 80 |
+
if input_ndim == 4:
|
| 81 |
+
hidden_states = hidden_states.transpose(-1, -2).reshape(
|
| 82 |
+
batch_size, channel, height, width
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
if attn.residual_connection:
|
| 86 |
+
hidden_states = hidden_states + residual
|
| 87 |
+
|
| 88 |
+
hidden_states = hidden_states / attn.rescale_output_factor
|
| 89 |
+
|
| 90 |
+
return hidden_states
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
class IPAttnProcessor(nn.Module):
|
| 94 |
+
r"""
|
| 95 |
+
Attention processor for IP-Adapater.
|
| 96 |
+
Args:
|
| 97 |
+
hidden_size (`int`):
|
| 98 |
+
The hidden size of the attention layer.
|
| 99 |
+
cross_attention_dim (`int`):
|
| 100 |
+
The number of channels in the `encoder_hidden_states`.
|
| 101 |
+
scale (`float`, defaults to 1.0):
|
| 102 |
+
the weight scale of image prompt.
|
| 103 |
+
num_tokens (`int`, defaults to 4 when do ip_adapter_plus it should be 16):
|
| 104 |
+
The context length of the image features.
|
| 105 |
+
"""
|
| 106 |
+
|
| 107 |
+
def __init__(self, hidden_size, cross_attention_dim=None, scale=1.0, num_tokens=4):
|
| 108 |
+
super().__init__()
|
| 109 |
+
|
| 110 |
+
self.hidden_size = hidden_size
|
| 111 |
+
self.cross_attention_dim = cross_attention_dim
|
| 112 |
+
self.scale = scale
|
| 113 |
+
self.num_tokens = num_tokens
|
| 114 |
+
|
| 115 |
+
self.to_k_ip = nn.Linear(
|
| 116 |
+
cross_attention_dim or hidden_size, hidden_size, bias=False
|
| 117 |
+
)
|
| 118 |
+
self.to_v_ip = nn.Linear(
|
| 119 |
+
cross_attention_dim or hidden_size, hidden_size, bias=False
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
def __call__(
|
| 123 |
+
self,
|
| 124 |
+
attn,
|
| 125 |
+
hidden_states,
|
| 126 |
+
encoder_hidden_states=None,
|
| 127 |
+
attention_mask=None,
|
| 128 |
+
temb=None,
|
| 129 |
+
):
|
| 130 |
+
residual = hidden_states
|
| 131 |
+
|
| 132 |
+
if attn.spatial_norm is not None:
|
| 133 |
+
hidden_states = attn.spatial_norm(hidden_states, temb)
|
| 134 |
+
|
| 135 |
+
input_ndim = hidden_states.ndim
|
| 136 |
+
|
| 137 |
+
if input_ndim == 4:
|
| 138 |
+
batch_size, channel, height, width = hidden_states.shape
|
| 139 |
+
hidden_states = hidden_states.view(
|
| 140 |
+
batch_size, channel, height * width
|
| 141 |
+
).transpose(1, 2)
|
| 142 |
+
|
| 143 |
+
batch_size, sequence_length, _ = (
|
| 144 |
+
hidden_states.shape
|
| 145 |
+
if encoder_hidden_states is None
|
| 146 |
+
else encoder_hidden_states.shape
|
| 147 |
+
)
|
| 148 |
+
attention_mask = attn.prepare_attention_mask(
|
| 149 |
+
attention_mask, sequence_length, batch_size
|
| 150 |
+
)
|
| 151 |
+
|
| 152 |
+
if attn.group_norm is not None:
|
| 153 |
+
hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(
|
| 154 |
+
1, 2
|
| 155 |
+
)
|
| 156 |
+
|
| 157 |
+
query = attn.to_q(hidden_states)
|
| 158 |
+
|
| 159 |
+
if encoder_hidden_states is None:
|
| 160 |
+
encoder_hidden_states = hidden_states
|
| 161 |
+
else:
|
| 162 |
+
# get encoder_hidden_states, ip_hidden_states
|
| 163 |
+
end_pos = encoder_hidden_states.shape[1] - self.num_tokens
|
| 164 |
+
encoder_hidden_states, ip_hidden_states = (
|
| 165 |
+
encoder_hidden_states[:, :end_pos, :],
|
| 166 |
+
encoder_hidden_states[:, end_pos:, :],
|
| 167 |
+
)
|
| 168 |
+
if attn.norm_cross:
|
| 169 |
+
encoder_hidden_states = attn.norm_encoder_hidden_states(
|
| 170 |
+
encoder_hidden_states
|
| 171 |
+
)
|
| 172 |
+
|
| 173 |
+
key = attn.to_k(encoder_hidden_states)
|
| 174 |
+
value = attn.to_v(encoder_hidden_states)
|
| 175 |
+
|
| 176 |
+
query = attn.head_to_batch_dim(query)
|
| 177 |
+
key = attn.head_to_batch_dim(key)
|
| 178 |
+
value = attn.head_to_batch_dim(value)
|
| 179 |
+
|
| 180 |
+
attention_probs = attn.get_attention_scores(query, key, attention_mask)
|
| 181 |
+
hidden_states = torch.bmm(attention_probs, value)
|
| 182 |
+
hidden_states = attn.batch_to_head_dim(hidden_states)
|
| 183 |
+
|
| 184 |
+
# for ip-adapter
|
| 185 |
+
ip_key = self.to_k_ip(ip_hidden_states)
|
| 186 |
+
ip_value = self.to_v_ip(ip_hidden_states)
|
| 187 |
+
|
| 188 |
+
ip_key = attn.head_to_batch_dim(ip_key)
|
| 189 |
+
ip_value = attn.head_to_batch_dim(ip_value)
|
| 190 |
+
|
| 191 |
+
ip_attention_probs = attn.get_attention_scores(query, ip_key, None)
|
| 192 |
+
ip_hidden_states = torch.bmm(ip_attention_probs, ip_value)
|
| 193 |
+
ip_hidden_states = attn.batch_to_head_dim(ip_hidden_states)
|
| 194 |
+
|
| 195 |
+
hidden_states = hidden_states + self.scale * ip_hidden_states
|
| 196 |
+
|
| 197 |
+
# linear proj
|
| 198 |
+
hidden_states = attn.to_out[0](hidden_states)
|
| 199 |
+
# dropout
|
| 200 |
+
hidden_states = attn.to_out[1](hidden_states)
|
| 201 |
+
|
| 202 |
+
if input_ndim == 4:
|
| 203 |
+
hidden_states = hidden_states.transpose(-1, -2).reshape(
|
| 204 |
+
batch_size, channel, height, width
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
if attn.residual_connection:
|
| 208 |
+
hidden_states = hidden_states + residual
|
| 209 |
+
|
| 210 |
+
hidden_states = hidden_states / attn.rescale_output_factor
|
| 211 |
+
|
| 212 |
+
return hidden_states
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
class AttnProcessor2_0(torch.nn.Module):
|
| 216 |
+
r"""
|
| 217 |
+
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0).
|
| 218 |
+
"""
|
| 219 |
+
|
| 220 |
+
def __init__(
|
| 221 |
+
self,
|
| 222 |
+
hidden_size=None,
|
| 223 |
+
cross_attention_dim=None,
|
| 224 |
+
frames=8,
|
| 225 |
+
):
|
| 226 |
+
super().__init__()
|
| 227 |
+
if not hasattr(F, "scaled_dot_product_attention"):
|
| 228 |
+
raise ImportError(
|
| 229 |
+
"AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0."
|
| 230 |
+
)
|
| 231 |
+
self.frames = frames
|
| 232 |
+
|
| 233 |
+
def __call__(
|
| 234 |
+
self,
|
| 235 |
+
attn,
|
| 236 |
+
hidden_states,
|
| 237 |
+
encoder_hidden_states=None,
|
| 238 |
+
attention_mask=None,
|
| 239 |
+
temb=None,
|
| 240 |
+
):
|
| 241 |
+
residual = hidden_states
|
| 242 |
+
|
| 243 |
+
if attn.spatial_norm is not None:
|
| 244 |
+
hidden_states = attn.spatial_norm(hidden_states, temb)
|
| 245 |
+
|
| 246 |
+
input_ndim = hidden_states.ndim
|
| 247 |
+
hidden_states = rearrange(
|
| 248 |
+
hidden_states, "(b f) n c -> b (f n) c", f=self.frames
|
| 249 |
+
)
|
| 250 |
+
if input_ndim == 4:
|
| 251 |
+
batch_size, channel, height, width = hidden_states.shape
|
| 252 |
+
hidden_states = hidden_states.view(
|
| 253 |
+
batch_size, channel, height * width
|
| 254 |
+
).transpose(1, 2)
|
| 255 |
+
|
| 256 |
+
batch_size, sequence_length, _ = (
|
| 257 |
+
hidden_states.shape
|
| 258 |
+
if encoder_hidden_states is None
|
| 259 |
+
else encoder_hidden_states.shape
|
| 260 |
+
)
|
| 261 |
+
|
| 262 |
+
if attention_mask is not None:
|
| 263 |
+
attention_mask = attn.prepare_attention_mask(
|
| 264 |
+
attention_mask, sequence_length, batch_size
|
| 265 |
+
)
|
| 266 |
+
# scaled_dot_product_attention expects attention_mask shape to be
|
| 267 |
+
# (batch, heads, source_length, target_length)
|
| 268 |
+
attention_mask = attention_mask.view(
|
| 269 |
+
batch_size, attn.heads, -1, attention_mask.shape[-1]
|
| 270 |
+
)
|
| 271 |
+
|
| 272 |
+
if attn.group_norm is not None:
|
| 273 |
+
hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(
|
| 274 |
+
1, 2
|
| 275 |
+
)
|
| 276 |
+
|
| 277 |
+
query = attn.to_q(hidden_states)
|
| 278 |
+
|
| 279 |
+
if encoder_hidden_states is None:
|
| 280 |
+
encoder_hidden_states = hidden_states
|
| 281 |
+
elif attn.norm_cross:
|
| 282 |
+
encoder_hidden_states = attn.norm_encoder_hidden_states(
|
| 283 |
+
encoder_hidden_states
|
| 284 |
+
)
|
| 285 |
+
|
| 286 |
+
key = attn.to_k(encoder_hidden_states)
|
| 287 |
+
value = attn.to_v(encoder_hidden_states)
|
| 288 |
+
|
| 289 |
+
inner_dim = key.shape[-1]
|
| 290 |
+
head_dim = inner_dim // attn.heads
|
| 291 |
+
|
| 292 |
+
query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
| 293 |
+
|
| 294 |
+
key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
| 295 |
+
value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
| 296 |
+
|
| 297 |
+
# the output of sdp = (batch, num_heads, seq_len, head_dim)
|
| 298 |
+
# TODO: add support for attn.scale when we move to Torch 2.1
|
| 299 |
+
hidden_states = F.scaled_dot_product_attention(
|
| 300 |
+
query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
|
| 301 |
+
)
|
| 302 |
+
|
| 303 |
+
hidden_states = hidden_states.transpose(1, 2).reshape(
|
| 304 |
+
batch_size, -1, attn.heads * head_dim
|
| 305 |
+
)
|
| 306 |
+
hidden_states = hidden_states.to(query.dtype)
|
| 307 |
+
|
| 308 |
+
# linear proj
|
| 309 |
+
hidden_states = attn.to_out[0](hidden_states)
|
| 310 |
+
# dropout
|
| 311 |
+
hidden_states = attn.to_out[1](hidden_states)
|
| 312 |
+
|
| 313 |
+
if input_ndim == 4:
|
| 314 |
+
hidden_states = hidden_states.transpose(-1, -2).reshape(
|
| 315 |
+
batch_size, channel, height, width
|
| 316 |
+
)
|
| 317 |
+
|
| 318 |
+
if attn.residual_connection:
|
| 319 |
+
hidden_states = hidden_states + residual
|
| 320 |
+
|
| 321 |
+
hidden_states = hidden_states / attn.rescale_output_factor
|
| 322 |
+
hidden_states = rearrange(
|
| 323 |
+
hidden_states, "b (f n) c -> (b f) n c", f=self.frames
|
| 324 |
+
)
|
| 325 |
+
return hidden_states
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
class IPAttnProcessor2_0(torch.nn.Module):
|
| 329 |
+
r"""
|
| 330 |
+
Attention processor for IP-Adapater for PyTorch 2.0.
|
| 331 |
+
Args:
|
| 332 |
+
hidden_size (`int`):
|
| 333 |
+
The hidden size of the attention layer.
|
| 334 |
+
cross_attention_dim (`int`):
|
| 335 |
+
The number of channels in the `encoder_hidden_states`.
|
| 336 |
+
scale (`float`, defaults to 1.0):
|
| 337 |
+
the weight scale of image prompt.
|
| 338 |
+
num_tokens (`int`, defaults to 4 when do ip_adapter_plus it should be 16):
|
| 339 |
+
The context length of the image features.
|
| 340 |
+
"""
|
| 341 |
+
|
| 342 |
+
def __init__(self, hidden_size, cross_attention_dim=None, scale=1.0, num_tokens=4):
|
| 343 |
+
super().__init__()
|
| 344 |
+
|
| 345 |
+
if not hasattr(F, "scaled_dot_product_attention"):
|
| 346 |
+
raise ImportError(
|
| 347 |
+
"AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0."
|
| 348 |
+
)
|
| 349 |
+
|
| 350 |
+
self.hidden_size = hidden_size
|
| 351 |
+
self.cross_attention_dim = cross_attention_dim
|
| 352 |
+
self.scale = scale
|
| 353 |
+
self.num_tokens = num_tokens
|
| 354 |
+
|
| 355 |
+
self.to_k_ip = nn.Linear(
|
| 356 |
+
cross_attention_dim or hidden_size, hidden_size, bias=False
|
| 357 |
+
)
|
| 358 |
+
self.to_v_ip = nn.Linear(
|
| 359 |
+
cross_attention_dim or hidden_size, hidden_size, bias=False
|
| 360 |
+
)
|
| 361 |
+
|
| 362 |
+
def __call__(
|
| 363 |
+
self,
|
| 364 |
+
attn,
|
| 365 |
+
hidden_states,
|
| 366 |
+
encoder_hidden_states=None,
|
| 367 |
+
attention_mask=None,
|
| 368 |
+
temb=None,
|
| 369 |
+
):
|
| 370 |
+
residual = hidden_states
|
| 371 |
+
|
| 372 |
+
if attn.spatial_norm is not None:
|
| 373 |
+
hidden_states = attn.spatial_norm(hidden_states, temb)
|
| 374 |
+
|
| 375 |
+
input_ndim = hidden_states.ndim
|
| 376 |
+
|
| 377 |
+
if input_ndim == 4:
|
| 378 |
+
batch_size, channel, height, width = hidden_states.shape
|
| 379 |
+
hidden_states = hidden_states.view(
|
| 380 |
+
batch_size, channel, height * width
|
| 381 |
+
).transpose(1, 2)
|
| 382 |
+
|
| 383 |
+
batch_size, sequence_length, _ = (
|
| 384 |
+
hidden_states.shape
|
| 385 |
+
if encoder_hidden_states is None
|
| 386 |
+
else encoder_hidden_states.shape
|
| 387 |
+
)
|
| 388 |
+
|
| 389 |
+
if attention_mask is not None:
|
| 390 |
+
attention_mask = attn.prepare_attention_mask(
|
| 391 |
+
attention_mask, sequence_length, batch_size
|
| 392 |
+
)
|
| 393 |
+
# scaled_dot_product_attention expects attention_mask shape to be
|
| 394 |
+
# (batch, heads, source_length, target_length)
|
| 395 |
+
attention_mask = attention_mask.view(
|
| 396 |
+
batch_size, attn.heads, -1, attention_mask.shape[-1]
|
| 397 |
+
)
|
| 398 |
+
|
| 399 |
+
if attn.group_norm is not None:
|
| 400 |
+
hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(
|
| 401 |
+
1, 2
|
| 402 |
+
)
|
| 403 |
+
|
| 404 |
+
query = attn.to_q(hidden_states)
|
| 405 |
+
|
| 406 |
+
if encoder_hidden_states is None:
|
| 407 |
+
encoder_hidden_states = hidden_states
|
| 408 |
+
else:
|
| 409 |
+
# get encoder_hidden_states, ip_hidden_states
|
| 410 |
+
end_pos = encoder_hidden_states.shape[1] - self.num_tokens
|
| 411 |
+
encoder_hidden_states, ip_hidden_states = (
|
| 412 |
+
encoder_hidden_states[:, :end_pos, :],
|
| 413 |
+
encoder_hidden_states[:, end_pos:, :],
|
| 414 |
+
)
|
| 415 |
+
if attn.norm_cross:
|
| 416 |
+
encoder_hidden_states = attn.norm_encoder_hidden_states(
|
| 417 |
+
encoder_hidden_states
|
| 418 |
+
)
|
| 419 |
+
|
| 420 |
+
key = attn.to_k(encoder_hidden_states)
|
| 421 |
+
value = attn.to_v(encoder_hidden_states)
|
| 422 |
+
|
| 423 |
+
inner_dim = key.shape[-1]
|
| 424 |
+
head_dim = inner_dim // attn.heads
|
| 425 |
+
|
| 426 |
+
query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
| 427 |
+
|
| 428 |
+
key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
| 429 |
+
value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
| 430 |
+
|
| 431 |
+
# the output of sdp = (batch, num_heads, seq_len, head_dim)
|
| 432 |
+
# TODO: add support for attn.scale when we move to Torch 2.1
|
| 433 |
+
hidden_states = F.scaled_dot_product_attention(
|
| 434 |
+
query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
|
| 435 |
+
)
|
| 436 |
+
|
| 437 |
+
hidden_states = hidden_states.transpose(1, 2).reshape(
|
| 438 |
+
batch_size, -1, attn.heads * head_dim
|
| 439 |
+
)
|
| 440 |
+
hidden_states = hidden_states.to(query.dtype)
|
| 441 |
+
|
| 442 |
+
# for ip-adapter
|
| 443 |
+
ip_key = self.to_k_ip(ip_hidden_states)
|
| 444 |
+
ip_value = self.to_v_ip(ip_hidden_states)
|
| 445 |
+
|
| 446 |
+
ip_key = ip_key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
| 447 |
+
ip_value = ip_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
| 448 |
+
|
| 449 |
+
# the output of sdp = (batch, num_heads, seq_len, head_dim)
|
| 450 |
+
# TODO: add support for attn.scale when we move to Torch 2.1
|
| 451 |
+
ip_hidden_states = F.scaled_dot_product_attention(
|
| 452 |
+
query, ip_key, ip_value, attn_mask=None, dropout_p=0.0, is_causal=False
|
| 453 |
+
)
|
| 454 |
+
|
| 455 |
+
ip_hidden_states = ip_hidden_states.transpose(1, 2).reshape(
|
| 456 |
+
batch_size, -1, attn.heads * head_dim
|
| 457 |
+
)
|
| 458 |
+
ip_hidden_states = ip_hidden_states.to(query.dtype)
|
| 459 |
+
|
| 460 |
+
hidden_states = hidden_states + self.scale * ip_hidden_states
|
| 461 |
+
|
| 462 |
+
# linear proj
|
| 463 |
+
hidden_states = attn.to_out[0](hidden_states)
|
| 464 |
+
# dropout
|
| 465 |
+
hidden_states = attn.to_out[1](hidden_states)
|
| 466 |
+
|
| 467 |
+
if input_ndim == 4:
|
| 468 |
+
hidden_states = hidden_states.transpose(-1, -2).reshape(
|
| 469 |
+
batch_size, channel, height, width
|
| 470 |
+
)
|
| 471 |
+
|
| 472 |
+
if attn.residual_connection:
|
| 473 |
+
hidden_states = hidden_states + residual
|
| 474 |
+
|
| 475 |
+
hidden_states = hidden_states / attn.rescale_output_factor
|
| 476 |
+
|
| 477 |
+
return hidden_states
|
| 478 |
+
|
| 479 |
+
|
| 480 |
+
## for controlnet
|
| 481 |
+
class CNAttnProcessor:
|
| 482 |
+
r"""
|
| 483 |
+
Default processor for performing attention-related computations.
|
| 484 |
+
"""
|
| 485 |
+
|
| 486 |
+
def __init__(self, num_tokens=4):
|
| 487 |
+
self.num_tokens = num_tokens
|
| 488 |
+
|
| 489 |
+
def __call__(
|
| 490 |
+
self,
|
| 491 |
+
attn,
|
| 492 |
+
hidden_states,
|
| 493 |
+
encoder_hidden_states=None,
|
| 494 |
+
attention_mask=None,
|
| 495 |
+
temb=None,
|
| 496 |
+
):
|
| 497 |
+
residual = hidden_states
|
| 498 |
+
|
| 499 |
+
if attn.spatial_norm is not None:
|
| 500 |
+
hidden_states = attn.spatial_norm(hidden_states, temb)
|
| 501 |
+
|
| 502 |
+
input_ndim = hidden_states.ndim
|
| 503 |
+
|
| 504 |
+
if input_ndim == 4:
|
| 505 |
+
batch_size, channel, height, width = hidden_states.shape
|
| 506 |
+
hidden_states = hidden_states.view(
|
| 507 |
+
batch_size, channel, height * width
|
| 508 |
+
).transpose(1, 2)
|
| 509 |
+
|
| 510 |
+
batch_size, sequence_length, _ = (
|
| 511 |
+
hidden_states.shape
|
| 512 |
+
if encoder_hidden_states is None
|
| 513 |
+
else encoder_hidden_states.shape
|
| 514 |
+
)
|
| 515 |
+
attention_mask = attn.prepare_attention_mask(
|
| 516 |
+
attention_mask, sequence_length, batch_size
|
| 517 |
+
)
|
| 518 |
+
|
| 519 |
+
if attn.group_norm is not None:
|
| 520 |
+
hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(
|
| 521 |
+
1, 2
|
| 522 |
+
)
|
| 523 |
+
|
| 524 |
+
query = attn.to_q(hidden_states)
|
| 525 |
+
|
| 526 |
+
if encoder_hidden_states is None:
|
| 527 |
+
encoder_hidden_states = hidden_states
|
| 528 |
+
else:
|
| 529 |
+
end_pos = encoder_hidden_states.shape[1] - self.num_tokens
|
| 530 |
+
encoder_hidden_states = encoder_hidden_states[:, :end_pos] # only use text
|
| 531 |
+
if attn.norm_cross:
|
| 532 |
+
encoder_hidden_states = attn.norm_encoder_hidden_states(
|
| 533 |
+
encoder_hidden_states
|
| 534 |
+
)
|
| 535 |
+
|
| 536 |
+
key = attn.to_k(encoder_hidden_states)
|
| 537 |
+
value = attn.to_v(encoder_hidden_states)
|
| 538 |
+
|
| 539 |
+
query = attn.head_to_batch_dim(query)
|
| 540 |
+
key = attn.head_to_batch_dim(key)
|
| 541 |
+
value = attn.head_to_batch_dim(value)
|
| 542 |
+
|
| 543 |
+
attention_probs = attn.get_attention_scores(query, key, attention_mask)
|
| 544 |
+
hidden_states = torch.bmm(attention_probs, value)
|
| 545 |
+
hidden_states = attn.batch_to_head_dim(hidden_states)
|
| 546 |
+
|
| 547 |
+
# linear proj
|
| 548 |
+
hidden_states = attn.to_out[0](hidden_states)
|
| 549 |
+
# dropout
|
| 550 |
+
hidden_states = attn.to_out[1](hidden_states)
|
| 551 |
+
|
| 552 |
+
if input_ndim == 4:
|
| 553 |
+
hidden_states = hidden_states.transpose(-1, -2).reshape(
|
| 554 |
+
batch_size, channel, height, width
|
| 555 |
+
)
|
| 556 |
+
|
| 557 |
+
if attn.residual_connection:
|
| 558 |
+
hidden_states = hidden_states + residual
|
| 559 |
+
|
| 560 |
+
hidden_states = hidden_states / attn.rescale_output_factor
|
| 561 |
+
|
| 562 |
+
return hidden_states
|
| 563 |
+
|
| 564 |
+
|
| 565 |
+
class CNAttnProcessor2_0:
|
| 566 |
+
r"""
|
| 567 |
+
Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0).
|
| 568 |
+
"""
|
| 569 |
+
|
| 570 |
+
def __init__(self, num_tokens=4):
|
| 571 |
+
if not hasattr(F, "scaled_dot_product_attention"):
|
| 572 |
+
raise ImportError(
|
| 573 |
+
"AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0."
|
| 574 |
+
)
|
| 575 |
+
self.num_tokens = num_tokens
|
| 576 |
+
|
| 577 |
+
def __call__(
|
| 578 |
+
self,
|
| 579 |
+
attn,
|
| 580 |
+
hidden_states,
|
| 581 |
+
encoder_hidden_states=None,
|
| 582 |
+
attention_mask=None,
|
| 583 |
+
temb=None,
|
| 584 |
+
):
|
| 585 |
+
residual = hidden_states
|
| 586 |
+
|
| 587 |
+
if attn.spatial_norm is not None:
|
| 588 |
+
hidden_states = attn.spatial_norm(hidden_states, temb)
|
| 589 |
+
|
| 590 |
+
input_ndim = hidden_states.ndim
|
| 591 |
+
|
| 592 |
+
if input_ndim == 4:
|
| 593 |
+
batch_size, channel, height, width = hidden_states.shape
|
| 594 |
+
hidden_states = hidden_states.view(
|
| 595 |
+
batch_size, channel, height * width
|
| 596 |
+
).transpose(1, 2)
|
| 597 |
+
|
| 598 |
+
batch_size, sequence_length, _ = (
|
| 599 |
+
hidden_states.shape
|
| 600 |
+
if encoder_hidden_states is None
|
| 601 |
+
else encoder_hidden_states.shape
|
| 602 |
+
)
|
| 603 |
+
|
| 604 |
+
if attention_mask is not None:
|
| 605 |
+
attention_mask = attn.prepare_attention_mask(
|
| 606 |
+
attention_mask, sequence_length, batch_size
|
| 607 |
+
)
|
| 608 |
+
# scaled_dot_product_attention expects attention_mask shape to be
|
| 609 |
+
# (batch, heads, source_length, target_length)
|
| 610 |
+
attention_mask = attention_mask.view(
|
| 611 |
+
batch_size, attn.heads, -1, attention_mask.shape[-1]
|
| 612 |
+
)
|
| 613 |
+
|
| 614 |
+
if attn.group_norm is not None:
|
| 615 |
+
hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(
|
| 616 |
+
1, 2
|
| 617 |
+
)
|
| 618 |
+
|
| 619 |
+
query = attn.to_q(hidden_states)
|
| 620 |
+
|
| 621 |
+
if encoder_hidden_states is None:
|
| 622 |
+
encoder_hidden_states = hidden_states
|
| 623 |
+
else:
|
| 624 |
+
end_pos = encoder_hidden_states.shape[1] - self.num_tokens
|
| 625 |
+
encoder_hidden_states = encoder_hidden_states[:, :end_pos] # only use text
|
| 626 |
+
if attn.norm_cross:
|
| 627 |
+
encoder_hidden_states = attn.norm_encoder_hidden_states(
|
| 628 |
+
encoder_hidden_states
|
| 629 |
+
)
|
| 630 |
+
|
| 631 |
+
key = attn.to_k(encoder_hidden_states)
|
| 632 |
+
value = attn.to_v(encoder_hidden_states)
|
| 633 |
+
|
| 634 |
+
inner_dim = key.shape[-1]
|
| 635 |
+
head_dim = inner_dim // attn.heads
|
| 636 |
+
|
| 637 |
+
query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
| 638 |
+
|
| 639 |
+
key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
| 640 |
+
value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
|
| 641 |
+
|
| 642 |
+
# the output of sdp = (batch, num_heads, seq_len, head_dim)
|
| 643 |
+
# TODO: add support for attn.scale when we move to Torch 2.1
|
| 644 |
+
hidden_states = F.scaled_dot_product_attention(
|
| 645 |
+
query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
|
| 646 |
+
)
|
| 647 |
+
|
| 648 |
+
hidden_states = hidden_states.transpose(1, 2).reshape(
|
| 649 |
+
batch_size, -1, attn.heads * head_dim
|
| 650 |
+
)
|
| 651 |
+
hidden_states = hidden_states.to(query.dtype)
|
| 652 |
+
|
| 653 |
+
# linear proj
|
| 654 |
+
hidden_states = attn.to_out[0](hidden_states)
|
| 655 |
+
# dropout
|
| 656 |
+
hidden_states = attn.to_out[1](hidden_states)
|
| 657 |
+
|
| 658 |
+
if input_ndim == 4:
|
| 659 |
+
hidden_states = hidden_states.transpose(-1, -2).reshape(
|
| 660 |
+
batch_size, channel, height, width
|
| 661 |
+
)
|
| 662 |
+
|
| 663 |
+
if attn.residual_connection:
|
| 664 |
+
hidden_states = hidden_states + residual
|
| 665 |
+
|
| 666 |
+
hidden_states = hidden_states / attn.rescale_output_factor
|
| 667 |
+
|
| 668 |
+
return hidden_states
|
| 669 |
+
|
| 670 |
+
|
| 671 |
+
from typing import Callable, Optional, Union
|
| 672 |
+
|
| 673 |
+
import xformers
|
| 674 |
+
import xformers.ops
|
| 675 |
+
|
| 676 |
+
|
| 677 |
+
class XFormersAttnProcessor:
|
| 678 |
+
r"""
|
| 679 |
+
Processor for implementing memory efficient attention using xFormers.
|
| 680 |
+
|
| 681 |
+
Args:
|
| 682 |
+
attention_op (`Callable`, *optional*, defaults to `None`):
|
| 683 |
+
The base
|
| 684 |
+
[operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to
|
| 685 |
+
use as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best
|
| 686 |
+
operator.
|
| 687 |
+
"""
|
| 688 |
+
|
| 689 |
+
def __init__(self, attention_op=None):
|
| 690 |
+
self.attention_op = attention_op
|
| 691 |
+
|
| 692 |
+
def __call__(
|
| 693 |
+
self,
|
| 694 |
+
attn,
|
| 695 |
+
hidden_states: torch.FloatTensor,
|
| 696 |
+
encoder_hidden_states: Optional[torch.FloatTensor] = None,
|
| 697 |
+
attention_mask: Optional[torch.FloatTensor] = None,
|
| 698 |
+
temb: Optional[torch.FloatTensor] = None,
|
| 699 |
+
scale: float = 1.0,
|
| 700 |
+
):
|
| 701 |
+
residual = hidden_states
|
| 702 |
+
|
| 703 |
+
if attn.spatial_norm is not None:
|
| 704 |
+
hidden_states = attn.spatial_norm(hidden_states, temb)
|
| 705 |
+
|
| 706 |
+
input_ndim = hidden_states.ndim
|
| 707 |
+
|
| 708 |
+
if input_ndim == 4:
|
| 709 |
+
batch_size, channel, height, width = hidden_states.shape
|
| 710 |
+
hidden_states = hidden_states.view(
|
| 711 |
+
batch_size, channel, height * width
|
| 712 |
+
).transpose(1, 2)
|
| 713 |
+
|
| 714 |
+
batch_size, key_tokens, _ = (
|
| 715 |
+
hidden_states.shape
|
| 716 |
+
if encoder_hidden_states is None
|
| 717 |
+
else encoder_hidden_states.shape
|
| 718 |
+
)
|
| 719 |
+
|
| 720 |
+
attention_mask = attn.prepare_attention_mask(
|
| 721 |
+
attention_mask, key_tokens, batch_size
|
| 722 |
+
)
|
| 723 |
+
if attention_mask is not None:
|
| 724 |
+
_, query_tokens, _ = hidden_states.shape
|
| 725 |
+
attention_mask = attention_mask.expand(-1, query_tokens, -1)
|
| 726 |
+
|
| 727 |
+
if attn.group_norm is not None:
|
| 728 |
+
hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(
|
| 729 |
+
1, 2
|
| 730 |
+
)
|
| 731 |
+
|
| 732 |
+
query = attn.to_q(hidden_states, scale=scale)
|
| 733 |
+
|
| 734 |
+
if encoder_hidden_states is None:
|
| 735 |
+
encoder_hidden_states = hidden_states
|
| 736 |
+
elif attn.norm_cross:
|
| 737 |
+
encoder_hidden_states = attn.norm_encoder_hidden_states(
|
| 738 |
+
encoder_hidden_states
|
| 739 |
+
)
|
| 740 |
+
|
| 741 |
+
key = attn.to_k(encoder_hidden_states, scale=scale)
|
| 742 |
+
value = attn.to_v(encoder_hidden_states, scale=scale)
|
| 743 |
+
|
| 744 |
+
query = attn.head_to_batch_dim(query).contiguous()
|
| 745 |
+
key = attn.head_to_batch_dim(key).contiguous()
|
| 746 |
+
value = attn.head_to_batch_dim(value).contiguous()
|
| 747 |
+
|
| 748 |
+
hidden_states = xformers.ops.memory_efficient_attention(
|
| 749 |
+
query,
|
| 750 |
+
key,
|
| 751 |
+
value,
|
| 752 |
+
attn_bias=attention_mask,
|
| 753 |
+
op=self.attention_op,
|
| 754 |
+
scale=attn.scale,
|
| 755 |
+
)
|
| 756 |
+
hidden_states = hidden_states.to(query.dtype)
|
| 757 |
+
hidden_states = attn.batch_to_head_dim(hidden_states)
|
| 758 |
+
|
| 759 |
+
# linear proj
|
| 760 |
+
hidden_states = attn.to_out[0](hidden_states, scale=scale)
|
| 761 |
+
# dropout
|
| 762 |
+
hidden_states = attn.to_out[1](hidden_states)
|
| 763 |
+
|
| 764 |
+
if input_ndim == 4:
|
| 765 |
+
hidden_states = hidden_states.transpose(-1, -2).reshape(
|
| 766 |
+
batch_size, channel, height, width
|
| 767 |
+
)
|
| 768 |
+
|
| 769 |
+
if attn.residual_connection:
|
| 770 |
+
hidden_states = hidden_states + residual
|
| 771 |
+
|
| 772 |
+
hidden_states = hidden_states / attn.rescale_output_factor
|
| 773 |
+
|
| 774 |
+
return hidden_states
|
| 775 |
+
|
| 776 |
+
|
| 777 |
+
class AttnProcessor3D(nn.Module):
|
| 778 |
+
r"""
|
| 779 |
+
Default processor for performing attention-related computations.
|
| 780 |
+
"""
|
| 781 |
+
|
| 782 |
+
def __init__(self, attention_op=None, video_length=16):
|
| 783 |
+
super().__init__()
|
| 784 |
+
self.video_length = video_length
|
| 785 |
+
self.attention_op = attention_op
|
| 786 |
+
|
| 787 |
+
def __call__(
|
| 788 |
+
self,
|
| 789 |
+
attn,
|
| 790 |
+
hidden_states,
|
| 791 |
+
encoder_hidden_states=None,
|
| 792 |
+
attention_mask=None,
|
| 793 |
+
temb=None,
|
| 794 |
+
):
|
| 795 |
+
residual = hidden_states
|
| 796 |
+
|
| 797 |
+
if attn.spatial_norm is not None:
|
| 798 |
+
hidden_states = attn.spatial_norm(hidden_states, temb)
|
| 799 |
+
|
| 800 |
+
input_ndim = hidden_states.ndim
|
| 801 |
+
|
| 802 |
+
if input_ndim == 4:
|
| 803 |
+
batch_size, channel, height, width = hidden_states.shape
|
| 804 |
+
hidden_states = hidden_states.view(
|
| 805 |
+
batch_size, channel, height * width
|
| 806 |
+
).transpose(1, 2)
|
| 807 |
+
|
| 808 |
+
batch_size, sequence_length, _ = (
|
| 809 |
+
hidden_states.shape
|
| 810 |
+
if encoder_hidden_states is None
|
| 811 |
+
else encoder_hidden_states.shape
|
| 812 |
+
)
|
| 813 |
+
attention_mask = attn.prepare_attention_mask(
|
| 814 |
+
attention_mask, sequence_length, batch_size
|
| 815 |
+
)
|
| 816 |
+
|
| 817 |
+
if attn.group_norm is not None:
|
| 818 |
+
hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(
|
| 819 |
+
1, 2
|
| 820 |
+
)
|
| 821 |
+
|
| 822 |
+
query = attn.to_q(hidden_states)
|
| 823 |
+
|
| 824 |
+
if encoder_hidden_states is None:
|
| 825 |
+
encoder_hidden_states = hidden_states
|
| 826 |
+
elif attn.norm_cross:
|
| 827 |
+
encoder_hidden_states = attn.norm_encoder_hidden_states(
|
| 828 |
+
encoder_hidden_states
|
| 829 |
+
)
|
| 830 |
+
|
| 831 |
+
key = attn.to_k(encoder_hidden_states)
|
| 832 |
+
value = attn.to_v(encoder_hidden_states)
|
| 833 |
+
|
| 834 |
+
video_length = self.video_length
|
| 835 |
+
query = rearrange(query, "(b f) d c -> b (f d) c", f=video_length)
|
| 836 |
+
key = rearrange(key, "(b f) d c -> b (f d) c", f=video_length)
|
| 837 |
+
value = rearrange(value, "(b f) d c -> b (f d) c", f=video_length)
|
| 838 |
+
|
| 839 |
+
query = attn.head_to_batch_dim(query).contiguous()
|
| 840 |
+
key = attn.head_to_batch_dim(key).contiguous()
|
| 841 |
+
value = attn.head_to_batch_dim(value).contiguous()
|
| 842 |
+
|
| 843 |
+
hidden_states = xformers.ops.memory_efficient_attention(
|
| 844 |
+
query,
|
| 845 |
+
key,
|
| 846 |
+
value,
|
| 847 |
+
attn_bias=attention_mask,
|
| 848 |
+
op=self.attention_op,
|
| 849 |
+
scale=attn.scale,
|
| 850 |
+
)
|
| 851 |
+
hidden_states = hidden_states.to(query.dtype)
|
| 852 |
+
hidden_states = attn.batch_to_head_dim(hidden_states)
|
| 853 |
+
|
| 854 |
+
# linear proj
|
| 855 |
+
hidden_states = attn.to_out[0](hidden_states)
|
| 856 |
+
# dropout
|
| 857 |
+
hidden_states = attn.to_out[1](hidden_states)
|
| 858 |
+
|
| 859 |
+
if input_ndim == 4:
|
| 860 |
+
hidden_states = hidden_states.transpose(-1, -2).reshape(
|
| 861 |
+
batch_size, channel, height, width
|
| 862 |
+
)
|
| 863 |
+
|
| 864 |
+
if attn.residual_connection:
|
| 865 |
+
hidden_states = hidden_states + residual
|
| 866 |
+
|
| 867 |
+
hidden_states = hidden_states / attn.rescale_output_factor
|
| 868 |
+
hidden_states = rearrange(
|
| 869 |
+
hidden_states, "b (f d) c -> (b f) d c", f=video_length
|
| 870 |
+
)
|
| 871 |
+
return hidden_states
|
loomnet/models/blocks/generic_blocks.py
ADDED
|
@@ -0,0 +1,295 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch.nn.functional as F
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
from typing import Union, Callable, Optional
|
| 5 |
+
import torch
|
| 6 |
+
from torch import Tensor
|
| 7 |
+
|
| 8 |
+
class ResnetBlock2D(nn.Module):
|
| 9 |
+
def __init__(self, in_channels, middle_channels, group_channels, eps=1e-5, kernel_size=3, stride =1, padding=1, dropout=0):
|
| 10 |
+
super().__init__()
|
| 11 |
+
|
| 12 |
+
self.norm1 = torch.nn.GroupNorm(num_groups=group_channels, num_channels=in_channels, eps=eps, affine=True)
|
| 13 |
+
self.nonlinearity = nn.SiLU()
|
| 14 |
+
self.conv1 = nn.Conv2d(in_channels, middle_channels, kernel_size=kernel_size, stride=stride, padding=padding)
|
| 15 |
+
self.dropout = torch.nn.Dropout(dropout)
|
| 16 |
+
self.conv2 = nn.Conv2d(middle_channels, in_channels, kernel_size=1, stride=stride, padding=padding)
|
| 17 |
+
|
| 18 |
+
def forward(self, x):
|
| 19 |
+
|
| 20 |
+
y = self.norm1(x)
|
| 21 |
+
y = self.nonlinearity(y)
|
| 22 |
+
y = self.conv1(y)
|
| 23 |
+
y_up = F.interpolate(y, size=(x.shape[-2], x.shape[-1]), mode="bilinear")
|
| 24 |
+
|
| 25 |
+
y2 = x + y_up
|
| 26 |
+
y2 = self.dropout(y2)
|
| 27 |
+
|
| 28 |
+
y3 = self.conv2(y2)
|
| 29 |
+
|
| 30 |
+
y4 = y3 + y2
|
| 31 |
+
|
| 32 |
+
return y4
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class UpSample2D(nn.Module):
|
| 41 |
+
def __init__(self, in_channels):
|
| 42 |
+
super().__init__()
|
| 43 |
+
|
| 44 |
+
self.scale_factor = 2
|
| 45 |
+
self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=(3,3), padding=(1,1))
|
| 46 |
+
|
| 47 |
+
def forward(self, x):
|
| 48 |
+
x = F.interpolate(x, scale_factor=self.scale_factor, mode="nearest")
|
| 49 |
+
x = self.conv(x)
|
| 50 |
+
return x
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class UpSampleDecoder(nn.Module):
|
| 55 |
+
def __init__(self, in_channels, middle_channels, up_factor = 1, group_channels = 32, eps=1e-5, kernel_size=3, dropout=0):
|
| 56 |
+
super().__init__()
|
| 57 |
+
|
| 58 |
+
assert in_channels % group_channels == 0, "in_channels must be divisible by group_channels"
|
| 59 |
+
|
| 60 |
+
self.blocks = nn.ModuleList()
|
| 61 |
+
|
| 62 |
+
for _ in range(up_factor):
|
| 63 |
+
self.blocks.append(nn.Sequential(
|
| 64 |
+
ResnetBlock2D(in_channels, middle_channels, group_channels, eps, kernel_size, dropout),
|
| 65 |
+
UpSample2D(in_channels))
|
| 66 |
+
)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def forward(self, x):
|
| 70 |
+
|
| 71 |
+
for block in self.blocks:
|
| 72 |
+
x = block(x)
|
| 73 |
+
|
| 74 |
+
return x
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
class GammaModulator(nn.Module):
|
| 78 |
+
def __init__(self, dim, init_values, mlp_hidden_dim, dropout=0.0):
|
| 79 |
+
super().__init__()
|
| 80 |
+
|
| 81 |
+
#self.norm1 = nn.LayerNorm(dim)
|
| 82 |
+
self.norm2 = nn.LayerNorm(dim)
|
| 83 |
+
#self.layer_scale1 = LayerScale(dim, init_values=init_values)
|
| 84 |
+
self.layer_scale2 = LayerScale(dim, init_values=init_values)
|
| 85 |
+
|
| 86 |
+
self.mlp = Mlp(
|
| 87 |
+
in_features=dim,
|
| 88 |
+
hidden_features=mlp_hidden_dim,
|
| 89 |
+
act_layer=nn.GELU,
|
| 90 |
+
drop=dropout,
|
| 91 |
+
bias=True,
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
'''
|
| 95 |
+
- original_x and new_x both with shape [b, num_tokens, dim])
|
| 96 |
+
'''
|
| 97 |
+
def forward(self, original_x, new_x):
|
| 98 |
+
|
| 99 |
+
#new_x = new_x + self.layer_scale1(self.norm1(new_x))
|
| 100 |
+
|
| 101 |
+
#modulate original feature map with the global info collected
|
| 102 |
+
original_x = original_x + self.layer_scale2(self.mlp(self.norm2(new_x)))
|
| 103 |
+
|
| 104 |
+
return original_x
|
| 105 |
+
|
| 106 |
+
#loomnet v3
|
| 107 |
+
# def forward(self, original_x, new_x):
|
| 108 |
+
|
| 109 |
+
# x = original_x + self.layer_scale1(self.norm1(new_x))
|
| 110 |
+
|
| 111 |
+
# x = x + self.layer_scale2(self.mlp(self.norm2(x)))
|
| 112 |
+
|
| 113 |
+
# return x
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
class LayerScale(nn.Module):
|
| 117 |
+
def __init__(
|
| 118 |
+
self,
|
| 119 |
+
dim: int,
|
| 120 |
+
init_values: Union[float, Tensor] = 1e-5,
|
| 121 |
+
inplace: bool = False,
|
| 122 |
+
) -> None:
|
| 123 |
+
super().__init__()
|
| 124 |
+
self.inplace = inplace
|
| 125 |
+
self.gamma = nn.Parameter(init_values * torch.ones(dim))
|
| 126 |
+
|
| 127 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 128 |
+
return x.mul_(self.gamma) if self.inplace else x * self.gamma
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
class Mlp(nn.Module):
|
| 134 |
+
def __init__(
|
| 135 |
+
self,
|
| 136 |
+
in_features: int,
|
| 137 |
+
hidden_features: Optional[int] = None,
|
| 138 |
+
out_features: Optional[int] = None,
|
| 139 |
+
act_layer: Callable[..., nn.Module] = nn.GELU,
|
| 140 |
+
drop: float = 0.0,
|
| 141 |
+
bias: bool = True,
|
| 142 |
+
) -> None:
|
| 143 |
+
super().__init__()
|
| 144 |
+
out_features = out_features or in_features
|
| 145 |
+
hidden_features = hidden_features or in_features
|
| 146 |
+
self.fc1 = nn.Linear(in_features, hidden_features, bias=bias)
|
| 147 |
+
self.act = act_layer()
|
| 148 |
+
self.fc2 = nn.Linear(hidden_features, out_features, bias=bias)
|
| 149 |
+
self.drop = nn.Dropout(drop)
|
| 150 |
+
|
| 151 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 152 |
+
x = self.fc1(x)
|
| 153 |
+
x = self.act(x)
|
| 154 |
+
x = self.drop(x)
|
| 155 |
+
x = self.fc2(x)
|
| 156 |
+
x = self.drop(x)
|
| 157 |
+
return x
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
class ConvBlock(nn.Module):
|
| 168 |
+
|
| 169 |
+
def __init__(self, dim, expansion=2, kernel_size=7):
|
| 170 |
+
super().__init__()
|
| 171 |
+
self.dwconv = nn.Conv2d(dim, dim, kernel_size, padding=kernel_size // 2, groups=dim) # depthwise conv
|
| 172 |
+
self.pwconv1 = nn.Conv2d(dim, dim * expansion, kernel_size=1)
|
| 173 |
+
self.act = nn.GELU()
|
| 174 |
+
self.pwconv2 = nn.Conv2d(dim * expansion, dim, kernel_size=1)
|
| 175 |
+
self.norm = nn.BatchNorm2d(dim)
|
| 176 |
+
|
| 177 |
+
def forward(self, x):
|
| 178 |
+
shortcut = x
|
| 179 |
+
x = self.dwconv(x)
|
| 180 |
+
x = self.norm(x)
|
| 181 |
+
x = self.pwconv1(x)
|
| 182 |
+
x = self.act(x)
|
| 183 |
+
x = self.pwconv2(x)
|
| 184 |
+
return x + shortcut # Residual connection
|
| 185 |
+
|
| 186 |
+
class FeatureEnhancer(nn.Module):
|
| 187 |
+
def __init__(self, grid_channels, feature_channels, mlp_hidden_dim, grid_encoder_hidden_dim, depth=2, gamma_init_values=1e-5):
|
| 188 |
+
super().__init__()
|
| 189 |
+
|
| 190 |
+
self.initial_proj = nn.Conv2d(grid_channels, grid_encoder_hidden_dim, kernel_size=1)
|
| 191 |
+
|
| 192 |
+
self.encoder = nn.Sequential(*[ConvBlock(grid_encoder_hidden_dim) for _ in range(depth)])
|
| 193 |
+
|
| 194 |
+
self.project_to_feat = nn.Conv2d(grid_encoder_hidden_dim, feature_channels, kernel_size=1)
|
| 195 |
+
|
| 196 |
+
# Gating mechanism (learnable modulation)
|
| 197 |
+
self.gamma_modulator = GammaModulator(dim=feature_channels, mlp_hidden_dim=mlp_hidden_dim, init_values=gamma_init_values)
|
| 198 |
+
|
| 199 |
+
def forward(self, original_feature_map, grid_features ):
|
| 200 |
+
|
| 201 |
+
B1, C1, H1, W1 = original_feature_map.shape
|
| 202 |
+
|
| 203 |
+
grid_features = F.interpolate(grid_features, size=(H1, W1), mode="nearest")
|
| 204 |
+
|
| 205 |
+
grid_features = self.initial_proj(grid_features) # [b, hidden_dim, N, N]
|
| 206 |
+
grid_features = self.encoder(grid_features) # deeper encoding
|
| 207 |
+
grid_features = self.project_to_feat(grid_features) # [b, C, N, N]
|
| 208 |
+
|
| 209 |
+
B2, C2, H2, W2 = grid_features.shape
|
| 210 |
+
|
| 211 |
+
original_feature_map_tokens = to_tokens(original_feature_map, B1, C1, H1, W1)
|
| 212 |
+
grid_features_tokens = to_tokens(grid_features, B2, C2, H2, W2)
|
| 213 |
+
|
| 214 |
+
new_feature_map = from_tokens(self.gamma_modulator(original_feature_map_tokens, grid_features_tokens), H1, W1)
|
| 215 |
+
|
| 216 |
+
return new_feature_map
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def to_tokens(x, B, C, H, W):
|
| 221 |
+
# x: [B, C, D, D]
|
| 222 |
+
x = x.view(B, C, H * W).contiguous() # [B, C, D*D]
|
| 223 |
+
x = x.permute(0, 2, 1).contiguous() # [B, D*D, C]
|
| 224 |
+
return x
|
| 225 |
+
|
| 226 |
+
def from_tokens(tokens, H, W):
|
| 227 |
+
# tokens: [B, D*D, C]
|
| 228 |
+
B, N, C = tokens.shape
|
| 229 |
+
x = tokens.permute(0, 2, 1).contiguous() # [B, C, D*D]
|
| 230 |
+
x = x.view(B, C, H, W).contiguous() # [B, C, D, D]
|
| 231 |
+
return x
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
class HarmonicEmbedding(torch.nn.Module):
|
| 237 |
+
def __init__(
|
| 238 |
+
self,
|
| 239 |
+
n_harmonic_functions: int = 6,
|
| 240 |
+
omega_0: float = 1.0,
|
| 241 |
+
logspace: bool = True,
|
| 242 |
+
append_input: bool = True,
|
| 243 |
+
) -> None:
|
| 244 |
+
super().__init__()
|
| 245 |
+
|
| 246 |
+
if logspace:
|
| 247 |
+
frequencies = 2.0 ** torch.arange(
|
| 248 |
+
n_harmonic_functions,
|
| 249 |
+
dtype=torch.float32,
|
| 250 |
+
)
|
| 251 |
+
else:
|
| 252 |
+
frequencies = torch.linspace(
|
| 253 |
+
1.0,
|
| 254 |
+
2.0 ** (n_harmonic_functions - 1),
|
| 255 |
+
n_harmonic_functions,
|
| 256 |
+
dtype=torch.float32,
|
| 257 |
+
)
|
| 258 |
+
|
| 259 |
+
self.register_buffer("_frequencies", frequencies * omega_0, persistent=False)
|
| 260 |
+
self.append_input = append_input
|
| 261 |
+
|
| 262 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 263 |
+
"""
|
| 264 |
+
Args:
|
| 265 |
+
x: tensor of shape [..., dim]
|
| 266 |
+
Returns:
|
| 267 |
+
embedding: a harmonic embedding of `x`
|
| 268 |
+
of shape [..., (n_harmonic_functions * 2 + int(append_input)) * dim]
|
| 269 |
+
"""
|
| 270 |
+
embed = (x[..., None] * self._frequencies).view(*x.shape[:-1], -1)
|
| 271 |
+
embed = torch.cat(
|
| 272 |
+
(embed.sin(), embed.cos(), x)
|
| 273 |
+
if self.append_input
|
| 274 |
+
else (embed.sin(), embed.cos()),
|
| 275 |
+
dim=-1,
|
| 276 |
+
)
|
| 277 |
+
return embed
|
| 278 |
+
|
| 279 |
+
@staticmethod
|
| 280 |
+
def get_output_dim_static(
|
| 281 |
+
input_dims: int,
|
| 282 |
+
n_harmonic_functions: int,
|
| 283 |
+
append_input: bool,
|
| 284 |
+
) -> int:
|
| 285 |
+
"""
|
| 286 |
+
Utility to help predict the shape of the output of `forward`.
|
| 287 |
+
Args:
|
| 288 |
+
input_dims: length of the last dimension of the input tensor
|
| 289 |
+
n_harmonic_functions: number of embedding frequencies
|
| 290 |
+
append_input: whether or not to concat the original
|
| 291 |
+
input to the harmonic embedding
|
| 292 |
+
Returns:
|
| 293 |
+
int: the length of the last dimension of the output tensor
|
| 294 |
+
"""
|
| 295 |
+
return input_dims * (2 * n_harmonic_functions + int(append_input))
|
loomnet/models/blocks/pointwise_spatial_transformer.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
#__________________________________ positional encoding _______________________________
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def make_sincos_pos_embed(embed_dim: int, pos: torch.Tensor, omega_0: float = 100) -> torch.Tensor:
|
| 8 |
+
"""
|
| 9 |
+
This function generates a 1D positional embedding from a given grid using sine and cosine functions.
|
| 10 |
+
|
| 11 |
+
Args:
|
| 12 |
+
- embed_dim: The embedding dimension.
|
| 13 |
+
- pos: The position to generate the embedding from.
|
| 14 |
+
|
| 15 |
+
Returns:
|
| 16 |
+
- emb: The generated 1D positional embedding.
|
| 17 |
+
"""
|
| 18 |
+
assert embed_dim % 2 == 0
|
| 19 |
+
omega = torch.arange(embed_dim // 2, dtype=torch.double, device=pos.device)
|
| 20 |
+
omega /= embed_dim / 2.0
|
| 21 |
+
omega = 1.0 / omega_0**omega # (D/2,)
|
| 22 |
+
|
| 23 |
+
pos = pos.reshape(-1) # (M,)
|
| 24 |
+
out = torch.einsum("m,d->md", pos, omega) # (M, D/2), outer product
|
| 25 |
+
|
| 26 |
+
emb_sin = torch.sin(out) # (M, D/2)
|
| 27 |
+
emb_cos = torch.cos(out) # (M, D/2)
|
| 28 |
+
|
| 29 |
+
emb = torch.cat([emb_sin, emb_cos], dim=1) # (M, D)
|
| 30 |
+
return emb.float()
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def position_grid_to_embed(pos_grid: torch.Tensor, embed_dim: int, omega_0: float = 100) -> torch.Tensor:
|
| 37 |
+
"""
|
| 38 |
+
Convert 2D position grid (HxWx2) to sinusoidal embeddings (HxWxC)
|
| 39 |
+
|
| 40 |
+
Args:
|
| 41 |
+
pos_grid: Tensor of shape (H*W, 2) containing 2D coordinates
|
| 42 |
+
embed_dim: Output channel dimension for embeddings
|
| 43 |
+
|
| 44 |
+
Returns:
|
| 45 |
+
Tensor of shape (H, W, embed_dim) with positional embeddings
|
| 46 |
+
"""
|
| 47 |
+
|
| 48 |
+
num_views, num_points, _ = pos_grid.shape
|
| 49 |
+
|
| 50 |
+
# Process x and y coordinates separately
|
| 51 |
+
emb_x = make_sincos_pos_embed(embed_dim // 2, pos_grid[..., 0], omega_0=omega_0) # [1, H*W, D/2]
|
| 52 |
+
emb_y = make_sincos_pos_embed(embed_dim // 2, pos_grid[..., 1], omega_0=omega_0) # [1, H*W, D/2]
|
| 53 |
+
|
| 54 |
+
# Combine and reshape
|
| 55 |
+
emb = torch.cat([emb_x, emb_y], dim=-1) # [1, H*W, D]
|
| 56 |
+
|
| 57 |
+
return emb.view(num_views, num_points, embed_dim) # [H, W, D]
|
| 58 |
+
|
| 59 |
+
def apply_pos_embed(pos_grid, feature_map_dimension: int=4, ratio: float = 0.1) -> torch.Tensor:
|
| 60 |
+
"""
|
| 61 |
+
Apply positional embedding to tensor x.
|
| 62 |
+
"""
|
| 63 |
+
#(num_grid_positions,2)
|
| 64 |
+
pos_embed = position_grid_to_embed(pos_grid, feature_map_dimension)
|
| 65 |
+
pos_embed = pos_embed * ratio
|
| 66 |
+
return pos_embed
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
#___________________________________ Point-wise Adaptive Spatial Transformer _____________________
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
# class PointWiseSpatialTransformer(nn.Module):
|
| 76 |
+
# def __init__(self, feature_map_spatial_dim, feature_map_channel_dim):
|
| 77 |
+
# super(PointWiseSpatialTransformer, self).__init__()
|
| 78 |
+
|
| 79 |
+
# # Spatial transformer localization-network
|
| 80 |
+
# self.localization = nn.Sequential(
|
| 81 |
+
# nn.Conv2d(feature_map_channel_dim, feature_map_channel_dim, kernel_size=1),
|
| 82 |
+
# nn.MaxPool2d(2, stride=2) if feature_map_spatial_dim in [8,16,32] else nn.Identity(),
|
| 83 |
+
# nn.ReLU(True),
|
| 84 |
+
# nn.Conv2d(feature_map_channel_dim, feature_map_channel_dim//2, kernel_size=1),
|
| 85 |
+
# nn.MaxPool2d(2, stride=2) if feature_map_spatial_dim in [16,32] else nn.Identity(),
|
| 86 |
+
# nn.ReLU(True),
|
| 87 |
+
# nn.Conv2d(feature_map_channel_dim//2, 16, kernel_size=1),
|
| 88 |
+
# nn.MaxPool2d(2, stride=2) if feature_map_spatial_dim in [32] else nn.Identity(),
|
| 89 |
+
# nn.ReLU(True)
|
| 90 |
+
# )
|
| 91 |
+
|
| 92 |
+
# #normalization layer before concatenatig pos embed in [-1, 1]
|
| 93 |
+
# self.loc_norm = torch.nn.LayerNorm(16 * 4 * 4)
|
| 94 |
+
|
| 95 |
+
# # Regressor for the 3 * 2 affine matrix
|
| 96 |
+
# self.fc_loc = nn.Sequential(
|
| 97 |
+
# nn.Linear(16 * 4 * 4 + feature_map_spatial_dim, 32),
|
| 98 |
+
# nn.ReLU(True),
|
| 99 |
+
# nn.Linear(32, 3 * 2)
|
| 100 |
+
# )
|
| 101 |
+
|
| 102 |
+
# # Initialize the weights/bias with identity transformation
|
| 103 |
+
# self.fc_loc[2].weight.data.zero_()
|
| 104 |
+
# self.fc_loc[2].bias.data.copy_(torch.tensor([1, 0, 0, 0, 1, 0], dtype=torch.float))
|
| 105 |
+
|
| 106 |
+
# # Spatial transformer network forward function
|
| 107 |
+
# def stn(self, x, pos_emb):
|
| 108 |
+
|
| 109 |
+
# b_and_num_views, num_points, _ = pos_emb.shape
|
| 110 |
+
|
| 111 |
+
# xs = self.localization(x)
|
| 112 |
+
# xs = xs.view(-1, 16 * 4 * 4)
|
| 113 |
+
# xs = self.loc_norm(xs)
|
| 114 |
+
# #replico la localization map per ogni punto
|
| 115 |
+
# xs = xs.unsqueeze(1).repeat(1, pos_emb.shape[1], 1)
|
| 116 |
+
# #aggiungo le posizioni all'embedding della mappa
|
| 117 |
+
# xs = torch.cat([xs, pos_emb], dim=-1)
|
| 118 |
+
|
| 119 |
+
# #predico affine transformation for each point
|
| 120 |
+
# theta = self.fc_loc(xs)
|
| 121 |
+
# theta = theta.view(b_and_num_views, num_points, 2, 3)
|
| 122 |
+
|
| 123 |
+
# return theta
|
| 124 |
+
|
| 125 |
+
# def forward(self, feature_map, pos_emb):
|
| 126 |
+
# # transform projected points
|
| 127 |
+
# theta = self.stn(feature_map, pos_emb)
|
| 128 |
+
|
| 129 |
+
# return theta
|
| 130 |
+
|
| 131 |
+
class PointWiseSpatialTransformer(nn.Module):
|
| 132 |
+
def __init__(self, feature_map_spatial_dim, feature_map_channel_dim):
|
| 133 |
+
super(PointWiseSpatialTransformer, self).__init__()
|
| 134 |
+
|
| 135 |
+
# Spatial transformer localization-network
|
| 136 |
+
self.localization = nn.Sequential(
|
| 137 |
+
nn.Conv2d(feature_map_channel_dim, feature_map_channel_dim, kernel_size=1),
|
| 138 |
+
nn.MaxPool2d(2, stride=2) if feature_map_spatial_dim in [8,16,32] else nn.Identity(),
|
| 139 |
+
nn.ReLU(True),
|
| 140 |
+
nn.Conv2d(feature_map_channel_dim, feature_map_channel_dim//2, kernel_size=1),
|
| 141 |
+
nn.MaxPool2d(2, stride=2) if feature_map_spatial_dim in [16,32] else nn.Identity(),
|
| 142 |
+
nn.ReLU(True),
|
| 143 |
+
nn.Conv2d(feature_map_channel_dim//2, 16, kernel_size=1),
|
| 144 |
+
nn.MaxPool2d(2, stride=2) if feature_map_spatial_dim in [32] else nn.Identity(),
|
| 145 |
+
nn.ReLU(True)
|
| 146 |
+
)
|
| 147 |
+
|
| 148 |
+
# Regressor for the 3 * 2 affine matrix
|
| 149 |
+
self.fc_loc = nn.Sequential(
|
| 150 |
+
nn.Linear(16 * 4 * 4, 32),
|
| 151 |
+
nn.ReLU(True),
|
| 152 |
+
nn.Linear(32, 3 * 2)
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
# Initialize the weights/bias with identity transformation
|
| 156 |
+
self.fc_loc[2].weight.data.zero_()
|
| 157 |
+
self.fc_loc[2].bias.data.copy_(torch.tensor([1, 0, 0, 0, 1, 0], dtype=torch.float))
|
| 158 |
+
|
| 159 |
+
# Spatial transformer network forward function
|
| 160 |
+
def stn(self, x, size_grid):
|
| 161 |
+
|
| 162 |
+
b_and_num_views, c, _, _ = x.size()
|
| 163 |
+
|
| 164 |
+
xs = self.localization(x)
|
| 165 |
+
xs = xs.view(-1, 16 * 4 * 4)
|
| 166 |
+
theta = self.fc_loc(xs)
|
| 167 |
+
theta = theta.view(-1, 2, 3)
|
| 168 |
+
|
| 169 |
+
grid = F.affine_grid(theta, [b_and_num_views, c, size_grid, size_grid])
|
| 170 |
+
ambassadors_features = F.grid_sample(x, grid)
|
| 171 |
+
|
| 172 |
+
return ambassadors_features, grid
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def forward(self, feature_map, size_grid):
|
| 176 |
+
# transform projected points
|
| 177 |
+
ambassadors_features, grid = self.stn(feature_map, size_grid)
|
| 178 |
+
|
| 179 |
+
return ambassadors_features, grid
|
loomnet/models/blocks/resnet.py
ADDED
|
@@ -0,0 +1,357 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Borrowed from https://github.com/Tangshitao/MVDiffusion/blob/main/src/models/modules/resnet.py
|
| 2 |
+
from abc import abstractmethod
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
import torch.nn as nn
|
| 6 |
+
import torch.nn.functional as F
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class BasicResNetBlock(nn.Module):
|
| 10 |
+
def __init__(self, in_channels, out_channels, stride=1, zero_init=False):
|
| 11 |
+
super(BasicResNetBlock, self).__init__()
|
| 12 |
+
self.conv1 = nn.Conv2d(
|
| 13 |
+
in_channels,
|
| 14 |
+
out_channels,
|
| 15 |
+
kernel_size=3,
|
| 16 |
+
stride=stride,
|
| 17 |
+
padding=1,
|
| 18 |
+
bias=False,
|
| 19 |
+
)
|
| 20 |
+
self.bn1 = nn.GroupNorm(32, out_channels)
|
| 21 |
+
self.relu = nn.ReLU(inplace=True)
|
| 22 |
+
self.conv2 = nn.Conv2d(
|
| 23 |
+
out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False
|
| 24 |
+
)
|
| 25 |
+
self.bn2 = nn.GroupNorm(32, out_channels)
|
| 26 |
+
|
| 27 |
+
self.shortcut = nn.Sequential()
|
| 28 |
+
if stride != 1 or in_channels != out_channels:
|
| 29 |
+
self.shortcut = nn.Sequential(
|
| 30 |
+
nn.Conv2d(
|
| 31 |
+
in_channels, out_channels, kernel_size=1, stride=stride, bias=False
|
| 32 |
+
),
|
| 33 |
+
nn.GroupNorm(32, out_channels),
|
| 34 |
+
)
|
| 35 |
+
if zero_init:
|
| 36 |
+
self.conv2.weight.data.zero_()
|
| 37 |
+
if self.conv2.bias is not None:
|
| 38 |
+
self.conv2.bias.data.zero_()
|
| 39 |
+
|
| 40 |
+
def forward(self, x):
|
| 41 |
+
identity = x
|
| 42 |
+
|
| 43 |
+
out = self.bn1(x)
|
| 44 |
+
out = self.relu(out)
|
| 45 |
+
out = self.conv1(out)
|
| 46 |
+
|
| 47 |
+
out = self.bn2(out)
|
| 48 |
+
out = self.relu(out)
|
| 49 |
+
out = self.conv2(out)
|
| 50 |
+
identity = self.shortcut(identity)
|
| 51 |
+
|
| 52 |
+
out += identity
|
| 53 |
+
|
| 54 |
+
return out
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
class SiLU(nn.Module):
|
| 58 |
+
def forward(self, x):
|
| 59 |
+
return x * torch.sigmoid(x)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class GroupNorm32(nn.GroupNorm):
|
| 63 |
+
def forward(self, x):
|
| 64 |
+
return super().forward(x.float()).type(x.dtype)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def conv_nd(dims, *args, **kwargs):
|
| 68 |
+
"""
|
| 69 |
+
Create a 1D, 2D, or 3D convolution module.
|
| 70 |
+
"""
|
| 71 |
+
if dims == 1:
|
| 72 |
+
return nn.Conv1d(*args, **kwargs)
|
| 73 |
+
elif dims == 2:
|
| 74 |
+
return nn.Conv2d(*args, **kwargs)
|
| 75 |
+
elif dims == 3:
|
| 76 |
+
return nn.Conv3d(*args, **kwargs)
|
| 77 |
+
raise ValueError(f"unsupported dimensions: {dims}")
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def linear(*args, **kwargs):
|
| 81 |
+
"""
|
| 82 |
+
Create a linear module.
|
| 83 |
+
"""
|
| 84 |
+
return nn.Linear(*args, **kwargs)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
class Upsample(nn.Module):
|
| 88 |
+
"""
|
| 89 |
+
An upsampling layer with an optional convolution.
|
| 90 |
+
:param channels: channels in the inputs and outputs.
|
| 91 |
+
:param use_conv: a bool determining if a convolution is applied.
|
| 92 |
+
:param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then
|
| 93 |
+
upsampling occurs in the inner-two dimensions.
|
| 94 |
+
"""
|
| 95 |
+
|
| 96 |
+
def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1):
|
| 97 |
+
super().__init__()
|
| 98 |
+
self.channels = channels
|
| 99 |
+
self.out_channels = out_channels or channels
|
| 100 |
+
self.use_conv = use_conv
|
| 101 |
+
self.dims = dims
|
| 102 |
+
if use_conv:
|
| 103 |
+
self.conv = conv_nd(
|
| 104 |
+
dims, self.channels, self.out_channels, 3, padding=padding
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
def forward(self, x):
|
| 108 |
+
assert x.shape[1] == self.channels
|
| 109 |
+
if self.dims == 3:
|
| 110 |
+
x = F.interpolate(
|
| 111 |
+
x, (x.shape[2], x.shape[3] * 2, x.shape[4] * 2), mode="nearest"
|
| 112 |
+
)
|
| 113 |
+
else:
|
| 114 |
+
x = F.interpolate(x, scale_factor=2, mode="nearest")
|
| 115 |
+
if self.use_conv:
|
| 116 |
+
x = self.conv(x)
|
| 117 |
+
return x
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def normalization(channels):
|
| 121 |
+
"""
|
| 122 |
+
Make a standard normalization layer.
|
| 123 |
+
:param channels: number of input channels.
|
| 124 |
+
:return: an nn.Module for normalization.
|
| 125 |
+
"""
|
| 126 |
+
return GroupNorm32(32, channels)
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def zero_module(module):
|
| 130 |
+
"""
|
| 131 |
+
Zero out the parameters of a module and return it.
|
| 132 |
+
"""
|
| 133 |
+
for p in module.parameters():
|
| 134 |
+
p.detach().zero_()
|
| 135 |
+
return module
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def avg_pool_nd(dims, *args, **kwargs):
|
| 139 |
+
"""
|
| 140 |
+
Create a 1D, 2D, or 3D average pooling module.
|
| 141 |
+
"""
|
| 142 |
+
if dims == 1:
|
| 143 |
+
return nn.AvgPool1d(*args, **kwargs)
|
| 144 |
+
elif dims == 2:
|
| 145 |
+
return nn.AvgPool2d(*args, **kwargs)
|
| 146 |
+
elif dims == 3:
|
| 147 |
+
return nn.AvgPool3d(*args, **kwargs)
|
| 148 |
+
raise ValueError(f"unsupported dimensions: {dims}")
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
class Downsample(nn.Module):
|
| 152 |
+
"""
|
| 153 |
+
A downsampling layer with an optional convolution.
|
| 154 |
+
:param channels: channels in the inputs and outputs.
|
| 155 |
+
:param use_conv: a bool determining if a convolution is applied.
|
| 156 |
+
:param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then
|
| 157 |
+
downsampling occurs in the inner-two dimensions.
|
| 158 |
+
"""
|
| 159 |
+
|
| 160 |
+
def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1):
|
| 161 |
+
super().__init__()
|
| 162 |
+
self.channels = channels
|
| 163 |
+
self.out_channels = out_channels or channels
|
| 164 |
+
self.use_conv = use_conv
|
| 165 |
+
self.dims = dims
|
| 166 |
+
stride = 2 if dims != 3 else (1, 2, 2)
|
| 167 |
+
if use_conv:
|
| 168 |
+
self.op = conv_nd(
|
| 169 |
+
dims,
|
| 170 |
+
self.channels,
|
| 171 |
+
self.out_channels,
|
| 172 |
+
3,
|
| 173 |
+
stride=stride,
|
| 174 |
+
padding=padding,
|
| 175 |
+
)
|
| 176 |
+
else:
|
| 177 |
+
assert self.channels == self.out_channels
|
| 178 |
+
self.op = avg_pool_nd(dims, kernel_size=stride, stride=stride)
|
| 179 |
+
|
| 180 |
+
def forward(self, x):
|
| 181 |
+
assert x.shape[1] == self.channels
|
| 182 |
+
return self.op(x)
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
class TimestepBlock(nn.Module):
|
| 186 |
+
"""
|
| 187 |
+
Any module where forward() takes timestep embeddings as a second argument.
|
| 188 |
+
"""
|
| 189 |
+
|
| 190 |
+
@abstractmethod
|
| 191 |
+
def forward(self, x, emb):
|
| 192 |
+
"""
|
| 193 |
+
Apply the module to `x` given `emb` timestep embeddings.
|
| 194 |
+
"""
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def checkpoint(func, inputs, params, flag):
|
| 198 |
+
"""
|
| 199 |
+
Evaluate a function without caching intermediate activations, allowing for
|
| 200 |
+
reduced memory at the expense of extra compute in the backward pass.
|
| 201 |
+
:param func: the function to evaluate.
|
| 202 |
+
:param inputs: the argument sequence to pass to `func`.
|
| 203 |
+
:param params: a sequence of parameters `func` depends on but does not
|
| 204 |
+
explicitly take as arguments.
|
| 205 |
+
:param flag: if False, disable gradient checkpointing.
|
| 206 |
+
"""
|
| 207 |
+
if flag:
|
| 208 |
+
args = tuple(inputs) + tuple(params)
|
| 209 |
+
return CheckpointFunction.apply(func, len(inputs), *args)
|
| 210 |
+
else:
|
| 211 |
+
return func(*inputs)
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
class CheckpointFunction(torch.autograd.Function):
|
| 215 |
+
@staticmethod
|
| 216 |
+
def forward(ctx, run_function, length, *args):
|
| 217 |
+
ctx.run_function = run_function
|
| 218 |
+
ctx.input_tensors = list(args[:length])
|
| 219 |
+
ctx.input_params = list(args[length:])
|
| 220 |
+
|
| 221 |
+
with torch.no_grad():
|
| 222 |
+
output_tensors = ctx.run_function(*ctx.input_tensors)
|
| 223 |
+
return output_tensors
|
| 224 |
+
|
| 225 |
+
@staticmethod
|
| 226 |
+
def backward(ctx, *output_grads):
|
| 227 |
+
ctx.input_tensors = [x.detach().requires_grad_(True) for x in ctx.input_tensors]
|
| 228 |
+
with torch.enable_grad():
|
| 229 |
+
# Fixes a bug where the first op in run_function modifies the
|
| 230 |
+
# Tensor storage in place, which is not allowed for detach()'d
|
| 231 |
+
# Tensors.
|
| 232 |
+
shallow_copies = [x.view_as(x) for x in ctx.input_tensors]
|
| 233 |
+
output_tensors = ctx.run_function(*shallow_copies)
|
| 234 |
+
input_grads = torch.autograd.grad(
|
| 235 |
+
output_tensors,
|
| 236 |
+
ctx.input_tensors + ctx.input_params,
|
| 237 |
+
output_grads,
|
| 238 |
+
allow_unused=True,
|
| 239 |
+
)
|
| 240 |
+
del ctx.input_tensors
|
| 241 |
+
del ctx.input_params
|
| 242 |
+
del output_tensors
|
| 243 |
+
return (None, None) + input_grads
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
class ResBlock(TimestepBlock):
|
| 247 |
+
"""
|
| 248 |
+
A residual block that can optionally change the number of channels.
|
| 249 |
+
:param channels: the number of input channels.
|
| 250 |
+
:param emb_channels: the number of timestep embedding channels.
|
| 251 |
+
:param dropout: the rate of dropout.
|
| 252 |
+
:param out_channels: if specified, the number of out channels.
|
| 253 |
+
:param use_conv: if True and out_channels is specified, use a spatial
|
| 254 |
+
convolution instead of a smaller 1x1 convolution to change the
|
| 255 |
+
channels in the skip connection.
|
| 256 |
+
:param dims: determines if the signal is 1D, 2D, or 3D.
|
| 257 |
+
:param use_checkpoint: if True, use gradient checkpointing on this module.
|
| 258 |
+
:param up: if True, use this block for upsampling.
|
| 259 |
+
:param down: if True, use this block for downsampling.
|
| 260 |
+
"""
|
| 261 |
+
|
| 262 |
+
def __init__(
|
| 263 |
+
self,
|
| 264 |
+
channels,
|
| 265 |
+
emb_channels,
|
| 266 |
+
dropout=0.0,
|
| 267 |
+
out_channels=None,
|
| 268 |
+
use_conv=False,
|
| 269 |
+
use_scale_shift_norm=False,
|
| 270 |
+
dims=2,
|
| 271 |
+
use_checkpoint=False,
|
| 272 |
+
up=False,
|
| 273 |
+
down=False,
|
| 274 |
+
):
|
| 275 |
+
super().__init__()
|
| 276 |
+
self.channels = channels
|
| 277 |
+
self.emb_channels = emb_channels
|
| 278 |
+
self.dropout = dropout
|
| 279 |
+
self.out_channels = out_channels or channels
|
| 280 |
+
self.use_conv = use_conv
|
| 281 |
+
self.use_checkpoint = use_checkpoint
|
| 282 |
+
self.use_scale_shift_norm = use_scale_shift_norm
|
| 283 |
+
|
| 284 |
+
self.in_layers = nn.Sequential(
|
| 285 |
+
normalization(channels),
|
| 286 |
+
nn.SiLU(),
|
| 287 |
+
conv_nd(dims, channels, self.out_channels, 3, padding=1),
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
self.updown = up or down
|
| 291 |
+
|
| 292 |
+
if up:
|
| 293 |
+
self.h_upd = Upsample(channels, False, dims)
|
| 294 |
+
self.x_upd = Upsample(channels, False, dims)
|
| 295 |
+
elif down:
|
| 296 |
+
self.h_upd = Downsample(channels, False, dims)
|
| 297 |
+
self.x_upd = Downsample(channels, False, dims)
|
| 298 |
+
else:
|
| 299 |
+
self.h_upd = self.x_upd = nn.Identity()
|
| 300 |
+
|
| 301 |
+
self.emb_layers = nn.Sequential(
|
| 302 |
+
nn.SiLU(),
|
| 303 |
+
linear(
|
| 304 |
+
emb_channels,
|
| 305 |
+
2 * self.out_channels if use_scale_shift_norm else self.out_channels,
|
| 306 |
+
),
|
| 307 |
+
)
|
| 308 |
+
self.out_layers = nn.Sequential(
|
| 309 |
+
normalization(self.out_channels),
|
| 310 |
+
nn.SiLU(),
|
| 311 |
+
nn.Dropout(p=dropout),
|
| 312 |
+
zero_module(
|
| 313 |
+
conv_nd(dims, self.out_channels, self.out_channels, 3, padding=1)
|
| 314 |
+
),
|
| 315 |
+
)
|
| 316 |
+
|
| 317 |
+
if self.out_channels == channels:
|
| 318 |
+
self.skip_connection = nn.Identity()
|
| 319 |
+
elif use_conv:
|
| 320 |
+
self.skip_connection = conv_nd(
|
| 321 |
+
dims, channels, self.out_channels, 3, padding=1
|
| 322 |
+
)
|
| 323 |
+
else:
|
| 324 |
+
self.skip_connection = conv_nd(dims, channels, self.out_channels, 1)
|
| 325 |
+
|
| 326 |
+
def forward(self, x, emb):
|
| 327 |
+
"""
|
| 328 |
+
Apply the block to a Tensor, conditioned on a timestep embedding.
|
| 329 |
+
:param x: an [N x C x ...] Tensor of features.
|
| 330 |
+
:param emb: an [N x emb_channels] Tensor of timestep embeddings.
|
| 331 |
+
:return: an [N x C x ...] Tensor of outputs.
|
| 332 |
+
"""
|
| 333 |
+
return checkpoint(
|
| 334 |
+
self._forward, (x, emb), self.parameters(), self.use_checkpoint
|
| 335 |
+
)
|
| 336 |
+
|
| 337 |
+
def _forward(self, x, emb):
|
| 338 |
+
if self.updown:
|
| 339 |
+
in_rest, in_conv = self.in_layers[:-1], self.in_layers[-1]
|
| 340 |
+
h = in_rest(x)
|
| 341 |
+
h = self.h_upd(h)
|
| 342 |
+
x = self.x_upd(x)
|
| 343 |
+
h = in_conv(h)
|
| 344 |
+
else:
|
| 345 |
+
h = self.in_layers(x)
|
| 346 |
+
emb_out = self.emb_layers(emb).type(h.dtype)
|
| 347 |
+
while len(emb_out.shape) < len(h.shape):
|
| 348 |
+
emb_out = emb_out[..., None]
|
| 349 |
+
if self.use_scale_shift_norm:
|
| 350 |
+
out_norm, out_rest = self.out_layers[0], self.out_layers[1:]
|
| 351 |
+
scale, shift = torch.chunk(emb_out, 2, dim=1)
|
| 352 |
+
h = out_norm(h) * (1 + scale) + shift
|
| 353 |
+
h = out_rest(h)
|
| 354 |
+
else:
|
| 355 |
+
h = h + emb_out
|
| 356 |
+
h = self.out_layers(h)
|
| 357 |
+
return self.skip_connection(x) + h
|
loomnet/models/blocks/resnet_fpn.py
ADDED
|
@@ -0,0 +1,372 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class BottleNeck(nn.Module):
|
| 7 |
+
# Scale factor of the number of output channels
|
| 8 |
+
expansion = 4
|
| 9 |
+
|
| 10 |
+
def __init__(self, in_channels, out_channels,
|
| 11 |
+
stride=1, is_first_block=False):
|
| 12 |
+
"""
|
| 13 |
+
Args:
|
| 14 |
+
in_channels: number of input channels
|
| 15 |
+
out_channels: number of output channels
|
| 16 |
+
stride: stride using in (a) 3x3 convolution and
|
| 17 |
+
(b) 1x1 convolution used for downsampling for skip connection
|
| 18 |
+
is_first_block: whether it is the first residual block of the layer
|
| 19 |
+
"""
|
| 20 |
+
super().__init__()
|
| 21 |
+
self.conv1 = nn.Conv2d(in_channels=in_channels,
|
| 22 |
+
out_channels=out_channels,
|
| 23 |
+
kernel_size=1, stride=1, padding=0)
|
| 24 |
+
self.bn1 = nn.BatchNorm2d(out_channels)
|
| 25 |
+
|
| 26 |
+
self.conv2 = nn.Conv2d(in_channels=out_channels,
|
| 27 |
+
out_channels=out_channels,
|
| 28 |
+
kernel_size=3, stride=stride, padding=1)
|
| 29 |
+
self.bn2 = nn.BatchNorm2d(out_channels)
|
| 30 |
+
|
| 31 |
+
self.conv3 = nn.Conv2d(in_channels=out_channels,
|
| 32 |
+
out_channels=out_channels*self.expansion,
|
| 33 |
+
kernel_size=1, stride=1, padding=0)
|
| 34 |
+
self.bn3 = nn.BatchNorm2d(out_channels*self.expansion)
|
| 35 |
+
|
| 36 |
+
self.relu = nn.ReLU()
|
| 37 |
+
|
| 38 |
+
# Skip connection goes through 1x1 convolution with stride=2 for
|
| 39 |
+
# the first blocks of conv3_x, conv4_x, and conv5_x layers for matching
|
| 40 |
+
# spatial dimension of feature maps and number of channels in order to
|
| 41 |
+
# perform the add operations.
|
| 42 |
+
self.downsample = None
|
| 43 |
+
if is_first_block:
|
| 44 |
+
self.downsample = nn.Sequential(nn.Conv2d(in_channels=in_channels,
|
| 45 |
+
out_channels=out_channels*self.expansion,
|
| 46 |
+
kernel_size=1,
|
| 47 |
+
stride=stride,
|
| 48 |
+
padding=0),
|
| 49 |
+
nn.BatchNorm2d(out_channels*self.expansion))
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def forward(self, x):
|
| 53 |
+
"""
|
| 54 |
+
Args:
|
| 55 |
+
x: input
|
| 56 |
+
Returns:
|
| 57 |
+
Residual block output
|
| 58 |
+
"""
|
| 59 |
+
identity = x.clone()
|
| 60 |
+
x = self.relu(self.bn1(self.conv1(x)))
|
| 61 |
+
x = self.relu(self.bn2(self.conv2(x)))
|
| 62 |
+
|
| 63 |
+
x = self.conv3(x)
|
| 64 |
+
x = self.bn3(x)
|
| 65 |
+
|
| 66 |
+
if self.downsample:
|
| 67 |
+
identity = self.downsample(identity)
|
| 68 |
+
|
| 69 |
+
x += identity
|
| 70 |
+
x = self.relu(x)
|
| 71 |
+
|
| 72 |
+
return x
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class BasicBlock(nn.Module):
|
| 76 |
+
# Scale factor of the number of output channels
|
| 77 |
+
expansion = 1
|
| 78 |
+
|
| 79 |
+
def __init__(self, in_channels, out_channels,
|
| 80 |
+
stride=1, is_first_block=False):
|
| 81 |
+
"""
|
| 82 |
+
Args:
|
| 83 |
+
in_channels: number of input channels
|
| 84 |
+
out_channels: number of output channels
|
| 85 |
+
stride: stride using in (a) the first 3x3 convolution and
|
| 86 |
+
(b) 1x1 convolution used for downsampling for skip connection
|
| 87 |
+
is_first_block: whether it is the first residual block of the layer
|
| 88 |
+
"""
|
| 89 |
+
super().__init__()
|
| 90 |
+
self.conv1 = nn.Conv2d(in_channels=in_channels,
|
| 91 |
+
out_channels=out_channels,
|
| 92 |
+
kernel_size=3,
|
| 93 |
+
stride=stride,
|
| 94 |
+
padding=1)
|
| 95 |
+
self.bn1 = nn.BatchNorm2d(out_channels)
|
| 96 |
+
self.conv2 = nn.Conv2d(in_channels=out_channels,
|
| 97 |
+
out_channels=out_channels,
|
| 98 |
+
kernel_size=3,
|
| 99 |
+
stride=1,
|
| 100 |
+
padding=1)
|
| 101 |
+
self.bn2 = nn.BatchNorm2d(out_channels)
|
| 102 |
+
|
| 103 |
+
self.relu = nn.ReLU()
|
| 104 |
+
|
| 105 |
+
# Skip connection goes through 1x1 convolution with stride=2 for
|
| 106 |
+
# the first blocks of conv3_x, conv4_x, and conv5_x layers for matching
|
| 107 |
+
# spatial dimension of feature maps and number of channels in order to
|
| 108 |
+
# perform the add operations.
|
| 109 |
+
self.downsample = None
|
| 110 |
+
if is_first_block and stride != 1:
|
| 111 |
+
self.downsample = nn.Sequential(nn.Conv2d(in_channels=in_channels,
|
| 112 |
+
out_channels=out_channels,
|
| 113 |
+
kernel_size=1,
|
| 114 |
+
stride=stride,
|
| 115 |
+
padding=0),
|
| 116 |
+
nn.BatchNorm2d(out_channels))
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def forward(self, x):
|
| 120 |
+
"""
|
| 121 |
+
Args:
|
| 122 |
+
x: input
|
| 123 |
+
Returns:
|
| 124 |
+
Residual block ouput
|
| 125 |
+
"""
|
| 126 |
+
identity = x.clone()
|
| 127 |
+
x = self.relu(self.bn1(self.conv1(x)))
|
| 128 |
+
x = self.bn2(self.conv2(x))
|
| 129 |
+
|
| 130 |
+
if self.downsample:
|
| 131 |
+
identity = self.downsample(identity)
|
| 132 |
+
x += identity
|
| 133 |
+
x = self.relu(x)
|
| 134 |
+
|
| 135 |
+
return x
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
class ResNet(nn.Module):
|
| 139 |
+
def __init__(self, ResBlock, n_blocks_list=[3, 4, 6, 3],
|
| 140 |
+
out_channels_list=[64, 128, 256, 512], num_channels=3, final_channel_dim=1280):
|
| 141 |
+
"""
|
| 142 |
+
Args:
|
| 143 |
+
ResBlock: residual block type, BasicBlock for ResNet-18, 34 or
|
| 144 |
+
BottleNeck for ResNet-50, 101, 152
|
| 145 |
+
n_block_lists: number of residual blocks for each conv layer (conv2_x - conv5_x)
|
| 146 |
+
out_channels_list: list of the output channel numbers for conv2_x - conv5_x
|
| 147 |
+
num_channels: the number of channels of input image
|
| 148 |
+
"""
|
| 149 |
+
super().__init__()
|
| 150 |
+
|
| 151 |
+
# Create four convoluiontal layers
|
| 152 |
+
in_channels = final_channel_dim // 5
|
| 153 |
+
|
| 154 |
+
# First layer
|
| 155 |
+
self.conv1 = nn.Sequential(nn.Conv2d(in_channels=num_channels,
|
| 156 |
+
out_channels=in_channels, kernel_size=7,
|
| 157 |
+
stride=2, padding=3),
|
| 158 |
+
nn.BatchNorm2d(in_channels),
|
| 159 |
+
nn.ReLU(),
|
| 160 |
+
nn.MaxPool2d(kernel_size=3,
|
| 161 |
+
stride=2, padding=1))
|
| 162 |
+
|
| 163 |
+
# For the first block of the second layer, do not downsample and use stride=1.
|
| 164 |
+
self.conv2_x = self.CreateLayer(ResBlock, n_blocks_list[0],
|
| 165 |
+
in_channels, out_channels_list[0], stride=1)
|
| 166 |
+
|
| 167 |
+
# For the first blocks of conv3_x - conv5_x layers, perform downsampling using stride=2.
|
| 168 |
+
# By default, ResBlock.expansion = 4 for ResNet-50, 101, 152,
|
| 169 |
+
# ResBlock.expansion = 1 for ResNet-18, 34.
|
| 170 |
+
self.conv3_x = self.CreateLayer(ResBlock, n_blocks_list[1],
|
| 171 |
+
out_channels_list[0]*ResBlock.expansion,
|
| 172 |
+
out_channels_list[1], stride=2)
|
| 173 |
+
self.conv4_x = self.CreateLayer(ResBlock, n_blocks_list[2],
|
| 174 |
+
out_channels_list[1]*ResBlock.expansion,
|
| 175 |
+
out_channels_list[2], stride=2)
|
| 176 |
+
self.conv5_x = self.CreateLayer(ResBlock, n_blocks_list[3],
|
| 177 |
+
out_channels_list[2]*ResBlock.expansion,
|
| 178 |
+
out_channels_list[3], stride=2)
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
def forward(self, x):
|
| 182 |
+
"""
|
| 183 |
+
Args:
|
| 184 |
+
x: input image
|
| 185 |
+
Returns:
|
| 186 |
+
C2: feature maps after conv2_x
|
| 187 |
+
C3: feature maps after conv3_x
|
| 188 |
+
C4: feature maps after conv4_x
|
| 189 |
+
C5: feature maps after conv5_x
|
| 190 |
+
y: output class
|
| 191 |
+
"""
|
| 192 |
+
x = self.conv1(x)
|
| 193 |
+
|
| 194 |
+
# Feature maps
|
| 195 |
+
C2 = self.conv2_x(x)
|
| 196 |
+
C3 = self.conv3_x(C2)
|
| 197 |
+
C4 = self.conv4_x(C3)
|
| 198 |
+
C5 = self.conv5_x(C4)
|
| 199 |
+
|
| 200 |
+
return C2, C3, C4, C5
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def CreateLayer(self, ResBlock, n_blocks, in_channels, out_channels, stride=1):
|
| 204 |
+
"""
|
| 205 |
+
Create a layer with specified type and number of residual blocks.
|
| 206 |
+
Args:
|
| 207 |
+
ResBlock: residual block type, BasicBlock for ResNet-18, 34 or
|
| 208 |
+
BottleNeck for ResNet-50, 101, 152
|
| 209 |
+
n_blocks: number of residual blocks
|
| 210 |
+
in_channels: number of input channels
|
| 211 |
+
out_channels: number of output channels
|
| 212 |
+
stride: stride used in the first 3x3 convolution of the first resdiual block
|
| 213 |
+
of the layer and 1x1 convolution for skip connection in that block
|
| 214 |
+
Returns:
|
| 215 |
+
Convolutional layer
|
| 216 |
+
"""
|
| 217 |
+
layer = []
|
| 218 |
+
for i in range(n_blocks):
|
| 219 |
+
if i == 0:
|
| 220 |
+
# Downsample the feature map using input stride for the first block of the layer.
|
| 221 |
+
layer.append(ResBlock(in_channels, out_channels,
|
| 222 |
+
stride=stride, is_first_block=True))
|
| 223 |
+
else:
|
| 224 |
+
# Keep the feature map size same for the rest three blocks of the layer.
|
| 225 |
+
# by setting stride=1 and is_first_block=False.
|
| 226 |
+
# By default, ResBlock.expansion = 4 for ResNet-50, 101, 152,
|
| 227 |
+
# ResBlock.expansion = 1 for ResNet-18, 34.
|
| 228 |
+
layer.append(ResBlock(out_channels*ResBlock.expansion, out_channels))
|
| 229 |
+
|
| 230 |
+
return nn.Sequential(*layer)
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
class FPNBlock(nn.Module):
|
| 234 |
+
def __init__(self, in_channels, out_channels=256, is_highest_block=False):
|
| 235 |
+
"""
|
| 236 |
+
Args:
|
| 237 |
+
in_channels: the number of input channels
|
| 238 |
+
out_channels: the number of output channels
|
| 239 |
+
is_highest_block: whether the block is at the highest level of pyramid
|
| 240 |
+
"""
|
| 241 |
+
super().__init__()
|
| 242 |
+
# 1x1 convolution to unify the number of feature map channels to
|
| 243 |
+
# a specified value for fusion (matching)
|
| 244 |
+
self.conv1 = nn.Conv2d(in_channels, out_channels,
|
| 245 |
+
kernel_size=1, stride=1, padding=0)
|
| 246 |
+
# apply 3x3 convolution for feature output
|
| 247 |
+
self.conv2 = nn.Conv2d(out_channels, out_channels,
|
| 248 |
+
kernel_size=3, stride=1, padding=1)
|
| 249 |
+
self.is_highest_block = is_highest_block
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
def forward(self, x, y):
|
| 253 |
+
"""
|
| 254 |
+
Args:
|
| 255 |
+
x: input tensor at the current level
|
| 256 |
+
y: input tesnor at the previous level (upper level)
|
| 257 |
+
Returns:
|
| 258 |
+
x: output tensor to the next level (lower level)
|
| 259 |
+
out: output tensor at the curret level (after 1x1 and 3x3 convolutions)
|
| 260 |
+
"""
|
| 261 |
+
x = self.conv1(x)
|
| 262 |
+
if not self.is_highest_block:
|
| 263 |
+
x += F.interpolate(y, scale_factor=2, mode="bilinear",
|
| 264 |
+
align_corners=True)
|
| 265 |
+
out = self.conv2(x)
|
| 266 |
+
return x, out
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
class FPN(nn.Module):
|
| 270 |
+
def __init__(self, expansion=2, in_channels_list=[64, 128, 256, 512],
|
| 271 |
+
out_channels=256):
|
| 272 |
+
"""
|
| 273 |
+
Args:
|
| 274 |
+
expansion: expansion rate of ResBlock (1 for BasicBlock or 4 for BottleNeck)
|
| 275 |
+
in_channels_list: list of the output channel numbers for conv2_x - conv5_x
|
| 276 |
+
out_channels: target number of channels (256 by default)
|
| 277 |
+
"""
|
| 278 |
+
super().__init__()
|
| 279 |
+
|
| 280 |
+
# Create layers to generate P2-P6
|
| 281 |
+
self.P2 = FPNBlock(in_channels_list[0]*expansion, out_channels=out_channels)
|
| 282 |
+
self.P3 = FPNBlock(in_channels_list[1]*expansion, out_channels=out_channels)
|
| 283 |
+
self.P4 = FPNBlock(in_channels_list[2]*expansion, out_channels=out_channels)
|
| 284 |
+
self.P5 = FPNBlock(in_channels_list[3]*expansion, out_channels=out_channels,
|
| 285 |
+
is_highest_block=True)
|
| 286 |
+
self.P6 = nn.MaxPool2d(kernel_size=1, stride=2, padding=0)
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
def forward(self, C2, C3, C4, C5):
|
| 290 |
+
"""
|
| 291 |
+
Args:
|
| 292 |
+
C2-C5: feature maps output by ResNet
|
| 293 |
+
Returns:
|
| 294 |
+
P2-P6: enchanced features by FPN
|
| 295 |
+
"""
|
| 296 |
+
x, P5 = self.P5(C5, None)
|
| 297 |
+
x, P4 = self.P4(C4, x)
|
| 298 |
+
x, P3 = self.P3(C3, x)
|
| 299 |
+
_, P2 = self.P2(C2, x)
|
| 300 |
+
P6 = self.P6(P5)
|
| 301 |
+
|
| 302 |
+
return P2, P3, P4, P5, P6
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
class ResnetFPN(nn.Module):
|
| 306 |
+
def __init__(self, resnet_arch='50', in_channels=3, final_channel_dim = 1280):
|
| 307 |
+
"""
|
| 308 |
+
Args:
|
| 309 |
+
resnet_arch: the type of ResNet architecture
|
| 310 |
+
in_channels: number of channels of the input feature map
|
| 311 |
+
"""
|
| 312 |
+
super().__init__()
|
| 313 |
+
|
| 314 |
+
assert(resnet_arch in ['18', '34', '50', '101', '152'])
|
| 315 |
+
assert final_channel_dim % 5 ==0, "final channel dim must be divisible by 5"
|
| 316 |
+
|
| 317 |
+
ResBlock = BasicBlock if resnet_arch == '18' or resnet_arch == '34' else BottleNeck
|
| 318 |
+
|
| 319 |
+
# Number of residual blocks for each conv layer (conv2_x - conv5_x)
|
| 320 |
+
if resnet_arch == '18':
|
| 321 |
+
n_blocks_list = [2, 2, 2, 2]
|
| 322 |
+
elif resnet_arch == '34' or resnet_arch == '50':
|
| 323 |
+
n_blocks_list = [3, 4, 6, 3]
|
| 324 |
+
elif resnet_arch == '101':
|
| 325 |
+
n_blocks_list = [3, 4, 23, 3]
|
| 326 |
+
else:
|
| 327 |
+
n_blocks_list = [3, 8, 36, 3]
|
| 328 |
+
|
| 329 |
+
# Create ResNet
|
| 330 |
+
self.resnet = ResNet(ResBlock, n_blocks_list=n_blocks_list, num_channels=in_channels, final_channel_dim=final_channel_dim)
|
| 331 |
+
|
| 332 |
+
# Create FPN
|
| 333 |
+
self.FPN = FPN(expansion=ResBlock.expansion)
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
def forward(self, x):
|
| 337 |
+
"""
|
| 338 |
+
Args:
|
| 339 |
+
x: input tensor
|
| 340 |
+
Returns:
|
| 341 |
+
P2-P6: enhanced features
|
| 342 |
+
"""
|
| 343 |
+
C2, C3, C4, C5 = self.resnet(x)
|
| 344 |
+
P2, P3, P4, P5, P6 = self.FPN(C2, C3, C4, C5)
|
| 345 |
+
return P2, P3, P4, P5, P6
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
if __name__ == "__main__":
|
| 349 |
+
### Resnet arch types
|
| 350 |
+
#resnet_arch = '18'
|
| 351 |
+
#resnet_arch = '34'
|
| 352 |
+
resnet_arch = '50'
|
| 353 |
+
#resnet_arch = '101'
|
| 354 |
+
#resnet_arch = '152'
|
| 355 |
+
|
| 356 |
+
### Create ResNet+FPN
|
| 357 |
+
net = ResnetFPN(resnet_arch, in_channels=3)
|
| 358 |
+
|
| 359 |
+
### Input
|
| 360 |
+
x = torch.randn((1, 3, 512, 800), dtype=torch.float32)
|
| 361 |
+
|
| 362 |
+
### Inference
|
| 363 |
+
P2, P3, P4, P5, P6 = net(x)
|
| 364 |
+
|
| 365 |
+
### Output
|
| 366 |
+
print(f"P2 shape is {P2.shape}")
|
| 367 |
+
print(f"P3 shape is {P3.shape}")
|
| 368 |
+
print(f"P4 shape is {P4.shape}")
|
| 369 |
+
print(f"P5 shape is {P5.shape}")
|
| 370 |
+
print(f"P6 shape is {P6.shape}")
|
| 371 |
+
|
| 372 |
+
print("Done!")
|
loomnet/models/blocks/transformer.py
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Borrowed from https://github.com/Tangshitao/MVDiffusion/blob/main/src/models/modules/transformer.py
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
from einops import rearrange
|
| 5 |
+
from torch import einsum, nn
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class GEGLU(nn.Module):
|
| 9 |
+
def __init__(self, dim_in, dim_out):
|
| 10 |
+
super().__init__()
|
| 11 |
+
self.proj = nn.Linear(dim_in, dim_out * 2)
|
| 12 |
+
|
| 13 |
+
def forward(self, x):
|
| 14 |
+
x, gate = self.proj(x).chunk(2, dim=-1)
|
| 15 |
+
return x * F.gelu(gate)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class FeedForward(nn.Module):
|
| 19 |
+
def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0.0):
|
| 20 |
+
super().__init__()
|
| 21 |
+
inner_dim = int(dim * mult)
|
| 22 |
+
dim_out = dim
|
| 23 |
+
project_in = (
|
| 24 |
+
nn.Sequential(nn.Linear(dim, inner_dim), nn.GELU())
|
| 25 |
+
if not glu
|
| 26 |
+
else GEGLU(dim, inner_dim)
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
linear = nn.Linear(inner_dim, dim_out)
|
| 30 |
+
linear.weight.data.fill_(0)
|
| 31 |
+
linear.bias.data.fill_(0)
|
| 32 |
+
self.net = nn.Sequential(project_in, nn.Dropout(dropout), linear)
|
| 33 |
+
|
| 34 |
+
def forward(self, x):
|
| 35 |
+
return self.net(x)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class CrossAttention(nn.Module):
|
| 39 |
+
def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.0):
|
| 40 |
+
super().__init__()
|
| 41 |
+
inner_dim = dim_head * heads
|
| 42 |
+
context_dim = context_dim
|
| 43 |
+
|
| 44 |
+
self.scale = dim_head**-0.5
|
| 45 |
+
self.heads = heads
|
| 46 |
+
|
| 47 |
+
self.to_q = nn.Linear(query_dim, inner_dim, bias=False)
|
| 48 |
+
self.to_k = nn.Linear(context_dim, inner_dim, bias=False)
|
| 49 |
+
self.to_v = nn.Linear(context_dim, inner_dim, bias=False)
|
| 50 |
+
|
| 51 |
+
self.to_out = nn.Linear(inner_dim, query_dim)
|
| 52 |
+
self.to_out.weight.data.fill_(0)
|
| 53 |
+
self.to_out.bias.data.fill_(0)
|
| 54 |
+
|
| 55 |
+
def forward(self, x, context=None, mask=None):
|
| 56 |
+
h = self.heads
|
| 57 |
+
|
| 58 |
+
q = self.to_q(x)
|
| 59 |
+
|
| 60 |
+
k = self.to_k(context)
|
| 61 |
+
v = self.to_v(context)
|
| 62 |
+
|
| 63 |
+
q, k, v = map(lambda t: rearrange(t, "b n (h d) -> (b h) n d", h=h), (q, k, v))
|
| 64 |
+
|
| 65 |
+
sim = einsum("b i d, b j d -> b i j", q, k) * self.scale
|
| 66 |
+
|
| 67 |
+
del q, k
|
| 68 |
+
|
| 69 |
+
sim = sim.softmax(dim=-1)
|
| 70 |
+
|
| 71 |
+
out = einsum("b i j, b j d -> b i d", sim, v)
|
| 72 |
+
out = rearrange(out, "(b h) n d -> b n (h d)", h=h)
|
| 73 |
+
return self.to_out(out)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def checkpoint(func, inputs, params, flag):
|
| 77 |
+
"""
|
| 78 |
+
Evaluate a function without caching intermediate activations, allowing for
|
| 79 |
+
reduced memory at the expense of extra compute in the backward pass.
|
| 80 |
+
:param func: the function to evaluate.
|
| 81 |
+
:param inputs: the argument sequence to pass to `func`.
|
| 82 |
+
:param params: a sequence of parameters `func` depends on but does not
|
| 83 |
+
explicitly take as arguments.
|
| 84 |
+
:param flag: if False, disable gradient checkpointing.
|
| 85 |
+
"""
|
| 86 |
+
if flag:
|
| 87 |
+
args = tuple(inputs) + tuple(params)
|
| 88 |
+
return CheckpointFunction.apply(func, len(inputs), *args)
|
| 89 |
+
else:
|
| 90 |
+
return func(*inputs)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
class CheckpointFunction(torch.autograd.Function):
|
| 94 |
+
@staticmethod
|
| 95 |
+
def forward(ctx, run_function, length, *args):
|
| 96 |
+
ctx.run_function = run_function
|
| 97 |
+
ctx.input_tensors = list(args[:length])
|
| 98 |
+
ctx.input_params = list(args[length:])
|
| 99 |
+
ctx.gpu_autocast_kwargs = {
|
| 100 |
+
"enabled": torch.is_autocast_enabled(),
|
| 101 |
+
"dtype": torch.get_autocast_gpu_dtype(),
|
| 102 |
+
"cache_enabled": torch.is_autocast_cache_enabled(),
|
| 103 |
+
}
|
| 104 |
+
with torch.no_grad():
|
| 105 |
+
output_tensors = ctx.run_function(*ctx.input_tensors)
|
| 106 |
+
return output_tensors
|
| 107 |
+
|
| 108 |
+
@staticmethod
|
| 109 |
+
def backward(ctx, *output_grads):
|
| 110 |
+
ctx.input_tensors = [x.detach().requires_grad_(True) for x in ctx.input_tensors]
|
| 111 |
+
with torch.enable_grad(), torch.cuda.amp.autocast(**ctx.gpu_autocast_kwargs):
|
| 112 |
+
# Fixes a bug where the first op in run_function modifies the
|
| 113 |
+
# Tensor storage in place, which is not allowed for detach()'d
|
| 114 |
+
# Tensors.
|
| 115 |
+
shallow_copies = [x.view_as(x) for x in ctx.input_tensors]
|
| 116 |
+
output_tensors = ctx.run_function(*shallow_copies)
|
| 117 |
+
input_grads = torch.autograd.grad(
|
| 118 |
+
output_tensors,
|
| 119 |
+
ctx.input_tensors + ctx.input_params,
|
| 120 |
+
output_grads,
|
| 121 |
+
allow_unused=True,
|
| 122 |
+
)
|
| 123 |
+
del ctx.input_tensors
|
| 124 |
+
del ctx.input_params
|
| 125 |
+
del output_tensors
|
| 126 |
+
return (None, None) + input_grads
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
class BasicTransformerBlock(nn.Module):
|
| 130 |
+
def __init__(
|
| 131 |
+
self,
|
| 132 |
+
dim,
|
| 133 |
+
n_heads,
|
| 134 |
+
d_head,
|
| 135 |
+
dropout=0.0,
|
| 136 |
+
context_dim=None,
|
| 137 |
+
gated_ff=True,
|
| 138 |
+
disable_self_attn=False,
|
| 139 |
+
use_checkpoint=True,
|
| 140 |
+
):
|
| 141 |
+
super().__init__()
|
| 142 |
+
|
| 143 |
+
self.attn1 = CrossAttention(
|
| 144 |
+
query_dim=dim,
|
| 145 |
+
heads=n_heads,
|
| 146 |
+
dim_head=d_head,
|
| 147 |
+
dropout=dropout,
|
| 148 |
+
context_dim=context_dim,
|
| 149 |
+
) # is a self-attention if not self.disable_self_attn
|
| 150 |
+
self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff)
|
| 151 |
+
self.norm1 = nn.LayerNorm(dim)
|
| 152 |
+
self.norm2 = nn.LayerNorm(dim)
|
| 153 |
+
|
| 154 |
+
self.checkpoint = checkpoint
|
| 155 |
+
self.use_checkpoint = use_checkpoint
|
| 156 |
+
|
| 157 |
+
def forward(self, x, context=None, query_pe=None):
|
| 158 |
+
if self.use_checkpoint:
|
| 159 |
+
return checkpoint(
|
| 160 |
+
self._forward,
|
| 161 |
+
(x, context, query_pe),
|
| 162 |
+
self.parameters(),
|
| 163 |
+
self.checkpoint,
|
| 164 |
+
)
|
| 165 |
+
else:
|
| 166 |
+
return self._forward(x, context, query_pe)
|
| 167 |
+
|
| 168 |
+
def _forward(self, x, context=None, query_pe=None):
|
| 169 |
+
if context is None:
|
| 170 |
+
context = x
|
| 171 |
+
query = x
|
| 172 |
+
if query_pe is not None:
|
| 173 |
+
query = query + query_pe
|
| 174 |
+
query = self.norm1(query)
|
| 175 |
+
context = self.norm1(context)
|
| 176 |
+
x = self.attn1(query, context=context) + x
|
| 177 |
+
x = self.ff(self.norm2(x)) + x
|
| 178 |
+
|
| 179 |
+
return x
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
class PosEmbedding(nn.Module):
|
| 183 |
+
def __init__(self, in_channels, N_freqs, logscale=True):
|
| 184 |
+
"""
|
| 185 |
+
Defines a function that embeds x to (x, sin(2^k x), cos(2^k x), ...)
|
| 186 |
+
in_channels: number of input channels (3 for both xyz and direction)
|
| 187 |
+
"""
|
| 188 |
+
super(PosEmbedding, self).__init__()
|
| 189 |
+
self.N_freqs = N_freqs
|
| 190 |
+
self.in_channels = in_channels
|
| 191 |
+
# self.funcs = [torch.sin, torch.cos]
|
| 192 |
+
# self.out_channels = in_channels*(len(self.funcs)*N_freqs)
|
| 193 |
+
if N_freqs <= 80:
|
| 194 |
+
base = 2
|
| 195 |
+
else:
|
| 196 |
+
base = 5000 ** (1 / (N_freqs / 2.5))
|
| 197 |
+
if logscale:
|
| 198 |
+
freq_bands = base ** torch.linspace(0, N_freqs - 1, N_freqs)[None, None]
|
| 199 |
+
else:
|
| 200 |
+
freq_bands = torch.linspace(1, 2 ** (N_freqs - 1), N_freqs)
|
| 201 |
+
self.register_buffer("freq_bands", freq_bands)
|
| 202 |
+
|
| 203 |
+
def forward(self, x):
|
| 204 |
+
"""
|
| 205 |
+
Embeds x to (x, sin(2^k x), cos(2^k x), ...)
|
| 206 |
+
Different from the paper, "x" is also in the output
|
| 207 |
+
See https://github.com/bmild/nerf/issues/12
|
| 208 |
+
Inputs:
|
| 209 |
+
x: (B, self.in_channels)
|
| 210 |
+
Outputs:
|
| 211 |
+
out: (B, self.out_channels)
|
| 212 |
+
"""
|
| 213 |
+
shape = x.shape[:-1]
|
| 214 |
+
x = x.reshape(-1, 2, 1)
|
| 215 |
+
|
| 216 |
+
encodings = x * self.freq_bands
|
| 217 |
+
sin_encodings = torch.sin(encodings) # (n, c, num_encoding_functions)
|
| 218 |
+
cos_encodings = torch.cos(encodings)
|
| 219 |
+
pe = torch.cat([sin_encodings, cos_encodings], dim=1)
|
| 220 |
+
pe = pe.reshape(*shape, -1)
|
| 221 |
+
return pe
|
loomnet/models/blocks/transformer_t.py
ADDED
|
@@ -0,0 +1,268 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
from einops import rearrange, repeat
|
| 5 |
+
|
| 6 |
+
from loomnet.utils.ops import default, exists, zero_module
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class CrossAttention(nn.Module):
|
| 10 |
+
def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.0):
|
| 11 |
+
super().__init__()
|
| 12 |
+
inner_dim = dim_head * heads
|
| 13 |
+
|
| 14 |
+
if context_dim is None:
|
| 15 |
+
context_dim = query_dim
|
| 16 |
+
|
| 17 |
+
self.scale = dim_head**-0.5
|
| 18 |
+
self.heads = heads
|
| 19 |
+
|
| 20 |
+
self.to_q = nn.Linear(query_dim, inner_dim, bias=False)
|
| 21 |
+
self.to_k = nn.Linear(context_dim, inner_dim, bias=False)
|
| 22 |
+
self.to_v = nn.Linear(context_dim, inner_dim, bias=False)
|
| 23 |
+
|
| 24 |
+
self.to_out = nn.Sequential(
|
| 25 |
+
nn.Linear(inner_dim, query_dim), nn.Dropout(dropout)
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
def forward(self, x, context=None, mask=None):
|
| 29 |
+
h = self.heads
|
| 30 |
+
|
| 31 |
+
q = self.to_q(x)
|
| 32 |
+
context = default(context, x)
|
| 33 |
+
k = self.to_k(context)
|
| 34 |
+
v = self.to_v(context)
|
| 35 |
+
q, k, v = map(lambda t: rearrange(t, "b n (h d) -> (b h) n d", h=h), (q, k, v))
|
| 36 |
+
|
| 37 |
+
sim = torch.einsum("b i d, b j d -> b i j", q, k) * self.scale
|
| 38 |
+
|
| 39 |
+
if exists(mask):
|
| 40 |
+
mask = rearrange(mask, "b ... -> b (...)")
|
| 41 |
+
max_neg_value = -torch.finfo(sim.dtype).max
|
| 42 |
+
mask = repeat(mask, "b j -> (b h) () j", h=h)
|
| 43 |
+
sim.masked_fill_(~mask, max_neg_value)
|
| 44 |
+
|
| 45 |
+
# attention, what we cannot get enough of
|
| 46 |
+
attn = sim.softmax(dim=-1)
|
| 47 |
+
|
| 48 |
+
out = torch.einsum("b i j, b j d -> b i d", attn, v)
|
| 49 |
+
out = rearrange(out, "(b h) n d -> b n (h d)", h=h)
|
| 50 |
+
return self.to_out(out)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class LayerScale(nn.Module):
|
| 54 |
+
def __init__(self, dim, init_values=1e-5, inplace=False):
|
| 55 |
+
super().__init__()
|
| 56 |
+
self.inplace = inplace
|
| 57 |
+
self.gamma = nn.Parameter(init_values * torch.ones(dim))
|
| 58 |
+
|
| 59 |
+
def forward(self, x):
|
| 60 |
+
return x.mul_(self.gamma) if self.inplace else x * self.gamma
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class GEGLU(nn.Module):
|
| 64 |
+
def __init__(self, dim_in, dim_out):
|
| 65 |
+
super().__init__()
|
| 66 |
+
self.proj = nn.Linear(dim_in, dim_out * 2)
|
| 67 |
+
|
| 68 |
+
def forward(self, x):
|
| 69 |
+
x, gate = self.proj(x).chunk(2, dim=-1)
|
| 70 |
+
return x * F.gelu(gate)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class FeedForward(nn.Module):
|
| 74 |
+
def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0.0):
|
| 75 |
+
super().__init__()
|
| 76 |
+
inner_dim = int(dim * mult)
|
| 77 |
+
if dim_out is None:
|
| 78 |
+
dim_out = dim
|
| 79 |
+
|
| 80 |
+
project_in = (
|
| 81 |
+
nn.Sequential(nn.Linear(dim, inner_dim), nn.GELU())
|
| 82 |
+
if not glu
|
| 83 |
+
else GEGLU(dim, inner_dim)
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
self.net = nn.Sequential(
|
| 87 |
+
project_in, nn.Dropout(dropout), nn.Linear(inner_dim, dim_out)
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
def forward(self, x):
|
| 91 |
+
return self.net(x)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
class AdaLayerNorm(nn.Module):
|
| 95 |
+
def __init__(self, n_embd):
|
| 96 |
+
super().__init__()
|
| 97 |
+
|
| 98 |
+
self.silu = nn.SiLU()
|
| 99 |
+
self.linear = nn.Linear(n_embd, n_embd * 2)
|
| 100 |
+
self.layernorm = nn.LayerNorm(n_embd, elementwise_affine=False)
|
| 101 |
+
|
| 102 |
+
def forward(self, x, timestep):
|
| 103 |
+
emb = self.linear(timestep)
|
| 104 |
+
scale, shift = torch.chunk(emb, 2, dim=-1)
|
| 105 |
+
x = self.layernorm(x) * (1 + scale) + shift
|
| 106 |
+
return x
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
class BasicTransformerBlock(nn.Module):
|
| 110 |
+
def __init__(
|
| 111 |
+
self,
|
| 112 |
+
dim,
|
| 113 |
+
n_heads,
|
| 114 |
+
d_head,
|
| 115 |
+
dropout=0.0,
|
| 116 |
+
context_dim=None,
|
| 117 |
+
gated_ff=True,
|
| 118 |
+
checkpoint=True,
|
| 119 |
+
):
|
| 120 |
+
super().__init__()
|
| 121 |
+
self.attn1 = CrossAttention(
|
| 122 |
+
query_dim=dim, heads=n_heads, dim_head=d_head, dropout=dropout
|
| 123 |
+
) # is a self-attention
|
| 124 |
+
self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff)
|
| 125 |
+
self.attn2 = CrossAttention(
|
| 126 |
+
query_dim=dim,
|
| 127 |
+
context_dim=context_dim,
|
| 128 |
+
heads=n_heads,
|
| 129 |
+
dim_head=d_head,
|
| 130 |
+
dropout=dropout,
|
| 131 |
+
) # is self-attn if context is none
|
| 132 |
+
self.norm1 = AdaLayerNorm(dim)
|
| 133 |
+
self.norm2 = AdaLayerNorm(dim)
|
| 134 |
+
self.norm3 = AdaLayerNorm(dim)
|
| 135 |
+
|
| 136 |
+
def forward(self, x, t, context=None, mask=None):
|
| 137 |
+
x = self.attn1(self.norm1(x, t)) + x
|
| 138 |
+
x = self.attn2(self.norm2(x, t), context=context, mask=mask) + x
|
| 139 |
+
x = self.ff(self.norm3(x, t)) + x
|
| 140 |
+
return x
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
class Transformer(nn.Module):
|
| 144 |
+
"""
|
| 145 |
+
Transformer block for image-like data.
|
| 146 |
+
First, project the input (aka embedding)
|
| 147 |
+
and reshape to b, t, d.
|
| 148 |
+
Then apply standard transformer action.
|
| 149 |
+
Finally, reshape to image
|
| 150 |
+
"""
|
| 151 |
+
|
| 152 |
+
def __init__(
|
| 153 |
+
self,
|
| 154 |
+
in_channels,
|
| 155 |
+
n_heads,
|
| 156 |
+
d_head,
|
| 157 |
+
out_channels=None,
|
| 158 |
+
depth=1,
|
| 159 |
+
dropout=0.0,
|
| 160 |
+
context_dim=None,
|
| 161 |
+
):
|
| 162 |
+
super().__init__()
|
| 163 |
+
self.in_channels = in_channels
|
| 164 |
+
inner_dim = n_heads * d_head
|
| 165 |
+
|
| 166 |
+
self.proj_in = nn.Linear(in_channels, inner_dim, bias=False)
|
| 167 |
+
|
| 168 |
+
self.transformer_blocks = nn.ModuleList(
|
| 169 |
+
[
|
| 170 |
+
BasicTransformerBlock(
|
| 171 |
+
inner_dim,
|
| 172 |
+
n_heads=n_heads,
|
| 173 |
+
d_head=d_head,
|
| 174 |
+
context_dim=context_dim,
|
| 175 |
+
dropout=dropout,
|
| 176 |
+
)
|
| 177 |
+
for _ in range(depth)
|
| 178 |
+
]
|
| 179 |
+
)
|
| 180 |
+
out_channels = default(out_channels, in_channels)
|
| 181 |
+
self.proj_out = zero_module(nn.Linear(inner_dim, out_channels, bias=False))
|
| 182 |
+
self.t_proj = nn.Linear(1280, inner_dim)
|
| 183 |
+
|
| 184 |
+
def forward(self, x, t, context=None, attention_mask=None):
|
| 185 |
+
x = self.proj_in(x)
|
| 186 |
+
t = self.t_proj(t).unsqueeze(1)
|
| 187 |
+
for block in self.transformer_blocks:
|
| 188 |
+
x = block(x, t, context=context, mask=attention_mask)
|
| 189 |
+
|
| 190 |
+
x = self.proj_out(x)
|
| 191 |
+
return x
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
class FuseTransformerBlock(nn.Module):
|
| 195 |
+
def __init__(
|
| 196 |
+
self,
|
| 197 |
+
dim,
|
| 198 |
+
n_heads,
|
| 199 |
+
d_head,
|
| 200 |
+
dropout=0.0,
|
| 201 |
+
context_dim=None,
|
| 202 |
+
gated_ff=True,
|
| 203 |
+
checkpoint=True,
|
| 204 |
+
):
|
| 205 |
+
super().__init__()
|
| 206 |
+
self.attn1 = CrossAttention(
|
| 207 |
+
query_dim=dim, heads=n_heads, dim_head=d_head, dropout=dropout
|
| 208 |
+
) # is a self-attention
|
| 209 |
+
self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff)
|
| 210 |
+
self.norm1 = AdaLayerNorm(dim)
|
| 211 |
+
self.norm3 = AdaLayerNorm(dim)
|
| 212 |
+
|
| 213 |
+
def forward(self, x, t, context=None, mask=None):
|
| 214 |
+
# (b f n) k s c
|
| 215 |
+
x = self.attn1(self.norm1(x, t), mask=mask) + x
|
| 216 |
+
x = self.ff(self.norm3(x, t)) + x
|
| 217 |
+
return x
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
class FuseTransformer(nn.Module):
|
| 221 |
+
"""
|
| 222 |
+
Transformer block for image-like data.
|
| 223 |
+
First, project the input (aka embedding)
|
| 224 |
+
and reshape to b, t, d.
|
| 225 |
+
Then apply standard transformer action.
|
| 226 |
+
Finally, reshape to image
|
| 227 |
+
"""
|
| 228 |
+
|
| 229 |
+
def __init__(
|
| 230 |
+
self,
|
| 231 |
+
in_channels,
|
| 232 |
+
n_heads,
|
| 233 |
+
d_head,
|
| 234 |
+
out_channels=None,
|
| 235 |
+
depth=1,
|
| 236 |
+
dropout=0.0,
|
| 237 |
+
context_dim=None,
|
| 238 |
+
):
|
| 239 |
+
super().__init__()
|
| 240 |
+
self.in_channels = in_channels
|
| 241 |
+
inner_dim = n_heads * d_head
|
| 242 |
+
|
| 243 |
+
self.proj_in = nn.Linear(in_channels, inner_dim, bias=False)
|
| 244 |
+
|
| 245 |
+
self.transformer_blocks = nn.ModuleList(
|
| 246 |
+
[
|
| 247 |
+
FuseTransformerBlock(
|
| 248 |
+
inner_dim,
|
| 249 |
+
n_heads=n_heads,
|
| 250 |
+
d_head=d_head,
|
| 251 |
+
context_dim=context_dim,
|
| 252 |
+
dropout=dropout,
|
| 253 |
+
)
|
| 254 |
+
for _ in range(depth)
|
| 255 |
+
]
|
| 256 |
+
)
|
| 257 |
+
|
| 258 |
+
out_channels = default(out_channels, in_channels)
|
| 259 |
+
self.proj_out = zero_module(nn.Linear(inner_dim, out_channels, bias=False))
|
| 260 |
+
self.t_proj = nn.Linear(1280, inner_dim)
|
| 261 |
+
|
| 262 |
+
def forward(self, x, t, context=None, attention_mask=None):
|
| 263 |
+
x = self.proj_in(x)
|
| 264 |
+
t = self.t_proj(t)
|
| 265 |
+
for block in self.transformer_blocks:
|
| 266 |
+
x = block(x, t, context=context, mask=attention_mask)
|
| 267 |
+
x = self.proj_out(x)
|
| 268 |
+
return x
|