Giulio Federico commited on
Commit
2039c18
·
1 Parent(s): 565d54b

Upload of the code without weights

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitignore +1 -0
  2. app.py +63 -0
  3. configs/baseline.yaml +125 -0
  4. configs/baseline_1_to_infinity.yaml +125 -0
  5. configs/baseline_for_ablation.yaml +122 -0
  6. configs/baseline_for_test.yaml +126 -0
  7. configs/baseline_geometric_loss.yaml +148 -0
  8. configs/baseline_global.yaml +125 -0
  9. configs/baseline_triplane.yaml +126 -0
  10. configs/baseline_triplane2.yaml +126 -0
  11. configs/baseline_triplane3.yaml +126 -0
  12. configs/baseline_triplane_with_3d_conv.yaml +126 -0
  13. configs/baseline_with_distillation.yaml +124 -0
  14. configs/baseline_with_projection.yaml +126 -0
  15. configs/loomnet_global2.log +2 -0
  16. configs/loomnet_triplane.yaml.save +1 -0
  17. configs/nerf.yaml +26 -0
  18. configs/neus.yaml +27 -0
  19. inference.py +141 -0
  20. loomnet/data/__init__.py +0 -0
  21. loomnet/data/__pycache__/__init__.cpython-311.pyc +0 -0
  22. loomnet/data/__pycache__/multiview.cpython-311.pyc +0 -0
  23. loomnet/data/__pycache__/multiview_infinity.cpython-311.pyc +0 -0
  24. loomnet/data/multiview.py +375 -0
  25. loomnet/data/multiview_infinity.py +410 -0
  26. loomnet/models/__init__.py +0 -0
  27. loomnet/models/__pycache__/__init__.cpython-311.pyc +0 -0
  28. loomnet/models/__pycache__/mv_model.cpython-311.pyc +0 -0
  29. loomnet/models/__pycache__/mv_model_1_to_infinity.cpython-311.pyc +0 -0
  30. loomnet/models/__pycache__/mv_model_infinity.cpython-311.pyc +0 -0
  31. loomnet/models/__pycache__/mv_model_triplane.cpython-311.pyc +0 -0
  32. loomnet/models/__pycache__/mv_model_triplane_with_3d_conv.cpython-311.pyc +0 -0
  33. loomnet/models/__pycache__/mv_model_v7.cpython-311.pyc +0 -0
  34. loomnet/models/__pycache__/mv_model_v8.cpython-311.pyc +0 -0
  35. loomnet/models/__pycache__/mv_model_with_distillation.cpython-311.pyc +0 -0
  36. loomnet/models/blocks/__init__.py +0 -0
  37. loomnet/models/blocks/__pycache__/__init__.cpython-311.pyc +0 -0
  38. loomnet/models/blocks/__pycache__/generic_blocks.cpython-311.pyc +0 -0
  39. loomnet/models/blocks/__pycache__/pointwise_spatial_transformer.cpython-311.pyc +0 -0
  40. loomnet/models/blocks/__pycache__/resnet.cpython-311.pyc +0 -0
  41. loomnet/models/blocks/__pycache__/resnet_fpn.cpython-311.pyc +0 -0
  42. loomnet/models/blocks/__pycache__/transformer_t.cpython-311.pyc +0 -0
  43. loomnet/models/blocks/attention.py +739 -0
  44. loomnet/models/blocks/attention_processor.py +871 -0
  45. loomnet/models/blocks/generic_blocks.py +295 -0
  46. loomnet/models/blocks/pointwise_spatial_transformer.py +179 -0
  47. loomnet/models/blocks/resnet.py +357 -0
  48. loomnet/models/blocks/resnet_fpn.py +372 -0
  49. loomnet/models/blocks/transformer.py +221 -0
  50. loomnet/models/blocks/transformer_t.py +268 -0
.gitignore ADDED
@@ -0,0 +1 @@
 
 
1
+ *.ckpt
app.py ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import spaces
3
+ import torch
4
+ import hydra
5
+ import lightning as L
6
+ from einops import rearrange
7
+ from PIL import Image
8
+ from pathlib import Path
9
+
10
+ # I tuoi import
11
+ from loomnet.utils import load_config
12
+ from inference import prepare_inputs
13
+
14
+ DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
15
+ OUTPUT_DIR = Path("outputs")
16
+ OUTPUT_DIR.mkdir(exist_ok=True)
17
+
18
+ # Il modello viene caricato nella RAM dello spazio cloud all'avvio
19
+ cfg = load_config("configs/baseline.yaml", cli_args=[])
20
+ model = hydra.utils.instantiate(cfg.system)
21
+ model.load_weights(cfg.resume)
22
+ model = model.to(DEVICE).eval()
23
+
24
+ @spaces.GPU
25
+ def run_demo(image_path, elevation, seed):
26
+ if image_path is None:
27
+ return None, None
28
+
29
+ L.seed_everything(int(seed), workers=True)
30
+ data = prepare_inputs(image_path, elevation, "ele30")
31
+ for k, v in data.items():
32
+ data[k] = v.unsqueeze(0).to(DEVICE)
33
+ if k not in ["k_near_indices"]: data[k] = data[k].float()
34
+
35
+ with torch.no_grad():
36
+ images_pred = model._generate_images(data)
37
+
38
+ gif_path = str(OUTPUT_DIR / "out.gif")
39
+ jpg_path = str(OUTPUT_DIR / "out.jpg")
40
+
41
+ image_list = [Image.fromarray(img) for img in images_pred[0]]
42
+ image_list[0].save(gif_path, save_all=True, append_images=image_list[1:], duration=100, loop=0)
43
+
44
+ full_image = rearrange(images_pred, "b m h w c -> (b h) (m w) c")
45
+ Image.fromarray(full_image).save(jpg_path)
46
+
47
+ return jpg_path, gif_path
48
+
49
+ with gr.Blocks() as demo:
50
+ with gr.Row():
51
+ with gr.Column():
52
+ img_in = gr.Image(type="filepath", label="Immagine Input")
53
+ elevation_in = gr.Slider(-90, 90, value=12.0, label="Elevation")
54
+ seed_in = gr.Number(value=0, label="Seed")
55
+ btn = gr.Button("Genera")
56
+ with gr.Column():
57
+ out_jpg = gr.Image(label="Viste Generate")
58
+ out_gif = gr.Image(label="GIF")
59
+
60
+ btn.click(fn=run_demo, inputs=[img_in, elevation_in, seed_in], outputs=[out_jpg, out_gif])
61
+
62
+ if __name__ == "__main__":
63
+ demo.launch()
configs/baseline.yaml ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: loomnet_global
2
+ tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
3
+ version: loomnet_v2-stable-zero123-n16f16-pabs
4
+ description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
5
+ output_dir: "outputs"
6
+
7
+ #NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
8
+ resume: epoch=47-step=37544.ckpt
9
+
10
+ extras:
11
+ # for dataset
12
+ # num_views: 17 # 1 reference view + 16 sampling views
13
+ # k_near_views: 16 # our baseline uses 16 nearest views
14
+ # sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
15
+ num_views: 17 # 1 reference view + 16 sampling views
16
+ k_near_views: 1 # our baseline uses 16 nearest views
17
+ sample_views_mode: "random"
18
+ img_wh: [256, 256]
19
+ bg_color: white
20
+ load_only_weights:
21
+
22
+ data:
23
+ _target_: loomnet.data.multiview.MultiViewDataModule
24
+ train_dataset:
25
+ _target_: loomnet.data.multiview.MultiViewDataset
26
+ type_of_dataset: "training"
27
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
28
+ img_wh: ${extras.img_wh}
29
+ bg_color: ${extras.bg_color}
30
+ num_views: ${extras.num_views}
31
+ k_near_views: ${extras.k_near_views}
32
+ ref_view_mode: "random" #64
33
+ sample_views_mode: ${extras.sample_views_mode}
34
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
35
+ use_abs_elevation: true # for stable zero123
36
+ train_batch_size: 1 #15
37
+ val_dataset:
38
+ _target_: loomnet.data.multiview.MultiViewDataset
39
+ type_of_dataset: "validation"
40
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
41
+ img_wh: ${extras.img_wh}
42
+ bg_color: ${extras.bg_color}
43
+ num_views: ${extras.num_views}
44
+ k_near_views: ${extras.k_near_views}
45
+ ref_view_mode: "random" #64
46
+ sample_views_mode: ${extras.sample_views_mode}
47
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
48
+ use_abs_elevation: true # for stable zero123
49
+ max_samples_per_category: 1
50
+ val_batch_size: 1 # [NB]: during validation we will use double val_batch_size due to cfg
51
+ test_dataset:
52
+ _target_: loomnet.data.multiview.MultiViewDataset
53
+ type_of_dataset: "testing"
54
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
55
+ img_wh: ${extras.img_wh}
56
+ bg_color: ${extras.bg_color}
57
+ num_views: ${extras.num_views}
58
+ k_near_views: ${extras.k_near_views}
59
+ ref_view_mode: 64
60
+ sample_views_mode: ${extras.sample_views_mode}
61
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
62
+ use_abs_elevation: true
63
+ test_batch_size: 1
64
+ num_workers: 16 #64
65
+ pin_memory: true
66
+
67
+ system:
68
+ _target_: loomnet.systems.i2mv_system_v8.I2MVSystem
69
+ base_model_id: "ashawkey/stable-zero123-diffusers"
70
+ mv_model:
71
+ _target_: loomnet.models.mv_model_v8.MVModel
72
+ _partial_: true
73
+ cp_block_model:
74
+ _target_: loomnet.models.extras.feature_aggregator_v8.FeatureAggregator
75
+ _partial_: true
76
+ num_views: ${extras.num_views}
77
+ n_samples_per_ray: 64 #number of samples M per ray
78
+ num_weaving_layers_per_decoder_block: [3, 4, 6, 6] #how many times the weaving stages is repeated for each decoder layer
79
+ num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
80
+ plane_channel_dim: ${system.mv_model.plane_channel_dim}
81
+ plane_resolution: ${system.mv_model.plane_resolution}
82
+ splat_plane_resolution: 64 #the resolution of each splatted plane
83
+ latent_rendering_stages: [False, False, False, True] #in what stage to use latent rendering
84
+ do_local_bilinear_splatting: True
85
+ do_global_rendering: False
86
+ use_positional_encoding: True
87
+
88
+ insert_stages: ["mid", "up"]
89
+ insert_up_layers: [0, 1, 2]
90
+ use_residual: false
91
+ plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
92
+ plane_channel_dim: 512 #the number of channels of each latent plane
93
+ lr: 1e-5
94
+ cfg: 0.1
95
+ tv_weight: 5e-4 #loss weight for triplane
96
+ report_to: none #TODO --> wandn
97
+
98
+ trainer:
99
+ _target_: lightning.pytorch.trainer.Trainer
100
+ default_root_dir: ${output_dir}
101
+ max_steps: 370001
102
+ check_val_every_n_epoch: 1
103
+ log_every_n_steps: 5
104
+ num_sanity_val_steps: 0
105
+ enable_progress_bar: true
106
+ strategy: ddp_find_unused_parameters_true
107
+ accelerator: gpu
108
+ precision: 16-mixed # mixed precision for extra speed-up
109
+ accumulate_grad_batches: 8
110
+ gradient_clip_val: 1.0
111
+
112
+ callbacks:
113
+ model_checkpoint:
114
+ _target_: lightning.pytorch.callbacks.ModelCheckpoint
115
+ save_top_k: 1
116
+ #every_n_train_steps: 10000
117
+ #every_n_epochs: 12
118
+ dirpath: "${output_dir}/${version}/checkpoints"
119
+
120
+ # logger:
121
+ # wandb:
122
+ # _target_: lightning.pytorch.loggers.wandb.WandbLogger
123
+ # project: "${name}"
124
+ # save_dir: ${output_dir}
125
+ # name: "${version}"
configs/baseline_1_to_infinity.yaml ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: loomnet_1_to_infinity_baseline_1
2
+ tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
3
+ version: loomnet_v2-stable-zero123-n16f16-pabs
4
+ description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
5
+ output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
6
+
7
+ #NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
8
+ resume: /raid/homes/giulio.federico/LoomNet/output_dir/loomnet_1_to_infinity_baseline_1/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=49-step=67001.ckpt
9
+
10
+ extras:
11
+ # for dataset
12
+ # num_views: 17 # 1 reference view + 16 sampling views
13
+ # k_near_views: 16 # our baseline uses 16 nearest views
14
+ # sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
15
+ num_views: 2 # 1 reference view + 32 sampling views
16
+ k_near_views: 1 # our baseline uses 16 nearest views
17
+ sample_views_mode: "random"
18
+ img_wh: [256, 256]
19
+ bg_color: white
20
+ load_only_weights:
21
+
22
+ data:
23
+ _target_: loomnet.data.multiview.MultiViewDataModule
24
+ train_dataset:
25
+ _target_: loomnet.data.multiview.MultiViewDataset
26
+ type_of_dataset: "training"
27
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
28
+ img_wh: ${extras.img_wh}
29
+ bg_color: ${extras.bg_color}
30
+ num_views: ${extras.num_views}
31
+ k_near_views: ${extras.k_near_views}
32
+ ref_view_mode: "random" #64
33
+ sample_views_mode: ${extras.sample_views_mode}
34
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
35
+ use_abs_elevation: true # for stable zero123
36
+ train_batch_size: 2 #15
37
+ val_dataset:
38
+ _target_: loomnet.data.multiview.MultiViewDataset
39
+ type_of_dataset: "validation"
40
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
41
+ img_wh: ${extras.img_wh}
42
+ bg_color: ${extras.bg_color}
43
+ num_views: ${extras.num_views}
44
+ k_near_views: ${extras.k_near_views}
45
+ ref_view_mode: "random" #64
46
+ sample_views_mode: ${extras.sample_views_mode}
47
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
48
+ use_abs_elevation: true # for stable zero123
49
+ max_samples_per_category: 1
50
+ val_batch_size: 2 # [NB]: during validation we will use double val_batch_size due to cfg
51
+ test_dataset:
52
+ _target_: loomnet.data.multiview.MultiViewDataset
53
+ type_of_dataset: "testing"
54
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
55
+ img_wh: ${extras.img_wh}
56
+ bg_color: ${extras.bg_color}
57
+ num_views: ${extras.num_views}
58
+ k_near_views: ${extras.k_near_views}
59
+ ref_view_mode: 64
60
+ sample_views_mode: ${extras.sample_views_mode}
61
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
62
+ use_abs_elevation: true
63
+ test_batch_size: 1
64
+ num_workers: 16 #64
65
+ pin_memory: true
66
+
67
+ system:
68
+ _target_: loomnet.systems.i2mv_system_1_to_infinity.I2MVSystem
69
+ base_model_id: "ashawkey/stable-zero123-diffusers"
70
+ mv_model:
71
+ _target_: loomnet.models.mv_model_1_to_infinity.MVModel
72
+ _partial_: true
73
+ cp_block_model:
74
+ _target_: loomnet.models.extras.feature_aggregator_1_to_infinity.FeatureAggregator
75
+ _partial_: true
76
+ num_views: ${extras.num_views}
77
+ n_samples_per_ray: 16 #number of samples M per ray
78
+ num_weaving_layers_per_decoder_block: [3, 4, 6, 8] #how many times the weaving stages is repeated for each decoder layer
79
+ num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
80
+ plane_channel_dim: ${system.mv_model.plane_channel_dim}
81
+ plane_resolution: ${system.mv_model.plane_resolution}
82
+ splat_plane_resolution: 64 #the resolution of each splatted plane
83
+ latent_rendering_stages: [False, False, False, True] #in what stage to use latent rendering
84
+ do_local_bilinear_splatting: True
85
+ use_positional_encoding: True
86
+ num_views_latent_space: 16 #num views for building the shared latent space
87
+
88
+ insert_stages: ["mid", "up"]
89
+ insert_up_layers: [0, 1, 2, 3]
90
+ use_residual: false
91
+ plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
92
+ plane_channel_dim: 512 #the number of channels of each latent plane
93
+ lr: 1e-5
94
+ cfg: 0.1
95
+ tv_weight: 5e-4 #loss weight for triplane
96
+ report_to: none #TODO --> wandn
97
+
98
+ trainer:
99
+ _target_: lightning.pytorch.trainer.Trainer
100
+ default_root_dir: ${output_dir}
101
+ max_steps: 300000
102
+ check_val_every_n_epoch: 1
103
+ log_every_n_steps: 5
104
+ num_sanity_val_steps: 0
105
+ enable_progress_bar: true
106
+ strategy: ddp_find_unused_parameters_true
107
+ accelerator: gpu
108
+ precision: 16-mixed # mixed precision for extra speed-up
109
+ accumulate_grad_batches: 4
110
+ gradient_clip_val: 1.0
111
+
112
+ callbacks:
113
+ model_checkpoint:
114
+ _target_: lightning.pytorch.callbacks.ModelCheckpoint
115
+ save_top_k: 1
116
+ #every_n_train_steps: 10000
117
+ #every_n_epochs: 12
118
+ dirpath: "${output_dir}/${version}/checkpoints"
119
+
120
+ # logger:
121
+ # wandb:
122
+ # _target_: lightning.pytorch.loggers.wandb.WandbLogger
123
+ # project: "${name}"
124
+ # save_dir: ${output_dir}
125
+ # name: "${version}"
configs/baseline_for_ablation.yaml ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: loomnet_v8_baseline_6
2
+ tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
3
+ version: loomnet_v2-stable-zero123-n16f16-pabs
4
+ description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
5
+ output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
6
+
7
+ resume: /raid/homes/giulio.federico/LoomNet/output_dir/loomnet_v8_baseline_6/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=10-step=21186.ckpt
8
+
9
+ extras:
10
+ # for dataset
11
+ # num_views: 17 # 1 reference view + 16 sampling views
12
+ # k_near_views: 16 # our baseline uses 16 nearest views
13
+ # sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
14
+ num_views: 17 # 1 reference view + 16 sampling views
15
+ k_near_views: 1 # our baseline uses 16 nearest views
16
+ sample_views_mode: "random"
17
+ img_wh: [256, 256]
18
+ bg_color: white
19
+
20
+ data:
21
+ _target_: loomnet.data.multiview.MultiViewDataModule
22
+ train_dataset:
23
+ _target_: loomnet.data.multiview.MultiViewDataset
24
+ type_of_dataset: "training"
25
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
26
+ img_wh: ${extras.img_wh}
27
+ bg_color: ${extras.bg_color}
28
+ num_views: ${extras.num_views}
29
+ k_near_views: ${extras.k_near_views}
30
+ ref_view_mode: "random" #64
31
+ sample_views_mode: ${extras.sample_views_mode}
32
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
33
+ use_abs_elevation: true # for stable zero123
34
+ train_batch_size: 2 #15
35
+ val_dataset:
36
+ _target_: loomnet.data.multiview.MultiViewDataset
37
+ type_of_dataset: "validation"
38
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
39
+ img_wh: ${extras.img_wh}
40
+ bg_color: ${extras.bg_color}
41
+ num_views: ${extras.num_views}
42
+ k_near_views: ${extras.k_near_views}
43
+ ref_view_mode: "random" #64
44
+ sample_views_mode: ${extras.sample_views_mode}
45
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
46
+ use_abs_elevation: true # for stable zero123
47
+ max_samples_per_category: 1
48
+ val_batch_size: 2 # [NB]: during validation we will use double val_batch_size due to cfg
49
+ test_dataset:
50
+ _target_: loomnet.data.multiview.MultiViewDataset
51
+ type_of_dataset: "testing"
52
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
53
+ img_wh: ${extras.img_wh}
54
+ bg_color: ${extras.bg_color}
55
+ num_views: ${extras.num_views}
56
+ k_near_views: ${extras.k_near_views}
57
+ ref_view_mode: 64
58
+ sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] #[60,67,74,81,88,95,4,9,14,19,24,31,38,45,52,59] #${extras.sample_views_mode}
59
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
60
+ use_abs_elevation: true
61
+ test_batch_size: 1
62
+ num_workers: 16 #64
63
+ pin_memory: true
64
+
65
+ system:
66
+ _target_: loomnet.systems.i2mv_system_v8.I2MVSystem
67
+ base_model_id: "ashawkey/stable-zero123-diffusers"
68
+ mv_model:
69
+ _target_: loomnet.models.mv_model_v8.MVModel
70
+ _partial_: true
71
+ cp_block_model:
72
+ _target_: loomnet.models.extras.feature_aggregator_v8.FeatureAggregator
73
+ _partial_: true
74
+ num_views: ${extras.num_views}
75
+ n_samples_per_ray: 16 #number of samples M per ray
76
+ num_weaving_layers_per_decoder_block: [3, 4, 6, 8] #how many times the weaving stages is repeated for each decoder layer
77
+ num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
78
+ plane_channel_dim: ${system.mv_model.plane_channel_dim}
79
+ plane_resolution: ${system.mv_model.plane_resolution}
80
+ splat_plane_resolution: 64 #the resolution of each splatted plane
81
+ latent_rendering_stages: [False, False, False, True] #in what stage to use latent rendering
82
+ do_local_bilinear_splatting: True
83
+ use_positional_encoding: False
84
+
85
+ insert_stages: ["mid", "up"]
86
+ insert_up_layers: [0, 1, 2, 3]
87
+ use_residual: false
88
+ plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
89
+ plane_channel_dim: 512 #the number of channels of each latent plane
90
+ lr: 1e-5
91
+ cfg: 0.1
92
+ tv_weight: 5e-4 #loss weight for triplane
93
+ report_to: none #TODO --> wandn
94
+
95
+ trainer:
96
+ _target_: lightning.pytorch.trainer.Trainer
97
+ default_root_dir: ${output_dir}
98
+ max_steps: 37001
99
+ check_val_every_n_epoch: 1
100
+ log_every_n_steps: 5
101
+ num_sanity_val_steps: 0
102
+ enable_progress_bar: true
103
+ strategy: ddp_find_unused_parameters_true
104
+ accelerator: gpu
105
+ precision: 16-mixed # mixed precision for extra speed-up
106
+ accumulate_grad_batches: 4
107
+ gradient_clip_val: 1.0
108
+
109
+ callbacks:
110
+ model_checkpoint:
111
+ _target_: lightning.pytorch.callbacks.ModelCheckpoint
112
+ save_top_k: 1
113
+ #every_n_train_steps: 10000
114
+ #every_n_epochs: 12
115
+ dirpath: "${output_dir}/${version}/checkpoints"
116
+
117
+ # logger:
118
+ # wandb:
119
+ # _target_: lightning.pytorch.loggers.wandb.WandbLogger
120
+ # project: "${name}"
121
+ # save_dir: ${output_dir}
122
+ # name: "${version}"
configs/baseline_for_test.yaml ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: loomnet_v8_da-eliminare_per_paper
2
+ tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
3
+ version: loomnet_v2-stable-zero123-n16f16-pabs
4
+ description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
5
+ output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
6
+
7
+ resume: /raid/homes/giulio.federico/LoomNet/output_dir/loomnet_v9/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=47-step=37544.ckpt
8
+
9
+ extras:
10
+ # for dataset
11
+ # num_views: 17 # 1 reference view + 16 sampling views
12
+ # k_near_views: 16 # our baseline uses 16 nearest views
13
+ # sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
14
+ num_views: 17 # 1 reference view + 16 sampling views
15
+ k_near_views: 1 # our baseline uses 16 nearest views
16
+ sample_views_mode: [4, 9, 14, 19, 24, 31, 38, 45, 52, 59, 60, 67, 74, 81, 88, 95] #[4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94]
17
+ img_wh: [256, 256]
18
+ bg_color: white
19
+
20
+ data:
21
+ _target_: loomnet.data.multiview.MultiViewDataModule
22
+ train_dataset:
23
+ _target_: loomnet.data.multiview.MultiViewDataset
24
+ type_of_dataset: "training"
25
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
26
+ img_wh: ${extras.img_wh}
27
+ bg_color: ${extras.bg_color}
28
+ num_views: ${extras.num_views}
29
+ k_near_views: ${extras.k_near_views}
30
+ ref_view_mode: "random" #64
31
+ sample_views_mode: ${extras.sample_views_mode}
32
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
33
+ use_abs_elevation: true # for stable zero123
34
+ train_batch_size: 2 #15
35
+ val_dataset:
36
+ _target_: loomnet.data.multiview.MultiViewDataset
37
+ type_of_dataset: "validation"
38
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
39
+ img_wh: ${extras.img_wh}
40
+ bg_color: ${extras.bg_color}
41
+ num_views: ${extras.num_views}
42
+ k_near_views: ${extras.k_near_views}
43
+ ref_view_mode: "random" #64
44
+ sample_views_mode: ${extras.sample_views_mode}
45
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
46
+ use_abs_elevation: true # for stable zero123
47
+ max_samples_per_category: 1
48
+ val_batch_size: 2 # [NB]: during validation we will use double val_batch_size due to cfg
49
+ test_dataset:
50
+ _target_: loomnet.data.multiview.MultiViewDataset
51
+ type_of_dataset: "testing"
52
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
53
+ img_wh: ${extras.img_wh}
54
+ bg_color: ${extras.bg_color}
55
+ num_views: ${extras.num_views}
56
+ k_near_views: ${extras.k_near_views}
57
+ ref_view_mode: 64
58
+ sample_views_mode: ${extras.sample_views_mode}
59
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
60
+ use_abs_elevation: true
61
+ test_batch_size: 1
62
+ num_workers: 16 #64
63
+ pin_memory: true
64
+
65
+ system:
66
+ _target_: loomnet.systems.i2mv_system_v8.I2MVSystem
67
+ base_model_id: "ashawkey/stable-zero123-diffusers"
68
+ use_geometric_loss: False
69
+ mv_model:
70
+ _target_: loomnet.models.mv_model_v8.MVModel
71
+ _partial_: true
72
+ cp_block_model:
73
+ # _target_: loomnet.models.extras.feature_aggregator.FeatureAggregator
74
+ # _partial_: true
75
+ # depth: 2
76
+ # use_resnet: true
77
+ # use_fhw_attn: false
78
+ # pos_encs: ["abs"] # ["abs", "ref"]
79
+ _target_: loomnet.models.extras.feature_aggregator_v8.FeatureAggregator
80
+ _partial_: true
81
+ num_views: ${extras.num_views}
82
+ # n_heads_transformers: [16, 16, 16] #quante teste
83
+ # d_heads_transformers: [64, 64, 64] #la dimensione di ogni testa
84
+ n_heads_transformers: [16, 16, 16] #quante teste
85
+ d_heads_transformers: [32, 32, 32] #la dimensione di ogni testa
86
+ depths_transformers: [2, None, 2]
87
+ ambassadors_feature_grid_resolution: {4: 2, 8: 4, 16: 8, 32: 16}
88
+ number_nearby_hubs_per_projected_point: {4: 4, 8: 4, 16: 4, 32: 4}
89
+ num_latent_points: {4: 32, 8: 64, 16: 128, 32: 256}
90
+ insert_stages: ["mid", "up"]
91
+ insert_up_layers: [0, 1, 2, 3]
92
+ use_residual: false
93
+ render_options:
94
+ n_samples: 8
95
+ lr: 1e-5
96
+ cfg: 0.1
97
+ report_to: none #TODO --> wandn
98
+
99
+ trainer:
100
+ _target_: lightning.pytorch.trainer.Trainer
101
+ default_root_dir: ${output_dir}
102
+ max_steps: 360001
103
+ check_val_every_n_epoch: 1
104
+ log_every_n_steps: 5
105
+ num_sanity_val_steps: 0
106
+ enable_progress_bar: true
107
+ strategy: ddp_find_unused_parameters_true
108
+ accelerator: gpu
109
+ precision: 16-mixed # mixed precision for extra speed-up
110
+ accumulate_grad_batches: 4
111
+ gradient_clip_val: 1.0
112
+
113
+ callbacks:
114
+ model_checkpoint:
115
+ _target_: lightning.pytorch.callbacks.ModelCheckpoint
116
+ save_top_k: 1
117
+ #every_n_train_steps: 10000
118
+ #every_n_epochs: 12
119
+ dirpath: "${output_dir}/${version}/checkpoints"
120
+
121
+ # logger:
122
+ # wandb:
123
+ # _target_: lightning.pytorch.loggers.wandb.WandbLogger
124
+ # project: "${name}"
125
+ # save_dir: ${output_dir}
126
+ # name: "${version}"
configs/baseline_geometric_loss.yaml ADDED
@@ -0,0 +1,148 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: loomnet_with_geometric_loss_point_matching_baseline_1
2
+ tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
3
+ version: loomnet_v2-stable-zero123-n16f16-pabs
4
+ description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
5
+ output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
6
+
7
+ resume: /raid/homes/giulio.federico/LoomNet/output_dir/loomnet_with_geometric_loss_point_matching_baseline_1/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=1-step=7702.ckpt
8
+
9
+
10
+ extras:
11
+ # for dataset
12
+ # num_views: 17 # 1 reference view + 16 sampling views
13
+ # k_near_views: 16 # our baseline uses 16 nearest views
14
+ # sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
15
+ num_views: 17 # 1 reference view + 16 sampling views
16
+ k_near_views: 4 # our baseline uses 16 nearest views
17
+ sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94]
18
+ img_wh: [256, 256]
19
+ bg_color: white
20
+ load_only_weights: #/raid/homes/giulio.federico/LoomNet/output_dir/loomnet_v9/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=47-step=37544.ckpt
21
+
22
+ token_based_geometric_loss:
23
+ max_geometric_loss_weight: 0.1
24
+ min_geometric_loss_weight: 0.005
25
+ geometric_loss_scheduler: "cosine"
26
+ target_vggt_layers: [4,11,17,23]
27
+
28
+ register_token_based_geometric_loss:
29
+ max_geometric_loss_weight: 0.1
30
+ min_geometric_loss_weight: 0.005
31
+ geometric_loss_scheduler: "cosine"
32
+ target_vggt_layers: [4,11,17,23]
33
+
34
+ point_matching_geometric_loss:
35
+ k_near_views: ${extras.k_near_views} #for each view to check we will use extras.k_near_views-1 near views
36
+ num_points: 32
37
+ num_views_to_check: 3 #how many random views to use to check consistency
38
+
39
+
40
+ data:
41
+ _target_: loomnet.data.multiview.MultiViewDataModule
42
+ train_dataset:
43
+ _target_: loomnet.data.multiview.MultiViewDataset
44
+ type_of_dataset: "training"
45
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
46
+ img_wh: ${extras.img_wh}
47
+ bg_color: ${extras.bg_color}
48
+ num_views: ${extras.num_views}
49
+ k_near_views: ${extras.k_near_views}
50
+ ref_view_mode: "random" #64
51
+ sample_views_mode: "random"
52
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
53
+ use_abs_elevation: true # for stable zero123
54
+ train_batch_size: 1 #USA SEMPRE BATCH SIZE 1
55
+ val_dataset:
56
+ _target_: loomnet.data.multiview.MultiViewDataset
57
+ type_of_dataset: "validation"
58
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
59
+ img_wh: ${extras.img_wh}
60
+ bg_color: ${extras.bg_color}
61
+ num_views: ${extras.num_views}
62
+ k_near_views: ${extras.k_near_views}
63
+ ref_view_mode: "random" #64
64
+ sample_views_mode: "random"
65
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
66
+ use_abs_elevation: true # for stable zero123
67
+ max_samples_per_category: 1
68
+ val_batch_size: 2 # [NB]: during validation we will use double val_batch_size due to cfg
69
+ test_dataset:
70
+ _target_: loomnet.data.multiview.MultiViewDataset
71
+ type_of_dataset: "testing"
72
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
73
+ img_wh: ${extras.img_wh}
74
+ bg_color: ${extras.bg_color}
75
+ num_views: ${extras.num_views}
76
+ k_near_views: ${extras.k_near_views}
77
+ ref_view_mode: 64
78
+ sample_views_mode: ${extras.sample_views_mode}
79
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
80
+ use_abs_elevation: true
81
+ test_batch_size: 1
82
+ num_workers: 16 #64
83
+ pin_memory: true
84
+
85
+ system:
86
+ _target_: loomnet.systems.i2mv_system_for_geometric_loss.I2MVSystem
87
+ base_model_id: "ashawkey/stable-zero123-diffusers"
88
+ mv_model:
89
+ _target_: loomnet.models.mv_model_v8.MVModel
90
+ _partial_: true
91
+ cp_block_model:
92
+ _target_: loomnet.models.extras.feature_aggregator_v8.FeatureAggregator
93
+ _partial_: true
94
+ num_views: ${extras.num_views}
95
+ n_samples_per_ray: 16 #number of samples M per ray
96
+ num_weaving_layers_per_decoder_block: [3, 4, 6, 8] #how many times the weaving stages is repeated for each decoder layer
97
+ num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
98
+ plane_channel_dim: ${system.mv_model.plane_channel_dim}
99
+ plane_resolution: ${system.mv_model.plane_resolution}
100
+ splat_plane_resolution: 64 #the resolution of each splatted plane
101
+ latent_rendering_stages: [False, False, False, True] #in what stage to use latent rendering
102
+ do_local_bilinear_splatting: True
103
+ use_positional_encoding: True
104
+
105
+ insert_stages: ["mid", "up"]
106
+ insert_up_layers: [0, 1, 2, 3]
107
+ use_residual: false
108
+ plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
109
+ plane_channel_dim: 512 #the number of channels of each latent plane
110
+
111
+ #for geometric (vggt) loss
112
+ type_of_geometric_loss: 'point_matching_geometric_loss' #['token_based_geometric_loss', 'register_token_based_geometric_loss']
113
+ use_vggt_gradient_checkpoint: True #If there are too many views or nearby views, it is better to use it to avoid going out of memory (even if it will be slower)
114
+ extras: ${extras}
115
+
116
+ lr: 1e-5
117
+ cfg: 0.1
118
+ tv_weight: 5e-4 #loss weight for triplane
119
+ report_to: none #TODO --> wandn
120
+
121
+ trainer:
122
+ _target_: lightning.pytorch.trainer.Trainer
123
+ default_root_dir: ${output_dir}
124
+ max_steps: 15000
125
+ check_val_every_n_epoch: 1
126
+ log_every_n_steps: 5
127
+ num_sanity_val_steps: 0
128
+ enable_progress_bar: true
129
+ strategy: ddp_find_unused_parameters_true
130
+ accelerator: gpu
131
+ precision: 16-mixed # mixed precision for extra speed-up
132
+ accumulate_grad_batches: 3
133
+ gradient_clip_val: 1.0
134
+
135
+ callbacks:
136
+ model_checkpoint:
137
+ _target_: lightning.pytorch.callbacks.ModelCheckpoint
138
+ save_top_k: 1
139
+ #every_n_train_steps: 10000
140
+ #every_n_epochs: 12
141
+ dirpath: "${output_dir}/${version}/checkpoints"
142
+
143
+ # logger:
144
+ # wandb:
145
+ # _target_: lightning.pytorch.loggers.wandb.WandbLogger
146
+ # project: "${name}"
147
+ # save_dir: ${output_dir}
148
+ # name: "${version}"
configs/baseline_global.yaml ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: loomnet_global
2
+ tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
3
+ version: loomnet_v2-stable-zero123-n16f16-pabs
4
+ description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
5
+ output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
6
+
7
+ #NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
8
+ resume: #/raid/homes/giulio.federico/LoomNet/output_dir/loomnet_v9/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=47-step=37544.ckpt
9
+
10
+ extras:
11
+ # for dataset
12
+ # num_views: 17 # 1 reference view + 16 sampling views
13
+ # k_near_views: 16 # our baseline uses 16 nearest views
14
+ # sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
15
+ num_views: 17 # 1 reference view + 16 sampling views
16
+ k_near_views: 1 # our baseline uses 16 nearest views
17
+ sample_views_mode: "random"
18
+ img_wh: [256, 256]
19
+ bg_color: white
20
+ load_only_weights:
21
+
22
+ data:
23
+ _target_: loomnet.data.multiview.MultiViewDataModule
24
+ train_dataset:
25
+ _target_: loomnet.data.multiview.MultiViewDataset
26
+ type_of_dataset: "training"
27
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
28
+ img_wh: ${extras.img_wh}
29
+ bg_color: ${extras.bg_color}
30
+ num_views: ${extras.num_views}
31
+ k_near_views: ${extras.k_near_views}
32
+ ref_view_mode: "random" #64
33
+ sample_views_mode: ${extras.sample_views_mode}
34
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
35
+ use_abs_elevation: true # for stable zero123
36
+ train_batch_size: 1 #15
37
+ val_dataset:
38
+ _target_: loomnet.data.multiview.MultiViewDataset
39
+ type_of_dataset: "validation"
40
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
41
+ img_wh: ${extras.img_wh}
42
+ bg_color: ${extras.bg_color}
43
+ num_views: ${extras.num_views}
44
+ k_near_views: ${extras.k_near_views}
45
+ ref_view_mode: "random" #64
46
+ sample_views_mode: ${extras.sample_views_mode}
47
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
48
+ use_abs_elevation: true # for stable zero123
49
+ max_samples_per_category: 1
50
+ val_batch_size: 1 # [NB]: during validation we will use double val_batch_size due to cfg
51
+ test_dataset:
52
+ _target_: loomnet.data.multiview.MultiViewDataset
53
+ type_of_dataset: "testing"
54
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
55
+ img_wh: ${extras.img_wh}
56
+ bg_color: ${extras.bg_color}
57
+ num_views: ${extras.num_views}
58
+ k_near_views: ${extras.k_near_views}
59
+ ref_view_mode: 64
60
+ sample_views_mode: ${extras.sample_views_mode}
61
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
62
+ use_abs_elevation: true
63
+ test_batch_size: 1
64
+ num_workers: 16 #64
65
+ pin_memory: true
66
+
67
+ system:
68
+ _target_: loomnet.systems.i2mv_system_global.I2MVSystem
69
+ base_model_id: "ashawkey/stable-zero123-diffusers"
70
+ mv_model:
71
+ _target_: loomnet.models.mv_model_v8.MVModel
72
+ _partial_: true
73
+ cp_block_model:
74
+ _target_: loomnet.models.extras.feature_aggregator_global.FeatureAggregator
75
+ _partial_: true
76
+ num_views: ${extras.num_views}
77
+ n_samples_per_ray: 64 #number of samples M per ray
78
+ num_weaving_layers_per_decoder_block: [3, 4, 6, 12] #how many times the weaving stages is repeated for each decoder layer
79
+ num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
80
+ plane_channel_dim: ${system.mv_model.plane_channel_dim}
81
+ plane_resolution: ${system.mv_model.plane_resolution}
82
+ splat_plane_resolution: 128 #the resolution of each splatted plane
83
+ triplane_upsample_channel_dim: 32
84
+ latent_rendering_stages: [False, False, False, True] #in what stage to use latent rendering
85
+ do_local_bilinear_splatting: True
86
+ use_positional_encoding: True
87
+
88
+ insert_stages: ["up"]
89
+ insert_up_layers: [3]
90
+ use_residual: false
91
+ plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
92
+ plane_channel_dim: 512 #the number of channels of each latent plane
93
+ lr: 1e-5
94
+ cfg: 0.1
95
+ tv_weight: 5e-4 #loss weight for triplane
96
+ report_to: none #TODO --> wandn
97
+
98
+ trainer:
99
+ _target_: lightning.pytorch.trainer.Trainer
100
+ default_root_dir: ${output_dir}
101
+ max_steps: 37001
102
+ check_val_every_n_epoch: 1
103
+ log_every_n_steps: 5
104
+ num_sanity_val_steps: 0
105
+ enable_progress_bar: true
106
+ strategy: ddp_find_unused_parameters_true
107
+ accelerator: gpu
108
+ precision: 16-mixed # mixed precision for extra speed-up
109
+ accumulate_grad_batches: 4
110
+ gradient_clip_val: 1.0
111
+
112
+ callbacks:
113
+ model_checkpoint:
114
+ _target_: lightning.pytorch.callbacks.ModelCheckpoint
115
+ save_top_k: 1
116
+ #every_n_train_steps: 10000
117
+ #every_n_epochs: 12
118
+ dirpath: "${output_dir}/${version}/checkpoints"
119
+
120
+ # logger:
121
+ # wandb:
122
+ # _target_: lightning.pytorch.loggers.wandb.WandbLogger
123
+ # project: "${name}"
124
+ # save_dir: ${output_dir}
125
+ # name: "${version}"
configs/baseline_triplane.yaml ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: loomnet_triplane_4
2
+ tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
3
+ version: loomnet_v2-stable-zero123-n16f16-pabs
4
+ description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
5
+ output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
6
+
7
+ #NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
8
+ resume:
9
+
10
+ extras:
11
+ # for dataset
12
+ # num_views: 17 # 1 reference view + 16 sampling views
13
+ # k_near_views: 16 # our baseline uses 16 nearest views
14
+ # sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
15
+ num_views: 2 # 1 reference view + 16 sampling views
16
+ k_near_views: 1 # our baseline uses 16 nearest views
17
+ sample_views_mode: "random"
18
+ img_wh: [256, 256]
19
+ bg_color: white
20
+ load_only_weights:
21
+
22
+ data:
23
+ _target_: loomnet.data.multiview.MultiViewDataModule
24
+ train_dataset:
25
+ _target_: loomnet.data.multiview.MultiViewDataset
26
+ type_of_dataset: "training"
27
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
28
+ img_wh: ${extras.img_wh}
29
+ bg_color: ${extras.bg_color}
30
+ num_views: ${extras.num_views}
31
+ k_near_views: ${extras.k_near_views}
32
+ ref_view_mode: "random" #64
33
+ sample_views_mode: ${extras.sample_views_mode}
34
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
35
+ use_abs_elevation: true # for stable zero123
36
+ train_batch_size: 6 #15
37
+ val_dataset:
38
+ _target_: loomnet.data.multiview.MultiViewDataset
39
+ type_of_dataset: "validation"
40
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
41
+ img_wh: ${extras.img_wh}
42
+ bg_color: ${extras.bg_color}
43
+ num_views: ${extras.num_views}
44
+ k_near_views: ${extras.k_near_views}
45
+ ref_view_mode: "random" #64
46
+ sample_views_mode: ${extras.sample_views_mode}
47
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
48
+ use_abs_elevation: true # for stable zero123
49
+ max_samples_per_category: 1
50
+ val_batch_size: 1 # [NB]: during validation we will use double val_batch_size due to cfg
51
+ test_dataset:
52
+ _target_: loomnet.data.multiview.MultiViewDataset
53
+ type_of_dataset: "testing"
54
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
55
+ img_wh: ${extras.img_wh}
56
+ bg_color: ${extras.bg_color}
57
+ num_views: ${extras.num_views}
58
+ k_near_views: ${extras.k_near_views}
59
+ ref_view_mode: 64
60
+ sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94]
61
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
62
+ use_abs_elevation: true
63
+ test_batch_size: 1
64
+ num_workers: 16 #64
65
+ pin_memory: true
66
+
67
+ system:
68
+ _target_: loomnet.systems.i2mv_system_v8.I2MVSystem
69
+ base_model_id: "ashawkey/stable-zero123-diffusers"
70
+ mv_model:
71
+ _target_: loomnet.models.mv_model_triplane.MVModel
72
+ _partial_: true
73
+ cp_block_model:
74
+ _target_: loomnet.models.extras.feature_aggregator_triplane.FeatureAggregator
75
+ _partial_: true
76
+ num_views: ${extras.num_views}
77
+ n_samples_per_ray: 48 #number of samples M per ray
78
+ num_weaving_layers_per_decoder_block: [4, 4, 5, 6] #how many times the weaving stages is repeated for each decoder layer
79
+ num_weaving_heads_per_decoder_block: [8, 8, 8, 16]
80
+ plane_channel_dim: ${system.mv_model.plane_channel_dim}
81
+ plane_resolution: ${system.mv_model.plane_resolution}
82
+ #plane_channel_dim_after_upsample: 240
83
+ splat_plane_resolutions: [64, 64, 64, 64] #the resolution of each splatted plane
84
+ latent_rendering_stages: [True, True, True, True] #in what stage to use latent rendering
85
+ do_local_bilinear_splatting: True
86
+ do_global_rendering: True
87
+ use_positional_encoding: True
88
+
89
+ insert_stages: ["mid", "up"]
90
+ insert_up_layers: [0, 1, 2, 3] #[0, 1, 2, 3]
91
+ use_residual: true
92
+ plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
93
+ plane_channel_dim: 512 #the number of channels of each latent plane
94
+ lr: 1e-5
95
+ cfg: 0.1
96
+ tv_weight: 5e-3 #loss weight for triplane
97
+ report_to: none #TODO --> wandn
98
+
99
+ trainer:
100
+ _target_: lightning.pytorch.trainer.Trainer
101
+ default_root_dir: ${output_dir}
102
+ max_steps: 946209
103
+ check_val_every_n_epoch: 1
104
+ log_every_n_steps: 50
105
+ num_sanity_val_steps: 0
106
+ enable_progress_bar: true
107
+ strategy: ddp_find_unused_parameters_true
108
+ accelerator: gpu
109
+ precision: 16-mixed # mixed precision for extra speed-up
110
+ accumulate_grad_batches: 16
111
+ gradient_clip_val: 1.0
112
+
113
+ callbacks:
114
+ model_checkpoint:
115
+ _target_: lightning.pytorch.callbacks.ModelCheckpoint
116
+ save_top_k: 1
117
+ #every_n_train_steps: 10000
118
+ #every_n_epochs: 12
119
+ dirpath: "${output_dir}/${version}/checkpoints"
120
+
121
+ # logger:
122
+ # wandb:
123
+ # _target_: lightning.pytorch.loggers.wandb.WandbLogger
124
+ # project: "${name}"
125
+ # save_dir: ${output_dir}
126
+ # name: "${version}"
configs/baseline_triplane2.yaml ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: loomnet_triplane_4
2
+ tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
3
+ version: loomnet_v2-stable-zero123-n16f16-pabs
4
+ description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
5
+ output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
6
+
7
+ #NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
8
+ resume: #/4/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=7-step=7704.ckpt
9
+
10
+ extras:
11
+ # for dataset
12
+ # num_views: 17 # 1 reference view + 16 sampling views
13
+ # k_near_views: 16 # our baseline uses 16 nearest views
14
+ # sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
15
+ num_views: 17 # 1 reference view + 16 sampling views
16
+ k_near_views: 1 # our baseline uses 16 nearest views
17
+ sample_views_mode: "random"
18
+ img_wh: [256, 256]
19
+ bg_color: white
20
+ load_only_weights:
21
+
22
+ data:
23
+ _target_: loomnet.data.multiview.MultiViewDataModule
24
+ train_dataset:
25
+ _target_: loomnet.data.multiview.MultiViewDataset
26
+ type_of_dataset: "training"
27
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
28
+ img_wh: ${extras.img_wh}
29
+ bg_color: ${extras.bg_color}
30
+ num_views: ${extras.num_views}
31
+ k_near_views: ${extras.k_near_views}
32
+ ref_view_mode: "random" #64
33
+ sample_views_mode: ${extras.sample_views_mode}
34
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
35
+ use_abs_elevation: true # for stable zero123
36
+ train_batch_size: 1 #15
37
+ val_dataset:
38
+ _target_: loomnet.data.multiview.MultiViewDataset
39
+ type_of_dataset: "validation"
40
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
41
+ img_wh: ${extras.img_wh}
42
+ bg_color: ${extras.bg_color}
43
+ num_views: ${extras.num_views}
44
+ k_near_views: ${extras.k_near_views}
45
+ ref_view_mode: "random" #64
46
+ sample_views_mode: ${extras.sample_views_mode}
47
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
48
+ use_abs_elevation: true # for stable zero123
49
+ max_samples_per_category: 1
50
+ val_batch_size: 1 # [NB]: during validation we will use double val_batch_size due to cfg
51
+ test_dataset:
52
+ _target_: loomnet.data.multiview.MultiViewDataset
53
+ type_of_dataset: "testing"
54
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
55
+ img_wh: ${extras.img_wh}
56
+ bg_color: ${extras.bg_color}
57
+ num_views: ${extras.num_views}
58
+ k_near_views: ${extras.k_near_views}
59
+ ref_view_mode: 64
60
+ sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94]
61
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
62
+ use_abs_elevation: true
63
+ test_batch_size: 1
64
+ num_workers: 16 #64
65
+ pin_memory: true
66
+
67
+ system:
68
+ _target_: loomnet.systems.i2mv_system_v8.I2MVSystem
69
+ base_model_id: "ashawkey/stable-zero123-diffusers"
70
+ mv_model:
71
+ _target_: loomnet.models.mv_model_triplane.MVModel
72
+ _partial_: true
73
+ cp_block_model:
74
+ _target_: loomnet.models.extras.feature_aggregator_triplane.FeatureAggregator
75
+ _partial_: true
76
+ num_views: ${extras.num_views}
77
+ n_samples_per_ray: 32 #number of samples M per ray
78
+ num_weaving_layers_per_decoder_block: [4, 4, 4, 4] #how many times the weaving stages is repeated for each decoder layer
79
+ num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
80
+ plane_channel_dim: ${system.mv_model.plane_channel_dim}
81
+ plane_resolution: ${system.mv_model.plane_resolution}
82
+ #plane_channel_dim_after_upsample: 240
83
+ splat_plane_resolutions: [64, 64, 64, 64] #the resolution of each splatted plane
84
+ latent_rendering_stages: [True, True, True, True] #in what stage to use latent rendering
85
+ do_local_bilinear_splatting: True
86
+ do_global_rendering: True
87
+ use_positional_encoding: True
88
+
89
+ insert_stages: ["mid", "up"]
90
+ insert_up_layers: [0, 1, 2, 3] #[0, 1, 2, 3]
91
+ use_residual: false
92
+ plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
93
+ plane_channel_dim: 512 #the number of channels of each latent plane
94
+ lr: 1e-5
95
+ cfg: 0.1
96
+ tv_weight: 5e-4 #loss weight for triplane
97
+ report_to: none #TODO --> wandn
98
+
99
+ trainer:
100
+ _target_: lightning.pytorch.trainer.Trainer
101
+ default_root_dir: ${output_dir}
102
+ max_steps: 946209
103
+ check_val_every_n_epoch: 1
104
+ log_every_n_steps: 50
105
+ num_sanity_val_steps: 0
106
+ enable_progress_bar: true
107
+ strategy: ddp_find_unused_parameters_true
108
+ accelerator: gpu
109
+ precision: 16-mixed # mixed precision for extra speed-up
110
+ accumulate_grad_batches: 16
111
+ gradient_clip_val: 1.0
112
+
113
+ callbacks:
114
+ model_checkpoint:
115
+ _target_: lightning.pytorch.callbacks.ModelCheckpoint
116
+ save_top_k: 1
117
+ #every_n_train_steps: 10000
118
+ #every_n_epochs: 12
119
+ dirpath: "${output_dir}/${version}/checkpoints"
120
+
121
+ # logger:
122
+ # wandb:
123
+ # _target_: lightning.pytorch.loggers.wandb.WandbLogger
124
+ # project: "${name}"
125
+ # save_dir: ${output_dir}
126
+ # name: "${version}"
configs/baseline_triplane3.yaml ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: loomnet_triplane_4
2
+ tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
3
+ version: loomnet_v2-stable-zero123-n16f16-pabs
4
+ description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
5
+ output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
6
+
7
+ #NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
8
+ resume: /raid/homes/giulio.federico/LoomNet/output_dir/loomnet_triplane_4/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=69-step=67410.ckpt
9
+
10
+ extras:
11
+ # for dataset
12
+ # num_views: 17 # 1 reference view + 16 sampling views
13
+ # k_near_views: 16 # our baseline uses 16 nearest views
14
+ # sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
15
+ num_views: 17 # 1 reference view + 16 sampling views
16
+ k_near_views: 1 # our baseline uses 16 nearest views
17
+ sample_views_mode: "random"
18
+ img_wh: [256, 256]
19
+ bg_color: white
20
+ load_only_weights:
21
+
22
+ data:
23
+ _target_: loomnet.data.multiview.MultiViewDataModule
24
+ train_dataset:
25
+ _target_: loomnet.data.multiview.MultiViewDataset
26
+ type_of_dataset: "training"
27
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
28
+ img_wh: ${extras.img_wh}
29
+ bg_color: ${extras.bg_color}
30
+ num_views: ${extras.num_views}
31
+ k_near_views: ${extras.k_near_views}
32
+ ref_view_mode: "random" #64
33
+ sample_views_mode: ${extras.sample_views_mode}
34
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
35
+ use_abs_elevation: true # for stable zero123
36
+ train_batch_size: 1 #15
37
+ val_dataset:
38
+ _target_: loomnet.data.multiview.MultiViewDataset
39
+ type_of_dataset: "validation"
40
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
41
+ img_wh: ${extras.img_wh}
42
+ bg_color: ${extras.bg_color}
43
+ num_views: ${extras.num_views}
44
+ k_near_views: ${extras.k_near_views}
45
+ ref_view_mode: "random" #64
46
+ sample_views_mode: ${extras.sample_views_mode}
47
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
48
+ use_abs_elevation: true # for stable zero123
49
+ max_samples_per_category: 1
50
+ val_batch_size: 1 # [NB]: during validation we will use double val_batch_size due to cfg
51
+ test_dataset:
52
+ _target_: loomnet.data.multiview.MultiViewDataset
53
+ type_of_dataset: "testing"
54
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
55
+ img_wh: ${extras.img_wh}
56
+ bg_color: ${extras.bg_color}
57
+ num_views: ${extras.num_views}
58
+ k_near_views: ${extras.k_near_views}
59
+ ref_view_mode: 64
60
+ sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94]
61
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
62
+ use_abs_elevation: true
63
+ test_batch_size: 1
64
+ num_workers: 16 #64
65
+ pin_memory: true
66
+
67
+ system:
68
+ _target_: loomnet.systems.i2mv_system_v8.I2MVSystem
69
+ base_model_id: "ashawkey/stable-zero123-diffusers"
70
+ mv_model:
71
+ _target_: loomnet.models.mv_model_triplane.MVModel
72
+ _partial_: true
73
+ cp_block_model:
74
+ _target_: loomnet.models.extras.feature_aggregator_triplane.FeatureAggregator
75
+ _partial_: true
76
+ num_views: ${extras.num_views}
77
+ n_samples_per_ray: 32 #number of samples M per ray
78
+ num_weaving_layers_per_decoder_block: [4, 4, 4, 4] #how many times the weaving stages is repeated for each decoder layer
79
+ num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
80
+ plane_channel_dim: ${system.mv_model.plane_channel_dim}
81
+ plane_resolution: ${system.mv_model.plane_resolution}
82
+ #plane_channel_dim_after_upsample: 240
83
+ splat_plane_resolutions: [64, 64, 64, 64] #the resolution of each splatted plane
84
+ latent_rendering_stages: [True, True, True, True] #in what stage to use latent rendering
85
+ do_local_bilinear_splatting: True
86
+ do_global_rendering: True
87
+ use_positional_encoding: True
88
+
89
+ insert_stages: ["mid", "up"]
90
+ insert_up_layers: [0, 1, 2, 3] #[0, 1, 2, 3]
91
+ use_residual: false
92
+ plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
93
+ plane_channel_dim: 512 #the number of channels of each latent plane
94
+ lr: 1e-5
95
+ cfg: 0.1
96
+ tv_weight: 5e-4 #loss weight for triplane
97
+ report_to: none #TODO --> wandn
98
+
99
+ trainer:
100
+ _target_: lightning.pytorch.trainer.Trainer
101
+ default_root_dir: ${output_dir}
102
+ max_steps: 946209
103
+ check_val_every_n_epoch: 1
104
+ log_every_n_steps: 50
105
+ num_sanity_val_steps: 0
106
+ enable_progress_bar: true
107
+ strategy: ddp_find_unused_parameters_true
108
+ accelerator: gpu
109
+ precision: 16-mixed # mixed precision for extra speed-up
110
+ accumulate_grad_batches: 16
111
+ gradient_clip_val: 1.0
112
+
113
+ callbacks:
114
+ model_checkpoint:
115
+ _target_: lightning.pytorch.callbacks.ModelCheckpoint
116
+ save_top_k: 1
117
+ #every_n_train_steps: 10000
118
+ #every_n_epochs: 12
119
+ dirpath: "${output_dir}/${version}/checkpoints"
120
+
121
+ # logger:
122
+ # wandb:
123
+ # _target_: lightning.pytorch.loggers.wandb.WandbLogger
124
+ # project: "${name}"
125
+ # save_dir: ${output_dir}
126
+ # name: "${version}"
configs/baseline_triplane_with_3d_conv.yaml ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: loomnet_triplane_4
2
+ tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
3
+ version: loomnet_v2-stable-zero123-n16f16-pabs
4
+ description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
5
+ output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
6
+
7
+ #NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
8
+ resume:
9
+
10
+ extras:
11
+ # for dataset
12
+ # num_views: 17 # 1 reference view + 16 sampling views
13
+ # k_near_views: 16 # our baseline uses 16 nearest views
14
+ # sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
15
+ num_views: 17 # 1 reference view + 16 sampling views
16
+ k_near_views: 1 # our baseline uses 16 nearest views
17
+ sample_views_mode: "random"
18
+ img_wh: [256, 256]
19
+ bg_color: white
20
+ load_only_weights:
21
+
22
+ data:
23
+ _target_: loomnet.data.multiview.MultiViewDataModule
24
+ train_dataset:
25
+ _target_: loomnet.data.multiview.MultiViewDataset
26
+ type_of_dataset: "training"
27
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
28
+ img_wh: ${extras.img_wh}
29
+ bg_color: ${extras.bg_color}
30
+ num_views: ${extras.num_views}
31
+ k_near_views: ${extras.k_near_views}
32
+ ref_view_mode: "random" #64
33
+ sample_views_mode: ${extras.sample_views_mode}
34
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
35
+ use_abs_elevation: true # for stable zero123
36
+ train_batch_size: 1 #15
37
+ val_dataset:
38
+ _target_: loomnet.data.multiview.MultiViewDataset
39
+ type_of_dataset: "validation"
40
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
41
+ img_wh: ${extras.img_wh}
42
+ bg_color: ${extras.bg_color}
43
+ num_views: ${extras.num_views}
44
+ k_near_views: ${extras.k_near_views}
45
+ ref_view_mode: "random" #64
46
+ sample_views_mode: ${extras.sample_views_mode}
47
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
48
+ use_abs_elevation: true # for stable zero123
49
+ max_samples_per_category: 1
50
+ val_batch_size: 1 # [NB]: during validation we will use double val_batch_size due to cfg
51
+ test_dataset:
52
+ _target_: loomnet.data.multiview.MultiViewDataset
53
+ type_of_dataset: "testing"
54
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
55
+ img_wh: ${extras.img_wh}
56
+ bg_color: ${extras.bg_color}
57
+ num_views: ${extras.num_views}
58
+ k_near_views: ${extras.k_near_views}
59
+ ref_view_mode: 64
60
+ sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94]
61
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
62
+ use_abs_elevation: true
63
+ test_batch_size: 1
64
+ num_workers: 16 #64
65
+ pin_memory: true
66
+
67
+ system:
68
+ _target_: loomnet.systems.i2mv_system_v8.I2MVSystem
69
+ base_model_id: "ashawkey/stable-zero123-diffusers"
70
+ mv_model:
71
+ _target_: loomnet.models.mv_model_triplane_with_3d_conv.MVModel
72
+ _partial_: true
73
+ cp_block_model:
74
+ _target_: loomnet.models.extras.feature_aggregator_triplane_with_3d_conv.FeatureAggregator
75
+ _partial_: true
76
+ num_views: ${extras.num_views}
77
+ n_samples_per_ray: 48 #number of samples M per ray
78
+ num_weaving_layers_per_decoder_block: [4, 4, 5, 6] #how many times the weaving stages is repeated for each decoder layer
79
+ num_weaving_heads_per_decoder_block: [8, 8, 8, 16]
80
+ plane_channel_dim: ${system.mv_model.plane_channel_dim}
81
+ plane_resolution: ${system.mv_model.plane_resolution}
82
+ #plane_channel_dim_after_upsample: 240
83
+ splat_plane_resolutions: [64, 64, 64, 64] #the resolution of each splatted plane
84
+ latent_rendering_stages: [True, True, True, True] #in what stage to use latent rendering
85
+ do_local_bilinear_splatting: True
86
+ do_global_rendering: True
87
+ use_positional_encoding: True
88
+
89
+ insert_stages: ["mid", "up"]
90
+ insert_up_layers: [0, 1, 2, 3] #[0, 1, 2, 3]
91
+ use_residual: true
92
+ plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
93
+ plane_channel_dim: 512 #the number of channels of each latent plane
94
+ lr: 1e-5
95
+ cfg: 0.1
96
+ tv_weight: 5e-3 #loss weight for triplane
97
+ report_to: none #TODO --> wandn
98
+
99
+ trainer:
100
+ _target_: lightning.pytorch.trainer.Trainer
101
+ default_root_dir: ${output_dir}
102
+ max_steps: 946209
103
+ check_val_every_n_epoch: 1
104
+ log_every_n_steps: 50
105
+ num_sanity_val_steps: 0
106
+ enable_progress_bar: true
107
+ strategy: ddp_find_unused_parameters_true
108
+ accelerator: gpu
109
+ precision: 16-mixed # mixed precision for extra speed-up
110
+ accumulate_grad_batches: 16
111
+ gradient_clip_val: 1.0
112
+
113
+ callbacks:
114
+ model_checkpoint:
115
+ _target_: lightning.pytorch.callbacks.ModelCheckpoint
116
+ save_top_k: 1
117
+ #every_n_train_steps: 10000
118
+ #every_n_epochs: 12
119
+ dirpath: "${output_dir}/${version}/checkpoints"
120
+
121
+ # logger:
122
+ # wandb:
123
+ # _target_: lightning.pytorch.loggers.wandb.WandbLogger
124
+ # project: "${name}"
125
+ # save_dir: ${output_dir}
126
+ # name: "${version}"
configs/baseline_with_distillation.yaml ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: loomnet_with_distillation_3
2
+ tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
3
+ version: loomnet_v2-stable-zero123-n16f16-pabs
4
+ description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
5
+ output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
6
+
7
+ #NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
8
+ resume: /raid/homes/giulio.federico/LoomNet/output_dir/loomnet_v9/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=47-step=37544.ckpt
9
+
10
+ extras:
11
+ # for dataset
12
+ # num_views: 17 # 1 reference view + 16 sampling views
13
+ # k_near_views: 16 # our baseline uses 16 nearest views
14
+ # sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
15
+ num_views: 17 # 1 reference view + 16 sampling views
16
+ k_near_views: 1 # our baseline uses 16 nearest views
17
+ sample_views_mode: "random"
18
+ img_wh: [256, 256]
19
+ bg_color: white
20
+ load_only_weights:
21
+
22
+ data:
23
+ _target_: loomnet.data.multiview.MultiViewDataModule
24
+ train_dataset:
25
+ _target_: loomnet.data.multiview.MultiViewDataset
26
+ type_of_dataset: "training"
27
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
28
+ img_wh: ${extras.img_wh}
29
+ bg_color: ${extras.bg_color}
30
+ num_views: ${extras.num_views}
31
+ k_near_views: ${extras.k_near_views}
32
+ ref_view_mode: "random" #64
33
+ sample_views_mode: ${extras.sample_views_mode}
34
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
35
+ use_abs_elevation: true # for stable zero123
36
+ train_batch_size: 2 #15
37
+ val_dataset:
38
+ _target_: loomnet.data.multiview.MultiViewDataset
39
+ type_of_dataset: "validation"
40
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
41
+ img_wh: ${extras.img_wh}
42
+ bg_color: ${extras.bg_color}
43
+ num_views: ${extras.num_views}
44
+ k_near_views: ${extras.k_near_views}
45
+ ref_view_mode: "random" #64
46
+ sample_views_mode: ${extras.sample_views_mode}
47
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
48
+ use_abs_elevation: true # for stable zero123
49
+ max_samples_per_category: 1
50
+ val_batch_size: 2 # [NB]: during validation we will use double val_batch_size due to cfg
51
+ test_dataset:
52
+ _target_: loomnet.data.multiview.MultiViewDataset
53
+ type_of_dataset: "testing"
54
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
55
+ img_wh: ${extras.img_wh}
56
+ bg_color: ${extras.bg_color}
57
+ num_views: ${extras.num_views}
58
+ k_near_views: ${extras.k_near_views}
59
+ ref_view_mode: 64
60
+ sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94]
61
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
62
+ use_abs_elevation: true
63
+ test_batch_size: 1
64
+ num_workers: 16 #64
65
+ pin_memory: true
66
+
67
+ system:
68
+ _target_: loomnet.systems.i2mv_system_with_distillation.I2MVSystem
69
+ base_model_id: "ashawkey/stable-zero123-diffusers"
70
+ mv_model:
71
+ _target_: loomnet.models.mv_model_with_distillation.MVModel
72
+ _partial_: true
73
+ cp_block_model:
74
+ _target_: loomnet.models.extras.feature_aggregator_with_distillation.FeatureAggregator
75
+ _partial_: true
76
+ num_views: ${extras.num_views}
77
+ n_samples_per_ray: 16 #number of samples M per ray
78
+ num_weaving_layers_per_decoder_block: [3, 4, 6, 8] #how many times the weaving stages is repeated for each decoder layer
79
+ num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
80
+ plane_channel_dim: ${system.mv_model.plane_channel_dim}
81
+ plane_resolution: ${system.mv_model.plane_resolution}
82
+ splat_plane_resolution: 64 #the resolution of each splatted plane
83
+ latent_rendering_stages: [False, False, False, True] #in what stage to use latent rendering
84
+ do_local_bilinear_splatting: True
85
+ use_positional_encoding: True
86
+
87
+ insert_stages: ["mid", "up"]
88
+ insert_up_layers: [0, 1, 2, 3]
89
+ use_residual: false
90
+ plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
91
+ plane_channel_dim: 512 #the number of channels of each latent plane
92
+ lr: 1e-5
93
+ cfg: 0.1
94
+ tv_weight: 5e-4 #loss weight for triplane
95
+ report_to: none #TODO --> wandn
96
+
97
+ trainer:
98
+ _target_: lightning.pytorch.trainer.Trainer
99
+ default_root_dir: ${output_dir}
100
+ max_steps: 52545
101
+ check_val_every_n_epoch: 1
102
+ log_every_n_steps: 5
103
+ num_sanity_val_steps: 0
104
+ enable_progress_bar: true
105
+ strategy: ddp_find_unused_parameters_true
106
+ accelerator: gpu
107
+ precision: 16-mixed # mixed precision for extra speed-up
108
+ accumulate_grad_batches: 4
109
+ gradient_clip_val: 1.0
110
+
111
+ callbacks:
112
+ model_checkpoint:
113
+ _target_: lightning.pytorch.callbacks.ModelCheckpoint
114
+ save_top_k: 1
115
+ #every_n_train_steps: 10000
116
+ #every_n_epochs: 12
117
+ dirpath: "${output_dir}/${version}/checkpoints"
118
+
119
+ # logger:
120
+ # wandb:
121
+ # _target_: lightning.pytorch.loggers.wandb.WandbLogger
122
+ # project: "${name}"
123
+ # save_dir: ${output_dir}
124
+ # name: "${version}"
configs/baseline_with_projection.yaml ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: loomnet
2
+ tags: ["lvis", "stable-zero123", "n16f16", "pabs"]
3
+ version: loomnet_v2-stable-zero123-n16f16-pabs
4
+ description: "Baseline config for loomnet. Trained on LVIS dataset with 16 fixed rendered views in a fixed elevation of 30. Epipolar constrained among 16 nearest views. Use absolute elevation for stable zero123."
5
+ output_dir: "/raid/homes/giulio.federico/LoomNet/output_dir/${name}"
6
+
7
+ #NB: if you want to resume all the training states, use 'resume'. If you want to load only weights, use 'extras.load_only_weights'
8
+ resume: /raid/homes/giulio.federico/LoomNet/output_dir/loomnet/loomnet_v2-stable-zero123-n16f16-pabs/checkpoints/epoch=60-step=39162.ckpt
9
+
10
+ extras:
11
+ # for dataset
12
+ # num_views: 17 # 1 reference view + 16 sampling views
13
+ # k_near_views: 16 # our baseline uses 16 nearest views
14
+ # sample_views_mode: [4, 10, 16, 22, 28, 34, 40, 46, 52, 58, 64, 70, 76, 82, 88, 94] # view indexs for sampling
15
+ num_views: 2 # 1 reference view + 16 sampling views
16
+ k_near_views: 1 # our baseline uses 16 nearest views
17
+ sample_views_mode: "random"
18
+ img_wh: [256, 256]
19
+ bg_color: white
20
+ load_only_weights:
21
+
22
+ data:
23
+ _target_: loomnet.data.multiview.MultiViewDataModule
24
+ train_dataset:
25
+ _target_: loomnet.data.multiview.MultiViewDataset
26
+ type_of_dataset: "training"
27
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
28
+ img_wh: ${extras.img_wh}
29
+ bg_color: ${extras.bg_color}
30
+ num_views: ${extras.num_views}
31
+ k_near_views: ${extras.k_near_views}
32
+ ref_view_mode: "random" #64
33
+ sample_views_mode: ${extras.sample_views_mode}
34
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_train.txt
35
+ use_abs_elevation: true # for stable zero123
36
+ train_batch_size: 1 #15
37
+ val_dataset:
38
+ _target_: loomnet.data.multiview.MultiViewDataset
39
+ type_of_dataset: "validation"
40
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview
41
+ img_wh: ${extras.img_wh}
42
+ bg_color: ${extras.bg_color}
43
+ num_views: ${extras.num_views}
44
+ k_near_views: ${extras.k_near_views}
45
+ ref_view_mode: "random" #64
46
+ sample_views_mode: ${extras.sample_views_mode}
47
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_val.txt
48
+ use_abs_elevation: true # for stable zero123
49
+ max_samples_per_category: 1
50
+ val_batch_size: 2 # [NB]: during validation we will use double val_batch_size due to cfg
51
+ test_dataset:
52
+ _target_: loomnet.data.multiview.MultiViewDataset
53
+ type_of_dataset: "testing"
54
+ root_dir: /raid/homes/giulio.federico/LoomNet/dataset/google_scanned_objects_multiview_evaluation
55
+ img_wh: ${extras.img_wh}
56
+ bg_color: ${extras.bg_color}
57
+ num_views: ${extras.num_views}
58
+ k_near_views: ${extras.k_near_views}
59
+ ref_view_mode: 64
60
+ sample_views_mode: ${extras.sample_views_mode}
61
+ caption_path: /raid/homes/giulio.federico/LoomNet/dataset/objaverse_lvis_multiview/caption_test.txt
62
+ use_abs_elevation: true
63
+ test_batch_size: 1
64
+ num_workers: 16 #64
65
+ pin_memory: true
66
+
67
+ system:
68
+ _target_: loomnet.systems.i2mv_system_with_projection.I2MVSystem
69
+ base_model_id: "ashawkey/stable-zero123-diffusers"
70
+ mv_model:
71
+ _target_: loomnet.models.mv_model_v8.MVModel
72
+ _partial_: true
73
+ cp_block_model:
74
+ _target_: loomnet.models.extras.feature_aggregator_with_projection.FeatureAggregator
75
+ _partial_: true
76
+ num_views: ${extras.num_views}
77
+ n_samples_per_ray: 64 #number of samples M per ray
78
+ num_weaving_layers_per_decoder_block: [4, 4, 6, 8] #how many times the weaving stages is repeated for each decoder layer
79
+ num_weaving_heads_per_decoder_block: [4, 8, 8, 8]
80
+ plane_channel_dim: ${system.mv_model.plane_channel_dim}
81
+ plane_resolution: ${system.mv_model.plane_resolution}
82
+ splat_plane_resolution: 128 #the resolution of each splatted plane
83
+ triplane_upsample_channel_dim: 32
84
+ latent_rendering_stages: [False, False, False, True] #in what stage to use latent rendering
85
+ do_local_bilinear_splatting: True
86
+ do_global_rendering: True
87
+ use_positional_encoding: True
88
+
89
+ insert_stages: ["up"]
90
+ insert_up_layers: [3]
91
+ use_residual: false
92
+ plane_resolution: 32 #the resolution of each latent plane (of latent triplane)
93
+ plane_channel_dim: 512 #the number of channels of each latent plane
94
+ lr: 1e-5
95
+ cfg: 0.1
96
+ tv_weight: 5e-4 #loss weight for triplane
97
+ report_to: none #TODO --> wandn
98
+
99
+ trainer:
100
+ _target_: lightning.pytorch.trainer.Trainer
101
+ default_root_dir: ${output_dir}
102
+ max_steps: 370001
103
+ check_val_every_n_epoch: 1
104
+ log_every_n_steps: 5
105
+ num_sanity_val_steps: 0
106
+ enable_progress_bar: true
107
+ strategy: ddp_find_unused_parameters_true
108
+ accelerator: gpu
109
+ precision: 16-mixed # mixed precision for extra speed-up
110
+ accumulate_grad_batches: 4
111
+ gradient_clip_val: 1.0
112
+
113
+ callbacks:
114
+ model_checkpoint:
115
+ _target_: lightning.pytorch.callbacks.ModelCheckpoint
116
+ save_top_k: 1
117
+ #every_n_train_steps: 10000
118
+ #every_n_epochs: 12
119
+ dirpath: "${output_dir}/${version}/checkpoints"
120
+
121
+ # logger:
122
+ # wandb:
123
+ # _target_: lightning.pytorch.loggers.wandb.WandbLogger
124
+ # project: "${name}"
125
+ # save_dir: ${output_dir}
126
+ # name: "${version}"
configs/loomnet_global2.log ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ nohup: ignoring input
2
+ nohup: failed to run command 'torchrun': No such file or directory
configs/loomnet_triplane.yaml.save ADDED
@@ -0,0 +1 @@
 
 
1
+ ^x
configs/nerf.yaml ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ base_lr: 1.0e-2
3
+ target: renderer.renderer.RendererTrainer
4
+ params:
5
+ total_steps: 2000
6
+ warm_up_steps: 100
7
+ train_batch_num: 40960
8
+ test_batch_num: 40960
9
+ renderer: ngp
10
+ cube_bound: 0.6
11
+ use_mask: true
12
+ lambda_rgb_loss: 0.5
13
+ lambda_mask_loss: 10.0
14
+ meta_camera_path: "/raid/homes/giulio.federico/LoomNet/src/meta/fixed_30_meta.pkl"
15
+
16
+ data:
17
+ target: renderer.dummy_dataset.DummyDataset
18
+ params: {}
19
+
20
+ callbacks:
21
+ save_interval: 5000
22
+
23
+ trainer:
24
+ val_check_interval: 500
25
+ max_steps: 2000
26
+
configs/neus.yaml ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ base_lr: 5.0e-4
3
+ target: renderer.renderer.RendererTrainer
4
+ params:
5
+ total_steps: 2000
6
+ warm_up_steps: 100
7
+ train_batch_num: 3584
8
+ train_batch_fg_num: 512
9
+ test_batch_num: 4096
10
+ use_mask: true
11
+ lambda_rgb_loss: 0.5
12
+ lambda_mask_loss: 1.0
13
+ lambda_eikonal_loss: 0.1
14
+ use_warm_up: true
15
+ meta_camera_path: /raid/homes/giulio.federico/LoomNet/competitors/SyncDreamer/meta_info/camera-16.pkl
16
+
17
+ data:
18
+ target: renderer.dummy_dataset.DummyDataset
19
+ params: {}
20
+
21
+ callbacks:
22
+ save_interval: 500
23
+
24
+ trainer:
25
+ val_check_interval: 500
26
+ max_steps: 2000
27
+
inference.py ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import json
3
+ import math
4
+ import os
5
+ from pathlib import Path
6
+ from typing import List
7
+
8
+ import hydra
9
+ import lightning as L
10
+ import numpy as np
11
+ import torch
12
+ from einops import rearrange
13
+ from lightning import LightningModule
14
+ from PIL import Image
15
+
16
+ from loomnet.utils import load_config
17
+ from loomnet.utils.media import get_bg_color, load_image
18
+ from loomnet.utils.pose import get_k_near_views
19
+
20
+ WIDTH, HEIGHT = 256, 256
21
+ K_NEAR_VIEWS = 16
22
+
23
+
24
+ def prepare_inputs(input_img: str, input_elevation: float, sample_views_mode: str):
25
+ bg_color = get_bg_color("white")
26
+ input_img = load_image(
27
+ input_img, (WIDTH, HEIGHT), bg_color, return_type="pt"
28
+ ).permute(2, 0, 1)
29
+
30
+ meta_fp = f"meta_info/transforms_{sample_views_mode}.json"
31
+ with open(meta_fp, "r") as f:
32
+ meta = json.load(f)
33
+
34
+ # Camera intrinsics
35
+ fov = meta["camera_angle_x"]
36
+ focal_length = 1 / (2 * np.tan(0.5 * fov))
37
+ intrinsics_4x4 = torch.tensor(
38
+ [
39
+ [focal_length, 0, 0.5, 0],
40
+ [0, focal_length, 0.5, 0],
41
+ [0, 0, 1, 0],
42
+ [0, 0, 0, 1],
43
+ ]
44
+ )
45
+
46
+ # Camera extrinsics
47
+ num_views = len(meta["frames"])
48
+ elevations, azimuths, c2w_matrixs = [], [], []
49
+ for frame in meta["frames"]:
50
+ elevations.append(frame["elevation"])
51
+ azimuths.append(frame["azimuth"])
52
+ c2w_matrixs.append(frame["transform_matrix"])
53
+ elevations = torch.tensor(elevations) # (N,)
54
+ azimuths = torch.tensor(azimuths) # (N,)
55
+ c2w_matrixs = torch.tensor(c2w_matrixs) # (N, 4, 4)
56
+ c2w_matrixs[:, :, 1:3] *= -1 # blender to opencv
57
+
58
+ # concat intrinsics and extrinsics
59
+ intrinsics_4x4 = (
60
+ intrinsics_4x4.unsqueeze(0).repeat(num_views, 1, 1).reshape(num_views, 16)
61
+ )
62
+ _c2w_matrixs = c2w_matrixs.reshape(num_views, 16)
63
+ camera_params = torch.cat([intrinsics_4x4, _c2w_matrixs], dim=1)
64
+
65
+ # find nearest views
66
+ k_near_indices = get_k_near_views(elevations, azimuths, K_NEAR_VIEWS, num_views)
67
+
68
+ # normalize elevations and azimuths
69
+ input_elevation = torch.tensor([input_elevation / 180 * math.pi])
70
+ d_elevations = (elevations - input_elevation).reshape(-1, 1)
71
+ d_azimuths = azimuths.reshape(-1, 1) % (2 * math.pi)
72
+
73
+ #distances = torch.zeros_like(d_elevations)
74
+ distances = torch.tensor(input_elevation).repeat(num_views).reshape(-1, 1)
75
+
76
+ return {
77
+ "image_0": input_img,
78
+ "elevations": d_elevations,
79
+ "azimuths": d_azimuths,
80
+ "distances": distances,
81
+ "c2w_matrixs": c2w_matrixs,
82
+ "cameras": camera_params,
83
+ "k_near_indices": k_near_indices,
84
+ }
85
+
86
+
87
+ def main():
88
+ parser = argparse.ArgumentParser()
89
+ parser.add_argument("--config", type=str, default="configs/baseline.yaml")
90
+ parser.add_argument("--input_img", type=str, default="/raid/homes/giulio.federico/LoomNet/da_eliminare/render_0065.png")
91
+ parser.add_argument("--output_dir", type=str, default="/raid/homes/giulio.federico/LoomNet/output_dir")
92
+ parser.add_argument("--elevation", type=float, default=0.0)
93
+ parser.add_argument(
94
+ "--sample_views_mode", type=str, choices=["ele30", "ele40"], default="ele30"
95
+ )
96
+ parser.add_argument("--seed", type=int, default=42)
97
+ parser.add_argument("--device", type=str, default="cuda")
98
+ args, extras = parser.parse_known_args()
99
+
100
+ output_dir = Path(args.output_dir)
101
+ output_dir.mkdir(exist_ok=True, parents=True)
102
+
103
+ L.seed_everything(args.seed, workers=True)
104
+ cfg = load_config(args.config, cli_args=extras)
105
+
106
+ # prepare model
107
+ model: LightningModule = hydra.utils.instantiate(cfg.system)
108
+ model.load_weights(cfg.resume)
109
+ model = model.to(args.device).eval()
110
+ print(f"Loaded model from {cfg.resume}")
111
+
112
+ # prepare data
113
+ data = prepare_inputs(args.input_img, args.elevation, args.sample_views_mode)
114
+ for k, v in data.items():
115
+ data[k] = v.unsqueeze(0).to(args.device)
116
+ if k not in ["k_near_indices"]:
117
+ data[k] = data[k].float()
118
+
119
+ # generate
120
+ with torch.no_grad():
121
+ images_pred = model._generate_images(data)
122
+
123
+ # save
124
+ image_base_name = os.path.basename(args.input_img).split(".")[0]
125
+ image_list = []
126
+ for image in images_pred[0]:
127
+ image_list.append(Image.fromarray(image))
128
+ image_list[0].save(
129
+ output_dir / f"{image_base_name}.gif",
130
+ save_all=True,
131
+ append_images=image_list[1:],
132
+ duration=100,
133
+ loop=0,
134
+ )
135
+
136
+ full_image = rearrange(images_pred, "b m h w c -> (b h) (m w) c")
137
+ Image.fromarray(full_image).save(output_dir / f"{image_base_name}.jpg")
138
+
139
+
140
+ if __name__ == "__main__":
141
+ main()
loomnet/data/__init__.py ADDED
File without changes
loomnet/data/__pycache__/__init__.cpython-311.pyc ADDED
Binary file (177 Bytes). View file
 
loomnet/data/__pycache__/multiview.cpython-311.pyc ADDED
Binary file (19.7 kB). View file
 
loomnet/data/__pycache__/multiview_infinity.cpython-311.pyc ADDED
Binary file (20.5 kB). View file
 
loomnet/data/multiview.py ADDED
@@ -0,0 +1,375 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import math
3
+ import os
4
+ from os.path import join as osp
5
+ from typing import Any, Dict, List, Optional, Tuple, Union
6
+ import random
7
+ import numpy as np
8
+ import torch
9
+ from lightning import LightningDataModule
10
+ from omegaconf import DictConfig, ListConfig
11
+ from PIL import Image
12
+ from torch.utils.data import DataLoader, Dataset
13
+
14
+
15
+ class MultiViewDataset(Dataset):
16
+ def __init__(
17
+ self,
18
+ type_of_dataset: str,
19
+ root_dir: str,
20
+ num_views: int,
21
+ bg_color: str,
22
+ img_wh: Tuple[int, int],
23
+ caption_path: str,
24
+ ref_view_mode: Union[str, List] = "random",
25
+ sample_views_mode: Union[str, List] = "random",
26
+ k_near_views: Optional[int] = None,
27
+ max_samples_per_category: Optional[int] = None,
28
+ use_abs_elevation: bool = False,
29
+ seed = 0
30
+ ):
31
+ #data = []
32
+
33
+ # # Loop per ogni cartella di categoria
34
+ # for category in os.listdir(root_dir):
35
+ # category_path = os.path.join(root_dir, category)
36
+
37
+ # if os.path.isdir(category_path):
38
+
39
+ # uids = os.listdir(category_path)
40
+ # #se validazione, prendiamo per semplicità solo due campioni
41
+ # if max_samples_per_category is not None:
42
+ # random.seed(seed)
43
+ # uids = random.sample(uids, min(max_samples_per_category, len(uids)))
44
+ # for uid in uids:
45
+
46
+ # uid_path = os.path.join(root_dir, category, uid)
47
+ # obj_data = {"obj_dir": uid_path, "caption": ""}
48
+
49
+ # data.append(obj_data)
50
+
51
+ data = []
52
+ # Loop per ogni cartella di categoria
53
+ for category in os.listdir(root_dir):
54
+ category_path = os.path.join(root_dir, category)
55
+
56
+ if os.path.isdir(category_path):
57
+
58
+ uids = [f for f in os.listdir(category_path) if not os.path.isfile(os.path.join(category_path, f))] #os.listdir(category_path)
59
+
60
+ if len(uids)!=0:
61
+ for uid in uids:
62
+
63
+ uid_path = os.path.join(root_dir, category, uid)
64
+ obj_data = {"obj_dir": uid_path, "caption": ""}
65
+
66
+ data.append(obj_data)
67
+ else:
68
+ uid_path = os.path.join(root_dir, category)
69
+ obj_data = {"obj_dir": uid_path, "caption": ""}
70
+
71
+ data.append(obj_data)
72
+
73
+
74
+ #da eliminare (per velocità prendo solo i primi 8)
75
+ if type_of_dataset=="validation":
76
+ random.seed(3) # esempio: random.seed(42)
77
+ indices = random.sample(range(len(data)), 16) # prendo 96 indici casuali unici da 0 a 999
78
+ data = [data[i] for i in indices]
79
+
80
+ #data = data[:50] #TODO ELIMINA
81
+
82
+ self.data = data
83
+ self.num_views = num_views
84
+ self.bg_color = bg_color
85
+ self.img_wh = img_wh
86
+ self.ref_view_mode = ref_view_mode
87
+ self.sample_views_mode = sample_views_mode
88
+ self.k_near_views = k_near_views if k_near_views else num_views
89
+ self.use_abs_elevation = use_abs_elevation
90
+
91
+ def get_bg_color(self, bg_color):
92
+ if bg_color == "white":
93
+ bg_color = np.array([1.0, 1.0, 1.0], dtype=np.float32)
94
+ elif bg_color == "black":
95
+ bg_color = np.array([0.0, 0.0, 0.0], dtype=np.float32)
96
+ elif bg_color == "gray":
97
+ bg_color = np.array([0.5, 0.5, 0.5], dtype=np.float32)
98
+ elif bg_color == "random":
99
+ bg_color = np.random.rand(3)
100
+ elif bg_color == "random_gray":
101
+ bg_color = np.random.rand(1) * 0.5 + 0.3
102
+ bg_color = np.array([bg_color] * 3, dtype=np.float32)
103
+ elif isinstance(bg_color, float):
104
+ bg_color = np.array([bg_color] * 3, dtype=np.float32)
105
+ else:
106
+ raise NotImplementedError
107
+ return bg_color
108
+
109
+ def load_image(self, img_path, bg_color, rescale=True, return_type="np"):
110
+ # not using cv2 as may load in uint16 format
111
+ # img = cv2.imread(img_path, cv2.IMREAD_UNCHANGED) # [0, 255]
112
+ # img = cv2.resize(img, self.img_wh, interpolation=cv2.INTER_CUBIC)
113
+ # pil always returns uint8
114
+ img = np.array(Image.open(img_path).resize(self.img_wh))
115
+ img = img.astype(np.float32) / 255.0 # [0, 1]
116
+
117
+ if img.shape[-1] == 4:
118
+ alpha = img[..., 3:4]
119
+ img = img[..., :3] * alpha + bg_color * (1 - alpha)
120
+
121
+ if rescale:
122
+ img = img * 2.0 - 1.0 # to -1 ~ 1
123
+
124
+ if return_type == "np":
125
+ pass
126
+ elif return_type == "pt":
127
+ img = torch.from_numpy(img)
128
+ else:
129
+ raise NotImplementedError
130
+
131
+ return img
132
+
133
+ def get_k_near_views(self, elevations, azimuths, k_near_views, num_views):
134
+ views = torch.cat((elevations.unsqueeze(1), azimuths.unsqueeze(1)), dim=1)
135
+ distances = torch.cdist(views, views)
136
+ torch.fill_(distances.diagonal(), 0.0)
137
+ k_nearest_indices = torch.topk(distances, k_near_views, largest=False).indices
138
+
139
+ return k_nearest_indices
140
+
141
+ def __len__(self):
142
+ return len(self.data)
143
+
144
+ def check_view_validaty(self, path_img):
145
+ img = np.array(Image.open(os.path.join(path_img)))
146
+
147
+ #check if transparent
148
+ alpha = img[..., 3]
149
+ is_fully_transparent = np.all(alpha == 0)
150
+
151
+ return not(is_fully_transparent)
152
+
153
+ def __getitem__(self, index):
154
+ #{'obj_dir': path to folder uid, 'caption': ""}
155
+ sample = self.data[index]
156
+ obj_dir = sample["obj_dir"]
157
+ with open(osp(obj_dir, "meta.json"), "r") as f:
158
+ meta = json.load(f)
159
+
160
+ # no use
161
+ caption = sample["caption"]
162
+
163
+ # camera intrinsics
164
+ fov = meta["camera_angle_x"]
165
+ focal_length = 0.5 * 1 / np.tan(0.5 * fov) # FIXME: hard-code
166
+ intrinsics = np.array(
167
+ [
168
+ [focal_length, 0, 0.5],
169
+ [0, focal_length, 0.5],
170
+ [0, 0, 1],
171
+ ],
172
+ dtype=np.float32,
173
+ )
174
+ intrinsics = torch.from_numpy(intrinsics)
175
+ intrinsics_4x4 = torch.zeros(4, 4)
176
+ intrinsics_4x4[:3, :3] = intrinsics
177
+ intrinsics_4x4[3, 3] = 1.0
178
+
179
+ # sample or select ref view ids
180
+ num_views_all = len(meta["locations"])
181
+ if self.ref_view_mode == "random":
182
+ check = False
183
+ c = 0
184
+ while not(check):
185
+ ref_view_id = np.random.randint(num_views_all)
186
+ #chech if ref view is not transparent
187
+ check = self.check_view_validaty(os.path.join(obj_dir, meta["locations"][ref_view_id]["frames"][0]["name"]))
188
+ c=c+1
189
+ if c == self.num_views: #DA ELIMINARE
190
+ print(f"The file at {obj_dir} directory do not contain any valid image....")
191
+
192
+ elif isinstance(self.ref_view_mode, int):
193
+ ref_view_id = self.ref_view_mode
194
+ else:
195
+ raise NotImplementedError
196
+
197
+ # sample or select view ids in a mode
198
+ num_views_all = len(meta["locations"])
199
+ if self.sample_views_mode == "random":
200
+ view_ids = np.random.choice(
201
+ num_views_all, self.num_views - 1, replace=False
202
+ )
203
+ elif isinstance(self.sample_views_mode, list) or isinstance(
204
+ self.sample_views_mode, ListConfig
205
+ ):
206
+ assert len(self.sample_views_mode) == self.num_views - 1, "Invalid view_ids"
207
+ view_ids = self.sample_views_mode
208
+ else:
209
+ raise NotImplementedError
210
+ view_ids = np.insert(view_ids, 0, ref_view_id)
211
+ locations = [meta["locations"][i] for i in view_ids]
212
+
213
+ # load images, elevations, azimuths, c2w_matrixs
214
+ bg_color = self.get_bg_color(self.bg_color)
215
+ img_paths, img_tensors, elevations, azimuths, c2w_matrixs = [], [], [], [], []
216
+ for loc in locations:
217
+ img_path = osp(obj_dir, loc["frames"][0]["name"])
218
+ img = self.load_image(img_path, bg_color, return_type="pt").permute(2, 0, 1)
219
+ img_tensors.append(img)
220
+ img_paths.append(img_path)
221
+ elevations.append(loc["elevation"])
222
+ azimuths.append(loc["azimuth"])
223
+ c2w_matrixs.append(loc["transform_matrix"])
224
+
225
+ # concat and stack
226
+ img_tensors = torch.stack(img_tensors, dim=0).float() # (Nv, 3, H, W)
227
+ elevations = torch.tensor(elevations).float() # (Nv,)
228
+ azimuths = torch.tensor(azimuths).float() # (Nv,)
229
+ c2w_matrixs = torch.tensor(c2w_matrixs).float() # (Nv, 4, 4)
230
+
231
+ # blender to opencv
232
+ c2w_matrixs[:, :, 1:3] *= -1
233
+
234
+ intrinsics_matrixs = intrinsics.unsqueeze(0).repeat(
235
+ self.num_views, 1, 1
236
+ ) # (Nv, 3, 3)
237
+ intrinsics_matrixs_4x4 = intrinsics_4x4.unsqueeze(0).repeat(
238
+ self.num_views, 1, 1
239
+ ) # (Nv, 4, 4)
240
+
241
+ # flatten intrinsics_matrixs_4x4 and c2w_matrixs to (Nv, 16), and concat them
242
+ intrinsics_matrixs_4x4 = intrinsics_matrixs_4x4.reshape(self.num_views, 16)
243
+ c2w_matrixs = c2w_matrixs.reshape(self.num_views, 16)
244
+ camera_params = torch.cat((intrinsics_matrixs_4x4, c2w_matrixs), dim=1)
245
+ c2w_matrixs = c2w_matrixs.reshape(self.num_views, 4, 4)
246
+
247
+ # find nearest views
248
+ if self.k_near_views > self.num_views:
249
+ raise ValueError("k_near_views should be no larger than num_views")
250
+ else:
251
+ k_near_indices = self.get_k_near_views(
252
+ elevations[1:], azimuths[1:], self.k_near_views, self.num_views - 1
253
+ ) # NOTE: we use the 1st view as the reference view
254
+ d_elevations = (elevations - elevations[0:1]).reshape(-1, 1)
255
+ d_azimuths = (azimuths - azimuths[0:1]).reshape(-1, 1) % (2 * math.pi)
256
+
257
+ if self.use_abs_elevation: # stable zero123 use abs_elevation as distance input
258
+ distances = elevations[0:1].repeat(self.num_views).reshape(-1, 1)
259
+ else:
260
+ distances = torch.zeros_like(d_elevations)
261
+
262
+ return {
263
+ "view_ids": torch.tensor(view_ids),
264
+ "images": img_tensors[1:],
265
+ "image_0": img_tensors[0],
266
+ "elevations": d_elevations[1:],
267
+ "azimuths": d_azimuths[1:],
268
+ "distances": distances[1:],
269
+ "c2w_matrixs": c2w_matrixs[1:],
270
+ "intrinsics_matrixs": intrinsics_matrixs[1:],
271
+ "cameras": camera_params[1:],
272
+ "caption": caption,
273
+ "k_near_indices": k_near_indices,
274
+ "obj_dir": obj_dir.split('/')[-1]
275
+ }
276
+
277
+
278
+ class MultiViewDataModule(LightningDataModule):
279
+ def __init__(
280
+ self,
281
+ train_dataset: Dataset[Any],
282
+ val_dataset: Dataset[Any],
283
+ test_dataset: Dataset[Any],
284
+ train_batch_size: int = 1,
285
+ val_batch_size: int = 1,
286
+ test_batch_size: int = 1,
287
+ num_workers: Optional[int] = None,
288
+ pin_memory: bool = True,
289
+ ):
290
+ super().__init__()
291
+
292
+ # this line allows to access init params with 'self.hparams' attribute
293
+ # also ensures init params will be stored in ckpt
294
+ self.save_hyperparameters(logger=False)
295
+
296
+ self.data_train = train_dataset
297
+ self.data_val = val_dataset
298
+ self.data_test = test_dataset
299
+
300
+ self.num_workers = num_workers if num_workers else train_batch_size * 2
301
+
302
+ def prepare_data(self) -> None:
303
+ # TODO: check if data is available
304
+ pass
305
+
306
+ def _dataloader(
307
+ self, dataset: Dataset, batch_size: int, shuffle: bool
308
+ ) -> DataLoader[Any]:
309
+ return DataLoader(
310
+ dataset,
311
+ batch_size=batch_size,
312
+ shuffle=shuffle,
313
+ num_workers=self.num_workers,
314
+ pin_memory=self.hparams.pin_memory,
315
+ )
316
+
317
+ def train_dataloader(self) -> DataLoader[Any]:
318
+ return DataLoader(
319
+ self.data_train,
320
+ batch_size=self.hparams.train_batch_size,
321
+ shuffle=True,
322
+ num_workers=self.num_workers,
323
+ pin_memory=self.hparams.pin_memory,
324
+ )
325
+
326
+ def val_dataloader(self) -> DataLoader[Any]:
327
+ # support multiple validation datasets
328
+ if isinstance(self.data_val, ListConfig):
329
+ return [
330
+ self._dataloader(dataset, self.hparams.val_batch_size, False)
331
+ for dataset in self.data_val
332
+ ]
333
+ elif isinstance(self.data_val, DictConfig):
334
+ return [
335
+ self._dataloader(dataset, self.hparams.val_batch_size, False)
336
+ for _, dataset in self.data_val.items()
337
+ ]
338
+ else:
339
+ return self._dataloader(self.data_val, self.hparams.val_batch_size, False)
340
+
341
+ def test_dataloader(self) -> DataLoader[Any]:
342
+ # support multiple test datasets
343
+ if isinstance(self.data_test, ListConfig):
344
+ return [
345
+ self._dataloader(dataset, self.hparams.test_batch_size, False)
346
+ for dataset in self.data_test
347
+ ]
348
+ elif isinstance(self.data_test, DictConfig):
349
+ return [
350
+ self._dataloader(dataset, self.hparams.test_batch_size, False)
351
+ for _, dataset in self.data_test.items()
352
+ ]
353
+ else:
354
+ return self._dataloader(self.data_test, self.hparams.test_batch_size, False)
355
+
356
+
357
+ if __name__ == "__main__":
358
+ from torchvision.utils import save_image
359
+
360
+ dataset = MultiViewDataset(
361
+ root_dir="/mnt/pfs/data/render_lvis_hzh",
362
+ num_views=17,
363
+ k_near_views=4,
364
+ bg_color="white",
365
+ img_wh=(256, 256),
366
+ ref_view_mode=2,
367
+ sample_views_mode="random",
368
+ caption_path="data/render_lvis_hzh/caption_train.txt",
369
+ use_abs_elevation=True,
370
+ )
371
+
372
+ sample = dataset[0]
373
+ print(sample["view_ids"])
374
+ save_image((sample["image_0"] + 1.0) / 2, "temp_0.jpg", nrow=1, value_range=(0, 1))
375
+ save_image((sample["images"] + 1.0) / 2, "temp.jpg", nrow=8, value_range=(0, 1))
loomnet/data/multiview_infinity.py ADDED
@@ -0,0 +1,410 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import math
3
+ import os
4
+ from os.path import join as osp
5
+ from typing import Any, Dict, List, Optional, Tuple, Union
6
+ import random
7
+ import numpy as np
8
+ import torch
9
+ from lightning import LightningDataModule
10
+ from omegaconf import DictConfig, ListConfig
11
+ from PIL import Image
12
+ from torch.utils.data import DataLoader, Dataset
13
+
14
+
15
+ class MultiViewDataset(Dataset):
16
+ def __init__(
17
+ self,
18
+ type_of_dataset: str,
19
+ root_dir: str,
20
+ num_views: int,
21
+ num_derived_views: int,
22
+ bg_color: str,
23
+ img_wh: Tuple[int, int],
24
+ caption_path: str,
25
+ ref_view_mode: Union[str, List] = "random",
26
+ sample_views_mode: Union[str, List] = "random",
27
+ sample_derived_views_mode: Union[str, List] = "random",
28
+ k_near_views: Optional[int] = None,
29
+ max_samples_per_category: Optional[int] = None,
30
+ use_abs_elevation: bool = False,
31
+ seed = 0
32
+ ):
33
+ #data = []
34
+
35
+ # # Loop per ogni cartella di categoria
36
+ # for category in os.listdir(root_dir):
37
+ # category_path = os.path.join(root_dir, category)
38
+
39
+ # if os.path.isdir(category_path):
40
+
41
+ # uids = os.listdir(category_path)
42
+ # #se validazione, prendiamo per semplicità solo due campioni
43
+ # if max_samples_per_category is not None:
44
+ # random.seed(seed)
45
+ # uids = random.sample(uids, min(max_samples_per_category, len(uids)))
46
+ # for uid in uids:
47
+
48
+ # uid_path = os.path.join(root_dir, category, uid)
49
+ # obj_data = {"obj_dir": uid_path, "caption": ""}
50
+
51
+ # data.append(obj_data)
52
+
53
+ data = []
54
+ # Loop per ogni cartella di categoria
55
+ for category in os.listdir(root_dir):
56
+ category_path = os.path.join(root_dir, category)
57
+
58
+ if os.path.isdir(category_path):
59
+
60
+ uids = [f for f in os.listdir(category_path) if not os.path.isfile(os.path.join(category_path, f))] #os.listdir(category_path)
61
+
62
+ if len(uids)!=0:
63
+ for uid in uids:
64
+
65
+ uid_path = os.path.join(root_dir, category, uid)
66
+ obj_data = {"obj_dir": uid_path, "caption": ""}
67
+
68
+ data.append(obj_data)
69
+ else:
70
+ uid_path = os.path.join(root_dir, category)
71
+ obj_data = {"obj_dir": uid_path, "caption": ""}
72
+
73
+ data.append(obj_data)
74
+
75
+
76
+ #da eliminare (per velocità prendo solo i primi 8)
77
+ if type_of_dataset=="validation":
78
+ data = data[:144]
79
+
80
+
81
+ self.data = data
82
+ self.num_views = num_views
83
+ self.num_derived_views = num_derived_views
84
+ self.bg_color = bg_color
85
+ self.img_wh = img_wh
86
+ self.ref_view_mode = ref_view_mode
87
+ self.sample_views_mode = sample_views_mode
88
+ self.sample_derived_views_mode = sample_derived_views_mode
89
+ self.k_near_views = k_near_views if k_near_views else num_views
90
+ self.use_abs_elevation = use_abs_elevation
91
+
92
+ def get_bg_color(self, bg_color):
93
+ if bg_color == "white":
94
+ bg_color = np.array([1.0, 1.0, 1.0], dtype=np.float32)
95
+ elif bg_color == "black":
96
+ bg_color = np.array([0.0, 0.0, 0.0], dtype=np.float32)
97
+ elif bg_color == "gray":
98
+ bg_color = np.array([0.5, 0.5, 0.5], dtype=np.float32)
99
+ elif bg_color == "random":
100
+ bg_color = np.random.rand(3)
101
+ elif bg_color == "random_gray":
102
+ bg_color = np.random.rand(1) * 0.5 + 0.3
103
+ bg_color = np.array([bg_color] * 3, dtype=np.float32)
104
+ elif isinstance(bg_color, float):
105
+ bg_color = np.array([bg_color] * 3, dtype=np.float32)
106
+ else:
107
+ raise NotImplementedError
108
+ return bg_color
109
+
110
+ def load_image(self, img_path, bg_color, rescale=True, return_type="np"):
111
+ # not using cv2 as may load in uint16 format
112
+ # img = cv2.imread(img_path, cv2.IMREAD_UNCHANGED) # [0, 255]
113
+ # img = cv2.resize(img, self.img_wh, interpolation=cv2.INTER_CUBIC)
114
+ # pil always returns uint8
115
+ img = np.array(Image.open(img_path).resize(self.img_wh))
116
+ img = img.astype(np.float32) / 255.0 # [0, 1]
117
+
118
+ if img.shape[-1] == 4:
119
+ alpha = img[..., 3:4]
120
+ img = img[..., :3] * alpha + bg_color * (1 - alpha)
121
+
122
+ if rescale:
123
+ img = img * 2.0 - 1.0 # to -1 ~ 1
124
+
125
+ if return_type == "np":
126
+ pass
127
+ elif return_type == "pt":
128
+ img = torch.from_numpy(img)
129
+ else:
130
+ raise NotImplementedError
131
+
132
+ return img
133
+
134
+ def get_k_near_anchors(self, elevations, azimuths, k_near_views, num_views, exclude_ids=None):
135
+ views = torch.cat((elevations.unsqueeze(1), azimuths.unsqueeze(1)), dim=1)
136
+ distances = torch.cdist(views, views)
137
+
138
+ # Imposta la diagonale a inf per non contare sé stessi
139
+ distances.fill_diagonal_(float('inf'))
140
+
141
+ if exclude_ids is not None:
142
+ # Esclude gli ID impostando la distanza a inf
143
+ distances[:, exclude_ids] = float('inf')
144
+
145
+ #escludi le ancore
146
+ distances = distances[num_views:]
147
+
148
+ k_nearest_indices = torch.topk(distances, k_near_views, largest=False).indices
149
+
150
+ return k_nearest_indices
151
+
152
+
153
+ def __len__(self):
154
+ return len(self.data)
155
+
156
+ def check_view_validaty(self, path_img):
157
+ img = np.array(Image.open(os.path.join(path_img)))
158
+
159
+ #check if transparent
160
+ alpha = img[..., 3]
161
+ is_fully_transparent = np.all(alpha == 0)
162
+
163
+ return not(is_fully_transparent)
164
+
165
+ def __getitem__(self, index):
166
+ #{'obj_dir': path to folder uid, 'caption': ""}
167
+ sample = self.data[index]
168
+ obj_dir = sample["obj_dir"]
169
+ with open(osp(obj_dir, "meta.json"), "r") as f:
170
+ meta = json.load(f)
171
+
172
+ # no use
173
+ caption = sample["caption"]
174
+
175
+ # camera intrinsics
176
+ fov = meta["camera_angle_x"]
177
+ focal_length = 0.5 * 1 / np.tan(0.5 * fov) # FIXME: hard-code
178
+ intrinsics = np.array(
179
+ [
180
+ [focal_length, 0, 0.5],
181
+ [0, focal_length, 0.5],
182
+ [0, 0, 1],
183
+ ],
184
+ dtype=np.float32,
185
+ )
186
+ intrinsics = torch.from_numpy(intrinsics)
187
+ intrinsics_4x4 = torch.zeros(4, 4)
188
+ intrinsics_4x4[:3, :3] = intrinsics
189
+ intrinsics_4x4[3, 3] = 1.0
190
+
191
+ # sample or select ref view ids
192
+ num_views_all = len(meta["locations"])
193
+ if self.ref_view_mode == "random":
194
+ check = False
195
+ c = 0
196
+ while not(check):
197
+ ref_view_id = np.random.randint(num_views_all)
198
+ #chech if ref view is not transparent
199
+ check = self.check_view_validaty(os.path.join(obj_dir, meta["locations"][ref_view_id]["frames"][0]["name"]))
200
+ c=c+1
201
+ if c == self.num_views: #DA ELIMINARE
202
+ print(f"The file at {obj_dir} directory do not contain any valid image....")
203
+
204
+ elif isinstance(self.ref_view_mode, int):
205
+ ref_view_id = self.ref_view_mode
206
+ else:
207
+ raise NotImplementedError
208
+
209
+ # sample or select anchor view ids
210
+ num_views_all = len(meta["locations"])
211
+ if self.sample_views_mode == "random":
212
+ view_ids = np.random.choice(
213
+ num_views_all, self.num_views - 1, replace=False
214
+ )
215
+ elif isinstance(self.sample_views_mode, list) or isinstance(
216
+ self.sample_views_mode, ListConfig
217
+ ):
218
+ assert len(self.sample_views_mode) == self.num_views - 1, "Invalid view_ids"
219
+ view_ids = self.sample_views_mode
220
+ else:
221
+ raise NotImplementedError
222
+
223
+
224
+ # sample or select derived view ids
225
+ if self.sample_derived_views_mode == "random":
226
+ derived_view_ids = np.random.choice(
227
+ num_views_all, self.num_derived_views, replace=False
228
+ )
229
+ elif isinstance(self.sample_derived_views_mode, list) or isinstance(
230
+ self.sample_derived_views_mode, ListConfig
231
+ ):
232
+ derived_view_ids = self.sample_derived_views_mode
233
+ else:
234
+ raise NotImplementedError
235
+
236
+
237
+
238
+ #insert reference view in the first position
239
+ view_ids = np.insert(view_ids, 0, ref_view_id)
240
+ #insert derived views in the last positions
241
+ view_ids = np.insert(view_ids, self.num_views, derived_view_ids)
242
+ locations = [meta["locations"][i] for i in view_ids]
243
+
244
+ # load images, elevations, azimuths, c2w_matrixs
245
+ bg_color = self.get_bg_color(self.bg_color)
246
+ img_paths, img_tensors, elevations, azimuths, c2w_matrixs = [], [], [], [], []
247
+ for loc in locations:
248
+ img_path = osp(obj_dir, loc["frames"][0]["name"])
249
+ img = self.load_image(img_path, bg_color, return_type="pt").permute(2, 0, 1)
250
+ img_tensors.append(img)
251
+ img_paths.append(img_path)
252
+ elevations.append(loc["elevation"])
253
+ azimuths.append(loc["azimuth"])
254
+ c2w_matrixs.append(loc["transform_matrix"])
255
+
256
+ # concat and stack
257
+ img_tensors = torch.stack(img_tensors, dim=0).float() # (Nv, 3, H, W)
258
+ elevations = torch.tensor(elevations).float() # (Nv,)
259
+ azimuths = torch.tensor(azimuths).float() # (Nv,)
260
+ c2w_matrixs = torch.tensor(c2w_matrixs).float() # (Nv, 4, 4)
261
+
262
+ # blender to opencv
263
+ c2w_matrixs[:, :, 1:3] *= -1
264
+
265
+ intrinsics_matrixs = intrinsics.unsqueeze(0).repeat(
266
+ self.num_views + self.num_derived_views, 1, 1
267
+ ) # (Nv, 3, 3)
268
+ intrinsics_matrixs_4x4 = intrinsics_4x4.unsqueeze(0).repeat(
269
+ self.num_views + self.num_derived_views, 1, 1
270
+ ) # (Nv, 4, 4)
271
+
272
+ # flatten intrinsics_matrixs_4x4 and c2w_matrixs to (Nv, 16), and concat them
273
+ intrinsics_matrixs_4x4 = intrinsics_matrixs_4x4.reshape(self.num_views + self.num_derived_views, 16)
274
+ c2w_matrixs = c2w_matrixs.reshape(self.num_views + self.num_derived_views, 16)
275
+ camera_params = torch.cat((intrinsics_matrixs_4x4, c2w_matrixs), dim=1)
276
+ c2w_matrixs = c2w_matrixs.reshape(self.num_views + self.num_derived_views, 4, 4)
277
+
278
+ # find nearest views
279
+ if self.k_near_views > self.num_views:
280
+ raise ValueError("k_near_views should be no larger than num_views")
281
+ else:
282
+ k_near_indices = self.get_k_near_anchors(
283
+ elevations[1:], azimuths[1:], self.k_near_views, self.num_views - 1, exclude_ids=np.arange(self.num_views-1, self.num_views-1 + self.num_derived_views)
284
+ ) # NOTE: we use the 1st view as the reference view
285
+ #k_near_indices = -1
286
+
287
+ d_elevations = (elevations - elevations[0:1]).reshape(-1, 1)
288
+ d_azimuths = (azimuths - azimuths[0:1]).reshape(-1, 1) % (2 * math.pi)
289
+
290
+ if self.use_abs_elevation: # stable zero123 use abs_elevation as distance input
291
+ distances = elevations[0:1].repeat(self.num_views + self.num_derived_views).reshape(-1, 1)
292
+ else:
293
+ distances = torch.zeros_like(d_elevations)
294
+
295
+ return {
296
+ "view_ids": torch.tensor(view_ids),
297
+ "images": img_tensors[1:],
298
+ "image_0": img_tensors[0],
299
+ "elevations": d_elevations[1:],
300
+ "azimuths": d_azimuths[1:],
301
+ "distances": distances[1:],
302
+ "c2w_matrixs": c2w_matrixs[1:],
303
+ "intrinsics_matrixs": intrinsics_matrixs[1:],
304
+ "cameras": camera_params[1:],
305
+ "caption": caption,
306
+ "k_near_indices": k_near_indices,
307
+ "obj_dir": obj_dir.split('/')[-1],
308
+ "num_anchor_images": self.num_views-1,
309
+ "num_derived_images": self.num_derived_views
310
+ }
311
+
312
+
313
+ class MultiViewDataModule(LightningDataModule):
314
+ def __init__(
315
+ self,
316
+ train_dataset: Dataset[Any],
317
+ val_dataset: Dataset[Any],
318
+ test_dataset: Dataset[Any],
319
+ train_batch_size: int = 1,
320
+ val_batch_size: int = 1,
321
+ test_batch_size: int = 1,
322
+ num_workers: Optional[int] = None,
323
+ pin_memory: bool = True,
324
+ ):
325
+ super().__init__()
326
+
327
+ # this line allows to access init params with 'self.hparams' attribute
328
+ # also ensures init params will be stored in ckpt
329
+ self.save_hyperparameters(logger=False)
330
+
331
+ self.data_train = train_dataset
332
+ self.data_val = val_dataset
333
+ self.data_test = test_dataset
334
+
335
+ self.num_workers = num_workers if num_workers else train_batch_size * 2
336
+
337
+ def prepare_data(self) -> None:
338
+ # TODO: check if data is available
339
+ pass
340
+
341
+ def _dataloader(
342
+ self, dataset: Dataset, batch_size: int, shuffle: bool
343
+ ) -> DataLoader[Any]:
344
+ return DataLoader(
345
+ dataset,
346
+ batch_size=batch_size,
347
+ shuffle=shuffle,
348
+ num_workers=self.num_workers,
349
+ pin_memory=self.hparams.pin_memory,
350
+ )
351
+
352
+ def train_dataloader(self) -> DataLoader[Any]:
353
+ return DataLoader(
354
+ self.data_train,
355
+ batch_size=self.hparams.train_batch_size,
356
+ shuffle=True,
357
+ num_workers=self.num_workers,
358
+ pin_memory=self.hparams.pin_memory,
359
+ )
360
+
361
+ def val_dataloader(self) -> DataLoader[Any]:
362
+ # support multiple validation datasets
363
+ if isinstance(self.data_val, ListConfig):
364
+ return [
365
+ self._dataloader(dataset, self.hparams.val_batch_size, False)
366
+ for dataset in self.data_val
367
+ ]
368
+ elif isinstance(self.data_val, DictConfig):
369
+ return [
370
+ self._dataloader(dataset, self.hparams.val_batch_size, False)
371
+ for _, dataset in self.data_val.items()
372
+ ]
373
+ else:
374
+ return self._dataloader(self.data_val, self.hparams.val_batch_size, False)
375
+
376
+ def test_dataloader(self) -> DataLoader[Any]:
377
+ # support multiple test datasets
378
+ if isinstance(self.data_test, ListConfig):
379
+ return [
380
+ self._dataloader(dataset, self.hparams.test_batch_size, False)
381
+ for dataset in self.data_test
382
+ ]
383
+ elif isinstance(self.data_test, DictConfig):
384
+ return [
385
+ self._dataloader(dataset, self.hparams.test_batch_size, False)
386
+ for _, dataset in self.data_test.items()
387
+ ]
388
+ else:
389
+ return self._dataloader(self.data_test, self.hparams.test_batch_size, False)
390
+
391
+
392
+ if __name__ == "__main__":
393
+ from torchvision.utils import save_image
394
+
395
+ dataset = MultiViewDataset(
396
+ root_dir="/mnt/pfs/data/render_lvis_hzh",
397
+ num_views=17,
398
+ k_near_views=4,
399
+ bg_color="white",
400
+ img_wh=(256, 256),
401
+ ref_view_mode=2,
402
+ sample_views_mode="random",
403
+ caption_path="data/render_lvis_hzh/caption_train.txt",
404
+ use_abs_elevation=True,
405
+ )
406
+
407
+ sample = dataset[0]
408
+ print(sample["view_ids"])
409
+ save_image((sample["image_0"] + 1.0) / 2, "temp_0.jpg", nrow=1, value_range=(0, 1))
410
+ save_image((sample["images"] + 1.0) / 2, "temp.jpg", nrow=8, value_range=(0, 1))
loomnet/models/__init__.py ADDED
File without changes
loomnet/models/__pycache__/__init__.cpython-311.pyc ADDED
Binary file (179 Bytes). View file
 
loomnet/models/__pycache__/mv_model.cpython-311.pyc ADDED
Binary file (12 kB). View file
 
loomnet/models/__pycache__/mv_model_1_to_infinity.cpython-311.pyc ADDED
Binary file (11.5 kB). View file
 
loomnet/models/__pycache__/mv_model_infinity.cpython-311.pyc ADDED
Binary file (12.5 kB). View file
 
loomnet/models/__pycache__/mv_model_triplane.cpython-311.pyc ADDED
Binary file (15.5 kB). View file
 
loomnet/models/__pycache__/mv_model_triplane_with_3d_conv.cpython-311.pyc ADDED
Binary file (15.6 kB). View file
 
loomnet/models/__pycache__/mv_model_v7.cpython-311.pyc ADDED
Binary file (11.1 kB). View file
 
loomnet/models/__pycache__/mv_model_v8.cpython-311.pyc ADDED
Binary file (11.6 kB). View file
 
loomnet/models/__pycache__/mv_model_with_distillation.cpython-311.pyc ADDED
Binary file (11.8 kB). View file
 
loomnet/models/blocks/__init__.py ADDED
File without changes
loomnet/models/blocks/__pycache__/__init__.cpython-311.pyc ADDED
Binary file (182 Bytes). View file
 
loomnet/models/blocks/__pycache__/generic_blocks.cpython-311.pyc ADDED
Binary file (16.5 kB). View file
 
loomnet/models/blocks/__pycache__/pointwise_spatial_transformer.cpython-311.pyc ADDED
Binary file (6.39 kB). View file
 
loomnet/models/blocks/__pycache__/resnet.cpython-311.pyc ADDED
Binary file (18.5 kB). View file
 
loomnet/models/blocks/__pycache__/resnet_fpn.cpython-311.pyc ADDED
Binary file (16.6 kB). View file
 
loomnet/models/blocks/__pycache__/transformer_t.cpython-311.pyc ADDED
Binary file (14.9 kB). View file
 
loomnet/models/blocks/attention.py ADDED
@@ -0,0 +1,739 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Adapted from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention.py
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Callable, Optional
5
+
6
+ import torch
7
+ import torch.nn.functional as F
8
+ from diffusers.configuration_utils import ConfigMixin, register_to_config
9
+ from diffusers.models.attention import AdaLayerNorm, FeedForward
10
+ from diffusers.models.modeling_utils import ModelMixin
11
+ from diffusers.utils import BaseOutput
12
+ from diffusers.utils.import_utils import is_xformers_available
13
+ from einops import rearrange, repeat
14
+ from torch import nn
15
+
16
+
17
+ @dataclass
18
+ class Transformer3DModelOutput(BaseOutput):
19
+ sample: torch.FloatTensor
20
+
21
+
22
+ if is_xformers_available():
23
+ import xformers
24
+ import xformers.ops
25
+ else:
26
+ xformers = None
27
+
28
+
29
+ class CrossAttention(nn.Module):
30
+ r"""
31
+ A cross attention layer.
32
+
33
+ Parameters:
34
+ query_dim (`int`): The number of channels in the query.
35
+ cross_attention_dim (`int`, *optional*):
36
+ The number of channels in the encoder_hidden_states. If not given, defaults to `query_dim`.
37
+ heads (`int`, *optional*, defaults to 8): The number of heads to use for multi-head attention.
38
+ dim_head (`int`, *optional*, defaults to 64): The number of channels in each head.
39
+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
40
+ bias (`bool`, *optional*, defaults to False):
41
+ Set to `True` for the query, key, and value linear layers to contain a bias parameter.
42
+ """
43
+
44
+ def __init__(
45
+ self,
46
+ query_dim: int,
47
+ cross_attention_dim: Optional[int] = None,
48
+ heads: int = 8,
49
+ dim_head: int = 64,
50
+ dropout: float = 0.0,
51
+ bias=False,
52
+ upcast_attention: bool = False,
53
+ upcast_softmax: bool = False,
54
+ added_kv_proj_dim: Optional[int] = None,
55
+ norm_num_groups: Optional[int] = None,
56
+ ):
57
+ super().__init__()
58
+ inner_dim = dim_head * heads
59
+ cross_attention_dim = (
60
+ cross_attention_dim if cross_attention_dim is not None else query_dim
61
+ )
62
+ self.upcast_attention = upcast_attention
63
+ self.upcast_softmax = upcast_softmax
64
+
65
+ self.scale = dim_head**-0.5
66
+
67
+ self.heads = heads
68
+ # for slice_size > 0 the attention score computation
69
+ # is split across the batch axis to save memory
70
+ # You can set slice_size with `set_attention_slice`
71
+ self.sliceable_head_dim = heads
72
+ self._slice_size = None
73
+ self._use_memory_efficient_attention_xformers = False
74
+ self.added_kv_proj_dim = added_kv_proj_dim
75
+
76
+ if norm_num_groups is not None:
77
+ self.group_norm = nn.GroupNorm(
78
+ num_channels=inner_dim,
79
+ num_groups=norm_num_groups,
80
+ eps=1e-5,
81
+ affine=True,
82
+ )
83
+ else:
84
+ self.group_norm = None
85
+
86
+ self.to_q = nn.Linear(query_dim, inner_dim, bias=bias)
87
+ self.to_k = nn.Linear(cross_attention_dim, inner_dim, bias=bias)
88
+ self.to_v = nn.Linear(cross_attention_dim, inner_dim, bias=bias)
89
+ if self.added_kv_proj_dim is not None:
90
+ self.add_k_proj = nn.Linear(added_kv_proj_dim, cross_attention_dim)
91
+ self.add_v_proj = nn.Linear(added_kv_proj_dim, cross_attention_dim)
92
+
93
+ self.to_out = nn.ModuleList([])
94
+ self.to_out.append(nn.Linear(inner_dim, query_dim))
95
+ self.to_out.append(nn.Dropout(dropout))
96
+
97
+ def reshape_heads_to_batch_dim(self, tensor):
98
+ batch_size, seq_len, dim = tensor.shape
99
+ head_size = self.heads
100
+ tensor = tensor.reshape(batch_size, seq_len, head_size, dim // head_size)
101
+ tensor = tensor.permute(0, 2, 1, 3).reshape(
102
+ batch_size * head_size, seq_len, dim // head_size
103
+ )
104
+ return tensor
105
+
106
+ def reshape_batch_dim_to_heads(self, tensor):
107
+ batch_size, seq_len, dim = tensor.shape
108
+ head_size = self.heads
109
+ tensor = tensor.reshape(batch_size // head_size, head_size, seq_len, dim)
110
+ tensor = tensor.permute(0, 2, 1, 3).reshape(
111
+ batch_size // head_size, seq_len, dim * head_size
112
+ )
113
+ return tensor
114
+
115
+ def set_attention_slice(self, slice_size):
116
+ if slice_size is not None and slice_size > self.sliceable_head_dim:
117
+ raise ValueError(
118
+ f"slice_size {slice_size} has to be smaller or equal to {self.sliceable_head_dim}."
119
+ )
120
+
121
+ self._slice_size = slice_size
122
+
123
+ def forward(
124
+ self, hidden_states, encoder_hidden_states=None, attention_mask=None, meta=None
125
+ ):
126
+ batch_size, sequence_length, _ = hidden_states.shape
127
+ encoder_hidden_states = encoder_hidden_states
128
+
129
+ if self.group_norm is not None:
130
+ hidden_states = self.group_norm(hidden_states.transpose(1, 2)).transpose(
131
+ 1, 2
132
+ )
133
+
134
+ query = self.to_q(hidden_states)
135
+ dim = query.shape[-1]
136
+ query = self.reshape_heads_to_batch_dim(query)
137
+
138
+ if self.added_kv_proj_dim is not None:
139
+ key = self.to_k(hidden_states)
140
+ value = self.to_v(hidden_states)
141
+ encoder_hidden_states_key_proj = self.add_k_proj(encoder_hidden_states)
142
+ encoder_hidden_states_value_proj = self.add_v_proj(encoder_hidden_states)
143
+
144
+ key = self.reshape_heads_to_batch_dim(key)
145
+ value = self.reshape_heads_to_batch_dim(value)
146
+ encoder_hidden_states_key_proj = self.reshape_heads_to_batch_dim(
147
+ encoder_hidden_states_key_proj
148
+ )
149
+ encoder_hidden_states_value_proj = self.reshape_heads_to_batch_dim(
150
+ encoder_hidden_states_value_proj
151
+ )
152
+
153
+ key = torch.concat([encoder_hidden_states_key_proj, key], dim=1)
154
+ value = torch.concat([encoder_hidden_states_value_proj, value], dim=1)
155
+ else:
156
+ encoder_hidden_states = (
157
+ encoder_hidden_states
158
+ if encoder_hidden_states is not None
159
+ else hidden_states
160
+ )
161
+ key = self.to_k(encoder_hidden_states)
162
+ value = self.to_v(encoder_hidden_states)
163
+
164
+ key = self.reshape_heads_to_batch_dim(key)
165
+ value = self.reshape_heads_to_batch_dim(value)
166
+
167
+ if attention_mask is not None:
168
+ if attention_mask.shape[-1] != query.shape[1]:
169
+ target_length = query.shape[1]
170
+ attention_mask = F.pad(attention_mask, (0, target_length), value=0.0)
171
+ attention_mask = attention_mask.repeat_interleave(self.heads, dim=0)
172
+
173
+ # attention, what we cannot get enough of
174
+ if self._use_memory_efficient_attention_xformers:
175
+ hidden_states = self._memory_efficient_attention_xformers(
176
+ query, key, value, attention_mask
177
+ )
178
+ # Some versions of xformers return output in fp32, cast it back to the dtype of the input
179
+ hidden_states = hidden_states.to(query.dtype)
180
+ else:
181
+ if self._slice_size is None or query.shape[0] // self._slice_size == 1:
182
+ hidden_states = self._attention(query, key, value, attention_mask)
183
+ else:
184
+ hidden_states = self._sliced_attention(
185
+ query, key, value, sequence_length, dim, attention_mask
186
+ )
187
+
188
+ # linear proj
189
+ hidden_states = self.to_out[0](hidden_states)
190
+
191
+ # dropout
192
+ hidden_states = self.to_out[1](hidden_states)
193
+ return hidden_states
194
+
195
+ def _attention(self, query, key, value, attention_mask=None):
196
+ if self.upcast_attention:
197
+ query = query.float()
198
+ key = key.float()
199
+
200
+ attention_scores = torch.baddbmm(
201
+ torch.empty(
202
+ query.shape[0],
203
+ query.shape[1],
204
+ key.shape[1],
205
+ dtype=query.dtype,
206
+ device=query.device,
207
+ ),
208
+ query,
209
+ key.transpose(-1, -2),
210
+ beta=0,
211
+ alpha=self.scale,
212
+ )
213
+
214
+ if attention_mask is not None:
215
+ attention_scores = attention_scores + attention_mask
216
+
217
+ if self.upcast_softmax:
218
+ attention_scores = attention_scores.float()
219
+
220
+ attention_probs = attention_scores.softmax(dim=-1)
221
+
222
+ # cast back to the original dtype
223
+ attention_probs = attention_probs.to(value.dtype)
224
+
225
+ # compute attention output
226
+ hidden_states = torch.bmm(attention_probs, value)
227
+
228
+ # reshape hidden_states
229
+ hidden_states = self.reshape_batch_dim_to_heads(hidden_states)
230
+ return hidden_states
231
+
232
+ def _sliced_attention(
233
+ self, query, key, value, sequence_length, dim, attention_mask
234
+ ):
235
+ batch_size_attention = query.shape[0]
236
+ hidden_states = torch.zeros(
237
+ (batch_size_attention, sequence_length, dim // self.heads),
238
+ device=query.device,
239
+ dtype=query.dtype,
240
+ )
241
+ slice_size = (
242
+ self._slice_size if self._slice_size is not None else hidden_states.shape[0]
243
+ )
244
+ for i in range(hidden_states.shape[0] // slice_size):
245
+ start_idx = i * slice_size
246
+ end_idx = (i + 1) * slice_size
247
+
248
+ query_slice = query[start_idx:end_idx]
249
+ key_slice = key[start_idx:end_idx]
250
+
251
+ if self.upcast_attention:
252
+ query_slice = query_slice.float()
253
+ key_slice = key_slice.float()
254
+
255
+ attn_slice = torch.baddbmm(
256
+ torch.empty(
257
+ slice_size,
258
+ query.shape[1],
259
+ key.shape[1],
260
+ dtype=query_slice.dtype,
261
+ device=query.device,
262
+ ),
263
+ query_slice,
264
+ key_slice.transpose(-1, -2),
265
+ beta=0,
266
+ alpha=self.scale,
267
+ )
268
+
269
+ if attention_mask is not None:
270
+ attn_slice = attn_slice + attention_mask[start_idx:end_idx]
271
+
272
+ if self.upcast_softmax:
273
+ attn_slice = attn_slice.float()
274
+
275
+ attn_slice = attn_slice.softmax(dim=-1)
276
+
277
+ # cast back to the original dtype
278
+ attn_slice = attn_slice.to(value.dtype)
279
+ attn_slice = torch.bmm(attn_slice, value[start_idx:end_idx])
280
+
281
+ hidden_states[start_idx:end_idx] = attn_slice
282
+
283
+ # reshape hidden_states
284
+ hidden_states = self.reshape_batch_dim_to_heads(hidden_states)
285
+ return hidden_states
286
+
287
+ def _memory_efficient_attention_xformers(self, query, key, value, attention_mask):
288
+ # TODO attention_mask
289
+ query = query.contiguous()
290
+ key = key.contiguous()
291
+ value = value.contiguous()
292
+ hidden_states = xformers.ops.memory_efficient_attention(
293
+ query, key, value, attn_bias=attention_mask
294
+ )
295
+ hidden_states = self.reshape_batch_dim_to_heads(hidden_states)
296
+ return hidden_states
297
+
298
+
299
+ class Transformer3DModel(ModelMixin, ConfigMixin):
300
+ @register_to_config
301
+ def __init__(
302
+ self,
303
+ num_attention_heads: int = 16,
304
+ attention_head_dim: int = 88,
305
+ in_channels: Optional[int] = None,
306
+ num_layers: int = 1,
307
+ dropout: float = 0.0,
308
+ norm_num_groups: int = 32,
309
+ cross_attention_dim: Optional[int] = None,
310
+ attention_bias: bool = False,
311
+ activation_fn: str = "geglu",
312
+ num_embeds_ada_norm: Optional[int] = None,
313
+ use_linear_projection: bool = False,
314
+ only_cross_attention: bool = False,
315
+ upcast_attention: bool = False,
316
+ ):
317
+ super().__init__()
318
+ self.use_linear_projection = use_linear_projection
319
+ self.num_attention_heads = num_attention_heads
320
+ self.attention_head_dim = attention_head_dim
321
+ inner_dim = num_attention_heads * attention_head_dim
322
+
323
+ # Define input layers
324
+ self.in_channels = in_channels
325
+
326
+ self.norm = torch.nn.GroupNorm(
327
+ num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True
328
+ )
329
+ if use_linear_projection:
330
+ self.proj_in = nn.Linear(in_channels, inner_dim)
331
+ else:
332
+ self.proj_in = nn.Conv2d(
333
+ in_channels, inner_dim, kernel_size=1, stride=1, padding=0
334
+ )
335
+
336
+ # Define transformers blocks
337
+ self.transformer_blocks = nn.ModuleList(
338
+ [
339
+ BasicTransformerBlock(
340
+ inner_dim,
341
+ num_attention_heads,
342
+ attention_head_dim,
343
+ dropout=dropout,
344
+ cross_attention_dim=cross_attention_dim,
345
+ activation_fn=activation_fn,
346
+ num_embeds_ada_norm=num_embeds_ada_norm,
347
+ attention_bias=attention_bias,
348
+ only_cross_attention=only_cross_attention,
349
+ upcast_attention=upcast_attention,
350
+ )
351
+ for d in range(num_layers)
352
+ ]
353
+ )
354
+
355
+ # Define output layers
356
+ if use_linear_projection:
357
+ self.proj_out = nn.Linear(in_channels, inner_dim)
358
+ else:
359
+ self.proj_out = nn.Conv2d(
360
+ inner_dim, in_channels, kernel_size=1, stride=1, padding=0
361
+ )
362
+
363
+ def forward(
364
+ self,
365
+ hidden_states: torch.Tensor,
366
+ encoder_hidden_states: Optional[torch.Tensor] = None,
367
+ timestep: Optional[torch.LongTensor] = None,
368
+ return_dict: bool = True,
369
+ meta=None,
370
+ ):
371
+ # Input
372
+ assert (
373
+ hidden_states.dim() == 5
374
+ ), f"Expected hidden_states to have ndim=5, but got ndim={hidden_states.dim()}."
375
+ video_length = hidden_states.shape[2]
376
+ hidden_states = rearrange(hidden_states, "b c f h w -> (b f) c h w")
377
+ batch, _, height, weight = hidden_states.shape
378
+ residual = hidden_states
379
+
380
+ hidden_states = self.norm(hidden_states)
381
+ if not self.use_linear_projection:
382
+ hidden_states = self.proj_in(hidden_states)
383
+ inner_dim = hidden_states.shape[1]
384
+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(
385
+ batch, height * weight, inner_dim
386
+ )
387
+ else:
388
+ inner_dim = hidden_states.shape[1]
389
+ hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(
390
+ batch, height * weight, inner_dim
391
+ )
392
+ hidden_states = self.proj_in(hidden_states)
393
+
394
+ # Blocks
395
+ for block in self.transformer_blocks:
396
+ hidden_states = block(
397
+ hidden_states,
398
+ encoder_hidden_states=encoder_hidden_states,
399
+ timestep=timestep,
400
+ meta=meta,
401
+ )
402
+
403
+ # Output
404
+ if not self.use_linear_projection:
405
+ hidden_states = (
406
+ hidden_states.reshape(batch, height, weight, inner_dim)
407
+ .permute(0, 3, 1, 2)
408
+ .contiguous()
409
+ )
410
+ hidden_states = self.proj_out(hidden_states)
411
+ else:
412
+ hidden_states = self.proj_out(hidden_states)
413
+ hidden_states = (
414
+ hidden_states.reshape(batch, height, weight, inner_dim)
415
+ .permute(0, 3, 1, 2)
416
+ .contiguous()
417
+ )
418
+
419
+ output = hidden_states + residual
420
+
421
+ output = rearrange(output, "(b f) c h w -> b c f h w", f=video_length)
422
+ if not return_dict:
423
+ return (output,)
424
+
425
+ return Transformer3DModelOutput(sample=output)
426
+
427
+
428
+ class BasicTransformerBlock(nn.Module):
429
+ def __init__(
430
+ self,
431
+ dim: int,
432
+ num_attention_heads: int,
433
+ attention_head_dim: int,
434
+ dropout=0.0,
435
+ cross_attention_dim: Optional[int] = None,
436
+ activation_fn: str = "geglu",
437
+ num_embeds_ada_norm: Optional[int] = None,
438
+ attention_bias: bool = False,
439
+ only_cross_attention: bool = False,
440
+ upcast_attention: bool = False,
441
+ ):
442
+ super().__init__()
443
+ self.only_cross_attention = only_cross_attention
444
+ self.use_ada_layer_norm = num_embeds_ada_norm is not None
445
+ # SC-Attn
446
+ self.attn1 = KNearsetAttention(
447
+ query_dim=dim,
448
+ heads=num_attention_heads,
449
+ dim_head=attention_head_dim,
450
+ dropout=dropout,
451
+ bias=attention_bias,
452
+ upcast_attention=upcast_attention,
453
+ )
454
+ # self.attn1= CrossAttention(
455
+ # query_dim=dim,
456
+ # heads=num_attention_heads,
457
+ # dim_head=attention_head_dim,
458
+ # dropout=dropout,
459
+ # bias=attention_bias,
460
+ # upcast_attention=upcast_attention,
461
+ # )
462
+ self.norm1 = (
463
+ AdaLayerNorm(dim, num_embeds_ada_norm)
464
+ if self.use_ada_layer_norm
465
+ else nn.LayerNorm(dim)
466
+ )
467
+
468
+ # Cross-Attn
469
+ if cross_attention_dim is not None:
470
+ self.attn2 = CrossAttention(
471
+ query_dim=dim,
472
+ cross_attention_dim=cross_attention_dim,
473
+ heads=num_attention_heads,
474
+ dim_head=attention_head_dim,
475
+ dropout=dropout,
476
+ bias=attention_bias,
477
+ upcast_attention=upcast_attention,
478
+ )
479
+ else:
480
+ self.attn2 = None
481
+
482
+ if cross_attention_dim is not None:
483
+ self.norm2 = (
484
+ AdaLayerNorm(dim, num_embeds_ada_norm)
485
+ if self.use_ada_layer_norm
486
+ else nn.LayerNorm(dim)
487
+ )
488
+ else:
489
+ self.norm2 = None
490
+
491
+ # Feed-forward
492
+ self.ff = FeedForward(dim, dropout=dropout, activation_fn=activation_fn)
493
+ self.norm3 = nn.LayerNorm(dim)
494
+
495
+ def set_use_memory_efficient_attention_xformers(
496
+ self,
497
+ use_memory_efficient_attention_xformers: bool,
498
+ attention_op: Optional[Callable] = None,
499
+ ):
500
+ if not is_xformers_available():
501
+ print("Here is how to install it")
502
+ raise ModuleNotFoundError(
503
+ "Refer to https://github.com/facebookresearch/xformers for more information on how to install"
504
+ " xformers",
505
+ name="xformers",
506
+ )
507
+ elif not torch.cuda.is_available():
508
+ raise ValueError(
509
+ "torch.cuda.is_available() should be True but is False. xformers' memory efficient attention is only"
510
+ " available for GPU "
511
+ )
512
+ else:
513
+ try:
514
+ # Make sure we can run the memory efficient attention
515
+ _ = xformers.ops.memory_efficient_attention(
516
+ torch.randn((1, 2, 40), device="cuda"),
517
+ torch.randn((1, 2, 40), device="cuda"),
518
+ torch.randn((1, 2, 40), device="cuda"),
519
+ )
520
+ except Exception as e:
521
+ raise e
522
+ self.attn1._use_memory_efficient_attention_xformers = (
523
+ use_memory_efficient_attention_xformers
524
+ )
525
+ if self.attn2 is not None:
526
+ self.attn2._use_memory_efficient_attention_xformers = (
527
+ use_memory_efficient_attention_xformers
528
+ )
529
+ # self.attn_temp._use_memory_efficient_attention_xformers = use_memory_efficient_attention_xformers
530
+
531
+ def forward(
532
+ self,
533
+ hidden_states,
534
+ encoder_hidden_states=None,
535
+ timestep=None,
536
+ attention_mask=None,
537
+ meta=None,
538
+ ):
539
+ # SparseCausal-Attention
540
+ norm_hidden_states = (
541
+ self.norm1(hidden_states, timestep)
542
+ if self.use_ada_layer_norm
543
+ else self.norm1(hidden_states)
544
+ )
545
+
546
+ if self.only_cross_attention:
547
+ hidden_states = (
548
+ self.attn1(
549
+ norm_hidden_states,
550
+ encoder_hidden_states,
551
+ attention_mask=attention_mask,
552
+ )
553
+ + hidden_states
554
+ )
555
+ else:
556
+ hidden_states = (
557
+ self.attn1(
558
+ norm_hidden_states,
559
+ attention_mask=attention_mask,
560
+ meta=meta,
561
+ )
562
+ + hidden_states
563
+ )
564
+ if self.attn2 is not None:
565
+ # Cross-Attention
566
+ norm_hidden_states = (
567
+ self.norm2(hidden_states, timestep)
568
+ if self.use_ada_layer_norm
569
+ else self.norm2(hidden_states)
570
+ )
571
+ hidden_states = (
572
+ self.attn2(
573
+ norm_hidden_states,
574
+ encoder_hidden_states=encoder_hidden_states,
575
+ attention_mask=attention_mask,
576
+ )
577
+ + hidden_states
578
+ )
579
+
580
+ # Feed-forward
581
+ hidden_states = self.ff(self.norm3(hidden_states)) + hidden_states
582
+ return hidden_states
583
+
584
+
585
+ class SparseCausalAttention(CrossAttention):
586
+ def forward(
587
+ self,
588
+ hidden_states,
589
+ encoder_hidden_states=None,
590
+ attention_mask=None,
591
+ video_length=None,
592
+ ):
593
+ batch_size, sequence_length, _ = hidden_states.shape
594
+
595
+ encoder_hidden_states = encoder_hidden_states
596
+
597
+ if self.group_norm is not None:
598
+ hidden_states = self.group_norm(hidden_states.transpose(1, 2)).transpose(
599
+ 1, 2
600
+ )
601
+
602
+ query = self.to_q(hidden_states)
603
+ dim = query.shape[-1]
604
+ query = self.reshape_heads_to_batch_dim(query)
605
+
606
+ if self.added_kv_proj_dim is not None:
607
+ raise NotImplementedError
608
+
609
+ encoder_hidden_states = (
610
+ encoder_hidden_states
611
+ if encoder_hidden_states is not None
612
+ else hidden_states
613
+ )
614
+ key = self.to_k(encoder_hidden_states)
615
+ value = self.to_v(encoder_hidden_states)
616
+
617
+ former_frame_index = torch.arange(video_length) - 1
618
+ former_frame_index[0] = 0
619
+
620
+ key = rearrange(key, "(b f) d c -> b f d c", f=video_length)
621
+ key = torch.cat([key[:, [0] * video_length], key[:, former_frame_index]], dim=2)
622
+ key = rearrange(key, "b f d c -> (b f) d c")
623
+
624
+ value = rearrange(value, "(b f) d c -> b f d c", f=video_length)
625
+ value = torch.cat(
626
+ [value[:, [0] * video_length], value[:, former_frame_index]], dim=2
627
+ )
628
+ value = rearrange(value, "b f d c -> (b f) d c")
629
+
630
+ key = self.reshape_heads_to_batch_dim(key)
631
+ value = self.reshape_heads_to_batch_dim(value)
632
+
633
+ if attention_mask is not None:
634
+ if attention_mask.shape[-1] != query.shape[1]:
635
+ target_length = query.shape[1]
636
+ attention_mask = F.pad(attention_mask, (0, target_length), value=0.0)
637
+ attention_mask = attention_mask.repeat_interleave(self.heads, dim=0)
638
+
639
+ # attention, what we cannot get enough of
640
+ if self._use_memory_efficient_attention_xformers:
641
+ hidden_states = self._memory_efficient_attention_xformers(
642
+ query, key, value, attention_mask
643
+ )
644
+ # Some versions of xformers return output in fp32, cast it back to the dtype of the input
645
+ hidden_states = hidden_states.to(query.dtype)
646
+ else:
647
+ if self._slice_size is None or query.shape[0] // self._slice_size == 1:
648
+ hidden_states = self._attention(query, key, value, attention_mask)
649
+ else:
650
+ hidden_states = self._sliced_attention(
651
+ query, key, value, sequence_length, dim, attention_mask
652
+ )
653
+
654
+ # linear proj
655
+ hidden_states = self.to_out[0](hidden_states)
656
+
657
+ # dropout
658
+ hidden_states = self.to_out[1](hidden_states)
659
+ return hidden_states
660
+
661
+
662
+ class KNearsetAttention(CrossAttention):
663
+ def forward(
664
+ self, hidden_states, encoder_hidden_states=None, attention_mask=None, meta=None
665
+ ):
666
+ video_length = meta["video_length"]
667
+ k_near_indices = meta["k_near_indices"] # B X F X K
668
+ batch_size, sequence_length, _ = hidden_states.shape
669
+
670
+ encoder_hidden_states = encoder_hidden_states
671
+
672
+ if self.group_norm is not None:
673
+ hidden_states = self.group_norm(hidden_states.transpose(1, 2)).transpose(
674
+ 1, 2
675
+ )
676
+
677
+ query = self.to_q(hidden_states)
678
+ dim = query.shape[-1]
679
+ query = self.reshape_heads_to_batch_dim(query)
680
+
681
+ if self.added_kv_proj_dim is not None:
682
+ raise NotImplementedError
683
+
684
+ encoder_hidden_states = (
685
+ encoder_hidden_states
686
+ if encoder_hidden_states is not None
687
+ else hidden_states
688
+ )
689
+ key = self.to_k(encoder_hidden_states)
690
+ value = self.to_v(encoder_hidden_states)
691
+
692
+ key = repeat(key, "(b f) d c -> b f k d c", f=video_length, k=video_length)
693
+
694
+ b, f_views, k_views = k_near_indices.shape
695
+ b_ids = (
696
+ torch.arange(b)[:, None, None].expand(-1, f_views, k_views).to(key.device)
697
+ )
698
+ f_ids = (
699
+ torch.arange(f_views)[None, :, None].expand(b, -1, k_views).to(key.device)
700
+ )
701
+
702
+ k_nearest_key = key[b_ids, f_ids, k_near_indices] # b x f x k x d x c
703
+
704
+ key = rearrange(key, "b f k d c -> (b f) (k d) c")
705
+
706
+ value = repeat(value, "(b f) d c -> b f k d c", f=video_length, k=video_length)
707
+ k_nearest_value = value[b_ids, f_ids, k_near_indices] # b x f x k x d x c
708
+ value = rearrange(value, "b f k d c -> (b f) (k d) c")
709
+
710
+ key = self.reshape_heads_to_batch_dim(key)
711
+ value = self.reshape_heads_to_batch_dim(value)
712
+
713
+ if attention_mask is not None:
714
+ if attention_mask.shape[-1] != query.shape[1]:
715
+ target_length = query.shape[1]
716
+ attention_mask = F.pad(attention_mask, (0, target_length), value=0.0)
717
+ attention_mask = attention_mask.repeat_interleave(self.heads, dim=0)
718
+
719
+ # attention, what we cannot get enough of
720
+ if self._use_memory_efficient_attention_xformers:
721
+ hidden_states = self._memory_efficient_attention_xformers(
722
+ query, key, value, attention_mask
723
+ )
724
+ # Some versions of xformers return output in fp32, cast it back to the dtype of the input
725
+ hidden_states = hidden_states.to(query.dtype)
726
+ else:
727
+ if self._slice_size is None or query.shape[0] // self._slice_size == 1:
728
+ hidden_states = self._attention(query, key, value, attention_mask)
729
+ else:
730
+ hidden_states = self._sliced_attention(
731
+ query, key, value, sequence_length, dim, attention_mask
732
+ )
733
+
734
+ # linear proj
735
+ hidden_states = self.to_out[0](hidden_states)
736
+
737
+ # dropout
738
+ hidden_states = self.to_out[1](hidden_states)
739
+ return hidden_states
loomnet/models/blocks/attention_processor.py ADDED
@@ -0,0 +1,871 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py
2
+ import torch
3
+ import torch.nn as nn
4
+ import torch.nn.functional as F
5
+ from einops import rearrange, repeat
6
+
7
+
8
+ class AttnProcessor(nn.Module):
9
+ r"""
10
+ Default processor for performing attention-related computations.
11
+ """
12
+
13
+ def __init__(
14
+ self,
15
+ hidden_size=None,
16
+ cross_attention_dim=None,
17
+ ):
18
+ super().__init__()
19
+
20
+ def __call__(
21
+ self,
22
+ attn,
23
+ hidden_states,
24
+ encoder_hidden_states=None,
25
+ attention_mask=None,
26
+ temb=None,
27
+ ):
28
+ residual = hidden_states
29
+
30
+ if attn.spatial_norm is not None:
31
+ hidden_states = attn.spatial_norm(hidden_states, temb)
32
+
33
+ input_ndim = hidden_states.ndim
34
+
35
+ if input_ndim == 4:
36
+ batch_size, channel, height, width = hidden_states.shape
37
+ hidden_states = hidden_states.view(
38
+ batch_size, channel, height * width
39
+ ).transpose(1, 2)
40
+
41
+ batch_size, sequence_length, _ = (
42
+ hidden_states.shape
43
+ if encoder_hidden_states is None
44
+ else encoder_hidden_states.shape
45
+ )
46
+ attention_mask = attn.prepare_attention_mask(
47
+ attention_mask, sequence_length, batch_size
48
+ )
49
+
50
+ if attn.group_norm is not None:
51
+ hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(
52
+ 1, 2
53
+ )
54
+
55
+ query = attn.to_q(hidden_states)
56
+
57
+ if encoder_hidden_states is None:
58
+ encoder_hidden_states = hidden_states
59
+ elif attn.norm_cross:
60
+ encoder_hidden_states = attn.norm_encoder_hidden_states(
61
+ encoder_hidden_states
62
+ )
63
+
64
+ key = attn.to_k(encoder_hidden_states)
65
+ value = attn.to_v(encoder_hidden_states)
66
+
67
+ query = attn.head_to_batch_dim(query)
68
+ key = attn.head_to_batch_dim(key)
69
+ value = attn.head_to_batch_dim(value)
70
+
71
+ attention_probs = attn.get_attention_scores(query, key, attention_mask)
72
+ hidden_states = torch.bmm(attention_probs, value)
73
+ hidden_states = attn.batch_to_head_dim(hidden_states)
74
+
75
+ # linear proj
76
+ hidden_states = attn.to_out[0](hidden_states)
77
+ # dropout
78
+ hidden_states = attn.to_out[1](hidden_states)
79
+
80
+ if input_ndim == 4:
81
+ hidden_states = hidden_states.transpose(-1, -2).reshape(
82
+ batch_size, channel, height, width
83
+ )
84
+
85
+ if attn.residual_connection:
86
+ hidden_states = hidden_states + residual
87
+
88
+ hidden_states = hidden_states / attn.rescale_output_factor
89
+
90
+ return hidden_states
91
+
92
+
93
+ class IPAttnProcessor(nn.Module):
94
+ r"""
95
+ Attention processor for IP-Adapater.
96
+ Args:
97
+ hidden_size (`int`):
98
+ The hidden size of the attention layer.
99
+ cross_attention_dim (`int`):
100
+ The number of channels in the `encoder_hidden_states`.
101
+ scale (`float`, defaults to 1.0):
102
+ the weight scale of image prompt.
103
+ num_tokens (`int`, defaults to 4 when do ip_adapter_plus it should be 16):
104
+ The context length of the image features.
105
+ """
106
+
107
+ def __init__(self, hidden_size, cross_attention_dim=None, scale=1.0, num_tokens=4):
108
+ super().__init__()
109
+
110
+ self.hidden_size = hidden_size
111
+ self.cross_attention_dim = cross_attention_dim
112
+ self.scale = scale
113
+ self.num_tokens = num_tokens
114
+
115
+ self.to_k_ip = nn.Linear(
116
+ cross_attention_dim or hidden_size, hidden_size, bias=False
117
+ )
118
+ self.to_v_ip = nn.Linear(
119
+ cross_attention_dim or hidden_size, hidden_size, bias=False
120
+ )
121
+
122
+ def __call__(
123
+ self,
124
+ attn,
125
+ hidden_states,
126
+ encoder_hidden_states=None,
127
+ attention_mask=None,
128
+ temb=None,
129
+ ):
130
+ residual = hidden_states
131
+
132
+ if attn.spatial_norm is not None:
133
+ hidden_states = attn.spatial_norm(hidden_states, temb)
134
+
135
+ input_ndim = hidden_states.ndim
136
+
137
+ if input_ndim == 4:
138
+ batch_size, channel, height, width = hidden_states.shape
139
+ hidden_states = hidden_states.view(
140
+ batch_size, channel, height * width
141
+ ).transpose(1, 2)
142
+
143
+ batch_size, sequence_length, _ = (
144
+ hidden_states.shape
145
+ if encoder_hidden_states is None
146
+ else encoder_hidden_states.shape
147
+ )
148
+ attention_mask = attn.prepare_attention_mask(
149
+ attention_mask, sequence_length, batch_size
150
+ )
151
+
152
+ if attn.group_norm is not None:
153
+ hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(
154
+ 1, 2
155
+ )
156
+
157
+ query = attn.to_q(hidden_states)
158
+
159
+ if encoder_hidden_states is None:
160
+ encoder_hidden_states = hidden_states
161
+ else:
162
+ # get encoder_hidden_states, ip_hidden_states
163
+ end_pos = encoder_hidden_states.shape[1] - self.num_tokens
164
+ encoder_hidden_states, ip_hidden_states = (
165
+ encoder_hidden_states[:, :end_pos, :],
166
+ encoder_hidden_states[:, end_pos:, :],
167
+ )
168
+ if attn.norm_cross:
169
+ encoder_hidden_states = attn.norm_encoder_hidden_states(
170
+ encoder_hidden_states
171
+ )
172
+
173
+ key = attn.to_k(encoder_hidden_states)
174
+ value = attn.to_v(encoder_hidden_states)
175
+
176
+ query = attn.head_to_batch_dim(query)
177
+ key = attn.head_to_batch_dim(key)
178
+ value = attn.head_to_batch_dim(value)
179
+
180
+ attention_probs = attn.get_attention_scores(query, key, attention_mask)
181
+ hidden_states = torch.bmm(attention_probs, value)
182
+ hidden_states = attn.batch_to_head_dim(hidden_states)
183
+
184
+ # for ip-adapter
185
+ ip_key = self.to_k_ip(ip_hidden_states)
186
+ ip_value = self.to_v_ip(ip_hidden_states)
187
+
188
+ ip_key = attn.head_to_batch_dim(ip_key)
189
+ ip_value = attn.head_to_batch_dim(ip_value)
190
+
191
+ ip_attention_probs = attn.get_attention_scores(query, ip_key, None)
192
+ ip_hidden_states = torch.bmm(ip_attention_probs, ip_value)
193
+ ip_hidden_states = attn.batch_to_head_dim(ip_hidden_states)
194
+
195
+ hidden_states = hidden_states + self.scale * ip_hidden_states
196
+
197
+ # linear proj
198
+ hidden_states = attn.to_out[0](hidden_states)
199
+ # dropout
200
+ hidden_states = attn.to_out[1](hidden_states)
201
+
202
+ if input_ndim == 4:
203
+ hidden_states = hidden_states.transpose(-1, -2).reshape(
204
+ batch_size, channel, height, width
205
+ )
206
+
207
+ if attn.residual_connection:
208
+ hidden_states = hidden_states + residual
209
+
210
+ hidden_states = hidden_states / attn.rescale_output_factor
211
+
212
+ return hidden_states
213
+
214
+
215
+ class AttnProcessor2_0(torch.nn.Module):
216
+ r"""
217
+ Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0).
218
+ """
219
+
220
+ def __init__(
221
+ self,
222
+ hidden_size=None,
223
+ cross_attention_dim=None,
224
+ frames=8,
225
+ ):
226
+ super().__init__()
227
+ if not hasattr(F, "scaled_dot_product_attention"):
228
+ raise ImportError(
229
+ "AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0."
230
+ )
231
+ self.frames = frames
232
+
233
+ def __call__(
234
+ self,
235
+ attn,
236
+ hidden_states,
237
+ encoder_hidden_states=None,
238
+ attention_mask=None,
239
+ temb=None,
240
+ ):
241
+ residual = hidden_states
242
+
243
+ if attn.spatial_norm is not None:
244
+ hidden_states = attn.spatial_norm(hidden_states, temb)
245
+
246
+ input_ndim = hidden_states.ndim
247
+ hidden_states = rearrange(
248
+ hidden_states, "(b f) n c -> b (f n) c", f=self.frames
249
+ )
250
+ if input_ndim == 4:
251
+ batch_size, channel, height, width = hidden_states.shape
252
+ hidden_states = hidden_states.view(
253
+ batch_size, channel, height * width
254
+ ).transpose(1, 2)
255
+
256
+ batch_size, sequence_length, _ = (
257
+ hidden_states.shape
258
+ if encoder_hidden_states is None
259
+ else encoder_hidden_states.shape
260
+ )
261
+
262
+ if attention_mask is not None:
263
+ attention_mask = attn.prepare_attention_mask(
264
+ attention_mask, sequence_length, batch_size
265
+ )
266
+ # scaled_dot_product_attention expects attention_mask shape to be
267
+ # (batch, heads, source_length, target_length)
268
+ attention_mask = attention_mask.view(
269
+ batch_size, attn.heads, -1, attention_mask.shape[-1]
270
+ )
271
+
272
+ if attn.group_norm is not None:
273
+ hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(
274
+ 1, 2
275
+ )
276
+
277
+ query = attn.to_q(hidden_states)
278
+
279
+ if encoder_hidden_states is None:
280
+ encoder_hidden_states = hidden_states
281
+ elif attn.norm_cross:
282
+ encoder_hidden_states = attn.norm_encoder_hidden_states(
283
+ encoder_hidden_states
284
+ )
285
+
286
+ key = attn.to_k(encoder_hidden_states)
287
+ value = attn.to_v(encoder_hidden_states)
288
+
289
+ inner_dim = key.shape[-1]
290
+ head_dim = inner_dim // attn.heads
291
+
292
+ query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
293
+
294
+ key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
295
+ value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
296
+
297
+ # the output of sdp = (batch, num_heads, seq_len, head_dim)
298
+ # TODO: add support for attn.scale when we move to Torch 2.1
299
+ hidden_states = F.scaled_dot_product_attention(
300
+ query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
301
+ )
302
+
303
+ hidden_states = hidden_states.transpose(1, 2).reshape(
304
+ batch_size, -1, attn.heads * head_dim
305
+ )
306
+ hidden_states = hidden_states.to(query.dtype)
307
+
308
+ # linear proj
309
+ hidden_states = attn.to_out[0](hidden_states)
310
+ # dropout
311
+ hidden_states = attn.to_out[1](hidden_states)
312
+
313
+ if input_ndim == 4:
314
+ hidden_states = hidden_states.transpose(-1, -2).reshape(
315
+ batch_size, channel, height, width
316
+ )
317
+
318
+ if attn.residual_connection:
319
+ hidden_states = hidden_states + residual
320
+
321
+ hidden_states = hidden_states / attn.rescale_output_factor
322
+ hidden_states = rearrange(
323
+ hidden_states, "b (f n) c -> (b f) n c", f=self.frames
324
+ )
325
+ return hidden_states
326
+
327
+
328
+ class IPAttnProcessor2_0(torch.nn.Module):
329
+ r"""
330
+ Attention processor for IP-Adapater for PyTorch 2.0.
331
+ Args:
332
+ hidden_size (`int`):
333
+ The hidden size of the attention layer.
334
+ cross_attention_dim (`int`):
335
+ The number of channels in the `encoder_hidden_states`.
336
+ scale (`float`, defaults to 1.0):
337
+ the weight scale of image prompt.
338
+ num_tokens (`int`, defaults to 4 when do ip_adapter_plus it should be 16):
339
+ The context length of the image features.
340
+ """
341
+
342
+ def __init__(self, hidden_size, cross_attention_dim=None, scale=1.0, num_tokens=4):
343
+ super().__init__()
344
+
345
+ if not hasattr(F, "scaled_dot_product_attention"):
346
+ raise ImportError(
347
+ "AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0."
348
+ )
349
+
350
+ self.hidden_size = hidden_size
351
+ self.cross_attention_dim = cross_attention_dim
352
+ self.scale = scale
353
+ self.num_tokens = num_tokens
354
+
355
+ self.to_k_ip = nn.Linear(
356
+ cross_attention_dim or hidden_size, hidden_size, bias=False
357
+ )
358
+ self.to_v_ip = nn.Linear(
359
+ cross_attention_dim or hidden_size, hidden_size, bias=False
360
+ )
361
+
362
+ def __call__(
363
+ self,
364
+ attn,
365
+ hidden_states,
366
+ encoder_hidden_states=None,
367
+ attention_mask=None,
368
+ temb=None,
369
+ ):
370
+ residual = hidden_states
371
+
372
+ if attn.spatial_norm is not None:
373
+ hidden_states = attn.spatial_norm(hidden_states, temb)
374
+
375
+ input_ndim = hidden_states.ndim
376
+
377
+ if input_ndim == 4:
378
+ batch_size, channel, height, width = hidden_states.shape
379
+ hidden_states = hidden_states.view(
380
+ batch_size, channel, height * width
381
+ ).transpose(1, 2)
382
+
383
+ batch_size, sequence_length, _ = (
384
+ hidden_states.shape
385
+ if encoder_hidden_states is None
386
+ else encoder_hidden_states.shape
387
+ )
388
+
389
+ if attention_mask is not None:
390
+ attention_mask = attn.prepare_attention_mask(
391
+ attention_mask, sequence_length, batch_size
392
+ )
393
+ # scaled_dot_product_attention expects attention_mask shape to be
394
+ # (batch, heads, source_length, target_length)
395
+ attention_mask = attention_mask.view(
396
+ batch_size, attn.heads, -1, attention_mask.shape[-1]
397
+ )
398
+
399
+ if attn.group_norm is not None:
400
+ hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(
401
+ 1, 2
402
+ )
403
+
404
+ query = attn.to_q(hidden_states)
405
+
406
+ if encoder_hidden_states is None:
407
+ encoder_hidden_states = hidden_states
408
+ else:
409
+ # get encoder_hidden_states, ip_hidden_states
410
+ end_pos = encoder_hidden_states.shape[1] - self.num_tokens
411
+ encoder_hidden_states, ip_hidden_states = (
412
+ encoder_hidden_states[:, :end_pos, :],
413
+ encoder_hidden_states[:, end_pos:, :],
414
+ )
415
+ if attn.norm_cross:
416
+ encoder_hidden_states = attn.norm_encoder_hidden_states(
417
+ encoder_hidden_states
418
+ )
419
+
420
+ key = attn.to_k(encoder_hidden_states)
421
+ value = attn.to_v(encoder_hidden_states)
422
+
423
+ inner_dim = key.shape[-1]
424
+ head_dim = inner_dim // attn.heads
425
+
426
+ query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
427
+
428
+ key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
429
+ value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
430
+
431
+ # the output of sdp = (batch, num_heads, seq_len, head_dim)
432
+ # TODO: add support for attn.scale when we move to Torch 2.1
433
+ hidden_states = F.scaled_dot_product_attention(
434
+ query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
435
+ )
436
+
437
+ hidden_states = hidden_states.transpose(1, 2).reshape(
438
+ batch_size, -1, attn.heads * head_dim
439
+ )
440
+ hidden_states = hidden_states.to(query.dtype)
441
+
442
+ # for ip-adapter
443
+ ip_key = self.to_k_ip(ip_hidden_states)
444
+ ip_value = self.to_v_ip(ip_hidden_states)
445
+
446
+ ip_key = ip_key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
447
+ ip_value = ip_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
448
+
449
+ # the output of sdp = (batch, num_heads, seq_len, head_dim)
450
+ # TODO: add support for attn.scale when we move to Torch 2.1
451
+ ip_hidden_states = F.scaled_dot_product_attention(
452
+ query, ip_key, ip_value, attn_mask=None, dropout_p=0.0, is_causal=False
453
+ )
454
+
455
+ ip_hidden_states = ip_hidden_states.transpose(1, 2).reshape(
456
+ batch_size, -1, attn.heads * head_dim
457
+ )
458
+ ip_hidden_states = ip_hidden_states.to(query.dtype)
459
+
460
+ hidden_states = hidden_states + self.scale * ip_hidden_states
461
+
462
+ # linear proj
463
+ hidden_states = attn.to_out[0](hidden_states)
464
+ # dropout
465
+ hidden_states = attn.to_out[1](hidden_states)
466
+
467
+ if input_ndim == 4:
468
+ hidden_states = hidden_states.transpose(-1, -2).reshape(
469
+ batch_size, channel, height, width
470
+ )
471
+
472
+ if attn.residual_connection:
473
+ hidden_states = hidden_states + residual
474
+
475
+ hidden_states = hidden_states / attn.rescale_output_factor
476
+
477
+ return hidden_states
478
+
479
+
480
+ ## for controlnet
481
+ class CNAttnProcessor:
482
+ r"""
483
+ Default processor for performing attention-related computations.
484
+ """
485
+
486
+ def __init__(self, num_tokens=4):
487
+ self.num_tokens = num_tokens
488
+
489
+ def __call__(
490
+ self,
491
+ attn,
492
+ hidden_states,
493
+ encoder_hidden_states=None,
494
+ attention_mask=None,
495
+ temb=None,
496
+ ):
497
+ residual = hidden_states
498
+
499
+ if attn.spatial_norm is not None:
500
+ hidden_states = attn.spatial_norm(hidden_states, temb)
501
+
502
+ input_ndim = hidden_states.ndim
503
+
504
+ if input_ndim == 4:
505
+ batch_size, channel, height, width = hidden_states.shape
506
+ hidden_states = hidden_states.view(
507
+ batch_size, channel, height * width
508
+ ).transpose(1, 2)
509
+
510
+ batch_size, sequence_length, _ = (
511
+ hidden_states.shape
512
+ if encoder_hidden_states is None
513
+ else encoder_hidden_states.shape
514
+ )
515
+ attention_mask = attn.prepare_attention_mask(
516
+ attention_mask, sequence_length, batch_size
517
+ )
518
+
519
+ if attn.group_norm is not None:
520
+ hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(
521
+ 1, 2
522
+ )
523
+
524
+ query = attn.to_q(hidden_states)
525
+
526
+ if encoder_hidden_states is None:
527
+ encoder_hidden_states = hidden_states
528
+ else:
529
+ end_pos = encoder_hidden_states.shape[1] - self.num_tokens
530
+ encoder_hidden_states = encoder_hidden_states[:, :end_pos] # only use text
531
+ if attn.norm_cross:
532
+ encoder_hidden_states = attn.norm_encoder_hidden_states(
533
+ encoder_hidden_states
534
+ )
535
+
536
+ key = attn.to_k(encoder_hidden_states)
537
+ value = attn.to_v(encoder_hidden_states)
538
+
539
+ query = attn.head_to_batch_dim(query)
540
+ key = attn.head_to_batch_dim(key)
541
+ value = attn.head_to_batch_dim(value)
542
+
543
+ attention_probs = attn.get_attention_scores(query, key, attention_mask)
544
+ hidden_states = torch.bmm(attention_probs, value)
545
+ hidden_states = attn.batch_to_head_dim(hidden_states)
546
+
547
+ # linear proj
548
+ hidden_states = attn.to_out[0](hidden_states)
549
+ # dropout
550
+ hidden_states = attn.to_out[1](hidden_states)
551
+
552
+ if input_ndim == 4:
553
+ hidden_states = hidden_states.transpose(-1, -2).reshape(
554
+ batch_size, channel, height, width
555
+ )
556
+
557
+ if attn.residual_connection:
558
+ hidden_states = hidden_states + residual
559
+
560
+ hidden_states = hidden_states / attn.rescale_output_factor
561
+
562
+ return hidden_states
563
+
564
+
565
+ class CNAttnProcessor2_0:
566
+ r"""
567
+ Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0).
568
+ """
569
+
570
+ def __init__(self, num_tokens=4):
571
+ if not hasattr(F, "scaled_dot_product_attention"):
572
+ raise ImportError(
573
+ "AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0."
574
+ )
575
+ self.num_tokens = num_tokens
576
+
577
+ def __call__(
578
+ self,
579
+ attn,
580
+ hidden_states,
581
+ encoder_hidden_states=None,
582
+ attention_mask=None,
583
+ temb=None,
584
+ ):
585
+ residual = hidden_states
586
+
587
+ if attn.spatial_norm is not None:
588
+ hidden_states = attn.spatial_norm(hidden_states, temb)
589
+
590
+ input_ndim = hidden_states.ndim
591
+
592
+ if input_ndim == 4:
593
+ batch_size, channel, height, width = hidden_states.shape
594
+ hidden_states = hidden_states.view(
595
+ batch_size, channel, height * width
596
+ ).transpose(1, 2)
597
+
598
+ batch_size, sequence_length, _ = (
599
+ hidden_states.shape
600
+ if encoder_hidden_states is None
601
+ else encoder_hidden_states.shape
602
+ )
603
+
604
+ if attention_mask is not None:
605
+ attention_mask = attn.prepare_attention_mask(
606
+ attention_mask, sequence_length, batch_size
607
+ )
608
+ # scaled_dot_product_attention expects attention_mask shape to be
609
+ # (batch, heads, source_length, target_length)
610
+ attention_mask = attention_mask.view(
611
+ batch_size, attn.heads, -1, attention_mask.shape[-1]
612
+ )
613
+
614
+ if attn.group_norm is not None:
615
+ hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(
616
+ 1, 2
617
+ )
618
+
619
+ query = attn.to_q(hidden_states)
620
+
621
+ if encoder_hidden_states is None:
622
+ encoder_hidden_states = hidden_states
623
+ else:
624
+ end_pos = encoder_hidden_states.shape[1] - self.num_tokens
625
+ encoder_hidden_states = encoder_hidden_states[:, :end_pos] # only use text
626
+ if attn.norm_cross:
627
+ encoder_hidden_states = attn.norm_encoder_hidden_states(
628
+ encoder_hidden_states
629
+ )
630
+
631
+ key = attn.to_k(encoder_hidden_states)
632
+ value = attn.to_v(encoder_hidden_states)
633
+
634
+ inner_dim = key.shape[-1]
635
+ head_dim = inner_dim // attn.heads
636
+
637
+ query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
638
+
639
+ key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
640
+ value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
641
+
642
+ # the output of sdp = (batch, num_heads, seq_len, head_dim)
643
+ # TODO: add support for attn.scale when we move to Torch 2.1
644
+ hidden_states = F.scaled_dot_product_attention(
645
+ query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
646
+ )
647
+
648
+ hidden_states = hidden_states.transpose(1, 2).reshape(
649
+ batch_size, -1, attn.heads * head_dim
650
+ )
651
+ hidden_states = hidden_states.to(query.dtype)
652
+
653
+ # linear proj
654
+ hidden_states = attn.to_out[0](hidden_states)
655
+ # dropout
656
+ hidden_states = attn.to_out[1](hidden_states)
657
+
658
+ if input_ndim == 4:
659
+ hidden_states = hidden_states.transpose(-1, -2).reshape(
660
+ batch_size, channel, height, width
661
+ )
662
+
663
+ if attn.residual_connection:
664
+ hidden_states = hidden_states + residual
665
+
666
+ hidden_states = hidden_states / attn.rescale_output_factor
667
+
668
+ return hidden_states
669
+
670
+
671
+ from typing import Callable, Optional, Union
672
+
673
+ import xformers
674
+ import xformers.ops
675
+
676
+
677
+ class XFormersAttnProcessor:
678
+ r"""
679
+ Processor for implementing memory efficient attention using xFormers.
680
+
681
+ Args:
682
+ attention_op (`Callable`, *optional*, defaults to `None`):
683
+ The base
684
+ [operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to
685
+ use as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best
686
+ operator.
687
+ """
688
+
689
+ def __init__(self, attention_op=None):
690
+ self.attention_op = attention_op
691
+
692
+ def __call__(
693
+ self,
694
+ attn,
695
+ hidden_states: torch.FloatTensor,
696
+ encoder_hidden_states: Optional[torch.FloatTensor] = None,
697
+ attention_mask: Optional[torch.FloatTensor] = None,
698
+ temb: Optional[torch.FloatTensor] = None,
699
+ scale: float = 1.0,
700
+ ):
701
+ residual = hidden_states
702
+
703
+ if attn.spatial_norm is not None:
704
+ hidden_states = attn.spatial_norm(hidden_states, temb)
705
+
706
+ input_ndim = hidden_states.ndim
707
+
708
+ if input_ndim == 4:
709
+ batch_size, channel, height, width = hidden_states.shape
710
+ hidden_states = hidden_states.view(
711
+ batch_size, channel, height * width
712
+ ).transpose(1, 2)
713
+
714
+ batch_size, key_tokens, _ = (
715
+ hidden_states.shape
716
+ if encoder_hidden_states is None
717
+ else encoder_hidden_states.shape
718
+ )
719
+
720
+ attention_mask = attn.prepare_attention_mask(
721
+ attention_mask, key_tokens, batch_size
722
+ )
723
+ if attention_mask is not None:
724
+ _, query_tokens, _ = hidden_states.shape
725
+ attention_mask = attention_mask.expand(-1, query_tokens, -1)
726
+
727
+ if attn.group_norm is not None:
728
+ hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(
729
+ 1, 2
730
+ )
731
+
732
+ query = attn.to_q(hidden_states, scale=scale)
733
+
734
+ if encoder_hidden_states is None:
735
+ encoder_hidden_states = hidden_states
736
+ elif attn.norm_cross:
737
+ encoder_hidden_states = attn.norm_encoder_hidden_states(
738
+ encoder_hidden_states
739
+ )
740
+
741
+ key = attn.to_k(encoder_hidden_states, scale=scale)
742
+ value = attn.to_v(encoder_hidden_states, scale=scale)
743
+
744
+ query = attn.head_to_batch_dim(query).contiguous()
745
+ key = attn.head_to_batch_dim(key).contiguous()
746
+ value = attn.head_to_batch_dim(value).contiguous()
747
+
748
+ hidden_states = xformers.ops.memory_efficient_attention(
749
+ query,
750
+ key,
751
+ value,
752
+ attn_bias=attention_mask,
753
+ op=self.attention_op,
754
+ scale=attn.scale,
755
+ )
756
+ hidden_states = hidden_states.to(query.dtype)
757
+ hidden_states = attn.batch_to_head_dim(hidden_states)
758
+
759
+ # linear proj
760
+ hidden_states = attn.to_out[0](hidden_states, scale=scale)
761
+ # dropout
762
+ hidden_states = attn.to_out[1](hidden_states)
763
+
764
+ if input_ndim == 4:
765
+ hidden_states = hidden_states.transpose(-1, -2).reshape(
766
+ batch_size, channel, height, width
767
+ )
768
+
769
+ if attn.residual_connection:
770
+ hidden_states = hidden_states + residual
771
+
772
+ hidden_states = hidden_states / attn.rescale_output_factor
773
+
774
+ return hidden_states
775
+
776
+
777
+ class AttnProcessor3D(nn.Module):
778
+ r"""
779
+ Default processor for performing attention-related computations.
780
+ """
781
+
782
+ def __init__(self, attention_op=None, video_length=16):
783
+ super().__init__()
784
+ self.video_length = video_length
785
+ self.attention_op = attention_op
786
+
787
+ def __call__(
788
+ self,
789
+ attn,
790
+ hidden_states,
791
+ encoder_hidden_states=None,
792
+ attention_mask=None,
793
+ temb=None,
794
+ ):
795
+ residual = hidden_states
796
+
797
+ if attn.spatial_norm is not None:
798
+ hidden_states = attn.spatial_norm(hidden_states, temb)
799
+
800
+ input_ndim = hidden_states.ndim
801
+
802
+ if input_ndim == 4:
803
+ batch_size, channel, height, width = hidden_states.shape
804
+ hidden_states = hidden_states.view(
805
+ batch_size, channel, height * width
806
+ ).transpose(1, 2)
807
+
808
+ batch_size, sequence_length, _ = (
809
+ hidden_states.shape
810
+ if encoder_hidden_states is None
811
+ else encoder_hidden_states.shape
812
+ )
813
+ attention_mask = attn.prepare_attention_mask(
814
+ attention_mask, sequence_length, batch_size
815
+ )
816
+
817
+ if attn.group_norm is not None:
818
+ hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(
819
+ 1, 2
820
+ )
821
+
822
+ query = attn.to_q(hidden_states)
823
+
824
+ if encoder_hidden_states is None:
825
+ encoder_hidden_states = hidden_states
826
+ elif attn.norm_cross:
827
+ encoder_hidden_states = attn.norm_encoder_hidden_states(
828
+ encoder_hidden_states
829
+ )
830
+
831
+ key = attn.to_k(encoder_hidden_states)
832
+ value = attn.to_v(encoder_hidden_states)
833
+
834
+ video_length = self.video_length
835
+ query = rearrange(query, "(b f) d c -> b (f d) c", f=video_length)
836
+ key = rearrange(key, "(b f) d c -> b (f d) c", f=video_length)
837
+ value = rearrange(value, "(b f) d c -> b (f d) c", f=video_length)
838
+
839
+ query = attn.head_to_batch_dim(query).contiguous()
840
+ key = attn.head_to_batch_dim(key).contiguous()
841
+ value = attn.head_to_batch_dim(value).contiguous()
842
+
843
+ hidden_states = xformers.ops.memory_efficient_attention(
844
+ query,
845
+ key,
846
+ value,
847
+ attn_bias=attention_mask,
848
+ op=self.attention_op,
849
+ scale=attn.scale,
850
+ )
851
+ hidden_states = hidden_states.to(query.dtype)
852
+ hidden_states = attn.batch_to_head_dim(hidden_states)
853
+
854
+ # linear proj
855
+ hidden_states = attn.to_out[0](hidden_states)
856
+ # dropout
857
+ hidden_states = attn.to_out[1](hidden_states)
858
+
859
+ if input_ndim == 4:
860
+ hidden_states = hidden_states.transpose(-1, -2).reshape(
861
+ batch_size, channel, height, width
862
+ )
863
+
864
+ if attn.residual_connection:
865
+ hidden_states = hidden_states + residual
866
+
867
+ hidden_states = hidden_states / attn.rescale_output_factor
868
+ hidden_states = rearrange(
869
+ hidden_states, "b (f d) c -> (b f) d c", f=video_length
870
+ )
871
+ return hidden_states
loomnet/models/blocks/generic_blocks.py ADDED
@@ -0,0 +1,295 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch.nn.functional as F
2
+ import torch
3
+ import torch.nn as nn
4
+ from typing import Union, Callable, Optional
5
+ import torch
6
+ from torch import Tensor
7
+
8
+ class ResnetBlock2D(nn.Module):
9
+ def __init__(self, in_channels, middle_channels, group_channels, eps=1e-5, kernel_size=3, stride =1, padding=1, dropout=0):
10
+ super().__init__()
11
+
12
+ self.norm1 = torch.nn.GroupNorm(num_groups=group_channels, num_channels=in_channels, eps=eps, affine=True)
13
+ self.nonlinearity = nn.SiLU()
14
+ self.conv1 = nn.Conv2d(in_channels, middle_channels, kernel_size=kernel_size, stride=stride, padding=padding)
15
+ self.dropout = torch.nn.Dropout(dropout)
16
+ self.conv2 = nn.Conv2d(middle_channels, in_channels, kernel_size=1, stride=stride, padding=padding)
17
+
18
+ def forward(self, x):
19
+
20
+ y = self.norm1(x)
21
+ y = self.nonlinearity(y)
22
+ y = self.conv1(y)
23
+ y_up = F.interpolate(y, size=(x.shape[-2], x.shape[-1]), mode="bilinear")
24
+
25
+ y2 = x + y_up
26
+ y2 = self.dropout(y2)
27
+
28
+ y3 = self.conv2(y2)
29
+
30
+ y4 = y3 + y2
31
+
32
+ return y4
33
+
34
+
35
+
36
+
37
+
38
+
39
+
40
+ class UpSample2D(nn.Module):
41
+ def __init__(self, in_channels):
42
+ super().__init__()
43
+
44
+ self.scale_factor = 2
45
+ self.conv = nn.Conv2d(in_channels, in_channels, kernel_size=(3,3), padding=(1,1))
46
+
47
+ def forward(self, x):
48
+ x = F.interpolate(x, scale_factor=self.scale_factor, mode="nearest")
49
+ x = self.conv(x)
50
+ return x
51
+
52
+
53
+
54
+ class UpSampleDecoder(nn.Module):
55
+ def __init__(self, in_channels, middle_channels, up_factor = 1, group_channels = 32, eps=1e-5, kernel_size=3, dropout=0):
56
+ super().__init__()
57
+
58
+ assert in_channels % group_channels == 0, "in_channels must be divisible by group_channels"
59
+
60
+ self.blocks = nn.ModuleList()
61
+
62
+ for _ in range(up_factor):
63
+ self.blocks.append(nn.Sequential(
64
+ ResnetBlock2D(in_channels, middle_channels, group_channels, eps, kernel_size, dropout),
65
+ UpSample2D(in_channels))
66
+ )
67
+
68
+
69
+ def forward(self, x):
70
+
71
+ for block in self.blocks:
72
+ x = block(x)
73
+
74
+ return x
75
+
76
+
77
+ class GammaModulator(nn.Module):
78
+ def __init__(self, dim, init_values, mlp_hidden_dim, dropout=0.0):
79
+ super().__init__()
80
+
81
+ #self.norm1 = nn.LayerNorm(dim)
82
+ self.norm2 = nn.LayerNorm(dim)
83
+ #self.layer_scale1 = LayerScale(dim, init_values=init_values)
84
+ self.layer_scale2 = LayerScale(dim, init_values=init_values)
85
+
86
+ self.mlp = Mlp(
87
+ in_features=dim,
88
+ hidden_features=mlp_hidden_dim,
89
+ act_layer=nn.GELU,
90
+ drop=dropout,
91
+ bias=True,
92
+ )
93
+
94
+ '''
95
+ - original_x and new_x both with shape [b, num_tokens, dim])
96
+ '''
97
+ def forward(self, original_x, new_x):
98
+
99
+ #new_x = new_x + self.layer_scale1(self.norm1(new_x))
100
+
101
+ #modulate original feature map with the global info collected
102
+ original_x = original_x + self.layer_scale2(self.mlp(self.norm2(new_x)))
103
+
104
+ return original_x
105
+
106
+ #loomnet v3
107
+ # def forward(self, original_x, new_x):
108
+
109
+ # x = original_x + self.layer_scale1(self.norm1(new_x))
110
+
111
+ # x = x + self.layer_scale2(self.mlp(self.norm2(x)))
112
+
113
+ # return x
114
+
115
+
116
+ class LayerScale(nn.Module):
117
+ def __init__(
118
+ self,
119
+ dim: int,
120
+ init_values: Union[float, Tensor] = 1e-5,
121
+ inplace: bool = False,
122
+ ) -> None:
123
+ super().__init__()
124
+ self.inplace = inplace
125
+ self.gamma = nn.Parameter(init_values * torch.ones(dim))
126
+
127
+ def forward(self, x: Tensor) -> Tensor:
128
+ return x.mul_(self.gamma) if self.inplace else x * self.gamma
129
+
130
+
131
+
132
+
133
+ class Mlp(nn.Module):
134
+ def __init__(
135
+ self,
136
+ in_features: int,
137
+ hidden_features: Optional[int] = None,
138
+ out_features: Optional[int] = None,
139
+ act_layer: Callable[..., nn.Module] = nn.GELU,
140
+ drop: float = 0.0,
141
+ bias: bool = True,
142
+ ) -> None:
143
+ super().__init__()
144
+ out_features = out_features or in_features
145
+ hidden_features = hidden_features or in_features
146
+ self.fc1 = nn.Linear(in_features, hidden_features, bias=bias)
147
+ self.act = act_layer()
148
+ self.fc2 = nn.Linear(hidden_features, out_features, bias=bias)
149
+ self.drop = nn.Dropout(drop)
150
+
151
+ def forward(self, x: Tensor) -> Tensor:
152
+ x = self.fc1(x)
153
+ x = self.act(x)
154
+ x = self.drop(x)
155
+ x = self.fc2(x)
156
+ x = self.drop(x)
157
+ return x
158
+
159
+
160
+
161
+
162
+
163
+
164
+
165
+
166
+
167
+ class ConvBlock(nn.Module):
168
+
169
+ def __init__(self, dim, expansion=2, kernel_size=7):
170
+ super().__init__()
171
+ self.dwconv = nn.Conv2d(dim, dim, kernel_size, padding=kernel_size // 2, groups=dim) # depthwise conv
172
+ self.pwconv1 = nn.Conv2d(dim, dim * expansion, kernel_size=1)
173
+ self.act = nn.GELU()
174
+ self.pwconv2 = nn.Conv2d(dim * expansion, dim, kernel_size=1)
175
+ self.norm = nn.BatchNorm2d(dim)
176
+
177
+ def forward(self, x):
178
+ shortcut = x
179
+ x = self.dwconv(x)
180
+ x = self.norm(x)
181
+ x = self.pwconv1(x)
182
+ x = self.act(x)
183
+ x = self.pwconv2(x)
184
+ return x + shortcut # Residual connection
185
+
186
+ class FeatureEnhancer(nn.Module):
187
+ def __init__(self, grid_channels, feature_channels, mlp_hidden_dim, grid_encoder_hidden_dim, depth=2, gamma_init_values=1e-5):
188
+ super().__init__()
189
+
190
+ self.initial_proj = nn.Conv2d(grid_channels, grid_encoder_hidden_dim, kernel_size=1)
191
+
192
+ self.encoder = nn.Sequential(*[ConvBlock(grid_encoder_hidden_dim) for _ in range(depth)])
193
+
194
+ self.project_to_feat = nn.Conv2d(grid_encoder_hidden_dim, feature_channels, kernel_size=1)
195
+
196
+ # Gating mechanism (learnable modulation)
197
+ self.gamma_modulator = GammaModulator(dim=feature_channels, mlp_hidden_dim=mlp_hidden_dim, init_values=gamma_init_values)
198
+
199
+ def forward(self, original_feature_map, grid_features ):
200
+
201
+ B1, C1, H1, W1 = original_feature_map.shape
202
+
203
+ grid_features = F.interpolate(grid_features, size=(H1, W1), mode="nearest")
204
+
205
+ grid_features = self.initial_proj(grid_features) # [b, hidden_dim, N, N]
206
+ grid_features = self.encoder(grid_features) # deeper encoding
207
+ grid_features = self.project_to_feat(grid_features) # [b, C, N, N]
208
+
209
+ B2, C2, H2, W2 = grid_features.shape
210
+
211
+ original_feature_map_tokens = to_tokens(original_feature_map, B1, C1, H1, W1)
212
+ grid_features_tokens = to_tokens(grid_features, B2, C2, H2, W2)
213
+
214
+ new_feature_map = from_tokens(self.gamma_modulator(original_feature_map_tokens, grid_features_tokens), H1, W1)
215
+
216
+ return new_feature_map
217
+
218
+
219
+
220
+ def to_tokens(x, B, C, H, W):
221
+ # x: [B, C, D, D]
222
+ x = x.view(B, C, H * W).contiguous() # [B, C, D*D]
223
+ x = x.permute(0, 2, 1).contiguous() # [B, D*D, C]
224
+ return x
225
+
226
+ def from_tokens(tokens, H, W):
227
+ # tokens: [B, D*D, C]
228
+ B, N, C = tokens.shape
229
+ x = tokens.permute(0, 2, 1).contiguous() # [B, C, D*D]
230
+ x = x.view(B, C, H, W).contiguous() # [B, C, D, D]
231
+ return x
232
+
233
+
234
+
235
+
236
+ class HarmonicEmbedding(torch.nn.Module):
237
+ def __init__(
238
+ self,
239
+ n_harmonic_functions: int = 6,
240
+ omega_0: float = 1.0,
241
+ logspace: bool = True,
242
+ append_input: bool = True,
243
+ ) -> None:
244
+ super().__init__()
245
+
246
+ if logspace:
247
+ frequencies = 2.0 ** torch.arange(
248
+ n_harmonic_functions,
249
+ dtype=torch.float32,
250
+ )
251
+ else:
252
+ frequencies = torch.linspace(
253
+ 1.0,
254
+ 2.0 ** (n_harmonic_functions - 1),
255
+ n_harmonic_functions,
256
+ dtype=torch.float32,
257
+ )
258
+
259
+ self.register_buffer("_frequencies", frequencies * omega_0, persistent=False)
260
+ self.append_input = append_input
261
+
262
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
263
+ """
264
+ Args:
265
+ x: tensor of shape [..., dim]
266
+ Returns:
267
+ embedding: a harmonic embedding of `x`
268
+ of shape [..., (n_harmonic_functions * 2 + int(append_input)) * dim]
269
+ """
270
+ embed = (x[..., None] * self._frequencies).view(*x.shape[:-1], -1)
271
+ embed = torch.cat(
272
+ (embed.sin(), embed.cos(), x)
273
+ if self.append_input
274
+ else (embed.sin(), embed.cos()),
275
+ dim=-1,
276
+ )
277
+ return embed
278
+
279
+ @staticmethod
280
+ def get_output_dim_static(
281
+ input_dims: int,
282
+ n_harmonic_functions: int,
283
+ append_input: bool,
284
+ ) -> int:
285
+ """
286
+ Utility to help predict the shape of the output of `forward`.
287
+ Args:
288
+ input_dims: length of the last dimension of the input tensor
289
+ n_harmonic_functions: number of embedding frequencies
290
+ append_input: whether or not to concat the original
291
+ input to the harmonic embedding
292
+ Returns:
293
+ int: the length of the last dimension of the output tensor
294
+ """
295
+ return input_dims * (2 * n_harmonic_functions + int(append_input))
loomnet/models/blocks/pointwise_spatial_transformer.py ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ import torch.nn.functional as F
4
+ #__________________________________ positional encoding _______________________________
5
+
6
+
7
+ def make_sincos_pos_embed(embed_dim: int, pos: torch.Tensor, omega_0: float = 100) -> torch.Tensor:
8
+ """
9
+ This function generates a 1D positional embedding from a given grid using sine and cosine functions.
10
+
11
+ Args:
12
+ - embed_dim: The embedding dimension.
13
+ - pos: The position to generate the embedding from.
14
+
15
+ Returns:
16
+ - emb: The generated 1D positional embedding.
17
+ """
18
+ assert embed_dim % 2 == 0
19
+ omega = torch.arange(embed_dim // 2, dtype=torch.double, device=pos.device)
20
+ omega /= embed_dim / 2.0
21
+ omega = 1.0 / omega_0**omega # (D/2,)
22
+
23
+ pos = pos.reshape(-1) # (M,)
24
+ out = torch.einsum("m,d->md", pos, omega) # (M, D/2), outer product
25
+
26
+ emb_sin = torch.sin(out) # (M, D/2)
27
+ emb_cos = torch.cos(out) # (M, D/2)
28
+
29
+ emb = torch.cat([emb_sin, emb_cos], dim=1) # (M, D)
30
+ return emb.float()
31
+
32
+
33
+
34
+
35
+
36
+ def position_grid_to_embed(pos_grid: torch.Tensor, embed_dim: int, omega_0: float = 100) -> torch.Tensor:
37
+ """
38
+ Convert 2D position grid (HxWx2) to sinusoidal embeddings (HxWxC)
39
+
40
+ Args:
41
+ pos_grid: Tensor of shape (H*W, 2) containing 2D coordinates
42
+ embed_dim: Output channel dimension for embeddings
43
+
44
+ Returns:
45
+ Tensor of shape (H, W, embed_dim) with positional embeddings
46
+ """
47
+
48
+ num_views, num_points, _ = pos_grid.shape
49
+
50
+ # Process x and y coordinates separately
51
+ emb_x = make_sincos_pos_embed(embed_dim // 2, pos_grid[..., 0], omega_0=omega_0) # [1, H*W, D/2]
52
+ emb_y = make_sincos_pos_embed(embed_dim // 2, pos_grid[..., 1], omega_0=omega_0) # [1, H*W, D/2]
53
+
54
+ # Combine and reshape
55
+ emb = torch.cat([emb_x, emb_y], dim=-1) # [1, H*W, D]
56
+
57
+ return emb.view(num_views, num_points, embed_dim) # [H, W, D]
58
+
59
+ def apply_pos_embed(pos_grid, feature_map_dimension: int=4, ratio: float = 0.1) -> torch.Tensor:
60
+ """
61
+ Apply positional embedding to tensor x.
62
+ """
63
+ #(num_grid_positions,2)
64
+ pos_embed = position_grid_to_embed(pos_grid, feature_map_dimension)
65
+ pos_embed = pos_embed * ratio
66
+ return pos_embed
67
+
68
+
69
+
70
+
71
+
72
+ #___________________________________ Point-wise Adaptive Spatial Transformer _____________________
73
+
74
+
75
+ # class PointWiseSpatialTransformer(nn.Module):
76
+ # def __init__(self, feature_map_spatial_dim, feature_map_channel_dim):
77
+ # super(PointWiseSpatialTransformer, self).__init__()
78
+
79
+ # # Spatial transformer localization-network
80
+ # self.localization = nn.Sequential(
81
+ # nn.Conv2d(feature_map_channel_dim, feature_map_channel_dim, kernel_size=1),
82
+ # nn.MaxPool2d(2, stride=2) if feature_map_spatial_dim in [8,16,32] else nn.Identity(),
83
+ # nn.ReLU(True),
84
+ # nn.Conv2d(feature_map_channel_dim, feature_map_channel_dim//2, kernel_size=1),
85
+ # nn.MaxPool2d(2, stride=2) if feature_map_spatial_dim in [16,32] else nn.Identity(),
86
+ # nn.ReLU(True),
87
+ # nn.Conv2d(feature_map_channel_dim//2, 16, kernel_size=1),
88
+ # nn.MaxPool2d(2, stride=2) if feature_map_spatial_dim in [32] else nn.Identity(),
89
+ # nn.ReLU(True)
90
+ # )
91
+
92
+ # #normalization layer before concatenatig pos embed in [-1, 1]
93
+ # self.loc_norm = torch.nn.LayerNorm(16 * 4 * 4)
94
+
95
+ # # Regressor for the 3 * 2 affine matrix
96
+ # self.fc_loc = nn.Sequential(
97
+ # nn.Linear(16 * 4 * 4 + feature_map_spatial_dim, 32),
98
+ # nn.ReLU(True),
99
+ # nn.Linear(32, 3 * 2)
100
+ # )
101
+
102
+ # # Initialize the weights/bias with identity transformation
103
+ # self.fc_loc[2].weight.data.zero_()
104
+ # self.fc_loc[2].bias.data.copy_(torch.tensor([1, 0, 0, 0, 1, 0], dtype=torch.float))
105
+
106
+ # # Spatial transformer network forward function
107
+ # def stn(self, x, pos_emb):
108
+
109
+ # b_and_num_views, num_points, _ = pos_emb.shape
110
+
111
+ # xs = self.localization(x)
112
+ # xs = xs.view(-1, 16 * 4 * 4)
113
+ # xs = self.loc_norm(xs)
114
+ # #replico la localization map per ogni punto
115
+ # xs = xs.unsqueeze(1).repeat(1, pos_emb.shape[1], 1)
116
+ # #aggiungo le posizioni all'embedding della mappa
117
+ # xs = torch.cat([xs, pos_emb], dim=-1)
118
+
119
+ # #predico affine transformation for each point
120
+ # theta = self.fc_loc(xs)
121
+ # theta = theta.view(b_and_num_views, num_points, 2, 3)
122
+
123
+ # return theta
124
+
125
+ # def forward(self, feature_map, pos_emb):
126
+ # # transform projected points
127
+ # theta = self.stn(feature_map, pos_emb)
128
+
129
+ # return theta
130
+
131
+ class PointWiseSpatialTransformer(nn.Module):
132
+ def __init__(self, feature_map_spatial_dim, feature_map_channel_dim):
133
+ super(PointWiseSpatialTransformer, self).__init__()
134
+
135
+ # Spatial transformer localization-network
136
+ self.localization = nn.Sequential(
137
+ nn.Conv2d(feature_map_channel_dim, feature_map_channel_dim, kernel_size=1),
138
+ nn.MaxPool2d(2, stride=2) if feature_map_spatial_dim in [8,16,32] else nn.Identity(),
139
+ nn.ReLU(True),
140
+ nn.Conv2d(feature_map_channel_dim, feature_map_channel_dim//2, kernel_size=1),
141
+ nn.MaxPool2d(2, stride=2) if feature_map_spatial_dim in [16,32] else nn.Identity(),
142
+ nn.ReLU(True),
143
+ nn.Conv2d(feature_map_channel_dim//2, 16, kernel_size=1),
144
+ nn.MaxPool2d(2, stride=2) if feature_map_spatial_dim in [32] else nn.Identity(),
145
+ nn.ReLU(True)
146
+ )
147
+
148
+ # Regressor for the 3 * 2 affine matrix
149
+ self.fc_loc = nn.Sequential(
150
+ nn.Linear(16 * 4 * 4, 32),
151
+ nn.ReLU(True),
152
+ nn.Linear(32, 3 * 2)
153
+ )
154
+
155
+ # Initialize the weights/bias with identity transformation
156
+ self.fc_loc[2].weight.data.zero_()
157
+ self.fc_loc[2].bias.data.copy_(torch.tensor([1, 0, 0, 0, 1, 0], dtype=torch.float))
158
+
159
+ # Spatial transformer network forward function
160
+ def stn(self, x, size_grid):
161
+
162
+ b_and_num_views, c, _, _ = x.size()
163
+
164
+ xs = self.localization(x)
165
+ xs = xs.view(-1, 16 * 4 * 4)
166
+ theta = self.fc_loc(xs)
167
+ theta = theta.view(-1, 2, 3)
168
+
169
+ grid = F.affine_grid(theta, [b_and_num_views, c, size_grid, size_grid])
170
+ ambassadors_features = F.grid_sample(x, grid)
171
+
172
+ return ambassadors_features, grid
173
+
174
+
175
+ def forward(self, feature_map, size_grid):
176
+ # transform projected points
177
+ ambassadors_features, grid = self.stn(feature_map, size_grid)
178
+
179
+ return ambassadors_features, grid
loomnet/models/blocks/resnet.py ADDED
@@ -0,0 +1,357 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Borrowed from https://github.com/Tangshitao/MVDiffusion/blob/main/src/models/modules/resnet.py
2
+ from abc import abstractmethod
3
+
4
+ import torch
5
+ import torch.nn as nn
6
+ import torch.nn.functional as F
7
+
8
+
9
+ class BasicResNetBlock(nn.Module):
10
+ def __init__(self, in_channels, out_channels, stride=1, zero_init=False):
11
+ super(BasicResNetBlock, self).__init__()
12
+ self.conv1 = nn.Conv2d(
13
+ in_channels,
14
+ out_channels,
15
+ kernel_size=3,
16
+ stride=stride,
17
+ padding=1,
18
+ bias=False,
19
+ )
20
+ self.bn1 = nn.GroupNorm(32, out_channels)
21
+ self.relu = nn.ReLU(inplace=True)
22
+ self.conv2 = nn.Conv2d(
23
+ out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False
24
+ )
25
+ self.bn2 = nn.GroupNorm(32, out_channels)
26
+
27
+ self.shortcut = nn.Sequential()
28
+ if stride != 1 or in_channels != out_channels:
29
+ self.shortcut = nn.Sequential(
30
+ nn.Conv2d(
31
+ in_channels, out_channels, kernel_size=1, stride=stride, bias=False
32
+ ),
33
+ nn.GroupNorm(32, out_channels),
34
+ )
35
+ if zero_init:
36
+ self.conv2.weight.data.zero_()
37
+ if self.conv2.bias is not None:
38
+ self.conv2.bias.data.zero_()
39
+
40
+ def forward(self, x):
41
+ identity = x
42
+
43
+ out = self.bn1(x)
44
+ out = self.relu(out)
45
+ out = self.conv1(out)
46
+
47
+ out = self.bn2(out)
48
+ out = self.relu(out)
49
+ out = self.conv2(out)
50
+ identity = self.shortcut(identity)
51
+
52
+ out += identity
53
+
54
+ return out
55
+
56
+
57
+ class SiLU(nn.Module):
58
+ def forward(self, x):
59
+ return x * torch.sigmoid(x)
60
+
61
+
62
+ class GroupNorm32(nn.GroupNorm):
63
+ def forward(self, x):
64
+ return super().forward(x.float()).type(x.dtype)
65
+
66
+
67
+ def conv_nd(dims, *args, **kwargs):
68
+ """
69
+ Create a 1D, 2D, or 3D convolution module.
70
+ """
71
+ if dims == 1:
72
+ return nn.Conv1d(*args, **kwargs)
73
+ elif dims == 2:
74
+ return nn.Conv2d(*args, **kwargs)
75
+ elif dims == 3:
76
+ return nn.Conv3d(*args, **kwargs)
77
+ raise ValueError(f"unsupported dimensions: {dims}")
78
+
79
+
80
+ def linear(*args, **kwargs):
81
+ """
82
+ Create a linear module.
83
+ """
84
+ return nn.Linear(*args, **kwargs)
85
+
86
+
87
+ class Upsample(nn.Module):
88
+ """
89
+ An upsampling layer with an optional convolution.
90
+ :param channels: channels in the inputs and outputs.
91
+ :param use_conv: a bool determining if a convolution is applied.
92
+ :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then
93
+ upsampling occurs in the inner-two dimensions.
94
+ """
95
+
96
+ def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1):
97
+ super().__init__()
98
+ self.channels = channels
99
+ self.out_channels = out_channels or channels
100
+ self.use_conv = use_conv
101
+ self.dims = dims
102
+ if use_conv:
103
+ self.conv = conv_nd(
104
+ dims, self.channels, self.out_channels, 3, padding=padding
105
+ )
106
+
107
+ def forward(self, x):
108
+ assert x.shape[1] == self.channels
109
+ if self.dims == 3:
110
+ x = F.interpolate(
111
+ x, (x.shape[2], x.shape[3] * 2, x.shape[4] * 2), mode="nearest"
112
+ )
113
+ else:
114
+ x = F.interpolate(x, scale_factor=2, mode="nearest")
115
+ if self.use_conv:
116
+ x = self.conv(x)
117
+ return x
118
+
119
+
120
+ def normalization(channels):
121
+ """
122
+ Make a standard normalization layer.
123
+ :param channels: number of input channels.
124
+ :return: an nn.Module for normalization.
125
+ """
126
+ return GroupNorm32(32, channels)
127
+
128
+
129
+ def zero_module(module):
130
+ """
131
+ Zero out the parameters of a module and return it.
132
+ """
133
+ for p in module.parameters():
134
+ p.detach().zero_()
135
+ return module
136
+
137
+
138
+ def avg_pool_nd(dims, *args, **kwargs):
139
+ """
140
+ Create a 1D, 2D, or 3D average pooling module.
141
+ """
142
+ if dims == 1:
143
+ return nn.AvgPool1d(*args, **kwargs)
144
+ elif dims == 2:
145
+ return nn.AvgPool2d(*args, **kwargs)
146
+ elif dims == 3:
147
+ return nn.AvgPool3d(*args, **kwargs)
148
+ raise ValueError(f"unsupported dimensions: {dims}")
149
+
150
+
151
+ class Downsample(nn.Module):
152
+ """
153
+ A downsampling layer with an optional convolution.
154
+ :param channels: channels in the inputs and outputs.
155
+ :param use_conv: a bool determining if a convolution is applied.
156
+ :param dims: determines if the signal is 1D, 2D, or 3D. If 3D, then
157
+ downsampling occurs in the inner-two dimensions.
158
+ """
159
+
160
+ def __init__(self, channels, use_conv, dims=2, out_channels=None, padding=1):
161
+ super().__init__()
162
+ self.channels = channels
163
+ self.out_channels = out_channels or channels
164
+ self.use_conv = use_conv
165
+ self.dims = dims
166
+ stride = 2 if dims != 3 else (1, 2, 2)
167
+ if use_conv:
168
+ self.op = conv_nd(
169
+ dims,
170
+ self.channels,
171
+ self.out_channels,
172
+ 3,
173
+ stride=stride,
174
+ padding=padding,
175
+ )
176
+ else:
177
+ assert self.channels == self.out_channels
178
+ self.op = avg_pool_nd(dims, kernel_size=stride, stride=stride)
179
+
180
+ def forward(self, x):
181
+ assert x.shape[1] == self.channels
182
+ return self.op(x)
183
+
184
+
185
+ class TimestepBlock(nn.Module):
186
+ """
187
+ Any module where forward() takes timestep embeddings as a second argument.
188
+ """
189
+
190
+ @abstractmethod
191
+ def forward(self, x, emb):
192
+ """
193
+ Apply the module to `x` given `emb` timestep embeddings.
194
+ """
195
+
196
+
197
+ def checkpoint(func, inputs, params, flag):
198
+ """
199
+ Evaluate a function without caching intermediate activations, allowing for
200
+ reduced memory at the expense of extra compute in the backward pass.
201
+ :param func: the function to evaluate.
202
+ :param inputs: the argument sequence to pass to `func`.
203
+ :param params: a sequence of parameters `func` depends on but does not
204
+ explicitly take as arguments.
205
+ :param flag: if False, disable gradient checkpointing.
206
+ """
207
+ if flag:
208
+ args = tuple(inputs) + tuple(params)
209
+ return CheckpointFunction.apply(func, len(inputs), *args)
210
+ else:
211
+ return func(*inputs)
212
+
213
+
214
+ class CheckpointFunction(torch.autograd.Function):
215
+ @staticmethod
216
+ def forward(ctx, run_function, length, *args):
217
+ ctx.run_function = run_function
218
+ ctx.input_tensors = list(args[:length])
219
+ ctx.input_params = list(args[length:])
220
+
221
+ with torch.no_grad():
222
+ output_tensors = ctx.run_function(*ctx.input_tensors)
223
+ return output_tensors
224
+
225
+ @staticmethod
226
+ def backward(ctx, *output_grads):
227
+ ctx.input_tensors = [x.detach().requires_grad_(True) for x in ctx.input_tensors]
228
+ with torch.enable_grad():
229
+ # Fixes a bug where the first op in run_function modifies the
230
+ # Tensor storage in place, which is not allowed for detach()'d
231
+ # Tensors.
232
+ shallow_copies = [x.view_as(x) for x in ctx.input_tensors]
233
+ output_tensors = ctx.run_function(*shallow_copies)
234
+ input_grads = torch.autograd.grad(
235
+ output_tensors,
236
+ ctx.input_tensors + ctx.input_params,
237
+ output_grads,
238
+ allow_unused=True,
239
+ )
240
+ del ctx.input_tensors
241
+ del ctx.input_params
242
+ del output_tensors
243
+ return (None, None) + input_grads
244
+
245
+
246
+ class ResBlock(TimestepBlock):
247
+ """
248
+ A residual block that can optionally change the number of channels.
249
+ :param channels: the number of input channels.
250
+ :param emb_channels: the number of timestep embedding channels.
251
+ :param dropout: the rate of dropout.
252
+ :param out_channels: if specified, the number of out channels.
253
+ :param use_conv: if True and out_channels is specified, use a spatial
254
+ convolution instead of a smaller 1x1 convolution to change the
255
+ channels in the skip connection.
256
+ :param dims: determines if the signal is 1D, 2D, or 3D.
257
+ :param use_checkpoint: if True, use gradient checkpointing on this module.
258
+ :param up: if True, use this block for upsampling.
259
+ :param down: if True, use this block for downsampling.
260
+ """
261
+
262
+ def __init__(
263
+ self,
264
+ channels,
265
+ emb_channels,
266
+ dropout=0.0,
267
+ out_channels=None,
268
+ use_conv=False,
269
+ use_scale_shift_norm=False,
270
+ dims=2,
271
+ use_checkpoint=False,
272
+ up=False,
273
+ down=False,
274
+ ):
275
+ super().__init__()
276
+ self.channels = channels
277
+ self.emb_channels = emb_channels
278
+ self.dropout = dropout
279
+ self.out_channels = out_channels or channels
280
+ self.use_conv = use_conv
281
+ self.use_checkpoint = use_checkpoint
282
+ self.use_scale_shift_norm = use_scale_shift_norm
283
+
284
+ self.in_layers = nn.Sequential(
285
+ normalization(channels),
286
+ nn.SiLU(),
287
+ conv_nd(dims, channels, self.out_channels, 3, padding=1),
288
+ )
289
+
290
+ self.updown = up or down
291
+
292
+ if up:
293
+ self.h_upd = Upsample(channels, False, dims)
294
+ self.x_upd = Upsample(channels, False, dims)
295
+ elif down:
296
+ self.h_upd = Downsample(channels, False, dims)
297
+ self.x_upd = Downsample(channels, False, dims)
298
+ else:
299
+ self.h_upd = self.x_upd = nn.Identity()
300
+
301
+ self.emb_layers = nn.Sequential(
302
+ nn.SiLU(),
303
+ linear(
304
+ emb_channels,
305
+ 2 * self.out_channels if use_scale_shift_norm else self.out_channels,
306
+ ),
307
+ )
308
+ self.out_layers = nn.Sequential(
309
+ normalization(self.out_channels),
310
+ nn.SiLU(),
311
+ nn.Dropout(p=dropout),
312
+ zero_module(
313
+ conv_nd(dims, self.out_channels, self.out_channels, 3, padding=1)
314
+ ),
315
+ )
316
+
317
+ if self.out_channels == channels:
318
+ self.skip_connection = nn.Identity()
319
+ elif use_conv:
320
+ self.skip_connection = conv_nd(
321
+ dims, channels, self.out_channels, 3, padding=1
322
+ )
323
+ else:
324
+ self.skip_connection = conv_nd(dims, channels, self.out_channels, 1)
325
+
326
+ def forward(self, x, emb):
327
+ """
328
+ Apply the block to a Tensor, conditioned on a timestep embedding.
329
+ :param x: an [N x C x ...] Tensor of features.
330
+ :param emb: an [N x emb_channels] Tensor of timestep embeddings.
331
+ :return: an [N x C x ...] Tensor of outputs.
332
+ """
333
+ return checkpoint(
334
+ self._forward, (x, emb), self.parameters(), self.use_checkpoint
335
+ )
336
+
337
+ def _forward(self, x, emb):
338
+ if self.updown:
339
+ in_rest, in_conv = self.in_layers[:-1], self.in_layers[-1]
340
+ h = in_rest(x)
341
+ h = self.h_upd(h)
342
+ x = self.x_upd(x)
343
+ h = in_conv(h)
344
+ else:
345
+ h = self.in_layers(x)
346
+ emb_out = self.emb_layers(emb).type(h.dtype)
347
+ while len(emb_out.shape) < len(h.shape):
348
+ emb_out = emb_out[..., None]
349
+ if self.use_scale_shift_norm:
350
+ out_norm, out_rest = self.out_layers[0], self.out_layers[1:]
351
+ scale, shift = torch.chunk(emb_out, 2, dim=1)
352
+ h = out_norm(h) * (1 + scale) + shift
353
+ h = out_rest(h)
354
+ else:
355
+ h = h + emb_out
356
+ h = self.out_layers(h)
357
+ return self.skip_connection(x) + h
loomnet/models/blocks/resnet_fpn.py ADDED
@@ -0,0 +1,372 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ import torch.nn.functional as F
4
+
5
+
6
+ class BottleNeck(nn.Module):
7
+ # Scale factor of the number of output channels
8
+ expansion = 4
9
+
10
+ def __init__(self, in_channels, out_channels,
11
+ stride=1, is_first_block=False):
12
+ """
13
+ Args:
14
+ in_channels: number of input channels
15
+ out_channels: number of output channels
16
+ stride: stride using in (a) 3x3 convolution and
17
+ (b) 1x1 convolution used for downsampling for skip connection
18
+ is_first_block: whether it is the first residual block of the layer
19
+ """
20
+ super().__init__()
21
+ self.conv1 = nn.Conv2d(in_channels=in_channels,
22
+ out_channels=out_channels,
23
+ kernel_size=1, stride=1, padding=0)
24
+ self.bn1 = nn.BatchNorm2d(out_channels)
25
+
26
+ self.conv2 = nn.Conv2d(in_channels=out_channels,
27
+ out_channels=out_channels,
28
+ kernel_size=3, stride=stride, padding=1)
29
+ self.bn2 = nn.BatchNorm2d(out_channels)
30
+
31
+ self.conv3 = nn.Conv2d(in_channels=out_channels,
32
+ out_channels=out_channels*self.expansion,
33
+ kernel_size=1, stride=1, padding=0)
34
+ self.bn3 = nn.BatchNorm2d(out_channels*self.expansion)
35
+
36
+ self.relu = nn.ReLU()
37
+
38
+ # Skip connection goes through 1x1 convolution with stride=2 for
39
+ # the first blocks of conv3_x, conv4_x, and conv5_x layers for matching
40
+ # spatial dimension of feature maps and number of channels in order to
41
+ # perform the add operations.
42
+ self.downsample = None
43
+ if is_first_block:
44
+ self.downsample = nn.Sequential(nn.Conv2d(in_channels=in_channels,
45
+ out_channels=out_channels*self.expansion,
46
+ kernel_size=1,
47
+ stride=stride,
48
+ padding=0),
49
+ nn.BatchNorm2d(out_channels*self.expansion))
50
+
51
+
52
+ def forward(self, x):
53
+ """
54
+ Args:
55
+ x: input
56
+ Returns:
57
+ Residual block output
58
+ """
59
+ identity = x.clone()
60
+ x = self.relu(self.bn1(self.conv1(x)))
61
+ x = self.relu(self.bn2(self.conv2(x)))
62
+
63
+ x = self.conv3(x)
64
+ x = self.bn3(x)
65
+
66
+ if self.downsample:
67
+ identity = self.downsample(identity)
68
+
69
+ x += identity
70
+ x = self.relu(x)
71
+
72
+ return x
73
+
74
+
75
+ class BasicBlock(nn.Module):
76
+ # Scale factor of the number of output channels
77
+ expansion = 1
78
+
79
+ def __init__(self, in_channels, out_channels,
80
+ stride=1, is_first_block=False):
81
+ """
82
+ Args:
83
+ in_channels: number of input channels
84
+ out_channels: number of output channels
85
+ stride: stride using in (a) the first 3x3 convolution and
86
+ (b) 1x1 convolution used for downsampling for skip connection
87
+ is_first_block: whether it is the first residual block of the layer
88
+ """
89
+ super().__init__()
90
+ self.conv1 = nn.Conv2d(in_channels=in_channels,
91
+ out_channels=out_channels,
92
+ kernel_size=3,
93
+ stride=stride,
94
+ padding=1)
95
+ self.bn1 = nn.BatchNorm2d(out_channels)
96
+ self.conv2 = nn.Conv2d(in_channels=out_channels,
97
+ out_channels=out_channels,
98
+ kernel_size=3,
99
+ stride=1,
100
+ padding=1)
101
+ self.bn2 = nn.BatchNorm2d(out_channels)
102
+
103
+ self.relu = nn.ReLU()
104
+
105
+ # Skip connection goes through 1x1 convolution with stride=2 for
106
+ # the first blocks of conv3_x, conv4_x, and conv5_x layers for matching
107
+ # spatial dimension of feature maps and number of channels in order to
108
+ # perform the add operations.
109
+ self.downsample = None
110
+ if is_first_block and stride != 1:
111
+ self.downsample = nn.Sequential(nn.Conv2d(in_channels=in_channels,
112
+ out_channels=out_channels,
113
+ kernel_size=1,
114
+ stride=stride,
115
+ padding=0),
116
+ nn.BatchNorm2d(out_channels))
117
+
118
+
119
+ def forward(self, x):
120
+ """
121
+ Args:
122
+ x: input
123
+ Returns:
124
+ Residual block ouput
125
+ """
126
+ identity = x.clone()
127
+ x = self.relu(self.bn1(self.conv1(x)))
128
+ x = self.bn2(self.conv2(x))
129
+
130
+ if self.downsample:
131
+ identity = self.downsample(identity)
132
+ x += identity
133
+ x = self.relu(x)
134
+
135
+ return x
136
+
137
+
138
+ class ResNet(nn.Module):
139
+ def __init__(self, ResBlock, n_blocks_list=[3, 4, 6, 3],
140
+ out_channels_list=[64, 128, 256, 512], num_channels=3, final_channel_dim=1280):
141
+ """
142
+ Args:
143
+ ResBlock: residual block type, BasicBlock for ResNet-18, 34 or
144
+ BottleNeck for ResNet-50, 101, 152
145
+ n_block_lists: number of residual blocks for each conv layer (conv2_x - conv5_x)
146
+ out_channels_list: list of the output channel numbers for conv2_x - conv5_x
147
+ num_channels: the number of channels of input image
148
+ """
149
+ super().__init__()
150
+
151
+ # Create four convoluiontal layers
152
+ in_channels = final_channel_dim // 5
153
+
154
+ # First layer
155
+ self.conv1 = nn.Sequential(nn.Conv2d(in_channels=num_channels,
156
+ out_channels=in_channels, kernel_size=7,
157
+ stride=2, padding=3),
158
+ nn.BatchNorm2d(in_channels),
159
+ nn.ReLU(),
160
+ nn.MaxPool2d(kernel_size=3,
161
+ stride=2, padding=1))
162
+
163
+ # For the first block of the second layer, do not downsample and use stride=1.
164
+ self.conv2_x = self.CreateLayer(ResBlock, n_blocks_list[0],
165
+ in_channels, out_channels_list[0], stride=1)
166
+
167
+ # For the first blocks of conv3_x - conv5_x layers, perform downsampling using stride=2.
168
+ # By default, ResBlock.expansion = 4 for ResNet-50, 101, 152,
169
+ # ResBlock.expansion = 1 for ResNet-18, 34.
170
+ self.conv3_x = self.CreateLayer(ResBlock, n_blocks_list[1],
171
+ out_channels_list[0]*ResBlock.expansion,
172
+ out_channels_list[1], stride=2)
173
+ self.conv4_x = self.CreateLayer(ResBlock, n_blocks_list[2],
174
+ out_channels_list[1]*ResBlock.expansion,
175
+ out_channels_list[2], stride=2)
176
+ self.conv5_x = self.CreateLayer(ResBlock, n_blocks_list[3],
177
+ out_channels_list[2]*ResBlock.expansion,
178
+ out_channels_list[3], stride=2)
179
+
180
+
181
+ def forward(self, x):
182
+ """
183
+ Args:
184
+ x: input image
185
+ Returns:
186
+ C2: feature maps after conv2_x
187
+ C3: feature maps after conv3_x
188
+ C4: feature maps after conv4_x
189
+ C5: feature maps after conv5_x
190
+ y: output class
191
+ """
192
+ x = self.conv1(x)
193
+
194
+ # Feature maps
195
+ C2 = self.conv2_x(x)
196
+ C3 = self.conv3_x(C2)
197
+ C4 = self.conv4_x(C3)
198
+ C5 = self.conv5_x(C4)
199
+
200
+ return C2, C3, C4, C5
201
+
202
+
203
+ def CreateLayer(self, ResBlock, n_blocks, in_channels, out_channels, stride=1):
204
+ """
205
+ Create a layer with specified type and number of residual blocks.
206
+ Args:
207
+ ResBlock: residual block type, BasicBlock for ResNet-18, 34 or
208
+ BottleNeck for ResNet-50, 101, 152
209
+ n_blocks: number of residual blocks
210
+ in_channels: number of input channels
211
+ out_channels: number of output channels
212
+ stride: stride used in the first 3x3 convolution of the first resdiual block
213
+ of the layer and 1x1 convolution for skip connection in that block
214
+ Returns:
215
+ Convolutional layer
216
+ """
217
+ layer = []
218
+ for i in range(n_blocks):
219
+ if i == 0:
220
+ # Downsample the feature map using input stride for the first block of the layer.
221
+ layer.append(ResBlock(in_channels, out_channels,
222
+ stride=stride, is_first_block=True))
223
+ else:
224
+ # Keep the feature map size same for the rest three blocks of the layer.
225
+ # by setting stride=1 and is_first_block=False.
226
+ # By default, ResBlock.expansion = 4 for ResNet-50, 101, 152,
227
+ # ResBlock.expansion = 1 for ResNet-18, 34.
228
+ layer.append(ResBlock(out_channels*ResBlock.expansion, out_channels))
229
+
230
+ return nn.Sequential(*layer)
231
+
232
+
233
+ class FPNBlock(nn.Module):
234
+ def __init__(self, in_channels, out_channels=256, is_highest_block=False):
235
+ """
236
+ Args:
237
+ in_channels: the number of input channels
238
+ out_channels: the number of output channels
239
+ is_highest_block: whether the block is at the highest level of pyramid
240
+ """
241
+ super().__init__()
242
+ # 1x1 convolution to unify the number of feature map channels to
243
+ # a specified value for fusion (matching)
244
+ self.conv1 = nn.Conv2d(in_channels, out_channels,
245
+ kernel_size=1, stride=1, padding=0)
246
+ # apply 3x3 convolution for feature output
247
+ self.conv2 = nn.Conv2d(out_channels, out_channels,
248
+ kernel_size=3, stride=1, padding=1)
249
+ self.is_highest_block = is_highest_block
250
+
251
+
252
+ def forward(self, x, y):
253
+ """
254
+ Args:
255
+ x: input tensor at the current level
256
+ y: input tesnor at the previous level (upper level)
257
+ Returns:
258
+ x: output tensor to the next level (lower level)
259
+ out: output tensor at the curret level (after 1x1 and 3x3 convolutions)
260
+ """
261
+ x = self.conv1(x)
262
+ if not self.is_highest_block:
263
+ x += F.interpolate(y, scale_factor=2, mode="bilinear",
264
+ align_corners=True)
265
+ out = self.conv2(x)
266
+ return x, out
267
+
268
+
269
+ class FPN(nn.Module):
270
+ def __init__(self, expansion=2, in_channels_list=[64, 128, 256, 512],
271
+ out_channels=256):
272
+ """
273
+ Args:
274
+ expansion: expansion rate of ResBlock (1 for BasicBlock or 4 for BottleNeck)
275
+ in_channels_list: list of the output channel numbers for conv2_x - conv5_x
276
+ out_channels: target number of channels (256 by default)
277
+ """
278
+ super().__init__()
279
+
280
+ # Create layers to generate P2-P6
281
+ self.P2 = FPNBlock(in_channels_list[0]*expansion, out_channels=out_channels)
282
+ self.P3 = FPNBlock(in_channels_list[1]*expansion, out_channels=out_channels)
283
+ self.P4 = FPNBlock(in_channels_list[2]*expansion, out_channels=out_channels)
284
+ self.P5 = FPNBlock(in_channels_list[3]*expansion, out_channels=out_channels,
285
+ is_highest_block=True)
286
+ self.P6 = nn.MaxPool2d(kernel_size=1, stride=2, padding=0)
287
+
288
+
289
+ def forward(self, C2, C3, C4, C5):
290
+ """
291
+ Args:
292
+ C2-C5: feature maps output by ResNet
293
+ Returns:
294
+ P2-P6: enchanced features by FPN
295
+ """
296
+ x, P5 = self.P5(C5, None)
297
+ x, P4 = self.P4(C4, x)
298
+ x, P3 = self.P3(C3, x)
299
+ _, P2 = self.P2(C2, x)
300
+ P6 = self.P6(P5)
301
+
302
+ return P2, P3, P4, P5, P6
303
+
304
+
305
+ class ResnetFPN(nn.Module):
306
+ def __init__(self, resnet_arch='50', in_channels=3, final_channel_dim = 1280):
307
+ """
308
+ Args:
309
+ resnet_arch: the type of ResNet architecture
310
+ in_channels: number of channels of the input feature map
311
+ """
312
+ super().__init__()
313
+
314
+ assert(resnet_arch in ['18', '34', '50', '101', '152'])
315
+ assert final_channel_dim % 5 ==0, "final channel dim must be divisible by 5"
316
+
317
+ ResBlock = BasicBlock if resnet_arch == '18' or resnet_arch == '34' else BottleNeck
318
+
319
+ # Number of residual blocks for each conv layer (conv2_x - conv5_x)
320
+ if resnet_arch == '18':
321
+ n_blocks_list = [2, 2, 2, 2]
322
+ elif resnet_arch == '34' or resnet_arch == '50':
323
+ n_blocks_list = [3, 4, 6, 3]
324
+ elif resnet_arch == '101':
325
+ n_blocks_list = [3, 4, 23, 3]
326
+ else:
327
+ n_blocks_list = [3, 8, 36, 3]
328
+
329
+ # Create ResNet
330
+ self.resnet = ResNet(ResBlock, n_blocks_list=n_blocks_list, num_channels=in_channels, final_channel_dim=final_channel_dim)
331
+
332
+ # Create FPN
333
+ self.FPN = FPN(expansion=ResBlock.expansion)
334
+
335
+
336
+ def forward(self, x):
337
+ """
338
+ Args:
339
+ x: input tensor
340
+ Returns:
341
+ P2-P6: enhanced features
342
+ """
343
+ C2, C3, C4, C5 = self.resnet(x)
344
+ P2, P3, P4, P5, P6 = self.FPN(C2, C3, C4, C5)
345
+ return P2, P3, P4, P5, P6
346
+
347
+
348
+ if __name__ == "__main__":
349
+ ### Resnet arch types
350
+ #resnet_arch = '18'
351
+ #resnet_arch = '34'
352
+ resnet_arch = '50'
353
+ #resnet_arch = '101'
354
+ #resnet_arch = '152'
355
+
356
+ ### Create ResNet+FPN
357
+ net = ResnetFPN(resnet_arch, in_channels=3)
358
+
359
+ ### Input
360
+ x = torch.randn((1, 3, 512, 800), dtype=torch.float32)
361
+
362
+ ### Inference
363
+ P2, P3, P4, P5, P6 = net(x)
364
+
365
+ ### Output
366
+ print(f"P2 shape is {P2.shape}")
367
+ print(f"P3 shape is {P3.shape}")
368
+ print(f"P4 shape is {P4.shape}")
369
+ print(f"P5 shape is {P5.shape}")
370
+ print(f"P6 shape is {P6.shape}")
371
+
372
+ print("Done!")
loomnet/models/blocks/transformer.py ADDED
@@ -0,0 +1,221 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Borrowed from https://github.com/Tangshitao/MVDiffusion/blob/main/src/models/modules/transformer.py
2
+ import torch
3
+ import torch.nn.functional as F
4
+ from einops import rearrange
5
+ from torch import einsum, nn
6
+
7
+
8
+ class GEGLU(nn.Module):
9
+ def __init__(self, dim_in, dim_out):
10
+ super().__init__()
11
+ self.proj = nn.Linear(dim_in, dim_out * 2)
12
+
13
+ def forward(self, x):
14
+ x, gate = self.proj(x).chunk(2, dim=-1)
15
+ return x * F.gelu(gate)
16
+
17
+
18
+ class FeedForward(nn.Module):
19
+ def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0.0):
20
+ super().__init__()
21
+ inner_dim = int(dim * mult)
22
+ dim_out = dim
23
+ project_in = (
24
+ nn.Sequential(nn.Linear(dim, inner_dim), nn.GELU())
25
+ if not glu
26
+ else GEGLU(dim, inner_dim)
27
+ )
28
+
29
+ linear = nn.Linear(inner_dim, dim_out)
30
+ linear.weight.data.fill_(0)
31
+ linear.bias.data.fill_(0)
32
+ self.net = nn.Sequential(project_in, nn.Dropout(dropout), linear)
33
+
34
+ def forward(self, x):
35
+ return self.net(x)
36
+
37
+
38
+ class CrossAttention(nn.Module):
39
+ def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.0):
40
+ super().__init__()
41
+ inner_dim = dim_head * heads
42
+ context_dim = context_dim
43
+
44
+ self.scale = dim_head**-0.5
45
+ self.heads = heads
46
+
47
+ self.to_q = nn.Linear(query_dim, inner_dim, bias=False)
48
+ self.to_k = nn.Linear(context_dim, inner_dim, bias=False)
49
+ self.to_v = nn.Linear(context_dim, inner_dim, bias=False)
50
+
51
+ self.to_out = nn.Linear(inner_dim, query_dim)
52
+ self.to_out.weight.data.fill_(0)
53
+ self.to_out.bias.data.fill_(0)
54
+
55
+ def forward(self, x, context=None, mask=None):
56
+ h = self.heads
57
+
58
+ q = self.to_q(x)
59
+
60
+ k = self.to_k(context)
61
+ v = self.to_v(context)
62
+
63
+ q, k, v = map(lambda t: rearrange(t, "b n (h d) -> (b h) n d", h=h), (q, k, v))
64
+
65
+ sim = einsum("b i d, b j d -> b i j", q, k) * self.scale
66
+
67
+ del q, k
68
+
69
+ sim = sim.softmax(dim=-1)
70
+
71
+ out = einsum("b i j, b j d -> b i d", sim, v)
72
+ out = rearrange(out, "(b h) n d -> b n (h d)", h=h)
73
+ return self.to_out(out)
74
+
75
+
76
+ def checkpoint(func, inputs, params, flag):
77
+ """
78
+ Evaluate a function without caching intermediate activations, allowing for
79
+ reduced memory at the expense of extra compute in the backward pass.
80
+ :param func: the function to evaluate.
81
+ :param inputs: the argument sequence to pass to `func`.
82
+ :param params: a sequence of parameters `func` depends on but does not
83
+ explicitly take as arguments.
84
+ :param flag: if False, disable gradient checkpointing.
85
+ """
86
+ if flag:
87
+ args = tuple(inputs) + tuple(params)
88
+ return CheckpointFunction.apply(func, len(inputs), *args)
89
+ else:
90
+ return func(*inputs)
91
+
92
+
93
+ class CheckpointFunction(torch.autograd.Function):
94
+ @staticmethod
95
+ def forward(ctx, run_function, length, *args):
96
+ ctx.run_function = run_function
97
+ ctx.input_tensors = list(args[:length])
98
+ ctx.input_params = list(args[length:])
99
+ ctx.gpu_autocast_kwargs = {
100
+ "enabled": torch.is_autocast_enabled(),
101
+ "dtype": torch.get_autocast_gpu_dtype(),
102
+ "cache_enabled": torch.is_autocast_cache_enabled(),
103
+ }
104
+ with torch.no_grad():
105
+ output_tensors = ctx.run_function(*ctx.input_tensors)
106
+ return output_tensors
107
+
108
+ @staticmethod
109
+ def backward(ctx, *output_grads):
110
+ ctx.input_tensors = [x.detach().requires_grad_(True) for x in ctx.input_tensors]
111
+ with torch.enable_grad(), torch.cuda.amp.autocast(**ctx.gpu_autocast_kwargs):
112
+ # Fixes a bug where the first op in run_function modifies the
113
+ # Tensor storage in place, which is not allowed for detach()'d
114
+ # Tensors.
115
+ shallow_copies = [x.view_as(x) for x in ctx.input_tensors]
116
+ output_tensors = ctx.run_function(*shallow_copies)
117
+ input_grads = torch.autograd.grad(
118
+ output_tensors,
119
+ ctx.input_tensors + ctx.input_params,
120
+ output_grads,
121
+ allow_unused=True,
122
+ )
123
+ del ctx.input_tensors
124
+ del ctx.input_params
125
+ del output_tensors
126
+ return (None, None) + input_grads
127
+
128
+
129
+ class BasicTransformerBlock(nn.Module):
130
+ def __init__(
131
+ self,
132
+ dim,
133
+ n_heads,
134
+ d_head,
135
+ dropout=0.0,
136
+ context_dim=None,
137
+ gated_ff=True,
138
+ disable_self_attn=False,
139
+ use_checkpoint=True,
140
+ ):
141
+ super().__init__()
142
+
143
+ self.attn1 = CrossAttention(
144
+ query_dim=dim,
145
+ heads=n_heads,
146
+ dim_head=d_head,
147
+ dropout=dropout,
148
+ context_dim=context_dim,
149
+ ) # is a self-attention if not self.disable_self_attn
150
+ self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff)
151
+ self.norm1 = nn.LayerNorm(dim)
152
+ self.norm2 = nn.LayerNorm(dim)
153
+
154
+ self.checkpoint = checkpoint
155
+ self.use_checkpoint = use_checkpoint
156
+
157
+ def forward(self, x, context=None, query_pe=None):
158
+ if self.use_checkpoint:
159
+ return checkpoint(
160
+ self._forward,
161
+ (x, context, query_pe),
162
+ self.parameters(),
163
+ self.checkpoint,
164
+ )
165
+ else:
166
+ return self._forward(x, context, query_pe)
167
+
168
+ def _forward(self, x, context=None, query_pe=None):
169
+ if context is None:
170
+ context = x
171
+ query = x
172
+ if query_pe is not None:
173
+ query = query + query_pe
174
+ query = self.norm1(query)
175
+ context = self.norm1(context)
176
+ x = self.attn1(query, context=context) + x
177
+ x = self.ff(self.norm2(x)) + x
178
+
179
+ return x
180
+
181
+
182
+ class PosEmbedding(nn.Module):
183
+ def __init__(self, in_channels, N_freqs, logscale=True):
184
+ """
185
+ Defines a function that embeds x to (x, sin(2^k x), cos(2^k x), ...)
186
+ in_channels: number of input channels (3 for both xyz and direction)
187
+ """
188
+ super(PosEmbedding, self).__init__()
189
+ self.N_freqs = N_freqs
190
+ self.in_channels = in_channels
191
+ # self.funcs = [torch.sin, torch.cos]
192
+ # self.out_channels = in_channels*(len(self.funcs)*N_freqs)
193
+ if N_freqs <= 80:
194
+ base = 2
195
+ else:
196
+ base = 5000 ** (1 / (N_freqs / 2.5))
197
+ if logscale:
198
+ freq_bands = base ** torch.linspace(0, N_freqs - 1, N_freqs)[None, None]
199
+ else:
200
+ freq_bands = torch.linspace(1, 2 ** (N_freqs - 1), N_freqs)
201
+ self.register_buffer("freq_bands", freq_bands)
202
+
203
+ def forward(self, x):
204
+ """
205
+ Embeds x to (x, sin(2^k x), cos(2^k x), ...)
206
+ Different from the paper, "x" is also in the output
207
+ See https://github.com/bmild/nerf/issues/12
208
+ Inputs:
209
+ x: (B, self.in_channels)
210
+ Outputs:
211
+ out: (B, self.out_channels)
212
+ """
213
+ shape = x.shape[:-1]
214
+ x = x.reshape(-1, 2, 1)
215
+
216
+ encodings = x * self.freq_bands
217
+ sin_encodings = torch.sin(encodings) # (n, c, num_encoding_functions)
218
+ cos_encodings = torch.cos(encodings)
219
+ pe = torch.cat([sin_encodings, cos_encodings], dim=1)
220
+ pe = pe.reshape(*shape, -1)
221
+ return pe
loomnet/models/blocks/transformer_t.py ADDED
@@ -0,0 +1,268 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ import torch.nn.functional as F
4
+ from einops import rearrange, repeat
5
+
6
+ from loomnet.utils.ops import default, exists, zero_module
7
+
8
+
9
+ class CrossAttention(nn.Module):
10
+ def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.0):
11
+ super().__init__()
12
+ inner_dim = dim_head * heads
13
+
14
+ if context_dim is None:
15
+ context_dim = query_dim
16
+
17
+ self.scale = dim_head**-0.5
18
+ self.heads = heads
19
+
20
+ self.to_q = nn.Linear(query_dim, inner_dim, bias=False)
21
+ self.to_k = nn.Linear(context_dim, inner_dim, bias=False)
22
+ self.to_v = nn.Linear(context_dim, inner_dim, bias=False)
23
+
24
+ self.to_out = nn.Sequential(
25
+ nn.Linear(inner_dim, query_dim), nn.Dropout(dropout)
26
+ )
27
+
28
+ def forward(self, x, context=None, mask=None):
29
+ h = self.heads
30
+
31
+ q = self.to_q(x)
32
+ context = default(context, x)
33
+ k = self.to_k(context)
34
+ v = self.to_v(context)
35
+ q, k, v = map(lambda t: rearrange(t, "b n (h d) -> (b h) n d", h=h), (q, k, v))
36
+
37
+ sim = torch.einsum("b i d, b j d -> b i j", q, k) * self.scale
38
+
39
+ if exists(mask):
40
+ mask = rearrange(mask, "b ... -> b (...)")
41
+ max_neg_value = -torch.finfo(sim.dtype).max
42
+ mask = repeat(mask, "b j -> (b h) () j", h=h)
43
+ sim.masked_fill_(~mask, max_neg_value)
44
+
45
+ # attention, what we cannot get enough of
46
+ attn = sim.softmax(dim=-1)
47
+
48
+ out = torch.einsum("b i j, b j d -> b i d", attn, v)
49
+ out = rearrange(out, "(b h) n d -> b n (h d)", h=h)
50
+ return self.to_out(out)
51
+
52
+
53
+ class LayerScale(nn.Module):
54
+ def __init__(self, dim, init_values=1e-5, inplace=False):
55
+ super().__init__()
56
+ self.inplace = inplace
57
+ self.gamma = nn.Parameter(init_values * torch.ones(dim))
58
+
59
+ def forward(self, x):
60
+ return x.mul_(self.gamma) if self.inplace else x * self.gamma
61
+
62
+
63
+ class GEGLU(nn.Module):
64
+ def __init__(self, dim_in, dim_out):
65
+ super().__init__()
66
+ self.proj = nn.Linear(dim_in, dim_out * 2)
67
+
68
+ def forward(self, x):
69
+ x, gate = self.proj(x).chunk(2, dim=-1)
70
+ return x * F.gelu(gate)
71
+
72
+
73
+ class FeedForward(nn.Module):
74
+ def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0.0):
75
+ super().__init__()
76
+ inner_dim = int(dim * mult)
77
+ if dim_out is None:
78
+ dim_out = dim
79
+
80
+ project_in = (
81
+ nn.Sequential(nn.Linear(dim, inner_dim), nn.GELU())
82
+ if not glu
83
+ else GEGLU(dim, inner_dim)
84
+ )
85
+
86
+ self.net = nn.Sequential(
87
+ project_in, nn.Dropout(dropout), nn.Linear(inner_dim, dim_out)
88
+ )
89
+
90
+ def forward(self, x):
91
+ return self.net(x)
92
+
93
+
94
+ class AdaLayerNorm(nn.Module):
95
+ def __init__(self, n_embd):
96
+ super().__init__()
97
+
98
+ self.silu = nn.SiLU()
99
+ self.linear = nn.Linear(n_embd, n_embd * 2)
100
+ self.layernorm = nn.LayerNorm(n_embd, elementwise_affine=False)
101
+
102
+ def forward(self, x, timestep):
103
+ emb = self.linear(timestep)
104
+ scale, shift = torch.chunk(emb, 2, dim=-1)
105
+ x = self.layernorm(x) * (1 + scale) + shift
106
+ return x
107
+
108
+
109
+ class BasicTransformerBlock(nn.Module):
110
+ def __init__(
111
+ self,
112
+ dim,
113
+ n_heads,
114
+ d_head,
115
+ dropout=0.0,
116
+ context_dim=None,
117
+ gated_ff=True,
118
+ checkpoint=True,
119
+ ):
120
+ super().__init__()
121
+ self.attn1 = CrossAttention(
122
+ query_dim=dim, heads=n_heads, dim_head=d_head, dropout=dropout
123
+ ) # is a self-attention
124
+ self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff)
125
+ self.attn2 = CrossAttention(
126
+ query_dim=dim,
127
+ context_dim=context_dim,
128
+ heads=n_heads,
129
+ dim_head=d_head,
130
+ dropout=dropout,
131
+ ) # is self-attn if context is none
132
+ self.norm1 = AdaLayerNorm(dim)
133
+ self.norm2 = AdaLayerNorm(dim)
134
+ self.norm3 = AdaLayerNorm(dim)
135
+
136
+ def forward(self, x, t, context=None, mask=None):
137
+ x = self.attn1(self.norm1(x, t)) + x
138
+ x = self.attn2(self.norm2(x, t), context=context, mask=mask) + x
139
+ x = self.ff(self.norm3(x, t)) + x
140
+ return x
141
+
142
+
143
+ class Transformer(nn.Module):
144
+ """
145
+ Transformer block for image-like data.
146
+ First, project the input (aka embedding)
147
+ and reshape to b, t, d.
148
+ Then apply standard transformer action.
149
+ Finally, reshape to image
150
+ """
151
+
152
+ def __init__(
153
+ self,
154
+ in_channels,
155
+ n_heads,
156
+ d_head,
157
+ out_channels=None,
158
+ depth=1,
159
+ dropout=0.0,
160
+ context_dim=None,
161
+ ):
162
+ super().__init__()
163
+ self.in_channels = in_channels
164
+ inner_dim = n_heads * d_head
165
+
166
+ self.proj_in = nn.Linear(in_channels, inner_dim, bias=False)
167
+
168
+ self.transformer_blocks = nn.ModuleList(
169
+ [
170
+ BasicTransformerBlock(
171
+ inner_dim,
172
+ n_heads=n_heads,
173
+ d_head=d_head,
174
+ context_dim=context_dim,
175
+ dropout=dropout,
176
+ )
177
+ for _ in range(depth)
178
+ ]
179
+ )
180
+ out_channels = default(out_channels, in_channels)
181
+ self.proj_out = zero_module(nn.Linear(inner_dim, out_channels, bias=False))
182
+ self.t_proj = nn.Linear(1280, inner_dim)
183
+
184
+ def forward(self, x, t, context=None, attention_mask=None):
185
+ x = self.proj_in(x)
186
+ t = self.t_proj(t).unsqueeze(1)
187
+ for block in self.transformer_blocks:
188
+ x = block(x, t, context=context, mask=attention_mask)
189
+
190
+ x = self.proj_out(x)
191
+ return x
192
+
193
+
194
+ class FuseTransformerBlock(nn.Module):
195
+ def __init__(
196
+ self,
197
+ dim,
198
+ n_heads,
199
+ d_head,
200
+ dropout=0.0,
201
+ context_dim=None,
202
+ gated_ff=True,
203
+ checkpoint=True,
204
+ ):
205
+ super().__init__()
206
+ self.attn1 = CrossAttention(
207
+ query_dim=dim, heads=n_heads, dim_head=d_head, dropout=dropout
208
+ ) # is a self-attention
209
+ self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff)
210
+ self.norm1 = AdaLayerNorm(dim)
211
+ self.norm3 = AdaLayerNorm(dim)
212
+
213
+ def forward(self, x, t, context=None, mask=None):
214
+ # (b f n) k s c
215
+ x = self.attn1(self.norm1(x, t), mask=mask) + x
216
+ x = self.ff(self.norm3(x, t)) + x
217
+ return x
218
+
219
+
220
+ class FuseTransformer(nn.Module):
221
+ """
222
+ Transformer block for image-like data.
223
+ First, project the input (aka embedding)
224
+ and reshape to b, t, d.
225
+ Then apply standard transformer action.
226
+ Finally, reshape to image
227
+ """
228
+
229
+ def __init__(
230
+ self,
231
+ in_channels,
232
+ n_heads,
233
+ d_head,
234
+ out_channels=None,
235
+ depth=1,
236
+ dropout=0.0,
237
+ context_dim=None,
238
+ ):
239
+ super().__init__()
240
+ self.in_channels = in_channels
241
+ inner_dim = n_heads * d_head
242
+
243
+ self.proj_in = nn.Linear(in_channels, inner_dim, bias=False)
244
+
245
+ self.transformer_blocks = nn.ModuleList(
246
+ [
247
+ FuseTransformerBlock(
248
+ inner_dim,
249
+ n_heads=n_heads,
250
+ d_head=d_head,
251
+ context_dim=context_dim,
252
+ dropout=dropout,
253
+ )
254
+ for _ in range(depth)
255
+ ]
256
+ )
257
+
258
+ out_channels = default(out_channels, in_channels)
259
+ self.proj_out = zero_module(nn.Linear(inner_dim, out_channels, bias=False))
260
+ self.t_proj = nn.Linear(1280, inner_dim)
261
+
262
+ def forward(self, x, t, context=None, attention_mask=None):
263
+ x = self.proj_in(x)
264
+ t = self.t_proj(t)
265
+ for block in self.transformer_blocks:
266
+ x = block(x, t, context=context, mask=attention_mask)
267
+ x = self.proj_out(x)
268
+ return x