Retain latest files; remove training checkpoint history

Co-authored-by: nousr <nousr@users.noreply.huggingface.co>
Co-authored-by: rom1504 <rom1504@users.noreply.huggingface.co>
Co-authored-by: Veldrovive <Veldrovive@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,27 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,3 @@
+---
+license: mit
+---

best.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bb6b8e748c7e13dfb522bebb7737906d0eb07c5d8c7ca1e9087f80e8a23220c
+size 4863029933

decoder.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be18b4e03694d0b02823f45b73ac85df7edd5ca7e72d633c4c4194577a7aad9b
+size 4490255033

decoder/1.5B/decoder_config.json

@@ -0,0 +1,118 @@
+{
+    "seed": 1,
+    
+    "decoder": {
+        "unets": [
+            {
+                "dim": 320,
+                "cond_dim": 512,
+                "image_embed_dim": 768,
+                "text_embed_dim": 768,
+                "cond_on_text_encodings": true,
+                "channels": 3,
+                "dim_mults": [1, 2, 3, 4],
+                "num_resnet_blocks": 4,
+                "attn_heads": 8,
+                "attn_dim_head": 64,
+                "sparse_attn": true,
+                "memory_efficient": true,
+		        "self_attn": [false, true, true, true]
+            }
+        ],
+        "clip": {
+            "make": "openai",
+            "model": "ViT-L/14"
+        },
+        "image_sizes": [64],
+        "channels": 3,
+        "timesteps": 1000,
+        "loss_type": "l2",
+        "beta_schedule": ["cosine"],
+        "learned_variance": true
+    },
+    "data": {
+        "webdataset_base_url": "pipe:aws s3 cp --quiet s3://s-datasets/laion-aesthetic/data/laion2B-en-aesthetic/{}.tar -", 
+        "embeddings_url": "s3://s-datasets/laion-aesthetic/ordered_embeddings/",
+        "num_workers": 12,
+        "batch_size": 26,
+        "start_shard": 0,
+        "end_shard": 5247,
+        "shard_width": 5,
+        "index_width": 4,
+        "splits": {
+            "train": 0.75,
+            "val": 0.15,
+            "test": 0.1
+        },
+        "shuffle_train": false,
+        "resample_train": true,
+        "preprocessing": {
+            "RandomResizedCrop": {
+                "size": [224, 224],
+                "scale": [0.75, 1.0],
+                "ratio": [1.0, 1.0]
+            },
+            "ToTensor": true
+        }
+    },
+    "train": {
+        "epochs": 1000,
+        "lr": 1e-4,
+        "wd": 0.01,
+        "max_grad_norm": 0.5,
+        "save_every_n_samples": 2000000,
+        "n_sample_images": 10,
+        "device": "cuda:0",
+        "epoch_samples": 10000000,
+        "validation_samples": 100000,
+        "use_ema": true,
+        "ema_beta": 0.99,
+        "save_all": false,
+        "save_latest": true,
+        "save_best": true,
+        "unet_training_mask": [true]
+    },
+    "evaluate": {
+        "n_evaluation_samples": 30,
+        "FID": {
+            "feature": 64
+        },
+        "LPIPS": {
+            "net_type": "vgg",
+            "reduction": "mean"
+        }
+    },
+    "tracker": {
+        "data_path": ".tracker-testing",
+        "overwrite_data_path": true,
+
+        "log": {
+            "log_type": "wandb",
+
+            "wandb_entity": "rom1504",
+            "wandb_project": "dalle2_train_decoder",
+            "wandb_resume": false,
+
+            "verbose": true
+        },
+
+        "load": {
+	    "resume": false,
+            "load_from": null
+        },
+
+        "save": [{
+            "save_to": "wandb"
+        },
+	{
+            "save_to": "huggingface",
+            "huggingface_repo": "laion/DALLE2-PyTorch",
+            "save_meta_to": "decoder/1.5B/",
+
+            "save_latest_to": "decoder/1.5B/latest.pth",
+
+            "save_type": "model"
+        }]
+    }
+}
+

decoder/1.5B/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81a0e9d0282be051461147b1c5988e02900bec7fcaf4972944947f0160d4bbdd
+size 4966221753

decoder/1.5B_laion2B/decoder_config.json

@@ -0,0 +1,117 @@
+{
+    "seed": 1,
+    
+    "decoder": {
+        "unets": [
+            {
+                "dim": 320,
+                "cond_dim": 512,
+                "image_embed_dim": 768,
+                "text_embed_dim": 768,
+                "cond_on_text_encodings": true,
+                "channels": 3,
+                "dim_mults": [1, 2, 3, 4],
+                "num_resnet_blocks": 4,
+                "attn_heads": 8,
+                "attn_dim_head": 64,
+                "sparse_attn": true,
+                "memory_efficient": true,
+		        "self_attn": [false, true, true, true]
+            }
+        ],
+        "clip": {
+            "make": "openai",
+            "model": "ViT-L/14"
+        },
+        "image_sizes": [64],
+        "channels": 3,
+        "timesteps": 1000,
+        "loss_type": "l2",
+        "beta_schedule": ["cosine"],
+        "learned_variance": true
+    },
+    "data": {
+        "webdataset_base_url": "pipe:aws s3 cp --quiet s3://s-datasets/laion5b/laion2B-data/{}.tar -", 
+        "num_workers": 12,
+        "batch_size": 26,
+        "start_shard": 0,
+        "end_shard": 231349,
+        "shard_width": 6,
+        "index_width": 4,
+        "splits": {
+            "train": 0.75,
+            "val": 0.15,
+            "test": 0.1
+        },
+        "shuffle_train": false,
+        "resample_train": true,
+        "preprocessing": {
+            "RandomResizedCrop": {
+                "size": [224, 224],
+                "scale": [0.75, 1.0],
+                "ratio": [1.0, 1.0]
+            },
+            "ToTensor": true
+        }
+    },
+    "train": {
+        "epochs": 1000,
+        "lr": 1e-4,
+        "wd": 0.01,
+        "max_grad_norm": 0.5,
+        "save_every_n_samples": 5000000,
+        "n_sample_images": 10,
+        "device": "cuda:0",
+        "epoch_samples": 10000000,
+        "validation_samples": 100000,
+        "use_ema": true,
+        "ema_beta": 0.99,
+        "save_all": false,
+        "save_latest": true,
+        "save_best": true,
+        "unet_training_mask": [true]
+    },
+    "evaluate": {
+        "n_evaluation_samples": 30,
+        "FID": {
+            "feature": 64
+        },
+        "LPIPS": {
+            "net_type": "vgg",
+            "reduction": "mean"
+        }
+    },
+    "tracker": {
+        "data_path": "/fsx/dalle2/.tracker-laion2B",
+        "overwrite_data_path": true,
+
+        "log": {
+            "log_type": "wandb",
+
+            "wandb_entity": "rom1504",
+            "wandb_project": "dalle2_train_decoder",
+            "wandb_resume": false,
+
+            "verbose": true
+        },
+
+        "load": {
+            "load_from": "local",
+	    "file_path": "/fsx/dalle2/1.5B.pth"
+        },
+
+        "save": [{
+            "save_to": "wandb"
+        },
+	{
+            "save_to": "huggingface",
+            "huggingface_repo": "laion/DALLE2-PyTorch",
+            "save_meta_to": "decoder/1.5B_laion2B/",
+
+            "save_latest_to": "decoder/1.5B_laion2B/latest.pth",
+
+            "save_type": "model"
+        }]
+    }
+}
+

decoder/1.5B_laion2B/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84d20bc9e9cfe1f8240b4d5815ab56e9fbdcaee97bbf552b13e5a62011b0e99c
+size 4966221753

decoder/2.4B_fp16/checkpoints/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7de079d8918990a80f2c4a23ac527161f3cfdd4e1944d27faebac550bd9bc83
+size 19169376093

decoder/2.4B_fp16/decoder_config.json

@@ -0,0 +1,122 @@
+{
+    "seed": 1,
+    
+    "decoder": {
+        "unets": [
+            {
+                "dim": 576,
+                "image_embed_dim": 768,
+                "text_embed_dim": 768,
+                "cond_on_text_encodings": false,
+                "channels": 3,
+                "dim_mults": [1, 2, 3, 4],
+                "self_attn": [false, true, true, true],
+                "num_resnet_blocks": 4,
+                "attn_heads": 16,
+                "attn_dim_head": 64,
+                "sparse_attn": false,
+                "memory_efficient": true
+            }
+        ],
+        "image_sizes": [64],
+        "channels": 3,
+        "timesteps": 1000,
+        "loss_type": "l2",
+        "beta_schedule": ["cosine"],
+        "learned_variance": false
+    },
+    "data": {
+        "webdataset_base_url": "pipe:aws s3 cp --quiet s3://s-datasets/laion-aesthetic/data/laion2B-en-aesthetic/{}.tar -", 
+        "img_embeddings_url": "s3://s-datasets/laion-aesthetic/ordered_embeddings/",
+        "num_workers": 6,
+        "batch_size": 40,
+        "start_shard": 0,
+        "end_shard": 5247,
+        "shard_width": 5,
+        "index_width": 4,
+        "splits": {
+            "train": 0.75,
+            "val": 0.15,
+            "test": 0.1
+        },
+        "shuffle_train": false,
+        "resample_train": true,
+        "preprocessing": {
+            "RandomResizedCrop": {
+                "size": [224, 224],
+                "scale": [0.75, 1.0],
+                "ratio": [1.0, 1.0]
+            },
+            "ToTensor": true
+        }
+    },
+    "train": {
+        "epochs": 1000,
+        "lr":1.2e-4,
+        "wd": 0.0,
+        "warmup_steps": 1000,
+        "max_grad_norm": 0.5,
+        "save_every_n_samples": 1500000,
+        "n_sample_images": 10,
+        "device": "cuda:0",
+        "epoch_samples": 3000000,
+        "validation_samples": 100000,
+        "use_ema": true,
+        "ema_beta": 0.9999,
+        "unet_training_mask": [true],
+        "find_unused_parameters": false
+    },
+    "evaluate": {
+        "n_evaluation_samples": 10,
+        "FID": {
+            "feature": 64
+        },
+        "LPIPS": {
+            "net_type": "vgg",
+            "reduction": "mean"
+        }
+    },
+    "tracker": {
+        "data_path": "/fsx/aidan/new/multinode/experiments/deepspeed_fp16/.tracker-data",
+        "overwrite_data_path": true,
+
+        "log": {
+            "log_type": "wandb",
+
+            "wandb_entity": "Veldrovive",
+            "wandb_project": "dalle2_train_decoder",
+            "wandb_run_name": "3B deepspeed fp16",
+
+            "auto_resume": true,
+            "verbose": true
+        },
+
+        "load": {
+            "load_from": "local",
+            "only_auto_resume": true,
+	        "file_path": "/fsx/aidan/new/multinode/experiments/deepspeed_fp16/models/checkpoints/latest.pth"
+        },
+
+        "save": [{
+            "save_to": "huggingface",
+            "huggingface_repo": "laion/DALLE2-PyTorch",
+            
+            "save_meta_to": "decoder/2.4B_fp16/",
+            "save_latest_to": "decoder/2.4B_fp16/latest.pth",
+
+            "save_type": "model"
+        },{
+            "save_to": "huggingface",
+            "huggingface_repo": "laion/DALLE2-PyTorch",
+            
+            "save_latest_to": "decoder/2.4B_fp16/checkpoints/latest.pth",
+
+            "save_type": "checkpoint"
+        },{
+            "save_to": "local",
+            "save_latest_to": "/fsx/aidan/new/multinode/experiments/deepspeed_fp16/models/checkpoints/latest.pth",
+
+            "save_type": "checkpoint"
+        }]
+    }
+}

decoder/2.4B_fp16/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92f8dd9fbd231de4401ddd4632991e89837e7ec24935c1390ba2e69e11dea4b5
+size 12392221457

decoder/2B/checkpoints/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03af89c41f07e1d57f57b7e9a8936376b4c2ef5f0967239fb750150fd533ed21
+size 19893880057

decoder/2B/decoder_config.json

@@ -0,0 +1,111 @@
+{
+    "seed": 1,
+    
+    "decoder": {
+        "unets": [
+            {
+                "dim": 512,
+                "cond_dim": 512,
+                "image_embed_dim": 768,
+                "text_embed_dim": 768,
+                "cond_on_text_encodings": false,
+                "channels": 3,
+                "dim_mults": [1, 2, 3, 4],
+                "self_attn": [false, true, true, true],
+                "num_resnet_blocks": 4,
+                "attn_heads": 8,
+                "attn_dim_head": 64,
+                "sparse_attn": true,
+                "memory_efficient": true
+            }
+        ],
+        "image_sizes": [64],
+        "channels": 3,
+        "timesteps": 1000,
+        "loss_type": "l2",
+        "beta_schedule": ["cosine"],
+        "learned_variance": false
+    },
+    "data": {
+        "webdataset_base_url": "pipe:aws s3 cp --quiet s3://s-datasets/laion-aesthetic/data/laion2B-en-aesthetic/{}.tar -", 
+        "img_embeddings_url": "s3://s-datasets/laion-aesthetic/ordered_embeddings/",
+        "num_workers": 6,
+        "batch_size": 1,
+        "start_shard": 0,
+        "end_shard": 5247,
+        "shard_width": 5,
+        "index_width": 4,
+        "splits": {
+            "train": 0.75,
+            "val": 0.15,
+            "test": 0.1
+        },
+        "shuffle_train": false,
+        "resample_train": true,
+        "preprocessing": {
+            "RandomResizedCrop": {
+                "size": [64, 64],
+                "scale": [0.75, 1.0],
+                "ratio": [1.0, 1.0]
+            },
+            "ToTensor": true
+        }
+    },
+    "train": {
+        "epochs": 1000,
+        "lr":1.2e-4,
+        "wd": 0.0,
+        "warmup_steps": 1000,
+        "max_grad_norm": 0.5,
+        "save_every_n_samples": 545400,
+        "n_sample_images": 10,
+        "device": "cuda:0",
+        "epoch_samples": 2100000,
+        "validation_samples": 100000,
+        "use_ema": true,
+        "ema_beta": 0.99,
+        "save_all": false,
+        "save_latest": true,
+        "save_best": true,
+        "unet_training_mask": [true],
+        "find_unused_parameters": true
+    },
+    "evaluate": {
+        "n_evaluation_samples": 10,
+        "FID": {
+            "feature": 64
+        },
+        "LPIPS": {
+            "net_type": "vgg",
+            "reduction": "mean"
+        }
+    },
+    "tracker": {
+        "data_path": "/fsx/aidan/new/multinode/experiments/deepspeed/.tracker-data",
+        "overwrite_data_path": true,
+
+        "log": {
+            "log_type": "console",
+
+            "wandb_entity": "Veldrovive",
+            "wandb_project": "dalle2_train_decoder",
+            "wandb_resume": false,
+
+            "auto_resume": true,
+            "verbose": true
+        },
+
+        "load": {
+            "load_from": null,
+            "only_auto_resume": true,
+	        "file_path": "/fsx/aidan/new/multinode/experiments/deepspeed/models/checkpoints/latest.pth"
+        },
+
+        "save": [{
+            "save_to": "local",
+            "save_latest_to": "/fsx/aidan/new/multinode/experiments/deepspeed/models/checkpoints/latest.pth",
+
+            "save_type": "checkpoint"
+        }]
+    }
+}

decoder/2B/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3201e8ce73e12bbc92530c6c9317ad29b7d6ee0ed21c52dafa42f2d9d768a285
+size 9793924217

decoder/3B_fp16/checkpoints/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6db755189f5bc595401766f4d3a62956b5db5ac6d7cb8a23688dcecf7d01612
+size 19169396811

decoder/3B_fp16/decoder_config.json

@@ -0,0 +1,125 @@
+{
+    "seed": 1,
+    
+    "decoder": {
+        "unets": [
+            {
+                "dim": 576,
+                "image_embed_dim": 768,
+                "text_embed_dim": 768,
+                "cond_on_text_encodings": false,
+                "channels": 3,
+                "dim_mults": [1, 2, 3, 4],
+                "self_attn": [false, true, true, true],
+                "num_resnet_blocks": 4,
+                "attn_heads": 16,
+                "attn_dim_head": 64,
+                "sparse_attn": false,
+                "memory_efficient": true
+            }
+        ],
+        "clip": {
+            "make": "openai",
+            "model": "ViT-L/14"
+        },
+        "image_sizes": [64],
+        "channels": 3,
+        "timesteps": 1000,
+        "loss_type": "l2",
+        "beta_schedule": ["cosine"],
+        "learned_variance": false
+    },
+    "data": {
+        "webdataset_base_url": "pipe:aws s3 cp --quiet s3://s-datasets/laion5b/laion2B-data/{}.tar -", 
+        "num_workers": 6,
+        "batch_size": 40,
+        "start_shard": 0,
+        "end_shard": 231349,
+        "shard_width": 6,
+        "index_width": 4,
+        "splits": {
+            "train": 0.75,
+            "val": 0.15,
+            "test": 0.1
+        },
+        "shuffle_train": false,
+        "resample_train": true,
+        "preprocessing": {
+            "RandomResizedCrop": {
+                "size": [224, 224],
+                "scale": [0.75, 1.0],
+                "ratio": [1.0, 1.0]
+            },
+            "ToTensor": true
+        }
+    },
+    "train": {
+        "epochs": 1000,
+        "lr":1.2e-4,
+        "wd": 0.0,
+        "warmup_steps": 1000,
+        "max_grad_norm": 0.5,
+        "save_every_n_samples": 1500000,
+        "n_sample_images": 10,
+        "device": "cuda:0",
+        "epoch_samples": 6000000,
+        "validation_samples": 100000,
+        "use_ema": true,
+        "ema_beta": 0.9999,
+        "unet_training_mask": [true],
+        "find_unused_parameters": false
+    },
+    "evaluate": {
+        "n_evaluation_samples": 10,
+        "FID": {
+            "feature": 64
+        },
+        "LPIPS": {
+            "net_type": "vgg",
+            "reduction": "mean"
+        }
+    },
+    "tracker": {
+        "data_path": "/fsx/aidan/new/multinode/experiments/deepspeed_fp16_2b/.tracker-data",
+        "overwrite_data_path": true,
+
+        "log": {
+            "log_type": "wandb",
+
+            "wandb_entity": "Veldrovive",
+            "wandb_project": "dalle2_train_decoder",
+            "wandb_run_name": "3B deepspeed fp16 2B",
+
+            "auto_resume": true,
+            "verbose": true
+        },
+
+        "load": {
+            "load_from": "local",
+            "only_auto_resume": true,
+	        "file_path": "/fsx/aidan/new/multinode/experiments/deepspeed_fp16_2b/models/checkpoints/latest.pth"
+        },
+
+        "save": [{
+            "save_to": "huggingface",
+            "huggingface_repo": "laion/DALLE2-PyTorch",
+            
+            "save_meta_to": "decoder/3B_fp16/",
+            "save_latest_to": "decoder/3B_fp16/latest.pth",
+
+            "save_type": "model"
+        },{
+            "save_to": "huggingface",
+            "huggingface_repo": "laion/DALLE2-PyTorch",
+            
+            "save_latest_to": "decoder/3B_fp16/checkpoints/latest.pth",
+
+            "save_type": "checkpoint"
+        },{
+            "save_to": "local",
+            "save_latest_to": "/fsx/aidan/new/multinode/experiments/deepspeed_fp16_2b/models/checkpoints/latest.pth",
+
+            "save_type": "checkpoint"
+        }]
+    }
+}

decoder/3B_fp16/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b717175ae98fa0cadb14e50d5bc7d9d5147d00a4b057f5793835594f5362a55
+size 12392221393

decoder/3B_fp16_highlr/decoder_config.json

@@ -0,0 +1,125 @@
+{
+    "seed": 1,
+    
+    "decoder": {
+        "unets": [
+            {
+                "dim": 576,
+                "image_embed_dim": 768,
+                "text_embed_dim": 768,
+                "cond_on_text_encodings": false,
+                "channels": 3,
+                "dim_mults": [1, 2, 3, 4],
+                "self_attn": [false, true, true, true],
+                "num_resnet_blocks": 4,
+                "attn_heads": 16,
+                "attn_dim_head": 64,
+                "sparse_attn": false,
+                "memory_efficient": true
+            }
+        ],
+        "clip": {
+            "make": "openai",
+            "model": "ViT-L/14"
+        },
+        "image_sizes": [64],
+        "channels": 3,
+        "timesteps": 1000,
+        "loss_type": "l2",
+        "beta_schedule": ["cosine"],
+        "learned_variance": false
+    },
+    "data": {
+        "webdataset_base_url": "pipe:aws s3 cp --quiet s3://s-datasets/laion5b/laion2B-data/{}.tar -", 
+        "num_workers": 6,
+        "batch_size": 40,
+        "start_shard": 0,
+        "end_shard": 231349,
+        "shard_width": 6,
+        "index_width": 4,
+        "splits": {
+            "train": 0.75,
+            "val": 0.15,
+            "test": 0.1
+        },
+        "shuffle_train": false,
+        "resample_train": true,
+        "preprocessing": {
+            "RandomResizedCrop": {
+                "size": [224, 224],
+                "scale": [0.75, 1.0],
+                "ratio": [1.0, 1.0]
+            },
+            "ToTensor": true
+        }
+    },
+    "train": {
+        "epochs": 1000,
+        "lr":9.6e-4,
+        "wd": 0.0,
+        "warmup_steps": 1000,
+        "max_grad_norm": 0.5,
+        "save_every_n_samples": 1500000,
+        "n_sample_images": 10,
+        "device": "cuda:0",
+        "epoch_samples": 6000000,
+        "validation_samples": 100000,
+        "use_ema": true,
+        "ema_beta": 0.9999,
+        "unet_training_mask": [true],
+        "find_unused_parameters": false
+    },
+    "evaluate": {
+        "n_evaluation_samples": 10,
+        "FID": {
+            "feature": 64
+        },
+        "LPIPS": {
+            "net_type": "vgg",
+            "reduction": "mean"
+        }
+    },
+    "tracker": {
+        "data_path": "/fsx/aidan/new/multinode/experiments/deepspeed_fp16_2b/.tracker-data-highlr",
+        "overwrite_data_path": true,
+
+        "log": {
+            "log_type": "wandb",
+
+            "wandb_entity": "Veldrovive",
+            "wandb_project": "dalle2_train_decoder",
+            "wandb_run_name": "High LR - 3B deepspeed fp16 2B",
+
+            "auto_resume": true,
+            "verbose": true
+        },
+
+        "load": {
+            "load_from": "local",
+            "only_auto_resume": true,
+	        "file_path": "/fsx/aidan/new/multinode/experiments/deepspeed_fp16_2b/models_highlr/checkpoints/latest.pth"
+        },
+
+        "save": [{
+            "save_to": "huggingface",
+            "huggingface_repo": "laion/DALLE2-PyTorch",
+            
+            "save_meta_to": "decoder/3B_fp16_highlr/",
+            "save_latest_to": "decoder/3B_fp16_highlr/latest.pth",
+
+            "save_type": "model"
+        },{
+            "save_to": "huggingface",
+            "huggingface_repo": "laion/DALLE2-PyTorch",
+            
+            "save_latest_to": "decoder/3B_fp16_highlr/checkpoints/latest.pth",
+
+            "save_type": "checkpoint"
+        },{
+            "save_to": "local",
+            "save_latest_to": "/fsx/aidan/new/multinode/experiments/deepspeed_fp16_2b/models_highlr/checkpoints/latest.pth",
+
+            "save_type": "checkpoint"
+        }]
+    }
+}

decoder/laion_august/decoder_config.json

@@ -0,0 +1,125 @@
+{
+    "seed": 1,
+    "decoder": {
+        "unets": [
+            {
+                "dim": 576,
+                "cond_dim": 512,
+                "image_embed_dim": 768,
+                "text_embed_dim": 768,
+                "cond_on_text_encodings": true,
+                "channels": 3,
+                "dim_mults": [1, 2, 3, 4],
+                "num_resnet_blocks": 4,
+                "attn_heads": 8,
+                "attn_dim_head": 64,
+                "sparse_attn": true,
+		"checkpoint_during_training": true,
+                "memory_efficient": true,
+		        "self_attn": [false, true, true, true]
+            }
+        ],
+        "clip": {
+            "make": "openai",
+            "model": "ViT-L/14"
+        },
+        "image_sizes": [64],
+        "channels": 3,
+        "timesteps": 1000,
+	"sample_timesteps": 250,
+        "loss_type": "l2",
+        "beta_schedule": ["cosine"],
+        "learned_variance": true
+    },
+    "data": {
+        "webdataset_base_url": "pipe:aws s3 cp --quiet s3://s-datasets/laion5b/laion2B-data/{}.tar -", 
+        "num_workers": 12,
+        "batch_size": 2,
+        "start_shard": 0,
+        "end_shard": 231349,
+        "shard_width": 6,
+        "index_width": 4,
+        "splits": {
+            "train": 0.75,
+            "val": 0.15,
+            "test": 0.1
+        },
+        "shuffle_train": false,
+        "resample_train": true,
+        "preprocessing": {
+            "RandomResizedCrop": {
+                "size": [224, 224],
+                "scale": [0.75, 1.0],
+                "ratio": [1.0, 1.0]
+            },
+            "ToTensor": true
+        }
+    },
+    "train": {
+        "epochs": 1000,
+        "lr": 1e-4,
+        "wd": 0.01,
+        "max_grad_norm": 0.5,
+        "save_every_n_samples": 5000000,
+        "n_sample_images": 10,
+        "device": "cuda:0",
+        "epoch_samples": 10000000,
+        "validation_samples": 100000,
+        "use_ema": true,
+        "ema_beta": 0.99,
+        "save_all": false,
+        "save_latest": true,
+        "save_best": true,
+        "unet_training_mask": [true]
+    },
+    "evaluate": {
+        "n_evaluation_samples": 30,
+        "FID": {
+            "feature": 64
+        },
+        "LPIPS": {
+            "net_type": "vgg",
+            "reduction": "mean"
+        }
+    },
+    "tracker": {
+        "data_path": "/fsx/dalle2/.tracker-laion-august",
+        "overwrite_data_path": true,
+
+        "log": {
+            "log_type": "wandb",
+
+	    "auto_resume": true,
+            "wandb_entity": "rom1504",
+            "wandb_project": "dalle2_train_decoder",
+            "wandb_resume": false,
+	    "auto_resume": false,
+
+            "verbose": true
+        },
+
+        "load": {
+            "load_from": null
+        },
+
+        "save": [
+	{
+            "save_to": "huggingface",
+            "huggingface_repo": "laion/DALLE2-PyTorch",
+            "save_meta_to": "decoder/laion_august/",
+
+            "save_latest_to": "decoder/laion_august/latest.pth",
+            "save_type": "model",
+	    "save_latest": true
+        },
+	{
+            "save_to": "huggingface",
+            "huggingface_repo": "laion/DALLE2-PyTorch",
+	    "save_latest": true,
+            "save_latest_to": "decoder/laion_august/checkpoints/latest.pth",
+            "save_type": "checkpoint"
+	}
+	]
+    }
+}
+

decoder/small_32gpus/decoder_config.json

@@ -0,0 +1,120 @@
+{
+    "seed": 1,
+    
+    "decoder": {
+        "unets": [
+            {
+                "dim": 256,
+                "cond_dim": 512,
+                "image_embed_dim": 768,
+                "text_embed_dim": 768,
+                "cond_on_text_encodings": true,
+                "channels": 3,
+                "dim_mults": [1, 2, 3, 4],
+                "num_resnet_blocks": 4,
+                "attn_heads": 8,
+                "attn_dim_head": 64,
+                "sparse_attn": true,
+                "memory_efficient": true,
+		        "self_attn": [false, true, true, true]
+            }
+        ],
+        "clip": {
+            "make": "openai",
+            "model": "ViT-L/14"
+        },
+        "image_sizes": [64],
+        "channels": 3,
+        "timesteps": 1000,
+        "loss_type": "l2",
+        "beta_schedule": ["cosine"],
+        "learned_variance": true
+    },
+    "data": {
+        "webdataset_base_url": "pipe:aws s3 cp --quiet s3://s-datasets/laion-aesthetic/data/laion2B-en-aesthetic/{}.tar -", 
+        "embeddings_url": "s3://s-datasets/laion-aesthetic/ordered_embeddings/",
+        "num_workers": 12,
+        "batch_size": 60,
+        "start_shard": 0,
+        "end_shard": 5247,
+        "shard_width": 5,
+        "index_width": 4,
+        "splits": {
+            "train": 0.75,
+            "val": 0.15,
+            "test": 0.1
+        },
+        "shuffle_train": false,
+        "resample_train": true,
+        "preprocessing": {
+            "RandomResizedCrop": {
+                "size": [64, 64],
+                "scale": [0.75, 1.0],
+                "ratio": [1.0, 1.0]
+            },
+            "ToTensor": true
+        }
+    },
+    "train": {
+        "epochs": 1000,
+        "lr": 5e-5,
+        "wd": 0.01,
+        "max_grad_norm": 0.5,
+        "save_every_n_samples": 2000000,
+        "n_sample_images": 10,
+        "device": "cuda:0",
+        "epoch_samples": 8000000,
+        "validation_samples": 100000,
+        "use_ema": true,
+        "ema_beta": 0.99,
+        "save_all": false,
+        "save_latest": true,
+        "save_best": true,
+        "unet_training_mask": [true]
+    },
+    "evaluate": {
+        "n_evaluation_samples": 30,
+        "FID": {
+            "feature": 64
+        },
+        "LPIPS": {
+            "net_type": "vgg",
+            "reduction": "mean"
+        }
+    },
+    "tracker": {
+        "data_path": ".tracker-data-2",
+        "overwrite_data_path": true,
+
+        "log": {
+            "log_type": "wandb",
+
+            "wandb_entity": "rom1504",
+            "wandb_project": "dalle2_train_decoder",
+            "wandb_resume": true,
+	    "wandb_run_id": "2yi3dyra",
+
+            "verbose": true
+        },
+
+        "load": {
+	    "resume": true,
+            "load_from": "url",
+            "url": "https://api.wandb.ai/files/rom1504/dalle2_train_decoder/2yi3dyra/latest.pth"
+        },
+
+        "save": [{
+            "save_to": "wandb"
+        },
+        {
+            "save_to": "huggingface",
+            "huggingface_repo": "laion/DALLE2-PyTorch",
+	    "save_meta_to": "decoder/small_32gpus/",
+
+            "save_latest_to": "decoder/small_32gpus/latest.pth",
+            
+            "save_type": "model"
+        }]
+    }
+}
+

decoder/small_32gpus/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19475c7b8f2a9d1feeb6221ef2372f9718058b62cb4f4b54a936028bd5cffd34
+size 3565749905

decoder/v1.0.2/checkpoints/checkpoint.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6ddd376df40c1112b81e245dd8e7b0a1dbdfbbe5f22996524bad6c82c0d4a0a
+size 15326138914

decoder/v1.0.2/checkpoints/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:536f1f2764789f631e9de92beeec76c5a742c9ac98673e1593a947d2355fcabf
+size 16655575706

decoder/v1.0.2/decoder_config.json

@@ -0,0 +1,125 @@
+{
+    "seed": 1,
+    
+    "decoder": {
+        "unets": [
+            {
+                "dim": 320,
+                "cond_dim": 512,
+                "image_embed_dim": 768,
+                "text_embed_dim": 768,
+                "cond_on_text_encodings": true,
+                "channels": 3,
+                "dim_mults": [1, 2, 3, 4],
+                "num_resnet_blocks": 4,
+                "attn_heads": 8,
+                "attn_dim_head": 64,
+                "sparse_attn": true,
+                "memory_efficient": true,
+		        "self_attn": [false, true, true, true]
+            }
+        ],
+        "clip": {
+            "make": "openai",
+            "model": "ViT-L/14"
+        },
+        "image_sizes": [64],
+        "channels": 3,
+        "timesteps": 1000,
+        "loss_type": "l2",
+        "beta_schedule": ["cosine"],
+        "learned_variance": true
+    },
+    "data": {
+        "webdataset_base_url": "pipe:aws s3 cp --quiet s3://s-datasets/laion5b/laion2B-data/{}.tar -", 
+        "num_workers": 12,
+        "batch_size": 26,
+        "start_shard": 0,
+        "end_shard": 231349,
+        "shard_width": 6,
+        "index_width": 4,
+        "splits": {
+            "train": 0.75,
+            "val": 0.15,
+            "test": 0.1
+        },
+        "shuffle_train": false,
+        "resample_train": true,
+        "preprocessing": {
+            "RandomResizedCrop": {
+                "size": [224, 224],
+                "scale": [0.75, 1.0],
+                "ratio": [1.0, 1.0]
+            },
+            "ToTensor": true
+        }
+    },
+    "train": {
+        "epochs": 1000,
+        "lr": 1e-4,
+        "wd": 0.01,
+        "max_grad_norm": 0.5,
+        "save_every_n_samples": 5000000,
+        "n_sample_images": 10,
+        "device": "cuda:0",
+        "epoch_samples": 10000000,
+        "validation_samples": 100000,
+        "use_ema": true,
+        "ema_beta": 0.99,
+        "unet_training_mask": [true]
+    },
+    "evaluate": {
+        "n_evaluation_samples": 30,
+        "FID": {
+            "feature": 64
+        },
+        "LPIPS": {
+            "net_type": "vgg",
+            "reduction": "mean"
+        }
+    },
+    "tracker": {
+        "data_path": "/fsx/aidan/new/multinode/experiments/decoder_first_stage/.tracker-data",
+        "overwrite_data_path": true,
+
+        "log": {
+            "log_type": "wandb",
+
+            "wandb_entity": "Veldrovive",
+            "wandb_project": "dalle2_train_decoder",
+            "wandb_resume": false,
+
+            "auto_resume": true,
+            "verbose": true
+        },
+
+        "load": {
+            "load_from": "local",
+            "only_auto_resume": true,
+	        "file_path": "/fsx/aidan/new/multinode/experiments/decoder_first_stage/models/checkpoints/latest.pth"
+        },
+
+        "save": [
+        {
+            "save_to": "huggingface",
+            "huggingface_repo": "laion/DALLE2-PyTorch",
+            
+            "save_meta_to": "decoder/v1.0.2/",
+            "save_latest_to": "decoder/v1.0.2/latest.pth",
+
+            "save_type": "model"
+        },{
+            "save_to": "huggingface",
+            "huggingface_repo": "laion/DALLE2-PyTorch",
+            
+            "save_latest_to": "decoder/v1.0.2/checkpoints/latest.pth",
+
+            "save_type": "checkpoint"
+        },{
+            "save_to": "local",
+            "save_latest_to": "/fsx/aidan/new/multinode/experiments/decoder_first_stage/models/checkpoints/latest.pth",
+
+            "save_type": "checkpoint"
+        }]
+    }
+}

decoder/v1.0.2/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5306218460f47d5f15344bf0f32919370b4994a482e2760733bd881a7efddd49
+size 3930800893

decoder_config.json

@@ -0,0 +1,118 @@
+{
+    "seed": 1,
+    
+    "decoder": {
+        "unets": [
+            {
+                "dim": 416,
+                "cond_dim": 512,
+                "image_embed_dim": 768,
+                "text_embed_dim": 768,
+                "cond_on_text_encodings": true,
+                "channels": 3,
+                "dim_mults": [1, 2, 3, 4],
+                "num_resnet_blocks": 4,
+                "attn_heads": 8,
+                "attn_dim_head": 64,
+                "sparse_attn": true,
+                "memory_efficient": true
+            }
+        ],
+        "clip": {
+            "make": "openai",
+            "model": "ViT-L/14"
+        },
+        "image_sizes": [64],
+        "channels": 3,
+        "timesteps": 1000,
+        "loss_type": "l2",
+        "beta_schedule": ["cosine"],
+        "learned_variance": true
+    },
+    "data": {
+        "webdataset_base_url": "pipe:aws s3 cp --quiet s3://s-datasets/laion-aesthetic/data/laion2B-en-aesthetic/{}.tar -", 
+        "embeddings_url": "s3://s-datasets/laion-aesthetic/ordered_embeddings/",
+        "num_workers": 12,
+        "batch_size": 21,
+        "start_shard": 0,
+        "end_shard": 5247,
+        "shard_width": 5,
+        "index_width": 4,
+        "splits": {
+            "train": 0.75,
+            "val": 0.15,
+            "test": 0.1
+        },
+        "shuffle_train": false,
+        "resample_train": true,
+        "preprocessing": {
+            "RandomResizedCrop": {
+                "size": [64, 64],
+                "scale": [0.75, 1.0],
+                "ratio": [1.0, 1.0]
+            },
+            "ToTensor": true
+        }
+    },
+    "train": {
+        "epochs": 1000,
+        "lr":1e-4,
+        "wd": 0.01,
+        "max_grad_norm": 0.5,
+        "save_every_n_samples": 200000,
+        "n_sample_images": 10,
+        "device": "cuda:0",
+        "epoch_samples": 2000000,
+        "validation_samples": 100000,
+        "use_ema": true,
+        "ema_beta": 0.99,
+        "save_all": false,
+        "save_latest": true,
+        "save_best": true,
+        "unet_training_mask": [true]
+    },
+    "evaluate": {
+        "n_evaluation_samples": 30,
+        "FID": {
+            "feature": 64
+        },
+        "LPIPS": {
+            "net_type": "vgg",
+            "reduction": "mean"
+        }
+    },
+    "tracker": {
+        "data_path": ".tracker-data-2",
+        "overwrite_data_path": true,
+
+        "log": {
+            "log_type": "wandb",
+
+            "wandb_entity": "nousr_laion",
+            "wandb_project": "dalle2_train_decoder",
+    
+            "wandb_run_id": "5ojoz4bw",
+
+            "wandb_resume": true, 
+
+            "verbose": true
+        },
+
+        "load": {
+            "load_from": "url",
+            "url": "https://huggingface.co/Veldrovive/test_model/resolve/main/eu_latest_checkpoint2.pth" 
+        },
+
+        "save": [{
+            "save_to": "wandb"
+        },
+        {
+            "save_to": "huggingface",
+            "huggingface_repo": "laion/DALLE2-PyTorch",
+
+            "save_latest_to": "latest_{epoch}.pth",
+            
+            "save_type": "model"
+        }]
+    }
+}

latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:231f2b3f0b65d09833e6e30cf8e1fe76cffabb2e2123d428c2b6de0da3ece73a
+size 5554894825

latest_10.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1742010fe2abce4ed9ec233a2ea911d9ee93b5095b73e2bd8e50f7ee6163f340
+size 5554894825

latest_11.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f13638d148ad64cbb922ba4b4703fc8cd5f3d3d8b79bb298ab2b2060473aa62d
+size 5554894825

latest_12.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ced8de62d60211da8459ca48ef4e01dbbc99b3489882a3eb4c6da128527ddaa
+size 5554894825

latest_13.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:165e838117029220223ac7778a52314f7d1c313928d36279f4c7ab34ddb4b1ef
+size 5554894825

latest_14.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b24da5e2e0920e1b10ed478324c24e471382736cb21c3c299ca586ec0a82b67
+size 5554894825

prior.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef5b66393bb1e0f9cf5f1bd8a3e007a2f85e23a7cabfdabc12226a8a04e930ca
+size 5853919809

prior/best.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e8da97ed73d4a10f81f0c7bc1ba0b4417c0aab32238a45eddf02fdd6c8f4c4d
+size 1339360429

prior/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3275ee517fd509fa3216bf781779b6b01e3bbf7ebf967fbdfa5e71c657ae3ff
+size 1339360429

prior/prior_config.json

@@ -0,0 +1,94 @@
+{
+    "prior": {
+        "clip": {
+            "make": "openai",
+            "model": "ViT-L/14"
+        },
+        "net": {
+            "dim": 768,
+            "depth": 12,
+            "num_timesteps": 1000,
+            "max_text_len": 77,
+            "num_time_embeds": 1,
+            "num_image_embeds": 1,
+            "num_text_embeds": 1,
+            "dim_head": 64,
+            "heads": 12,
+            "ff_mult": 4,
+            "norm_out": true,
+            "attn_dropout": 0.05,
+            "ff_dropout": 0.05,
+            "final_proj": true,
+            "normformer": true,
+            "rotary_emb": true
+        },
+        "image_embed_dim": 768,
+        "image_size": 224,
+        "image_channels": 3,
+        "timesteps": 1000,
+        "sample_timesteps": 64,
+        "cond_drop_prob": 0.1,
+        "loss_type": "l2",
+        "predict_x_start": true,
+        "beta_schedule": "cosine",
+        "condition_on_text_encodings": true
+    },
+    "data": {
+        "batch_size": 225,
+        "num_data_points": 500000000,
+        "eval_every_seconds": 1600,
+        "image_url": "s3://s-laion5b/embeddings/vit-l-14/laion2B-en/img_emb/",
+        "meta_url": "s3://s-laion5b/embeddings/vit-l-14/laion2B-en/metadata/",
+        "splits": {
+            "train": 0.9,
+            "val":  1.2e-5,
+            "test": 0.09998799999999998
+        }
+    },
+    "train": {
+        "epochs": 5,
+        "lr": 0.5e-5,
+        "wd": 6.02e-2,
+        "max_grad_norm": 0.5,
+        "use_ema": true,
+        "ema_beta": 0.9999,
+        "ema_update_after_step": 50,
+        "warmup_steps": 50,
+        "amp": false,
+        "save_every_seconds": 3600,
+        "eval_timesteps": [64, 1000],
+        "random_seed": 84513
+    },
+    "tracker": {
+        "data_path": ".prior",
+        "overwrite_data_path": true,
+        "log": {
+            "log_type": "wandb",
+            "wandb_entity": "nousr_laion",
+            "wandb_project": "dalle2_diffusion_prior",
+            "wandb_resume": true,
+            "wandb_run_id": "20jdn8i2",
+            "verbose": true
+        },
+        "load": {
+            "load_from": "local",
+            "file_path": "/fsx/nousr/.prior/latest_checkpoint.pth"
+        },
+        "save": [
+            {
+                "save_to": "local",
+                "save_type": "checkpoint",
+                "save_latest_to": ".prior/latest_checkpoint.pth",
+                "save_best_to": ".prior/best_checkpoint.pth"
+            },
+            {
+                "save_to": "huggingface",
+                "huggingface_repo": "laion/DALLE2-PyTorch",
+                "save_meta_to": "prior/",
+                "save_latest_to": "prior/latest.pth",
+                "save_best_to": "prior/best.pth",
+                "save_type": "model"
+            }
+        ]
+    }
+}

prior_ema_fp16.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7d6ec6be12f285c0cd128655974b544885a49a3ff17feb5346226fea59757fa
+size 1354273725

upsampler/1024/v1.0.2/checkpoints/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79ce607c4da1180c909eeadbe60437538352a11eea3cb8eccec239fe2f7d2335
+size 5784890776

upsampler/1024/v1.0.3/decoder_config.json

@@ -0,0 +1,159 @@
+{
+    "seed": 1,
+    
+    "decoder": {
+        "unets": [
+            {
+                "dim": 32,
+                "cond_dim": 512,
+                "image_embed_dim": 768,
+                "text_embed_dim": 768,
+                "cond_on_text_encodings": true,
+                "channels": 3,
+                "dim_mults": [1, 2, 3, 4],
+                "num_resnet_blocks": 4,
+                "attn_heads": 8,
+                "attn_dim_head": 64,
+                "sparse_attn": true,
+                "memory_efficient": true,
+		        "self_attn": [false, true, true, true]
+            },
+            {
+                "dim": 32,
+                "cond_dim": 512,
+                "image_embed_dim": 768,
+                "text_embed_dim": 768,
+                "cond_on_text_encodings": true,
+                "channels": 3,
+                "dim_mults": [1, 2, 3, 4],
+                "num_resnet_blocks": 4,
+                "attn_heads": 8,
+                "attn_dim_head": 64,
+                "sparse_attn": true,
+                "memory_efficient": true,
+		        "self_attn": [false, true, true, true]
+            },
+            {
+                "dim": 192,
+                "cond_dim": 512,
+                "image_embed_dim": 768,
+                "text_embed_dim": 768,
+                "cond_on_text_encodings": true,
+                "init_cross_embed": false,
+                "channels": 3,
+                "dim_mults": [1, 2, 3, 4],
+                "num_resnet_blocks": 3,
+                "attn_heads": 8,
+                "attn_dim_head": 64,
+                "sparse_attn": false,
+                "memory_efficient": true,
+                "self_attn": [false, false, false, false]
+            }
+        ],
+        "clip": {
+            "make": "openai",
+            "model": "ViT-L/14"
+        },
+        "image_sizes": [64, 256, 1024],
+        "random_crop_sizes": [null, null, 256],
+        "channels": 3,
+        "timesteps": 1000,
+        "loss_type": "l2",
+        "beta_schedule": ["cosine", "cosine", "cosine"],
+        "learned_variance": true,
+        "text_cond_drop_prob": 0.0,
+        "image_cond_drop_prob": 0.0
+    },
+    "data": {
+        "webdataset_base_url": "pipe:aws s3 cp --quiet s3://s-datasets/laion-high-resolution/{}.tar -",
+        "num_workers": 6,
+        "batch_size": 8,
+        "start_shard": 0,
+        "end_shard": 17535,
+        "shard_width": 5,
+        "index_width": 4,
+        "splits": {
+            "train": 0.75,
+            "val": 0.15,
+            "test": 0.1
+        },
+        "shuffle_train": false,
+        "resample_train": true,
+        "preprocessing": {
+            "RandomResizedCrop": {
+                "size": [1024, 1024],
+                "scale": [0.75, 1.0],
+                "ratio": [1.0, 1.0]
+            },
+            "ToTensor": true
+        }
+    },
+    "train": {
+        "epochs": 1000,
+        "lr": 1.2e-4,
+        "wd": 0.0,
+        "max_grad_norm": 0.5,
+        "save_every_n_samples": 2000000,
+        "n_sample_images": 2,
+        "device": "cuda:0",
+        "epoch_samples": 10000000,
+        "validation_samples": 100000,
+        "use_ema": true,
+        "ema_beta": 0.9999,
+        "unet_training_mask": [false, false, true]
+    },
+    "evaluate": {
+        "n_evaluation_samples": 2,
+        "FID": {
+            "feature": 64
+        },
+        "LPIPS": {
+            "net_type": "vgg",
+            "reduction": "mean"
+        }
+    },
+    "tracker": {
+        "data_path": "/fsx/aidan/new/multinode/experiments/decoder_1024/.tracker-data",
+        "overwrite_data_path": true,
+
+        "log": {
+            "log_type": "wandb",
+
+            "wandb_entity": "Veldrovive",
+            "wandb_project": "upsamplers_1024",
+            "wandb_resume": false,
+
+            "auto_resume": true,
+            "verbose": true
+        },
+
+        "load": {
+            "load_from": null,
+            "only_auto_resume": true,
+	        "file_path": "/fsx/aidan/new/multinode/experiments/decoder_1024/models/checkpoints/latest.pth"
+        },
+
+        "save": [
+        {
+            "save_to": "huggingface",
+            "huggingface_repo": "laion/DALLE2-PyTorch",
+            
+            "save_meta_to": "upsampler/1024/v1.0.3/",
+            "save_latest_to": "upsampler/1024/v1.0.3/latest.pth",
+
+            "save_type": "model"
+        },{
+            "save_to": "huggingface",
+            "huggingface_repo": "laion/DALLE2-PyTorch",
+            
+            "save_latest_to": "upsampler/1024/v1.0.2/checkpoints/latest.pth",
+
+            "save_type": "checkpoint"
+        },{
+            "save_to": "local",
+            "save_latest_to": "/fsx/aidan/new/multinode/experiments/decoder_1024/models/checkpoints/latest.pth",
+
+            "save_type": "checkpoint"
+        }]
+    }
+}

upsampler/1024/v1.0.3/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bbc25b1f5734634f439dfe805b7f5944a4a78be5e33467f3cc3ffaa630681be
+size 1213254057