Add files using upload-large-folder tool

anole/vqgan.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ede986bf6b171db3081ce171ad88e4ac970793cea14c180b3e5ac5105f4cb43
+size 281270377

anole/vqgan.yaml

@@ -0,0 +1,57 @@
+model:
+  base_learning_rate: 4.5e-06
+  target: taming.models.vqgan.VQModel
+  params:
+    embed_dim: 256
+    n_embed: 8192
+    ddconfig:
+      double_z: false
+      z_channels: 256
+      resolution: 512
+      in_channels: 3
+      out_ch: 3
+      ch: 128
+      ch_mult:
+      - 1
+      - 1
+      - 2
+      - 2
+      - 4
+      num_res_blocks: 2
+      attn_resolutions: []
+      dropout: 0.0
+    lossconfig:
+      target: taming.modules.losses.vqperceptual_vit_vqgan.VQLPIPSWithDiscriminator
+      params:
+        disc_start: 100001
+        perceptual_weight: 1.0
+        adversarial_weight: 0.5
+        disc_params:
+          size: 512
+    ckpt_path: manifold://fair_onellm_checkpoints/tree/v2/tokenizer/vqgan_wm_0209.ckpt
+data:
+  target: main.DataModuleFromConfig
+  params:
+    batch_size: 4
+    num_workers: 10
+    image_size: 512
+    filter_image_size: 512
+    dataset: coco
+    aesthetics_th: 0
+    clipsim_th: 0
+--distributed-world-size: null
+'32': null
+--distributed-port: null
+'17338': null
+--save-dir: null
+/checkpoint/shellysheynin/shutterstock/512x512_1024tokens_4node_shutterstock_laion_no_attn_styleGAN:
+  log_every-500:
+    ngpu32: null
+--tensorboard-logdir: null
+/checkpoint/shellysheynin/tensorboard_logs/2023-03-30/512x512_1024tokens_4node_shutterstock_laion_no_attn_styleGAN:
+  log_every-500:
+    ngpu32: null
+'14561': null
+/checkpoint/shellysheynin/tensorboard_logs/2023-04-02/512x512_1024tokens_4node_shutterstock_laion_no_attn_styleGAN:
+  log_every-500:
+    ngpu32: null

bsqvit/checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b11e147819d7633e1b569ccb367c5df43f0ba683c2bbe640f6dfaad0d235a34
+size 3144485248

bsqvit/config.yaml

@@ -0,0 +1,80 @@
+model:
+  target: transcoder.models.bsqvit.VITBSQModel
+  params:
+    embed_dim: 36
+    embed_group_size: 1
+    l2_norm: True
+    persample_entropy_compute: 'analytical'
+    post_q_l2_norm: True
+    logit_laplace: False
+    beta: 0.
+    vitconfig:
+      image_size: 256
+      patch_size: 8
+      width: 768
+      layers: 12
+      heads: 12
+      mlp_ratio: 4
+      drop_rate: 0.
+  # grad_checkpointing: True
+loss:
+  target: transcoder.losses.vqperceptual.VQLPIPSWithDiscriminator
+  params:
+    disc_type: 'stylegan'
+    disc_input_size: 256
+    disc_loss: 'vanilla'
+    disc_reg_freq: 16
+    disc_conditional: False
+    disc_in_channels: 3
+    disc_start: 0
+    disc_weight: 0.1
+    codebook_weight: 0.1
+    codebook_rampup_multiplier: 3.0
+    codebook_rampup_steps: 2_000
+    perceptual_weight: 0.1
+    use_adaptive_disc_weight: False
+
+data:
+  image_size: 256
+  batch_size: 32
+  num_workers: 8
+  train:
+    target: torchvision.datasets.ImageFolder
+    params:
+      root: '/storage/Datasets/ILSVRC2012/train'
+  val:
+    target: torchvision.datasets.ImageFolder
+    params:
+      root: '/storage/Datasets/ILSVRC2012/val'
+        # root: '/storage/Datasets/ILSVRC2012/coco/'   # for coco2017val
+  zero_mean: True
+
+optimizer:
+  disable_amp: False
+  use_bf16: True
+  base_lr: 4e-7
+  max_iter: 1_000_000
+  lr_scheduler_config:
+      target: transcoder.optim.schedulers.LambdaWarmUpCosineScheduler
+      params:
+        warm_up_steps: 5_000
+        max_decay_steps: 1_000_000
+        lr_start: 0.1
+        lr_max: 1.0
+        lr_min: 0.5
+  target: torch.optim.AdamW
+  params:
+    weight_decay: 1e-4
+    betas: [0.9, 0.99]
+    eps: 1e-8
+evaluation:
+  interpolation: 'lanczos'
+  fid:
+    dims: 2048
+    num_samples: 50_000
+    # num_samples: 5_000   # for coco2017val
+    groundtruth_npz: 'imagenet_val_256x256_lanczos.npz'
+    # groundtruth_npz: 'coco_val_256x256_lanczos.npz'   # for coco2017val
+wandb:
+  project: transcoder
+  run: imagenet_128x128_bsqvit_b18g18_stylegan_f8_fp16

chameleon/vqgan.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ede986bf6b171db3081ce171ad88e4ac970793cea14c180b3e5ac5105f4cb43
+size 281270377

chameleon/vqgan.yaml

@@ -0,0 +1,57 @@
+model:
+  base_learning_rate: 4.5e-06
+  target: taming.models.vqgan.VQModel
+  params:
+    embed_dim: 256
+    n_embed: 8192
+    ddconfig:
+      double_z: false
+      z_channels: 256
+      resolution: 512
+      in_channels: 3
+      out_ch: 3
+      ch: 128
+      ch_mult:
+      - 1
+      - 1
+      - 2
+      - 2
+      - 4
+      num_res_blocks: 2
+      attn_resolutions: []
+      dropout: 0.0
+    lossconfig:
+      target: taming.modules.losses.vqperceptual_vit_vqgan.VQLPIPSWithDiscriminator
+      params:
+        disc_start: 100001
+        perceptual_weight: 1.0
+        adversarial_weight: 0.5
+        disc_params:
+          size: 512
+    ckpt_path: manifold://fair_onellm_checkpoints/tree/v2/tokenizer/vqgan_wm_0209.ckpt
+data:
+  target: main.DataModuleFromConfig
+  params:
+    batch_size: 4
+    num_workers: 10
+    image_size: 512
+    filter_image_size: 512
+    dataset: coco
+    aesthetics_th: 0
+    clipsim_th: 0
+--distributed-world-size: null
+'32': null
+--distributed-port: null
+'17338': null
+--save-dir: null
+/checkpoint/shellysheynin/shutterstock/512x512_1024tokens_4node_shutterstock_laion_no_attn_styleGAN:
+  log_every-500:
+    ngpu32: null
+--tensorboard-logdir: null
+/checkpoint/shellysheynin/tensorboard_logs/2023-03-30/512x512_1024tokens_4node_shutterstock_laion_no_attn_styleGAN:
+  log_every-500:
+    ngpu32: null
+'14561': null
+/checkpoint/shellysheynin/tensorboard_logs/2023-04-02/512x512_1024tokens_4node_shutterstock_laion_no_attn_styleGAN:
+  log_every-500:
+    ngpu32: null

flowmo_hi/base.yaml

@@ -0,0 +1,77 @@
+global_seed: 0
+
+data:
+  batch_size: 16
+  eval_batch_size: 5
+  image_size: 256
+  num_workers: 8
+
+  imagenet_train_index: "imagenet_train_index_overall.json"
+  imagenet_train_tar: "ILSVRC2012_img_train.tar"
+  imagenet_val_index: "imagenet_val_index_overall.json"
+  imagenet_val_tar: "ILSVRC2012_img_val.tar"
+
+model:
+  enable_mup: True
+  patch_size: 4
+  mup_width: 6
+  enc_mup_width: 4
+
+  fix_initial_norms: True
+  fix_norm_mode: l2
+
+  quantization_type: lfq
+  code_length: 256
+  context_dim: 18
+  codebook_size_for_entropy: 9
+  entropy_loss_weight: 0.0025
+  commit_loss_weight: 0.000625
+
+  enc_depth: 8
+  dec_depth: 16
+  enable_cfg: True
+
+  ema_decay: 0.9999
+
+  posttrain_sample: False
+  posttrain_sample_enable_cfg: False
+  posttrain_sample_k: 8
+
+opt:
+  # Floats need to be specified with decimals to get loaded as such
+  lr: 1.0e-4
+  n_grad_acc: 2
+  schedule: fat_lognormal
+  freeze_encoder: False
+  freeze_encoder_after: 200000
+  weight_decay: 0.0
+  log_norms: True
+  lpips_weight: 0.1
+  lpips_mode: "vgg"
+
+  beta1: .9
+  beta2: .95
+
+trainer:
+  enable_bfloat16: True
+  log_every: 100
+  checkpoint_every: 5000
+  max_steps: 10000000000000
+  keep_every: 200000
+  gs_checkpoint_bucket: ""
+
+eval:
+  reconstruction: True
+  state_dict_key: model_ema_state_dict
+  eval_dir: ""
+  eval_baseline: ""
+  continuous: True
+  force_ckpt_path: null
+  subsample_rate: 1
+  
+  sampling:
+    sample_steps: 25
+    schedule: "pow_0.25"
+    cfg: 1.5
+    mode: 'rf'
+    cfg_interval: "(.17,1.02)"

flowmo_hi/flowmo_hi.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa8278ec492028aa54915b061c27300550731bc1d4ddb543dea2ec54d432a137
+size 7564557830

flowmo_lo/base.yaml

@@ -0,0 +1,77 @@
+global_seed: 0
+
+data:
+  batch_size: 16
+  eval_batch_size: 5
+  image_size: 256
+  num_workers: 8
+
+  imagenet_train_index: "imagenet_train_index_overall.json"
+  imagenet_train_tar: "ILSVRC2012_img_train.tar"
+  imagenet_val_index: "imagenet_val_index_overall.json"
+  imagenet_val_tar: "ILSVRC2012_img_val.tar"
+
+model:
+  enable_mup: True
+  patch_size: 4
+  mup_width: 6
+  enc_mup_width: 4
+
+  fix_initial_norms: True
+  fix_norm_mode: l2
+
+  quantization_type: lfq
+  code_length: 256
+  context_dim: 18
+  codebook_size_for_entropy: 9
+  entropy_loss_weight: 0.0025
+  commit_loss_weight: 0.000625
+
+  enc_depth: 8
+  dec_depth: 16
+  enable_cfg: True
+
+  ema_decay: 0.9999
+
+  posttrain_sample: False
+  posttrain_sample_enable_cfg: False
+  posttrain_sample_k: 8
+
+opt:
+  # Floats need to be specified with decimals to get loaded as such
+  lr: 1.0e-4
+  n_grad_acc: 2
+  schedule: fat_lognormal
+  freeze_encoder: False
+  freeze_encoder_after: 200000
+  weight_decay: 0.0
+  log_norms: True
+  lpips_weight: 0.1
+  lpips_mode: "vgg"
+
+  beta1: .9
+  beta2: .95
+
+trainer:
+  enable_bfloat16: True
+  log_every: 100
+  checkpoint_every: 5000
+  max_steps: 10000000000000
+  keep_every: 200000
+  gs_checkpoint_bucket: ""
+
+eval:
+  reconstruction: True
+  state_dict_key: model_ema_state_dict
+  eval_dir: ""
+  eval_baseline: ""
+  continuous: True
+  force_ckpt_path: null
+  subsample_rate: 1
+  
+  sampling:
+    sample_steps: 25
+    schedule: "pow_0.25"
+    cfg: 1.5
+    mode: 'rf'
+    cfg_interval: "(.17,1.02)"

flowmo_lo/flowmo_lo.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12ee996b780057ef986260e6ed5a87d9036c08c29bcee3f479e5d0c8208636fc
+size 7563389958

infinity-d32/infinity_vae_d32.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a669bca347d46dc1b10ca601ab67d4f231ad9bad07339e80b22f807ee36ad3e4
+size 1557325341

infinity-d64/infinity_vae_d64.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab1d29c91d52aece43bae1e9c3f11ae993427a315db0be7c26780c5d1e2dbe28
+size 1560864925

llamagen-ds16-t2i/vq_ds16_t2i.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e21fc1318e2e9ee641a07bdad0e20675e9ec35e6e3eb911d58b5d7a2cd8d4cb
+size 287920306

llamagen-ds16/vq_ds16_c2i.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:109aa8afb2cf3761eec23cdc8644154cb498f5ab7eef2a35264d25e5e0499f7d
+size 287920306

llamagen-ds8/vq_ds8_c2i.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c70d2966ec012f378793214ff0755087e7c5fe6f035b469995143325dca4a4e3
+size 280809450

maskbit-16bit/maskbit_tokenizer_16bit.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44e3ff7c3bf15eda4e1e25586ff8f657be9291d5e2bf3b755505394e11765b06
+size 222258958

maskbit-16bit/maskbit_tokenizer_16bit.yaml

@@ -0,0 +1,104 @@
+experiment:
+    project: "MaskBit"
+    name: "maskbit_tokenizer_16bit"
+    max_train_examples: 1_281_167 # total number of imagenet examples
+    save_every: 20_000
+    eval_every: 20_000
+    generate_every: 2000
+    log_every: 50
+    log_grad_norm_every: 100_000
+    logger: "tensorboard"
+    resume: True
+    init_checkpoint: ""
+    # vqgan_checkpoint: "MODEL_PATH/maskbit_tokenizer_16bit.bin"  # Only for evaluating a trained model
+
+model:
+    vq_model:
+        model_class: "vqgan+"
+        quantizer_type: "lookup-free"
+        codebook_size: 4096
+        token_size: 16
+        commitment_cost: 0.25
+        entropy_loss_weight: 0.02
+        entropy_loss_temperature: 0.01
+        entropy_gamma: 1.0
+        num_channels: 3  # rgb
+        hidden_channels: 128
+        channel_mult: [1,1,2,2,4]
+        num_resolutions: 5
+        num_res_blocks: 2
+        sample_with_conv: True
+
+    discriminator:
+        name: "VQGAN+Discriminator"
+        num_channels: 3
+        num_stages: 4
+        hidden_channels: 128
+        blur_resample: True
+        blur_kernel_size: 4
+
+
+losses:
+    quantizer_weight: 1.0
+    perceptual_loss: "resnet50"
+    perceptual_weight: 0.1
+    perceptual_loss_on_logits: True
+    reconstruction_loss: "l2"
+    reconstruction_weight: 4.0
+    discriminator_start: 20_000
+    discriminator_loss: "hinge"
+    discriminator_factor: 1.0
+    discriminator_weight: 0.02
+    discriminator_gradient_penalty: "none"
+    discriminator_penalty_cost: 10.0
+    lecam_regularization_weight: 0.001
+    entropy_annealing_steps: 2000
+    entropy_annealing_factor: 2.0
+
+dataset:
+    params:
+        train_shards_path_or_url: "DATA_PATH/imagenet_shards/train/imagenet-train-{0000..0252}.tar"
+        eval_shards_path_or_url: "DATA_PATH/imagenet_shards/val/imagenet-val-{0000..0009}.tar"
+        shuffle_buffer_size: 1000
+        num_workers_per_gpu: 8
+        pin_memory: True
+        persistent_workers: True
+    preprocessing:
+        resolution: 256
+        use_aspect_ratio_aug: True
+        use_random_crop: True
+        min_scale: 0.8
+        interpolation: "bilinear"
+
+
+optimizer:
+    name: adamw 
+    params: # default adamw params
+        learning_rate: 1e-4
+        discriminator_learning_rate: 1e-4
+        scale_lr: False # scale learning rate by total batch size
+        beta1: 0.9
+        beta2: 0.999
+        weight_decay: 1e-4
+        epsilon: 1e-8
+
+
+lr_scheduler:
+    scheduler: "cosine_with_minimum"
+    params:
+        learning_rate: ${optimizer.params.learning_rate}
+        warmup_steps: 5_000
+
+
+training:
+    gradient_accumulation_steps: 1
+    per_gpu_batch_size: 16
+    mixed_precision: "no"  # "bf16"
+    enable_tf32: True
+    use_ema: True
+    seed: 42
+    max_train_steps: 1_350_000
+    overfit_batch: False
+    overfit_batch_num: 1
+    num_generated_images: 2  # Must be smaller than or equal to per_gpu_batch_size
+    max_grad_norm: 1.0

maskbit-18bit/maskbit_tokenizer_18bit.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c672c2508e14c4d7cd8621c1cb8939610478632725bb26521cf3c4bb6fc3f78
+size 236979982

maskbit-18bit/maskbit_tokenizer_18bit.yaml

@@ -0,0 +1,104 @@
+experiment:
+    project: "MaskBit"
+    name: "maskbit_tokenizer_18bit"
+    max_train_examples: 1_281_167 # total number of imagenet examples
+    save_every: 20_000
+    eval_every: 20_000
+    generate_every: 2000
+    log_every: 50
+    log_grad_norm_every: 100_000
+    logger: "tensorboard"
+    resume: True
+    init_checkpoint: ""
+    # vqgan_checkpoint: ""MODEL_PATH/maskbit_tokenizer_18bit.bin"  # Only for evaluating a trained model
+
+model:
+    vq_model:
+        model_class: "vqgan+"
+        quantizer_type: "lookup-free"
+        codebook_size: 4096
+        token_size: 18
+        commitment_cost: 0.25
+        entropy_loss_weight: 0.02
+        entropy_loss_temperature: 0.01
+        entropy_gamma: 1.0
+        num_channels: 3  # rgb
+        hidden_channels: 128
+        channel_mult: [1,1,2,2,4]
+        num_resolutions: 5
+        num_res_blocks: 2
+        sample_with_conv: True
+
+    discriminator:
+        name: "VQGAN+Discriminator"
+        num_channels: 3
+        num_stages: 4
+        hidden_channels: 128
+        blur_resample: True
+        blur_kernel_size: 4
+
+
+losses:
+    quantizer_weight: 1.0
+    perceptual_loss: "resnet50"
+    perceptual_weight: 0.1
+    perceptual_loss_on_logits: True
+    reconstruction_loss: "l2"
+    reconstruction_weight: 4.0
+    discriminator_start: 20_000
+    discriminator_loss: "hinge"
+    discriminator_factor: 1.0
+    discriminator_weight: 0.02
+    discriminator_gradient_penalty: "none"
+    discriminator_penalty_cost: 10.0
+    lecam_regularization_weight: 0.001
+    entropy_annealing_steps: 2000
+    entropy_annealing_factor: 2.0
+
+dataset:
+    params:
+        train_shards_path_or_url: "DATA_PATH/imagenet_shards/train/imagenet-train-{0000..0252}.tar"
+        eval_shards_path_or_url: "DATA_PATHimagenet_shards/val/imagenet-val-{0000..0009}.tar"
+        shuffle_buffer_size: 1000
+        num_workers_per_gpu: 8
+        pin_memory: True
+        persistent_workers: True
+    preprocessing:
+        resolution: 256
+        use_aspect_ratio_aug: True
+        use_random_crop: True
+        min_scale: 0.8
+        interpolation: "bilinear"
+
+
+optimizer:
+    name: adamw 
+    params: # default adamw params
+        learning_rate: 1e-4
+        discriminator_learning_rate: 1e-4
+        scale_lr: False # scale learning rate by total batch size
+        beta1: 0.9
+        beta2: 0.999
+        weight_decay: 1e-4
+        epsilon: 1e-8
+
+
+lr_scheduler:
+    scheduler: "cosine_with_minimum"
+    params:
+        learning_rate: ${optimizer.params.learning_rate}
+        warmup_steps: 5_000
+
+
+training:
+    gradient_accumulation_steps: 1
+    per_gpu_batch_size: 16
+    mixed_precision: "no"  # "bf16"
+    enable_tf32: True
+    use_ema: True
+    seed: 42
+    max_train_steps: 1_350_000
+    overfit_batch: False
+    overfit_batch_num: 1
+    num_generated_images: 2  # Must be smaller than or equal to per_gpu_batch_size
+    max_grad_norm: 1.0

open_magvit2/imagenet_256_L.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0e27c4a5ab21db0f38cba484e717584e8a8ea7d9fda9a729d058bc8d68d0922
+size 921240234

open_magvit2/imagenet_lfqgan_256_L.yaml

@@ -0,0 +1,88 @@
+seed_everything: true
+trainer:
+  accelerator: gpu
+  strategy: ddp_find_unused_parameters_true
+  devices: 8
+  num_nodes: 4
+  precision: 16-mixed
+  max_epochs: 270
+  check_val_every_n_epoch: 1
+  num_sanity_val_steps: -1
+  log_every_n_steps: 100
+  callbacks:
+    - class_path: lightning.pytorch.callbacks.ModelCheckpoint
+      init_args:
+        dirpath: "../../checkpoints/vqgan/test"
+        save_top_k: -1 # save all checkpoints
+    - class_path: lightning.pytorch.callbacks.LearningRateMonitor
+      init_args:
+        logging_interval: step
+  logger:
+    class_path: lightning.pytorch.loggers.TensorBoardLogger
+    init_args:
+      save_dir: "../../results/vqgan/"
+      version: "test"
+      name:
+
+model:
+  class_path: src.Open_MAGVIT2.models.lfqgan.VQModel
+  init_args:
+    ddconfig:
+      double_z: False
+      z_channels: 18
+      resolution: 128
+      in_channels: 3
+      out_ch: 3
+      ch: 128
+      ch_mult: [1,1,2,2,4]  # num_down = len(ch_mult)-1
+      num_res_blocks: 4
+
+    lossconfig:
+      target: src.Open_MAGVIT2.modules.losses.vqperceptual.VQLPIPSWithDiscriminator
+      params:
+        disc_conditional: False
+        disc_in_channels: 3
+        disc_start: 0 # from 0 epoch
+        disc_weight: 0.8
+        gen_loss_weight: 0.1
+        lecam_loss_weight: 0.05
+        codebook_weight: 0.1
+        commit_weight: 0.25
+        codebook_enlarge_ratio: 0
+        codebook_enlarge_steps: 2000
+
+    n_embed: 262144
+    embed_dim: 18
+    learning_rate: 1e-4
+    sample_minimization_weight: 1.0
+    batch_maximization_weight: 1.0
+    scheduler_type: "None"
+    use_ema: True
+    resume_lr:
+    lr_drop_epoch: [200, 250]
+
+data:
+  class_path: main.DataModuleFromConfig
+  init_args:
+    batch_size: 8
+    num_workers: 16
+    train:
+      target: src.Open_MAGVIT2.data.imagenet.ImageNetTrain
+      params:
+        config:
+          size: 256
+          subset:
+    validation:
+      target: src.Open_MAGVIT2.data.imagenet.ImageNetValidation
+      params:
+        config:
+          size: 256
+          subset:
+    test:
+      target: src.Open_MAGVIT2.data.imagenet.ImageNetValidation
+      params:
+        config:
+          size: 256
+          subset:
+
+ckpt_path: null # to resume

var/vae_ch160v4096z32.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c3ec27ae28a3f87055e83211ea8cc8558bd1985d7b51742d074fb4c2fcf186c
+size 436075834