upload t2v-version-1-1 models

models/base_t2v/model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8870651704d6bfd2476c37e2bb8296711638fd35812292b2d0e99f98b6427e08
+size 4663057434

models/base_t2v/model_config.yaml

@@ -0,0 +1,119 @@
+model:
+  load_from_pretrained_img_model: true
+  ckpt_path: /apdcephfs_cq2/share_1290939/yingqinghe/dependencies/stable_diffusion/compvis-sd-v1-4-original/sd-v1-4-full-ema.ckpt
+  config_path: configs/latent-diffusion/txt2img-1p4B-eval-Clipembedder.yaml
+  load_from_checkpoint: /apdcephfs/share_1290939/yingqinghe/results/latent_diffusion/text2video/tv_054_NoFPSEmbd_NoMotionAdapter_FS32_basedon050_2_8nodes_e0_V/checkpoints/trainstep_checkpoints/epoch=000003-step=000020000.ckpt
+  base_learning_rate: 5.0e-07
+  scale_lr: false
+  target: lvdm.models.ddpm3d.LatentDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.012
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: video
+    cond_stage_key: caption
+    image_size:
+    - 32
+    - 32
+    video_length: 16
+    channels: 4
+    cond_stage_trainable: false
+    conditioning_key: crossattn
+    monitor: train/loss_simple_step
+    scale_by_std: false
+    scale_factor: 0.18215
+    use_ema: false
+    loss_type: l2-consistency
+    val_prompt_file: info/prompts/magicvideo_mini.txt
+    seed: 23
+    val_fvd_interval: 5000
+    unet_config:
+      target: lvdm.models.modules.openaimodel3d.UNetModel
+      params:
+        image_size: 32
+        in_channels: 4
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions:
+        - 4
+        - 2
+        - 1
+        num_res_blocks: 2
+        channel_mult:
+        - 1
+        - 2
+        - 4
+        - 4
+        num_heads: 8
+        transformer_depth: 1
+        context_dim: 768
+        use_checkpoint: true
+        legacy: false
+        kernel_size_t: 1
+        padding_t: 0
+        use_temporal_transformer: true
+        temporal_length: 16
+        use_relative_position: true
+        img_video_joint_train: false
+        image_length: null
+        temporal_crossattn_type: selfattn
+        motion_adaptor: false
+        n_mix_channels: 1
+    first_stage_config:
+      target: lvdm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+    cond_stage_config:
+      target: lvdm.models.modules.condition_modules.FrozenCLIPEmbedder
+    logdir: /apdcephfs/share_1290939/yingqinghe/results/latent_diffusion/text2video/tv_056_2_0.01_lr_ConsistencyLoss_resumefrom054_8nodes_e0_V
+  ckptdir: /apdcephfs/share_1290939/yingqinghe/results/latent_diffusion/text2video/tv_056_2_0.01_lr_ConsistencyLoss_resumefrom054_8nodes_e0_V/checkpoints
+data:
+  auto_cal_bs: true
+  target: main.DataModuleFromConfig
+  params:
+    batch_size: 3
+    num_workers: 12
+    wrap: false
+    train:
+      target: lvdm.data.webvid.WebVid
+      params:
+        data_dir: /apdcephfs/share_1290939/0_public_datasets/WebVid
+        meta_path: /apdcephfs/share_1290939/0_public_datasets/WebVid/metadata/results_2M_train.csv
+        video_length: 16
+        frame_stride: 32
+        load_raw_resolution: true
+        resolution: 256
+        spatial_transform: resize_center_crop
+        fps_max: 199
+    validation:
+      target: lvdm.data.webvid.WebVid
+      params:
+        data_dir: /apdcephfs/share_1290939/0_public_datasets/WebVid
+        meta_path: /apdcephfs_cq2/share_1290939/yingqinghe/datasets/webvid/metadata_2048_val.csv
+        video_length: 16
+        frame_stride: 32
+        load_raw_resolution: true
+        resolution: 256
+        spatial_transform: resize_center_crop
+        fps_max: 199
+--local_rank: 0