Image to video?

README.md

@@ -78,54 +78,38 @@ pipeline_tag: text-to-video
 
 # Quick start
 
-1. Install the necessary requirements.
+1. Download the [Allegro GitHub code](https://github.com/rhymes-ai/Allegro).
+   
+2. Install the necessary requirements.
      
-   - Ensure Python >= 3.10, PyTorch >= 2.4, CUDA >= 12.4.
+   - Ensure Python >= 3.10, PyTorch >= 2.4, CUDA >= 12.4. For details, see [requirements.txt](https://github.com/rhymes-ai/Allegro/blob/main/requirements.txt).  
        
-   - It is recommended to use Anaconda to create a new environment (Python >= 3.10) `conda create -n rllegro python=3.10 -y` to run the following example.
-  
-   - run `pip install git+https://github.com/huggingface/diffusers.git torch==2.4.1 transformers==4.40.1 accelerate sentencepiece imageio imageio-ffmpeg beautifulsoup4` 
+   - It is recommended to use Anaconda to create a new environment (Python >= 3.10) to run the following example.  
+
+3. Download the [Allegro model weights](https://huggingface.co/rhymes-ai/Allegro). Before diffuser integration, use git lfs or snapshot_download.
  
-2. Run inference.
+4. Run inference.
+   
     ```python
-    import torch
-    from diffusers import AutoencoderKLAllegro, AllegroPipeline
-    from diffusers.utils import export_to_video
-    vae = AutoencoderKLAllegro.from_pretrained("rhymes-ai/Allegro", subfolder="vae", torch_dtype=torch.float32)
-    pipe = AllegroPipeline.from_pretrained(
-        "rhymes-ai/Allegro", vae=vae, torch_dtype=torch.bfloat16
-    )
-    pipe.to("cuda")
-    pipe.vae.enable_tiling()
-    prompt = "A seaside harbor with bright sunlight and sparkling seawater, with many boats in the water. From an aerial view, the boats vary in size and color, some moving and some stationary. Fishing boats in the water suggest that this location might be a popular spot for docking fishing boats."
-    
-    positive_prompt = """
-    (masterpiece), (best quality), (ultra-detailed), (unwatermarked), 
-    {} 
-    emotional, harmonious, vignette, 4k epic detailed, shot on kodak, 35mm photo, 
-    sharp focus, high budget, cinemascope, moody, epic, gorgeous
-    """
-    
-    negative_prompt = """
-    nsfw, lowres, bad anatomy, bad hands, text, error, missing fingers, extra digit, fewer digits, cropped, worst quality, 
-    low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry.
-    """
-    
-    prompt = prompt.format(prompt.lower().strip())
-    
-    video = pipe(prompt, negative_prompt=negative_prompt, guidance_scale=7.5, max_sequence_length=512, num_inference_steps=100, generator = torch.Generator(device="cuda:0").manual_seed(42)).frames[0]
-    export_to_video(video, "output.mp4", fps=15)
+    python single_inference.py \
+    --user_prompt 'A seaside harbor with bright sunlight and sparkling seawater, with many boats in the water. From an aerial view, the boats vary in size and color, some moving and some stationary. Fishing boats in the water suggest that this location might be a popular spot for docking fishing boats.' \
+    --save_path ./output_videos/test_video.mp4
+    --vae your/path/to/vae \
+    --dit your/path/to/transformer \
+    --text_encoder your/path/to/text_encoder \
+    --tokenizer your/path/to/tokenizer \
+    --guidance_scale 7.5 \
+    --num_sampling_steps 100 \
+    --seed 42
     ```
-      
-    Use `pipe.enable_sequential_cpu_offload()` to offload the model into CPU for less GPU memory cost (about 9.3G, compared to 27.5G if CPU offload is not enabled), but the inference time will increase significantly.
+  
+    Use '--enable_cpu_offload' to offload the model into CPU for less GPU memory cost (about 9.3G, compared to 27.5G if CPU offload is not enabled), but the inference time will increase significantly.
 
-3. (Optional) Interpolate the video to 30 FPS.
+5. (Optional) Interpolate the video to 30 FPS.
 
     It is recommended to use [EMA-VFI](https://github.com/MCG-NJU/EMA-VFI) to interpolate the video from 15 FPS to 30 FPS.
   
     For better visual quality, please use imageio to save the video.
 
-4. For faster inference such Context Parallel, PAB, please refer to our [github repo](https://github.com/rhymes-ai/Allegro). 
-
 # License
 This repo is released under the Apache 2.0 License.

model_index.json

@@ -1,24 +0,0 @@
-{
-  "_class_name": "AllegroPipeline",
-  "_diffusers_version": "0.31.0.dev0",
-  "scheduler": [
-    "diffusers",
-    "EulerAncestralDiscreteScheduler"
-  ],
-  "text_encoder": [
-    "transformers",
-    "T5EncoderModel"
-  ],
-  "tokenizer": [
-    "transformers",
-    "T5Tokenizer"
-  ],
-  "transformer": [
-    "diffusers",
-    "AllegroTransformer3DModel"
-  ],
-  "vae": [
-    "diffusers",
-    "AutoencoderKLAllegro"
-  ]
-}

text_encoder/model-00001-of-00004.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7a68b2c8c080696a10109612a649bc69330991ecfea65930ccfdfbdb011f2686
-size 4989319680

text_encoder/model-00002-of-00004.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b8ed6556d7507e38af5b428c605fb2a6f2bdb7e80bd481308b865f7a40c551ca
-size 4999830656

text_encoder/model-00003-of-00004.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c831635f83041f83faf0024b39c6ecb21b45d70dd38a63ea5bac6c7c6e5e558c
-size 4865612720

text_encoder/model-00004-of-00004.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:02a5f2d69205be92ad48fe5d712d38c2ff55627969116aeffc58bd75a28da468
-size 4194506688

text_encoder/model.safetensors.index.json

@@ -1,226 +0,0 @@
-{
-  "metadata": {
-    "total_size": 19049242624
-  },
-  "weight_map": {
-    "encoder.block.0.layer.0.SelfAttention.k.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.0.layer.0.SelfAttention.o.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.0.layer.0.SelfAttention.q.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.0.layer.0.SelfAttention.v.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.0.layer.0.layer_norm.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.0.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.0.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.0.layer.1.DenseReluDense.wo.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.0.layer.1.layer_norm.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.1.layer.0.SelfAttention.k.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.1.layer.0.SelfAttention.o.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.1.layer.0.SelfAttention.q.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.1.layer.0.SelfAttention.v.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.1.layer.0.layer_norm.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.1.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.1.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.1.layer.1.DenseReluDense.wo.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.1.layer.1.layer_norm.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.10.layer.0.SelfAttention.k.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.10.layer.0.SelfAttention.o.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.10.layer.0.SelfAttention.q.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.10.layer.0.SelfAttention.v.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.10.layer.0.layer_norm.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.10.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.10.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.10.layer.1.DenseReluDense.wo.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.10.layer.1.layer_norm.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.11.layer.0.SelfAttention.k.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.11.layer.0.SelfAttention.o.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.11.layer.0.SelfAttention.q.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.11.layer.0.SelfAttention.v.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.11.layer.0.layer_norm.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.11.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.11.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.11.layer.1.DenseReluDense.wo.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.11.layer.1.layer_norm.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.12.layer.0.SelfAttention.k.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.12.layer.0.SelfAttention.o.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.12.layer.0.SelfAttention.q.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.12.layer.0.SelfAttention.v.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.12.layer.0.layer_norm.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.12.layer.1.DenseReluDense.wi_0.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.12.layer.1.DenseReluDense.wi_1.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.12.layer.1.DenseReluDense.wo.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.12.layer.1.layer_norm.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.13.layer.0.SelfAttention.k.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.13.layer.0.SelfAttention.o.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.13.layer.0.SelfAttention.q.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.13.layer.0.SelfAttention.v.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.13.layer.0.layer_norm.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.13.layer.1.DenseReluDense.wi_0.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.13.layer.1.DenseReluDense.wi_1.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.13.layer.1.DenseReluDense.wo.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.13.layer.1.layer_norm.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.14.layer.0.SelfAttention.k.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.14.layer.0.SelfAttention.o.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.14.layer.0.SelfAttention.q.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.14.layer.0.SelfAttention.v.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.14.layer.0.layer_norm.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.14.layer.1.DenseReluDense.wi_0.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.14.layer.1.DenseReluDense.wi_1.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.14.layer.1.DenseReluDense.wo.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.14.layer.1.layer_norm.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.15.layer.0.SelfAttention.k.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.15.layer.0.SelfAttention.o.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.15.layer.0.SelfAttention.q.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.15.layer.0.SelfAttention.v.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.15.layer.0.layer_norm.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.15.layer.1.DenseReluDense.wi_0.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.15.layer.1.DenseReluDense.wi_1.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.15.layer.1.DenseReluDense.wo.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.15.layer.1.layer_norm.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.16.layer.0.SelfAttention.k.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.16.layer.0.SelfAttention.o.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.16.layer.0.SelfAttention.q.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.16.layer.0.SelfAttention.v.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.16.layer.0.layer_norm.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.16.layer.1.DenseReluDense.wi_0.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.16.layer.1.DenseReluDense.wi_1.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.16.layer.1.DenseReluDense.wo.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.16.layer.1.layer_norm.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.17.layer.0.SelfAttention.k.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.17.layer.0.SelfAttention.o.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.17.layer.0.SelfAttention.q.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.17.layer.0.SelfAttention.v.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.17.layer.0.layer_norm.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.17.layer.1.DenseReluDense.wi_0.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.17.layer.1.DenseReluDense.wi_1.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.17.layer.1.DenseReluDense.wo.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.17.layer.1.layer_norm.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.18.layer.0.SelfAttention.k.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.18.layer.0.SelfAttention.o.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.18.layer.0.SelfAttention.q.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.18.layer.0.SelfAttention.v.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.18.layer.0.layer_norm.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.18.layer.1.DenseReluDense.wi_0.weight": "model-00003-of-00004.safetensors",
-    "encoder.block.18.layer.1.DenseReluDense.wi_1.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.18.layer.1.DenseReluDense.wo.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.18.layer.1.layer_norm.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.19.layer.0.SelfAttention.k.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.19.layer.0.SelfAttention.o.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.19.layer.0.SelfAttention.q.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.19.layer.0.SelfAttention.v.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.19.layer.0.layer_norm.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.19.layer.1.DenseReluDense.wi_0.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.19.layer.1.DenseReluDense.wi_1.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.19.layer.1.DenseReluDense.wo.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.19.layer.1.layer_norm.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.2.layer.0.SelfAttention.k.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.2.layer.0.SelfAttention.o.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.2.layer.0.SelfAttention.q.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.2.layer.0.SelfAttention.v.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.2.layer.0.layer_norm.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.2.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.2.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.2.layer.1.DenseReluDense.wo.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.2.layer.1.layer_norm.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.20.layer.0.SelfAttention.k.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.20.layer.0.SelfAttention.o.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.20.layer.0.SelfAttention.q.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.20.layer.0.SelfAttention.v.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.20.layer.0.layer_norm.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.20.layer.1.DenseReluDense.wi_0.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.20.layer.1.DenseReluDense.wi_1.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.20.layer.1.DenseReluDense.wo.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.20.layer.1.layer_norm.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.21.layer.0.SelfAttention.k.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.21.layer.0.SelfAttention.o.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.21.layer.0.SelfAttention.q.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.21.layer.0.SelfAttention.v.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.21.layer.0.layer_norm.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.21.layer.1.DenseReluDense.wi_0.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.21.layer.1.DenseReluDense.wi_1.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.21.layer.1.DenseReluDense.wo.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.21.layer.1.layer_norm.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.22.layer.0.SelfAttention.k.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.22.layer.0.SelfAttention.o.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.22.layer.0.SelfAttention.q.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.22.layer.0.SelfAttention.v.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.22.layer.0.layer_norm.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.22.layer.1.DenseReluDense.wi_0.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.22.layer.1.DenseReluDense.wi_1.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.22.layer.1.DenseReluDense.wo.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.22.layer.1.layer_norm.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.23.layer.0.SelfAttention.k.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.23.layer.0.SelfAttention.o.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.23.layer.0.SelfAttention.q.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.23.layer.0.SelfAttention.v.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.23.layer.0.layer_norm.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.23.layer.1.DenseReluDense.wi_0.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.23.layer.1.DenseReluDense.wi_1.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.23.layer.1.DenseReluDense.wo.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.23.layer.1.layer_norm.weight": "model-00004-of-00004.safetensors",
-    "encoder.block.3.layer.0.SelfAttention.k.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.3.layer.0.SelfAttention.o.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.3.layer.0.SelfAttention.q.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.3.layer.0.SelfAttention.v.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.3.layer.0.layer_norm.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.3.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.3.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.3.layer.1.DenseReluDense.wo.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.3.layer.1.layer_norm.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.4.layer.0.SelfAttention.k.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.4.layer.0.SelfAttention.o.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.4.layer.0.SelfAttention.q.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.4.layer.0.SelfAttention.v.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.4.layer.0.layer_norm.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.4.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.4.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.4.layer.1.DenseReluDense.wo.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.4.layer.1.layer_norm.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.5.layer.0.SelfAttention.k.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.5.layer.0.SelfAttention.o.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.5.layer.0.SelfAttention.q.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.5.layer.0.SelfAttention.v.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.5.layer.0.layer_norm.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.5.layer.1.DenseReluDense.wi_0.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.5.layer.1.DenseReluDense.wi_1.weight": "model-00001-of-00004.safetensors",
-    "encoder.block.5.layer.1.DenseReluDense.wo.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.5.layer.1.layer_norm.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.6.layer.0.SelfAttention.k.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.6.layer.0.SelfAttention.o.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.6.layer.0.SelfAttention.q.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.6.layer.0.SelfAttention.v.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.6.layer.0.layer_norm.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.6.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.6.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.6.layer.1.DenseReluDense.wo.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.6.layer.1.layer_norm.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.7.layer.0.SelfAttention.k.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.7.layer.0.SelfAttention.o.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.7.layer.0.SelfAttention.q.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.7.layer.0.SelfAttention.v.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.7.layer.0.layer_norm.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.7.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.7.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.7.layer.1.DenseReluDense.wo.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.7.layer.1.layer_norm.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.8.layer.0.SelfAttention.k.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.8.layer.0.SelfAttention.o.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.8.layer.0.SelfAttention.q.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.8.layer.0.SelfAttention.v.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.8.layer.0.layer_norm.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.8.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.8.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.8.layer.1.DenseReluDense.wo.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.8.layer.1.layer_norm.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.9.layer.0.SelfAttention.k.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.9.layer.0.SelfAttention.o.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.9.layer.0.SelfAttention.q.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.9.layer.0.SelfAttention.v.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.9.layer.0.layer_norm.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.9.layer.1.DenseReluDense.wi_0.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.9.layer.1.DenseReluDense.wi_1.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.9.layer.1.DenseReluDense.wo.weight": "model-00002-of-00004.safetensors",
-    "encoder.block.9.layer.1.layer_norm.weight": "model-00002-of-00004.safetensors",
-    "encoder.final_layer_norm.weight": "model-00004-of-00004.safetensors",
-    "shared.weight": "model-00001-of-00004.safetensors"
-  }
-}

transformer/config.json

@@ -1,30 +1,38 @@
 {
   "_class_name": "AllegroTransformer3DModel",
-  "_diffusers_version": "0.31.0.dev0",
+  "_diffusers_version": "0.28.0",
   "activation_fn": "gelu-approximate",
   "attention_bias": true,
   "attention_head_dim": 96,
+  "ca_attention_mode": "xformers",
   "caption_channels": 4096,
   "cross_attention_dim": 2304,
+  "double_self_attention": false,
+  "downsampler": null,
   "dropout": 0.0,
   "in_channels": 4,
   "interpolation_scale_h": 2.0,
   "interpolation_scale_t": 2.2,
   "interpolation_scale_w": 2.0,
+  "model_max_length": 300,
   "norm_elementwise_affine": false,
   "norm_eps": 1e-06,
   "norm_type": "ada_norm_single",
   "num_attention_heads": 24,
+  "num_embeds_ada_norm": 1000,
   "num_layers": 32,
+  "only_cross_attention": false,
   "out_channels": 4,
   "patch_size": 2,
   "patch_size_t": 1,
-  "sample_frames": 22,
-  "sample_height": 90,
+  "sa_attention_mode": "flash",
   "sample_size": [
     90,
     160
   ],
   "sample_size_t": 22,
-  "sample_width": 160
+  "upcast_attention": false,
+  "use_additional_conditions": null,
+  "use_linear_projection": false,
+  "use_rope": true
 }

vae/config.json

@@ -1,6 +1,6 @@
 {
-  "_class_name": "AutoencoderKLAllegro",
-  "_diffusers_version": "0.31.0.dev0",
+  "_class_name": "AllegroAutoencoderKL3D",
+  "_diffusers_version": "0.28.0",
   "act_fn": "silu",
   "block_out_channels": [
     128,
@@ -8,37 +8,33 @@
     512,
     512
   ],
-  "down_block_types": [
-    "AllegroDownBlock3D",
-    "AllegroDownBlock3D",
-    "AllegroDownBlock3D",
-    "AllegroDownBlock3D"
-  ],
-  "force_upcast": true,
-  "in_channels": 3,
-  "latent_channels": 4,
-  "layers_per_block": 2,
-  "norm_num_groups": 32,
-  "out_channels": 3,
-  "sample_size": 320,
-  "scaling_factor": 0.13,
-  "temporal_compression_ratio": 4,
-  "temporal_downsample_blocks": [
+  "blocks_tempdown_li": [
     true,
     true,
     false,
     false
   ],
-  "temporal_upsample_blocks": [
+  "blocks_tempup_li": [
     false,
     true,
     true,
     false
   ],
-  "up_block_types": [
-    "AllegroUpBlock3D",
-    "AllegroUpBlock3D",
-    "AllegroUpBlock3D",
-    "AllegroUpBlock3D"
-  ]
+  "chunk_len": 24,
+  "down_block_num": 4,
+  "force_upcast": true,
+  "in_channels": 3,
+  "latent_channels": 4,
+  "layers_per_block": 2,
+  "load_mode": "full",
+  "norm_num_groups": 32,
+  "out_channels": 3,
+  "sample_size": 320,
+  "scale_factor": 0.13,
+  "t_over": 8,
+  "tile_overlap": [
+    120,
+    80
+  ],
+  "up_block_num": 4
 }