model upload

.gitattributes

@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/Neodragon_pipeline.png filter=lfs diff=lfs merge=lfs -text
+assets/neodragon_title_transparent_bkgd.png filter=lfs diff=lfs merge=lfs -text
+assets/Neodragon_title.jpg filter=lfs diff=lfs merge=lfs -text
+assets/showcase_video_banner_fallback.png filter=lfs diff=lfs merge=lfs -text
+assets/showcase_video_banner.webp filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,174 @@
 ---
-license: bsd-3-clause-clear
+license: other
+license_name: qualcomm-ai-hub-proprietary-license
+license_link: >-
+  https://qaihub-public-assets.s3.us-west-2.amazonaws.com/qai-hub-models/Qualcomm+AI+Hub+Proprietary+License.pdf
+pipeline_tag: text-to-video
+tags:
+- efficient
+- mobile video generation
+- dit
+- pyramidal diffusion
+language:
+- en
+base_model:
+- rain1011/pyramid-flow-sd3
 ---
+<script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
+<div align="center" style="padding: 20px; border-radius: 10px;">
+  <div style="display: flex; align-items: center; justify-content: center; gap: 20px;">
+    <img src="assets/Neodragon_title.jpg" alt="neodragon logo"/>
+  </div>
+  
+
+  <!-- Animated banner (WebP with fallback) -->
+  <p align="center">
+    <img src="assets/showcase_video_banner.webp" alt="Neodragon showcase banner">
+  </p>  
+  <h1> Neodragon: Mobile Video Generation Using Diffusion Transformer </h1>
+
+  <!-- Badges -->
+  <a href="https://qualcomm-ai-research.github.io/neodragon">
+    <img src="https://img.shields.io/badge/Project-Page-Green" alt="Project Page">
+  </a>
+  <a href="https://arxiv.org/abs/2511.06055">
+    <img src="https://img.shields.io/badge/arXiv-2511.06055-b31b1b.svg" alt="arXiv">
+  </a>
+  <a href="https://huggingface.co/karnewar/Neodragon">
+    <img src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Model-blue" alt="Hugging Face Model">
+  </a>
+  <a href="https://openreview.net/forum?id=XBzIhhwv8d">
+    <img src="https://img.shields.io/badge/ICLR%202026-OpenReview-8A2BE2" alt="ICLR 2026 OpenReview">
+  </a>
+  <a href="https://github.com/qualcomm-ai-research/neodragon">
+    <img src="https://img.shields.io/badge/GitHub-Code-181717?logo=github&logoColor=white" alt="GitHub Code">
+  </a>
+
+  **[Qualcomm AI Research](https://www.qualcomm.com/research/artificial-intelligence)**
+
+  [Animesh Karnewar](https://akanimax.github.io), 
+  [Denis Korzhenkov](https://scholar.google.com/citations?user=ypspak0AAAAJ), 
+  [Ioannis Lelekas](https://nl.linkedin.com/in/ioannis-lelekas-609bb5151), 
+  [Noor Fathima](https://scholar.google.com/citations?user=M9BUCaUAAAAJ&hl=en), 
+  [Adil Karjauv](https://scholar.google.com/citations?user=bN7UGiYAAAAJ&hl=en), 
+  [Hanwen Xiong](#), 
+  [Vancheeswaran Vaidyanathan](https://www.linkedin.com/in/vancheeswaran-vaidyanathan), 
+  [Will Zeng](https://scholar.google.com/citations?user=B_fh4ioAAAAJ&hl=en), 
+  [Rafael Esteves](https://www.linkedin.com/in/rafael-esteves-124353145), 
+  [Tushar Singhal](https://www.linkedin.com/in/tushar-singhal), 
+  [Fatih Porikli](https://scholar.google.com/citations?user=VpB8NZ8AAAAJ&hl=en), 
+  [Mohsen Ghafoorian](https://mohsenghafoorian.github.io), 
+  [Amirhossein Habibian](https://habibian.github.io/)
+
+</div>
+
+```bibtex
+
+@inproceedings{
+  karnewar2026neodragon,
+  title={Neodragon: Mobile Video Generation Using Diffusion Transformer},
+  author={Animesh Karnewar and Denis Korzhenkov and Ioannis Lelekas and Noor Fathima and Adil Karjauv and Mohsen Ghafoorian and Amir Habibian},
+  booktitle={The Fourteenth International Conference on Learning Representations},
+  year={2026},
+  url={https://openreview.net/forum?id=XBzIhhwv8d}
+}
+@article{karnewar2025neodragonTR,
+  title={Neodragon: Mobile Video Generation using Diffusion Transformer},
+  author={Karnewar, Animesh and Korzhenkov, Denis and Lelekas, Ioannis and Karjauv, Adil and Fathima, Noor and Xiong, Hanwen and Vaidyanathan, Vancheeswaran and Zeng, Will and Esteves, Rafael and Singhal, Tushar and Porikli, Fatih and Ghafoorian, Mohsen and Habibian Amirhossein},
+  journal={arXiv preprint arXiv:2511.06055},
+  url={https://qualcomm-ai-research.github.io/neodragon},
+  year={2025}
+}
+
+```
+
+<section class="section hero is-light">
+  <div class="container is-max-widescreen">
+    <div class="columns is-centered has-text-centered">
+      <div class="column is-11">
+        <div class="content has-text-justified">
+          <p>
+            We introduce Neodragon, a text-to-video system capable of generating 2s (49 frames @24 fps) videos
+            at a resolution of <code>[640×1024]</code> directly on a <strong>Qualcomm Hexagon NPU</strong> in a
+            record <strong>~6.7s</strong> (7 FPS). Differing from existing transformer-based offline text-to-video
+            generation models, <strong>Neodragon</strong> is the first to have been specifically optimized for mobile
+            hardware to achieve efficient, low-cost, and high-fidelity video synthesis.
+          </p>
+          <ul>
+            <li>
+              <strong>Replacing the original large 4.762B <em>T5</em><sub>XXL</sub> Text-Encoder</strong>
+              with a much smaller 0.2B <em>DT5</em> (DistilT5) with minimal quality loss, enabling the entire model
+              to run without CPU offloading. This is enabled through a novel Text-Encoder Distillation
+              procedure which uses only generative text-prompt data and <em>does not</em> require any image or video data.
+            </li>
+            <li>
+              <strong>Proposing an Asymmetric Decoder Distillation approach</strong> which allows us to replace the native
+              codec-latent-VAE decoder with a more efficient one, without disturbing the generative latent-space of the
+              video generation pipeline.
+            </li>
+            <li>
+              <strong>Pruning of MMDiT blocks</strong> within the denoiser backbone based on their relative importance,
+              with recovery of original performance through a two-stage distillation process.
+            </li>
+            <li>
+              <strong>Reducing the NFE (Neural Functional Evaluation) requirement</strong> of the denoiser by performing
+              step distillation using a technique adapted from DMD for <em>pyramidal</em> flow-matching, thereby significantly
+              accelerating video generation.
+            </li>
+          </ul>
+          <p>
+            When paired with an optimized SSD1B first-frame image generator and QuickSRNet for 2&times;
+            super-resolution, our end-to-end <strong>Neodragon</strong> system becomes a highly parameter
+            (<strong>4.945B</strong> full model), memory (<strong>3.5GB</strong> peak RAM usage), and
+            runtime (<strong>6.7s</strong> E2E latency) efficient mobile-friendly model, while achieving a <em>VBench</em>
+            total score of <strong>81.61</strong>, yielding high-fidelity generated videos.
+          </p>
+          <p>
+            By enabling low-cost, private, and on-device text-to-video synthesis, <strong>Neodragon</strong> democratizes
+            AI-based video content creation, empowering creators to generate high-quality videos without reliance on cloud services.
+          </p>
+          <p>
+            Inference code is available at:
+            <a href="https://github.com/qualcomm-ai-research/neodragon">
+              https://github.com/qualcomm-ai-research/neodragon
+            </a>
+          </p>
+        </div>
+      </div>
+    </div>
+  </div>
+</section>
+
+
+# How to Inference
+Please Refer to: https://github.com/qualcomm-ai-research/neodragon
+
+### Model Description
+
+- **Developed by:** Qualcomm AI Research, Generative Vision group, Amsterdam, Netherlands
+- **Model type:** Mobile Video Generation with efficient pyramidal Diffusion Transformer
+- **Model size:** 4.945B parameters (full package)
+- **Model precision:** torch.bfloat16 (BF16)
+- **Model resolution:** This model is developed to generate [320 x 512] resolution 49(2s @ 24fps) frames videos directly on a Snapdragon powered mobile phone.
+- **Model Description:** This is a model that can be used to generate videos based on the provided text prompts. 
+It is a Diffusion Transformer that uses our finetuned TinyAEHV Auto-Encoder with 8x8x8x spatio-temporal-compressed latent features ([TinyAEHV](https://github.com/madebyollin/taehv)).
+- **Resources for more information:** Check out our [GitHub Repository](https://github.com/qualcomm-ai-research/Neodragon) and the [Technical-report on arXiv](https://arxiv.org/abs/2511.06055) and the [ICLR 2026 Openreview](https://openreview.net/forum?id=XBzIhhwv8d).
+
+## License/Terms of Use
+This model is released under the terms-and-conditions laid out in the [Qualcomm-AI-Hub-proprietory License](https://qaihub-public-assets.s3.us-west-2.amazonaws.com/qai-hub-models/Qualcomm+AI+Hub+Proprietary+License.pdf).
+## Uses
+The model is intended for research purposes. Possible research areas and tasks include: 
+- Research on Efficient Transformer or non-Transformer based Backbone Architectures for Video Generation.
+- Generation of Image/Video based artworks and use in design and other artistic processes.
+- Applications in educational or creative tools.
+- Research on generative models.
+- Safe deployment of models which have the potential to generate harmful content.
+- Probing and understanding the limitations and biases of generative models.
+Excluded uses are described below.
+## Limitations and Bias
+### Limitations
+- The model does not achieve perfect photorealism
+- The model cannot render complex legible text
+- The model cannot produce videos with accurate physically compliant motion
+### Bias
+While the capabilities of the presented mobile video generation model are impressive, they can also reinforce or exacerbate social biases strictly based on our foundational-base model [Pyramidal-Flow](https://arxiv.org/abs/2410.05954).

causal_video_vae/config.json

@@ -0,0 +1,63 @@
+{
+  "_class_name": "AsymmetricCausalVideoVAE",
+  "_diffusers_version": "0.34.0.dev0",
+  "decoder_num_features": [
+    256,
+    128,
+    64,
+    64
+  ],
+  "decoder_space_upscale": [ 
+    true, 
+    true, 
+    true 
+  ],
+  "decoder_time_upscale": [ 
+    true, 
+    true, 
+    true 
+  ],
+  "encoder_act_fn": "silu",
+  "encoder_block_dropout": [
+    0.0,
+    0.0,
+    0.0,
+    0.0
+  ],
+  "encoder_block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "encoder_down_block_types": [
+    "DownEncoderBlockCausal3D",
+    "DownEncoderBlockCausal3D",
+    "DownEncoderBlockCausal3D",
+    "DownEncoderBlockCausal3D"
+  ],
+  "encoder_in_channels": 3,
+  "encoder_layers_per_block": [
+    2,
+    2,
+    2,
+    2
+  ],
+  "encoder_norm_num_groups": 32,
+  "encoder_out_channels": 16,
+  "encoder_spatial_down_sample": [
+    true,
+    true,
+    true,
+    false
+  ],
+  "encoder_temporal_down_sample": [
+    true,
+    true,
+    true,
+    false
+  ],
+  "sample_size": 256,
+  "spatial_downsample_scale": 8,
+  "temporal_downsample_scale": 8
+}

causal_video_vae/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:413f1e98fe0611bf78853acad28ca653ce2768577cfe9822420f4eafd3cb1cf1
+size 239107038

context_adapter/config.json

@@ -0,0 +1,22 @@
+{
+  "_class_name": "SkipMLP",
+  "_diffusers_version": "0.34.0.dev0",
+  "activation_fn": "gelu",
+  "input_dims": 4096,
+  "layer_depths": [
+    4096,
+    4096,
+    4096,
+    4096
+  ],
+  "out_activation_fn": "linear",
+  "output_dims": 1536,
+  "pre_activation": false,
+  "skips": [
+    true,
+    true,
+    true,
+    true
+  ],
+  "use_biases": true
+}

context_adapter/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c9e2ccbed009c5fd94856c933ef492a9045f28bf04cb51b74fbe08698229d7f
+size 520166344

context_adapter_multistep_t2v/config.json

@@ -0,0 +1,22 @@
+{
+  "_class_name": "SkipMLP",
+  "_diffusers_version": "0.34.0.dev0",
+  "activation_fn": "gelu",
+  "input_dims": 4096,
+  "layer_depths": [
+    4096,
+    4096,
+    4096,
+    4096
+  ],
+  "out_activation_fn": "linear",
+  "output_dims": 1536,
+  "pre_activation": false,
+  "skips": [
+    true,
+    true,
+    true,
+    true
+  ],
+  "use_biases": true
+}

context_adapter_multistep_t2v/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea0096d1f6ca72bca0811a927e98a88a1a1f02c7dc4f7ed75c69c3a74928f08b
+size 520166344

diffusion_transformer_320p/config.json

@@ -0,0 +1,24 @@
+{
+  "_class_name": "PyramidDiffusionMMDiT",
+  "_diffusers_version": "0.34.0.dev0",
+  "add_temp_pos_embed": true,
+  "attention_head_dim": 64,
+  "caption_projection_dim": 1536,
+  "gradient_checkpointing_ratio": 0.6,
+  "in_channels": 16,
+  "interp_condition_pos": true,
+  "joint_attention_dim": 4096,
+  "max_num_frames": 200,
+  "num_attention_heads": 24,
+  "num_layers": 18,
+  "patch_size": 2,
+  "pooled_projection_dim": 2048,
+  "pos_embed_max_size": 192,
+  "pos_embed_type": "sincos",
+  "qk_norm": "rms_norm",
+  "sample_size": 128,
+  "temp_pos_embed_type": "rope",
+  "use_gradient_checkpointing": false,
+  "use_t5_mask": true,
+  "use_temporal_causal": true
+}

diffusion_transformer_320p/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccc929881ee6a0474325998b2b80dcab199a39282ae773abf17a18c7fde0e31d
+size 3137624552

diffusion_transformer_320p_multistep_t2v/config.json

@@ -0,0 +1,24 @@
+{
+  "_class_name": "PyramidDiffusionMMDiT",
+  "_diffusers_version": "0.34.0.dev0",
+  "add_temp_pos_embed": true,
+  "attention_head_dim": 64,
+  "caption_projection_dim": 1536,
+  "gradient_checkpointing_ratio": 0.6,
+  "in_channels": 16,
+  "interp_condition_pos": true,
+  "joint_attention_dim": 4096,
+  "max_num_frames": 200,
+  "num_attention_heads": 24,
+  "num_layers": 18,
+  "patch_size": 2,
+  "pooled_projection_dim": 2048,
+  "pos_embed_max_size": 192,
+  "pos_embed_type": "sincos",
+  "qk_norm": "rms_norm",
+  "sample_size": 128,
+  "temp_pos_embed_type": "rope",
+  "use_gradient_checkpointing": false,
+  "use_t5_mask": true,
+  "use_temporal_causal": true
+}

diffusion_transformer_320p_multistep_t2v/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13b9eba1a507b4a31a8bdbe7279de5a5f1738669dc44e1e2f13821c2238014cd
+size 3137624552

ssd_1b_text_encoder/config.json

@@ -0,0 +1,24 @@
+{
+  "architectures": [
+    "CLIPTextModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "quick_gelu",
+  "hidden_size": 768,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "projection_dim": 768,
+  "torch_dtype": "float32",
+  "transformers_version": "4.29.2",
+  "vocab_size": 49408
+}

ssd_1b_text_encoder/model.fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5487ea0eee9c9a9bff8abd097908d4deff3ae1fa87b3b67397f8b9538139d447
+size 246144864

ssd_1b_text_encoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8377b1ca9d88fe06ec483dd7b3cfc62e5e8dbf8ddd252f455e79d659fa0553c5
+size 492265880

ssd_1b_text_encoder_2/config.json

@@ -0,0 +1,24 @@
+{
+  "architectures": [
+    "CLIPTextModelWithProjection"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_size": 1280,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 5120,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 20,
+  "num_hidden_layers": 32,
+  "pad_token_id": 1,
+  "projection_dim": 1280,
+  "torch_dtype": "float32",
+  "transformers_version": "4.29.2",
+  "vocab_size": 49408
+}

ssd_1b_text_encoder_2/model.fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3df577f6e3799c8e1bd9b40e30133710e02e8e25d0ce48cdcc790e7dfe12d6d
+size 1389382880

ssd_1b_text_encoder_2/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b84f413eebecbd049b72874c1df533a516510cb5a2489ae58c7e320209cf0ebe
+size 2778702976

ssd_1b_tokenizer/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

ssd_1b_tokenizer/tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "do_lower_case": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 77,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

ssd_1b_tokenizer_2/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "!",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

ssd_1b_tokenizer_2/tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "do_lower_case": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 77,
+  "pad_token": "!",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

ssd_1b_unet/config.json

@@ -0,0 +1,92 @@
+{
+  "_class_name": "UNet2DConditionModel",
+  "_diffusers_version": "0.34.0.dev0",
+  "act_fn": "silu",
+  "addition_embed_type": "text_time",
+  "addition_embed_type_num_heads": 64,
+  "addition_time_embed_dim": 256,
+  "attention_head_dim": [
+    5,
+    10,
+    20
+  ],
+  "attention_type": "default",
+  "block_out_channels": [
+    320,
+    640,
+    1280
+  ],
+  "center_input_sample": false,
+  "class_embed_type": null,
+  "class_embeddings_concat": false,
+  "conv_in_kernel": 3,
+  "conv_out_kernel": 3,
+  "cross_attention_dim": 2048,
+  "cross_attention_norm": null,
+  "down_block_types": [
+    "DownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "dropout": 0.0,
+  "dual_cross_attention": false,
+  "encoder_hid_dim": null,
+  "encoder_hid_dim_type": null,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_only_cross_attention": null,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlock2D",
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_attention_heads": null,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "out_channels": 4,
+  "projection_class_embeddings_input_dim": 2816,
+  "resnet_out_scale_factor": 1.0,
+  "resnet_skip_time_act": false,
+  "resnet_time_scale_shift": "default",
+  "reverse_transformer_layers_per_block": [
+    [
+      4,
+      4,
+      10
+    ],
+    [
+      2,
+      1,
+      1
+    ],
+    1
+  ],
+  "sample_size": 128,
+  "time_cond_proj_dim": null,
+  "time_embedding_act_fn": null,
+  "time_embedding_dim": null,
+  "time_embedding_type": "positional",
+  "timestep_post_act": null,
+  "transformer_layers_per_block": [
+    [
+      1
+    ],
+    [
+      2,
+      2
+    ],
+    [
+      4,
+      4
+    ]
+  ],
+  "up_block_types": [
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D",
+    "UpBlock2D"
+  ],
+  "upcast_attention": null,
+  "use_linear_projection": true
+}

ssd_1b_unet/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62a020501c81355bd616ce9b491458ea632636bd3051dde8d572171b1c46d840
+size 2662791552

ssd_1b_vae/config.json

@@ -0,0 +1,31 @@
+{
+  "_class_name": "AutoencoderKL",
+  "_diffusers_version": "0.19.0",
+  "act_fn": "silu",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "force_upcast": true,
+  "in_channels": 3,
+  "latent_channels": 4,
+  "layers_per_block": 2,
+  "norm_num_groups": 32,
+  "out_channels": 3,
+  "sample_size": 1024,
+  "scaling_factor": 0.13025,
+  "up_block_types": [
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D"
+  ]
+}

ssd_1b_vae/diffusion_pytorch_model.fp16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6353737672c94b96174cb590f711eac6edf2fcce5b6e91aa9d73c5adc589ee48
+size 167335342

ssd_1b_vae/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78f6189c8492013e3cac81637a1f657f790a237387f8a9dfd6bfa5fee28eb646
+size 334643268

text_encoder/config.json

@@ -0,0 +1,24 @@
+{
+  "architectures": [
+    "CLIPTextModelWithProjection"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "quick_gelu",
+  "hidden_size": 768,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "projection_dim": 768,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.3",
+  "vocab_size": 49408
+}

text_encoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4ddf499861727e83713b525ba1e087202f248a7ebb2268b04d1de09de5b67c2
+size 247324096

text_encoder_2/config.json

@@ -0,0 +1,24 @@
+{
+  "architectures": [
+    "CLIPTextModelWithProjection"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_size": 1280,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 5120,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 20,
+  "num_hidden_layers": 32,
+  "pad_token_id": 1,
+  "projection_dim": 1280,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.3",
+  "vocab_size": 49408
+}

text_encoder_2/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4ba596d8c78b5509aafd523f619e13aa129ae1a886a1e92766d7cddd9706f3c
+size 1389382688

text_encoder_3/config.json

@@ -0,0 +1,63 @@
+{
+  "architectures": [
+    "T5EncoderWithProjection"
+  ],
+  "classifier_dropout": 0.0,
+  "d_ff": 2048,
+  "d_kv": 64,
+  "d_model": 768,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.1,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "t5",
+  "n_positions": 512,
+  "num_decoder_layers": 12,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "project_in_dim": 768,
+  "project_out_dim": 4096,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "task_specific_params": {
+    "summarization": {
+      "early_stopping": true,
+      "length_penalty": 2.0,
+      "max_length": 200,
+      "min_length": 30,
+      "no_repeat_ngram_size": 3,
+      "num_beams": 4,
+      "prefix": "summarize: "
+    },
+    "translation_en_to_de": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to German: "
+    },
+    "translation_en_to_fr": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to French: "
+    },
+    "translation_en_to_ro": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to Romanian: "
+    }
+  },
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.3",
+  "use_cache": true,
+  "vocab_size": 32128
+}

text_encoder_3/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:604e3fce339b87c4f34a1ab97fc05dee34149438218af1ce1d769d568938ef5c
+size 259117320

tokenizer/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer/tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "49406": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49407": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "do_lower_case": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 77,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": "<|endoftext|>"
+}

tokenizer_2/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "!",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_2/tokenizer_config.json

@@ -0,0 +1,38 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "!",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49406": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49407": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "do_lower_case": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 77,
+  "pad_token": "!",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": "<|endoftext|>"
+}