Duplicate from Soul-AILab/SoulX-FlashHead-1_3B

Co-authored-by: JokerZ <JokerZhou@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,41 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/chengdu.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/einstein.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/flashhead_logo.png filter=lfs diff=lfs merge=lfs -text
+assets/qitiandasheng.mp4 filter=lfs diff=lfs merge=lfs -text
+assets/soul_event_link.png filter=lfs diff=lfs merge=lfs -text
+assets/wechat_group.png filter=lfs diff=lfs merge=lfs -text

Model_Lite/config.json

@@ -0,0 +1,24 @@
+{
+  "_class_name": "WanModelAudioProject",
+  "_diffusers_version": "0.36.0",
+  "dim": 1536,
+  "eps": 1e-06,
+  "ffn_dim": 8960,
+  "freq_dim": 256,
+  "has_image_input": false,
+  "in_dim": 256,
+  "num_heads": 12,
+  "num_layers": 30,
+  "out_dim": 128,
+  "patch_size": [
+    1,
+    1,
+    1
+  ],
+  "text_dim": 4096,
+  "vae_stride": [
+    8,
+    32,
+    32
+  ]
+}

Model_Lite/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaf1cde6e80ca23f740aae236c47954249f65b151db133cc0f77d3a138ccdf6e
+size 6107542048

Model_Pro/config.json

@@ -0,0 +1,24 @@
+{
+  "_class_name": "WanModelAudioProject",
+  "_diffusers_version": "0.36.0",
+  "dim": 1536,
+  "eps": 1e-06,
+  "ffn_dim": 8960,
+  "freq_dim": 256,
+  "has_image_input": false,
+  "in_dim": 32,
+  "num_heads": 12,
+  "num_layers": 30,
+  "out_dim": 16,
+  "patch_size": [
+    1,
+    2,
+    2
+  ],
+  "text_dim": 4096,
+  "vae_stride": [
+    4,
+    8,
+    8
+  ]
+}

Model_Pro/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e47e61b9023ea1aac60c0c0fff077289bc6cca443c71907e5a76a411480af250
+size 6030864656

README.md

@@ -0,0 +1,196 @@
+---
+license: apache-2.0
+datasets:
+- Soul-AILab/VividHead
+pipeline_tag: image-to-video
+---
+Soul-AILab/SoulX-FlashHead-1_3B
+<div align="center">
+
+
+<h1>SoulX-FlashHead: Oracle-guided Generation of Infinite Real-time Streaming Talking Heads</h1>
+
+[Tan Yu*](https://jiayoujiayoujiayoua.github.io/), [Qian Qiao*](https://qianqiaoai.github.io/)<sup>&#9993;</sup>, [Le Shen*](https://openreview.net/profile?id=%7ELe_Shen3), [Ke Zhou](https://github.com/jokerz0624), [Jincheng Hu](#), [Dian Sheng](#), [Bo Hu](#), [Haoming Qin](#), [Jun Gao](#), [Changhai Zhou](#), [Shunshun Yin](#), [Siyuan Liu](#) <sup>&#9993;</sup>
+
+
+<sup>*</sup>Equal Contribution
+<sup>&#9993;</sup>Corresponding Author
+
+
+<a href='https://soul-ailab.github.io/soulx-flashhead/' target="_blank"><img src='https://img.shields.io/badge/Project-Page-green'></a> <a href='https://arxiv.org/pdf/2602.07449' target="_blank"><img src='https://img.shields.io/badge/Technical-Report-red'></a>
+<a href='https://huggingface.co/Soul-AILab/SoulX-FlashHead-1_3B' target="_blank"><img src='https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Model-blue'></a>
+<a href="https://huggingface.co/datasets/Soul-AILab/VividHead" target="_blank"><img src="https://img.shields.io/badge/🤗 Hugging Face-Dataset-blue" alt="Dataset"></a>&nbsp;
+</div>
+
+## ⚡ Highlights
+- **Model_Lite** [Released](https://huggingface.co/Soul-AILab/SoulX-FlashHead-1_3B/tree/main/Model_Lite) get 96 FPS, or 3-concurrent real-time(25+ FPS) streaming on single RTX4090.
+- **Model_Pro** [Released](https://huggingface.co/Soul-AILab/SoulX-FlashHead-1_3B/tree/main/Model_Pro) can generate high-quality videos with 10.8 FPS on single RTX4090, or real-time(25+ FPS) on two RTX5090.
+- **Model_Pretrained** is coming soon, providing high-performance weights and experimental foundations for community research.
+
+## 🔥 News
+- **2026.02.12** - The [online demo](#online-experience-qr) is now available via the Soul App. Download it today to try it out. Download it today to try it out.
+- **2026.02.12** - We have released the [inference code](https://github.com/Soul-AILab/SoulX-FlashHead), and the [model weights](https://huggingface.co/Soul-AILab/SoulX-FlashHead-1_3B).
+- **2026.02.12** - We released **Project page** on [SoulX-FlashHead](https://soul-ailab.github.io/soulx-flashhead/).
+- **2026.02.07** - We released [Dataset](https://huggingface.co/datasets/Soul-AILab/VividHead).
+- **2026.02.07** - We released **SoulX-FlashHead Technical Report** on [Arxiv](https://arxiv.org/pdf/2602.07449) and [GitHub repository](./assets/SoulX_FlashHead.pdf).
+
+
+## 📑 Todo List
+- [x] Technical report 
+- [x] Project Page
+- [x] Inference code
+- [x] Distilled Checkpoint of Pro-Model & Lite-Model release
+- [ ] Pretrained Checkpoint release
+
+## 🌰 Examples
+More examples are available in the project.
+
+<table>
+  <tbody>
+    <!-- Row 1: Videos 1-5 -->
+    <tr>
+      <td width="30%"><video src="https://huggingface.co/Soul-AILab/SoulX-FlashHead-1_3B/resolve/main/assets/qitiandasheng.mp4" style="width:100%; aspect-ratio:512/512; object-fit:cover;" controls loop></video></td>
+      <td width="30%"><video src="https://huggingface.co/Soul-AILab/SoulX-FlashHead-1_3B/resolve/main/assets/chengdu.mp4" style="width:100%; aspect-ratio:512/512; object-fit:cover;" controls loop></video></td>
+      <td width="30%"><video src="https://huggingface.co/Soul-AILab/SoulX-FlashHead-1_3B/resolve/main/assets/einstein.mp4" style="width:100%; aspect-ratio:512/512; object-fit:cover;" controls loop></video></td>
+    </tr>
+
+  </tbody>
+</table>
+
+
+
+## 📖 Quickstart
+###  🔧 Installation
+#### 1. Create a Conda environment
+```bash
+conda create -n flashhead python=3.10
+conda activate flashhead
+```
+#### 2. Install PyTorch on CUDA
+```bash
+pip install torch==2.7.1 torchvision==0.22.1 --index-url https://download.pytorch.org/whl/cu128
+```
+#### 3. Install other dependencies
+```bash
+pip install -r requirements.txt
+```
+#### 4. FlashAttention installation:
+```bash
+pip install ninja
+pip install flash_attn==2.8.0.post2 --no-build-isolation
+```
+
+-- If it takes a long time, we recommend the way below.
+1. download wheel file from [here](https://github.com/Dao-AILab/flash-attention/releases/tag/v2.8.0.post2)
+2. pip install xxx.whl
+
+#### 5. SageAttention installation (Optional)
+```bash
+pip install sageattention==2.2.0 --no-build-isolation
+```
+
+#### 5. FFmpeg installation
+```bash
+# Ubuntu / Debian
+apt-get install ffmpeg
+# CentOS / RHEL
+yum install ffmpeg ffmpeg-devel
+```
+or
+```bash
+# Conda (no root required) 
+conda install -c conda-forge ffmpeg==7
+```
+### 🤗 Model download
+```bash
+# If you are in china mainland, run this first: export HF_ENDPOINT=https://hf-mirror.com
+pip install "huggingface_hub[cli]"
+huggingface-cli download Soul-AILab/SoulX-FlashHead-1_3B --local-dir ./models/SoulX-FlashHead-1_3B
+huggingface-cli download facebook/wav2vec2-base-960h --local-dir ./models/wav2vec2-base-960h
+```
+### 🚀 Inference
+```bash
+# Infer with [Pro-Model] on single GPU
+bash inference_script_single_gpu_pro.sh
+
+
+# Infer with [Pro-Model] on multy GPUs
+bash inference_script_multi_gpu_pro.sh
+# Real-time inference speed of Pro-Model can only be supported on two RTX-5090 with SageAttention.
+
+
+# Infer with [Lite-Model] on single GPU
+bash inference_script_single_gpu_lite.sh
+# Real-time inference speed can be supported on single RTX-4090 (up to 3 concurrent).
+```
+
+### 👋 Online Experience 
+For a real-time interactive experience, scan the QR code to enter the event link. [2026.2.12~2026.3.11]
+<a id="online-experience-qr"></a>
+<div align="center">
+  <table>
+    <tr>
+      <td align="center">
+        <img src="assets/soul_event_link.png" width="200" alt="SoulApp event QR Code"/>
+        <br />
+        <strong>Real-time Online Experience<br>(SoulApp 实时在线体验)</strong>
+      </td>
+    </tr>
+  </table>
+</div>
+
+## 📧 Contact Us
+If you are interested in leaving a message to our work, feel free to email yutan@soulapp.cn or qiaoqian@soulapp.cn or le.shen@mail.dhu.edu.cn or zhouke@soulapp.cn or liusiyuan@soulapp.cn
+
+We have opened a WeChat group. Additionally, we represent **SoulApp** and warmly welcome everyone to download the app and join our Soul group for further technical discussions and updates!
+
+<div align="center">
+  <table>
+    <tr>
+      <td align="center">
+        <img src="assets/wechat_group.png" width="300" alt="WeChat Group QR Code"/>
+        <br />
+        <strong>Join WeChat Group<br>(加入微信技术群)</strong>
+      </td>
+      <td width="100"></td>
+      <td align="center">
+        <img src="assets/soul_group.png" width="300" alt="Soul App Group QR Code"/>
+        <br />
+        <strong>Download SoulApp & Join Group<br>(下载SoulApp加入群组)</strong>
+      </td>
+    </tr>
+  </table>
+</div>
+
+ 
+## 📚 Citation
+
+If you find our work useful in your research, please consider citing:
+
+```
+@misc{yu2026soulxflashheadoracleguidedgenerationinfinite,
+      title={SoulX-FlashHead: Oracle-guided Generation of Infinite Real-time Streaming Talking Heads}, 
+      author={Tan Yu and Qian Qiao and Le Shen and Ke Zhou and Jincheng Hu and Dian Sheng and Bo Hu and Haoming Qin and Jun Gao and Changhai Zhou and Shunshun Yin and Siyuan Liu},
+      year={2026},
+      eprint={2602.07449},
+      archivePrefix={arXiv},
+      primaryClass={cs.CV},
+      url={https://arxiv.org/abs/2602.07449}, 
+}
+```
+
+## 🙇 Acknowledgement
+- [Wan](https://github.com/Wan-Video/Wan2.1): the base model we built upon.
+- [LTX-Video](https://github.com/Lightricks/LTX-Video): the VAE of our Lite-Model.
+- [Self forcing](https://github.com/guandeh17/Self-Forcing): the codebase we built upon.
+- [DMD](https://github.com/tianweiy/DMD2) and [Self forcing++](https://github.com/justincui03/Self-Forcing-Plus-Plus): the key distillation technique used by our method.
+- [SoulX-FlashTalk](https://github.com/Soul-AILab/SoulX-FlashTalk/) is another model developed by our team, featuring 14B parameters and real-time capabilities.
+> [!TIP]
+> If you find our work useful, please also consider starring the original repositories of these foundational methods.
+
+## 💡 Star History
+<p align="center">
+  <a href="https://star-history.com/#Soul-AILab/SoulX-FlashHead&Date">
+    <img src="https://api.star-history.com/svg?repos=Soul-AILab/SoulX-FlashHead&type=Date" alt="Star History Chart" width="100%">
+  </a>
+</p>

VAE_LTX/config.json

@@ -0,0 +1,32 @@
+{
+  "_class_name": "AutoencoderKLLTXVideo",
+  "_diffusers_version": "0.32.0.dev0",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "decoder_causal": false,
+  "encoder_causal": true,
+  "in_channels": 3,
+  "latent_channels": 128,
+  "layers_per_block": [
+    4,
+    3,
+    3,
+    3,
+    4
+  ],
+  "out_channels": 3,
+  "patch_size": 4,
+  "patch_size_t": 1,
+  "resnet_norm_eps": 1e-06,
+  "scaling_factor": 1.0,
+  "spatio_temporal_scaling": [
+    true,
+    true,
+    true,
+    false
+  ]
+}

VAE_LTX/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:265ca87cb5dff5e37f924286e957324e282fe7710a952a7dafc0df43883e2010
+size 1676798532

VAE_Wan/Wan2.1_VAE.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38071ab59bd94681c686fa51d75a1968f64e470262043be31f7a094e442fd981
+size 507609880

assets/chengdu.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95bab76c675fa176b1c2b1be54fe4ac5cb2ad4fb7868196be8a380b608e7b49d
+size 2170506

assets/einstein.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c10118b169916df66ff4ac4258ae3b8e959f51b2ba52f6551e1442125655f899
+size 5901287

assets/qitiandasheng.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72a340f8b74936961c4525592e8f07280f1cc771efa3e0dbc84b731ece6e3198
+size 1682357

config.json

@@ -0,0 +1,5 @@
+{
+  "model_type": "wan",
+  "architectures": ["WanVideoModel"],
+  "transformers_version": "4.48.0"
+}

model_index.json

@@ -0,0 +1,4 @@
+{
+  "_class_name": "WanModelAudioProject",
+  "_diffusers_version": "0.36.0"
+}