Duplicate from HeartMuLa/HeartMuLa-oss-3B

Co-authored-by: HeartMuLa <HeartMuLa@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,60 @@
+---
+license: apache-2.0
+language:
+- zh
+- en
+- ja
+- ko
+- es
+pipeline_tag: text-to-audio
+tags:
+- music
+- art
+---
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+- **Developed by:** [HeartMuLa Team]
+- **License:** [Apache 2.0]
+
+### Model Sources
+
+<!-- Provide the basic links for the model. -->
+
+- **Github Repo:** https://github.com/HeartMuLa/heartlib
+- **Paper:** https://arxiv.org/abs/2601.10547
+- **Demo:** https://heartmula.github.io/
+- **HeartMuLa-oss-3B:** https://huggingface.co/HeartMuLa/HeartMuLa-oss-3B
+- **HeartCodec-oss:** https://huggingface.co/HeartMuLa/HeartCodec-oss
+- **HeartTranscriptor-oss:** https://huggingface.co/HeartMuLa/HeartTranscriptor-oss
+
+
+## Get Started
+
+Check our github repo https://github.com/HeartMuLa/heartlib for a quickstart and local deployment of HeartMuLa.
+
+## Citation
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+If you find HeartMuLa useful, please cite:
+
+```bibtex
+@misc{yang2026heartmulafamilyopensourced,
+      title={HeartMuLa: A Family of Open Sourced Music Foundation Models}, 
+      author={Dongchao Yang and Yuxin Xie and Yuguo Yin and Zheyu Wang and Xiaoyu Yi and Gongxi Zhu and Xiaolong Weng and Zihan Xiong and Yingzhe Ma and Dading Cong and Jingliang Liu and Zihang Huang and Jinghan Ru and Rongjie Huang and Haoran Wan and Peixu Wang and Kuoxi Yu and Helin Wang and Liming Liang and Xianwei Zhuang and Yuanyuan Wang and Haohan Guo and Junjie Cao and Zeqian Ju and Songxiang Liu and Yuewen Cao and Heming Weng and Yuexian Zou},
+      year={2026},
+      eprint={2601.10547},
+      archivePrefix={arXiv},
+      primaryClass={cs.SD},
+      url={https://arxiv.org/abs/2601.10547}, 
+}
+```
+
+
+## Contact
+If you are interested in HeartMuLa, feel free to reach us at heartmula.ai@gmail.com

config.json

@@ -0,0 +1,14 @@
+{
+  "architectures": [
+    "HeartMuLa"
+  ],
+  "audio_num_codebooks": 8,
+  "audio_vocab_size": 8197,
+  "backbone_flavor": "llama-3B",
+  "decoder_flavor": "llama-300M",
+  "model_type": "heartmula",
+  "muq_dim": 512,
+  "text_vocab_size": 128256,
+  "torch_dtype": "float32",
+  "transformers_version": "4.45.1"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:211968cbe5178c7c7c0e26c79d41a49e53b906a7416b0a94e00ba20f8eaf5b26
+size 4933188000

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7035a3f488acdcbdb95cad247c37c3acb7d0066c7458a6c58dc341ce186c359
+size 4932808592

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ac4b0bc001498f267e077392d813bcd63ece2d9d230d1e1e5e4cc6f38d3636a
+size 4935901576

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b934da1102acca846cdcbc7abd38a33be80a44891a9d84e03eff786738c4df6
+size 950588000

model.safetensors.index.json

@@ -0,0 +1,296 @@
+{
+  "metadata": {
+    "total_size": 15752454144
+  },
+  "weight_map": {
+    "audio_embeddings.weight": "model-00004-of-00004.safetensors",
+    "audio_head": "model-00001-of-00004.safetensors",
+    "backbone.layers.0.attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.0.attn.output_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.0.attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.0.attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.0.mlp.w1.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.0.mlp.w2.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.0.mlp.w3.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.0.mlp_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.0.sa_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.1.attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.1.attn.output_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.1.attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.1.attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.1.mlp.w1.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.1.mlp.w2.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.1.mlp.w3.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.1.mlp_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.1.sa_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.10.attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.10.attn.output_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.10.attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.10.attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.10.mlp.w1.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.10.mlp.w2.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.10.mlp.w3.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.10.mlp_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.10.sa_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.11.attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.11.attn.output_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.11.attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.11.attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.11.mlp.w1.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.11.mlp.w2.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.11.mlp.w3.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.11.mlp_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.11.sa_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.12.attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.12.attn.output_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.12.attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.12.attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.12.mlp.w1.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.12.mlp.w2.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.12.mlp.w3.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.12.mlp_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.12.sa_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.13.attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.13.attn.output_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.13.attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.13.attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.13.mlp.w1.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.13.mlp.w2.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.13.mlp.w3.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.13.mlp_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.13.sa_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.14.attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.14.attn.output_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.14.attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.14.attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.14.mlp.w1.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.14.mlp.w2.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.14.mlp.w3.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.14.mlp_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.14.sa_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.15.attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.15.attn.output_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.15.attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.15.attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.15.mlp.w1.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.15.mlp.w2.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.15.mlp.w3.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.15.mlp_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.15.sa_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.16.attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.16.attn.output_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.16.attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.16.attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.16.mlp.w1.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.16.mlp.w2.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.16.mlp.w3.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.16.mlp_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.16.sa_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.17.attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.17.attn.output_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.17.attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.17.attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.17.mlp.w1.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.17.mlp.w2.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.17.mlp.w3.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.17.mlp_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.17.sa_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.18.attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.18.attn.output_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.18.attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.18.attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.18.mlp.w1.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.18.mlp.w2.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.18.mlp.w3.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.18.mlp_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.18.sa_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.19.attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.19.attn.output_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.19.attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.19.attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.19.mlp.w1.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.19.mlp.w2.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.19.mlp.w3.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.19.mlp_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.19.sa_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.2.attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.2.attn.output_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.2.attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.2.attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.2.mlp.w1.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.2.mlp.w2.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.2.mlp.w3.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.2.mlp_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.2.sa_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.20.attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.20.attn.output_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.20.attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.20.attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.20.mlp.w1.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.20.mlp.w2.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.20.mlp.w3.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.20.mlp_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.20.sa_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.21.attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.21.attn.output_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.21.attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.21.attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.21.mlp.w1.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.21.mlp.w2.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.21.mlp.w3.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.21.mlp_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.21.sa_norm.scale": "model-00002-of-00004.safetensors",
+    "backbone.layers.22.attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.22.attn.output_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.22.attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.22.attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.22.mlp.w1.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.22.mlp.w2.weight": "model-00002-of-00004.safetensors",
+    "backbone.layers.22.mlp.w3.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.22.mlp_norm.scale": "model-00003-of-00004.safetensors",
+    "backbone.layers.22.sa_norm.scale": "model-00003-of-00004.safetensors",
+    "backbone.layers.23.attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.23.attn.output_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.23.attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.23.attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.23.mlp.w1.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.23.mlp.w2.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.23.mlp.w3.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.23.mlp_norm.scale": "model-00003-of-00004.safetensors",
+    "backbone.layers.23.sa_norm.scale": "model-00003-of-00004.safetensors",
+    "backbone.layers.24.attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.24.attn.output_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.24.attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.24.attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.24.mlp.w1.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.24.mlp.w2.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.24.mlp.w3.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.24.mlp_norm.scale": "model-00003-of-00004.safetensors",
+    "backbone.layers.24.sa_norm.scale": "model-00003-of-00004.safetensors",
+    "backbone.layers.25.attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.25.attn.output_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.25.attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.25.attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.25.mlp.w1.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.25.mlp.w2.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.25.mlp.w3.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.25.mlp_norm.scale": "model-00003-of-00004.safetensors",
+    "backbone.layers.25.sa_norm.scale": "model-00003-of-00004.safetensors",
+    "backbone.layers.26.attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.26.attn.output_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.26.attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.26.attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.26.mlp.w1.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.26.mlp.w2.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.26.mlp.w3.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.26.mlp_norm.scale": "model-00003-of-00004.safetensors",
+    "backbone.layers.26.sa_norm.scale": "model-00003-of-00004.safetensors",
+    "backbone.layers.27.attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.27.attn.output_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.27.attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.27.attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.27.mlp.w1.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.27.mlp.w2.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.27.mlp.w3.weight": "model-00003-of-00004.safetensors",
+    "backbone.layers.27.mlp_norm.scale": "model-00003-of-00004.safetensors",
+    "backbone.layers.27.sa_norm.scale": "model-00003-of-00004.safetensors",
+    "backbone.layers.3.attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.3.attn.output_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.3.attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.3.attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.3.mlp.w1.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.3.mlp.w2.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.3.mlp.w3.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.3.mlp_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.3.sa_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.4.attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.4.attn.output_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.4.attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.4.attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.4.mlp.w1.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.4.mlp.w2.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.4.mlp.w3.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.4.mlp_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.4.sa_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.5.attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.5.attn.output_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.5.attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.5.attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.5.mlp.w1.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.5.mlp.w2.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.5.mlp.w3.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.5.mlp_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.5.sa_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.6.attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.6.attn.output_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.6.attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.6.attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.6.mlp.w1.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.6.mlp.w2.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.6.mlp.w3.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.6.mlp_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.6.sa_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.7.attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.7.attn.output_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.7.attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.7.attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.7.mlp.w1.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.7.mlp.w2.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.7.mlp.w3.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.7.mlp_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.7.sa_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.8.attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.8.attn.output_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.8.attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.8.attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.8.mlp.w1.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.8.mlp.w2.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.8.mlp.w3.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.8.mlp_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.8.sa_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.9.attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.9.attn.output_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.9.attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.9.attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.9.mlp.w1.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.9.mlp.w2.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.9.mlp.w3.weight": "model-00001-of-00004.safetensors",
+    "backbone.layers.9.mlp_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.layers.9.sa_norm.scale": "model-00001-of-00004.safetensors",
+    "backbone.norm.scale": "model-00003-of-00004.safetensors",
+    "codebook0_head.weight": "model-00004-of-00004.safetensors",
+    "decoder.layers.0.attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.0.attn.output_proj.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.0.attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.0.attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.0.mlp.w1.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.0.mlp.w2.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.0.mlp.w3.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.0.mlp_norm.scale": "model-00003-of-00004.safetensors",
+    "decoder.layers.0.sa_norm.scale": "model-00003-of-00004.safetensors",
+    "decoder.layers.1.attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.1.attn.output_proj.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.1.attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.1.attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.1.mlp.w1.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.1.mlp.w2.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.1.mlp.w3.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.1.mlp_norm.scale": "model-00003-of-00004.safetensors",
+    "decoder.layers.1.sa_norm.scale": "model-00003-of-00004.safetensors",
+    "decoder.layers.2.attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.2.attn.output_proj.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.2.attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.2.attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.2.mlp.w1.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.2.mlp.w2.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.2.mlp.w3.weight": "model-00003-of-00004.safetensors",
+    "decoder.layers.2.mlp_norm.scale": "model-00003-of-00004.safetensors",
+    "decoder.layers.2.sa_norm.scale": "model-00003-of-00004.safetensors",
+    "decoder.norm.scale": "model-00003-of-00004.safetensors",
+    "muq_linear.bias": "model-00004-of-00004.safetensors",
+    "muq_linear.weight": "model-00004-of-00004.safetensors",
+    "projection.weight": "model-00004-of-00004.safetensors",
+    "text_embeddings.weight": "model-00003-of-00004.safetensors",
+    "unconditional_text_embedding.weight": "model-00004-of-00004.safetensors"
+  }
+}