Duplicate from deepseek-ai/Janus-Pro-1B

Co-authored-by: Xiaokang Chen <CharlesCXK@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,61 @@
+---
+license: mit
+license_name: deepseek
+license_link: LICENSE
+pipeline_tag: any-to-any
+library_name: transformers
+tags:
+- muiltimodal
+- text-to-image
+- unified-model
+---
+
+## 1. Introduction
+
+Janus-Pro is a novel autoregressive framework that unifies multimodal understanding and generation. 
+It addresses the limitations of previous approaches by decoupling visual encoding into separate pathways, while still utilizing a single, unified transformer architecture for processing. The decoupling not only alleviates the conflict between the visual encoder’s roles in understanding and generation, but also enhances the framework’s flexibility. 
+Janus-Pro surpasses previous unified model and matches or exceeds the performance of task-specific models. 
+The simplicity, high flexibility, and effectiveness of Janus-Pro make it a strong candidate for next-generation unified multimodal models.
+
+[**Github Repository**](https://github.com/deepseek-ai/Janus)
+
+<div align="center">
+<img alt="image" src="janus_pro_teaser1.png" style="width:90%;">
+</div>
+
+<div align="center">
+<img alt="image" src="janus_pro_teaser2.png" style="width:90%;">
+</div>
+
+
+### 2. Model Summary
+
+Janus-Pro is a unified understanding and generation MLLM, which decouples visual encoding for multimodal understanding and generation. 
+Janus-Pro is constructed based on the DeepSeek-LLM-1.5b-base/DeepSeek-LLM-7b-base.
+
+For multimodal understanding, it uses the [SigLIP-L](https://huggingface.co/timm/ViT-L-16-SigLIP-384) as the vision encoder, which supports 384 x 384 image input. For image generation, Janus-Pro uses the tokenizer from [here](https://github.com/FoundationVision/LlamaGen) with a downsample rate of 16.
+
+
+
+## 3. Quick Start
+
+Please refer to [**Github Repository**](https://github.com/deepseek-ai/Janus)
+
+
+## 4. License
+
+This code repository is licensed under [the MIT License](https://github.com/deepseek-ai/DeepSeek-LLM/blob/HEAD/LICENSE-CODE). The use of Janus-Pro models is subject to [DeepSeek Model License](https://github.com/deepseek-ai/DeepSeek-LLM/blob/HEAD/LICENSE-MODEL).
+## 5. Citation
+
+```
+@article{chen2025janus,
+  title={Janus-Pro: Unified Multimodal Understanding and Generation with Data and Model Scaling},
+  author={Chen, Xiaokang and Wu, Zhiyu and Liu, Xingchao and Pan, Zizheng and Liu, Wen and Xie, Zhenda and Yu, Xingkai and Ruan, Chong},
+  journal={arXiv preprint arXiv:2501.17811},
+  year={2025}
+}
+```
+
+## 6. Contact
+
+If you have any questions, please raise an issue or contact us at [service@deepseek.com](mailto:service@deepseek.com).

config.json

@@ -0,0 +1,66 @@
+{
+  "aligner_config": {
+    "cls": "MlpProjector",
+    "model_type": "aligner",
+    "params": {
+      "depth": 2,
+      "input_dim": 1024,
+      "n_embed": 2048,
+      "projector_type": "mlp_gelu"
+    }
+  },
+  "architectures": [
+    "MultiModalityCausalLM"
+  ],
+  "gen_aligner_config": {
+    "cls": "MlpProjector",
+    "model_type": "gen_aligner",
+    "params": {
+      "depth": 2,
+      "input_dim": 8,
+      "n_embed": 2048,
+      "projector_type": "mlp_gelu"
+    }
+  },
+  "gen_head_config": {
+    "cls": "vision_head",
+    "model_type": "gen_head",
+    "params": {
+      "image_token_embed": 2048,
+      "image_token_size": 16384,
+      "n_embed": 2048
+    }
+  },
+  "gen_vision_config": {
+    "cls": "VQ-16",
+    "model_type": "gen_vision",
+    "params": {
+      "image_token_size": 16384,
+      "n_embed": 8
+    }
+  },
+  "language_config": {
+    "hidden_size": 2048,
+    "intermediate_size": 5632,
+    "max_position_embeddings": 16384,
+    "model_type": "llama",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 16,
+    "torch_dtype": "bfloat16",
+    "vocab_size": 102400
+  },
+  "model_type": "multi_modality",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.33.1",
+  "vision_config": {
+    "cls": "CLIPVisionTower",
+    "model_type": "vision",
+    "params": {
+      "image_size": 384,
+      "model_name": "siglip_large_patch16_384",
+      "select_feature": "same",
+      "select_layer": -1
+    }
+  }
+}

preprocessor_config.json

@@ -0,0 +1,23 @@
+{
+  "background_color": [
+    127,
+    127,
+    127
+  ],
+  "do_normalize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "VLMImageProcessor",
+  "image_size": 384,
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "min_size": 14,
+  "processor_class": "VLChatProcessor",
+  "rescale_factor": 0.00392156862745098
+}

processor_config.json

@@ -0,0 +1,9 @@
+{
+  "add_special_token": false,
+  "ignore_id": -100,
+  "image_tag": "<image_placeholder>",
+  "mask_prompt": true,
+  "num_image_tokens": 576,
+  "processor_class": "VLChatProcessor",
+  "sft_format": "deepseek"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea7cf164cbed272be2a9999bc4c314da6a6f23ef51871ddef3afc2c0c430cc3f
+size 4178890389

special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "additional_special_tokens": [
+    "<image_placeholder>",
+    "<patch_placeholder>",
+    "<|ref|>",
+    "<|/ref|>",
+    "<|det|>",
+    "<|/det|>",
+    "<|grounding|>",
+    "<|User|>",
+    "<|Assistant|>"
+  ],
+  "bos_token": "<｜begin▁of▁sentence｜>",
+  "eos_token": "<｜end▁of▁sentence｜>",
+  "pad_token": "<｜▁pad▁｜>"
+}

tokenizer_config.json

@@ -0,0 +1,10 @@
+{
+  "bos_token": "<｜begin▁of▁sentence｜>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<｜end▁of▁sentence｜>",
+  "model_max_length": 16384,
+  "pad_token": null,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": null,
+  "use_default_system_prompt": true
+}