Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+processor/tokenizer.json filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,198 @@
+{
+  "architectures": [
+    "DMLLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_dmllm.DMLLMConfig",
+    "AutoModel": "modeling_dmllm.DMLLM",
+    "AutoModelForCausalLM": "modeling_dmllm.DMLLM"
+  },
+  "downsample_ratio": 0.5,
+  "image_size": 512,
+  "image_token_id": 151655,
+  "language_model_config": {
+    "_name_or_path": "./hf_models/JetLM/SDAR-1.7B-Chat-b32/",
+    "add_cross_attention": false,
+    "architectures": [
+      "SDARForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "auto_map": {
+      "AutoConfig": "configuration_sdar.SDARConfig",
+      "AutoModel": "modeling_sdar.SDARModel",
+      "AutoModelForCausalLM": "modeling_sdar.SDARForCausalLM"
+    },
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "block_size": 32,
+    "bos_token_id": 151643,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "enable_block_generation": true,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 151643,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "fuse_cross_entropy": false,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 2048,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 6144,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 32768,
+    "max_window_layers": 28,
+    "min_length": 0,
+    "model_type": "sdar",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 16,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": null,
+    "rope_theta": 1000000,
+    "sep_token_id": null,
+    "sliding_window": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "model_type": "dmllm",
+  "patch_size": 16,
+  "rm_vit_merger": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.52.1",
+  "vision_abstractor_config": {
+    "projection_type": "patch_merger2x"
+  },
+  "vision_model_config": {
+    "_name_or_path": "./hf_models/shilinxu/Qwen2-VL-7B-ViT/",
+    "add_cross_attention": false,
+    "architectures": [
+      "Qwen2VisionTransformerPretrainedModel"
+    ],
+    "auto_map": {
+      "AutoConfig": "configuration_qwen2_vl.Qwen2VLVisionConfig",
+      "AutoModel": "modeling_qwen2_vl.Qwen2VisionTransformerPretrainedModel"
+    },
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "depth": 32,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "embed_dim": 1280,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "quick_gelu",
+    "hidden_size": 3584,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "in_channels": 3,
+    "in_chans": 3,
+    "initializer_range": 0.02,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "mlp_ratio": 4,
+    "model_type": "qwen2_vl",
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_heads": 16,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 14,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "spatial_merge_size": 2,
+    "spatial_patch_size": 14,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "temporal_patch_size": 2,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false
+  },
+  "vision_model_type": "qwen2vit",
+  "vision_output_key": null,
+  "vision_select_layer": -2
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f1a9405096e60187d14f7425ff75ec7d3f5ecfc4cd913230bb61a56aa46002f
+size 4777020264

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e809785cfe255beacba63d6d6e22c270063519167d127f08cd45f91572186c3
+size 622330000

model.safetensors.index.json

@@ -0,0 +1,709 @@
+{
+  "metadata": {
+    "total_size": 5399266304
+  },
+  "weight_map": {
+    "language_model.lm_head.weight": "model-00002-of-00002.safetensors",
+    "language_model.model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.18.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.18.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.19.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.19.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.20.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.20.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.21.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.21.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.22.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.22.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.23.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.23.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.23.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.23.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.23.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.23.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.24.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.24.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.24.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.24.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.24.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.24.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.24.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.24.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.24.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.24.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.24.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.25.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.25.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.25.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.25.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.25.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.25.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.25.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.25.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.25.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.25.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.25.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.26.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.26.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.26.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.26.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.26.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.26.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.26.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.26.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.26.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.26.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.26.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.27.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.27.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.27.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.27.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.27.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.27.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.27.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.27.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.27.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.27.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.27.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.self_attn.k_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.self_attn.q_norm.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "language_model.model.norm.weight": "model-00001-of-00002.safetensors",
+    "vision_abstractor.projection.ln_q.bias": "model-00001-of-00002.safetensors",
+    "vision_abstractor.projection.ln_q.weight": "model-00001-of-00002.safetensors",
+    "vision_abstractor.projection.mlp.0.bias": "model-00001-of-00002.safetensors",
+    "vision_abstractor.projection.mlp.0.weight": "model-00001-of-00002.safetensors",
+    "vision_abstractor.projection.mlp.2.bias": "model-00001-of-00002.safetensors",
+    "vision_abstractor.projection.mlp.2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.0.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.0.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.0.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.0.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.0.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.0.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.0.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.0.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.0.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.0.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.0.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.0.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.1.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.1.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.1.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.1.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.1.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.1.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.1.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.1.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.1.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.1.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.1.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.1.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.10.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.10.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.10.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.10.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.10.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.10.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.10.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.10.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.10.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.10.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.10.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.10.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.11.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.11.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.11.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.11.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.11.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.11.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.11.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.11.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.11.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.11.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.11.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.11.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.12.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.12.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.12.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.12.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.12.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.12.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.12.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.12.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.12.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.12.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.12.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.12.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.13.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.13.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.13.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.13.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.13.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.13.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.13.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.13.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.13.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.13.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.13.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.13.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.14.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.14.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.14.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.14.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.14.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.14.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.14.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.14.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.14.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.14.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.14.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.14.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.15.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.15.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.15.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.15.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.15.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.15.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.15.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.15.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.15.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.15.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.15.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.15.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.16.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.16.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.16.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.16.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.16.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.16.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.16.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.16.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.16.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.16.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.16.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.16.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.17.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.17.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.17.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.17.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.17.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.17.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.17.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.17.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.17.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.17.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.17.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.17.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.18.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.18.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.18.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.18.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.18.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.18.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.18.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.18.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.18.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.18.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.18.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.18.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.19.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.19.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.19.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.19.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.19.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.19.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.19.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.19.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.19.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.19.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.19.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.19.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.2.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.2.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.2.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.2.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.2.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.2.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.2.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.2.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.2.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.2.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.2.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.2.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.20.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.20.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.20.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.20.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.20.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.20.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.20.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.20.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.20.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.20.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.20.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.20.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.21.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.21.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.21.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.21.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.21.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.21.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.21.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.21.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.21.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.21.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.21.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.21.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.22.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.22.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.22.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.22.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.22.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.22.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.22.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.22.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.22.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.22.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.22.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.22.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.23.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.23.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.23.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.23.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.23.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.23.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.23.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.23.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.23.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.23.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.23.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.23.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.24.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.24.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.24.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.24.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.24.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.24.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.24.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.24.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.24.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.24.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.24.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.24.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.25.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.25.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.25.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.25.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.25.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.25.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.25.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.25.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.25.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.25.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.25.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.25.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.26.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.26.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.26.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.26.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.26.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.26.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.26.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.26.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.26.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.26.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.26.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.26.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.27.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.27.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.27.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.27.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.27.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.27.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.27.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.27.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.27.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.27.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.27.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.27.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.28.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.28.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.28.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.28.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.28.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.28.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.28.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.28.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.28.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.28.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.28.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.28.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.29.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.29.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.29.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.29.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.29.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.29.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.29.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.29.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.29.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.29.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.29.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.29.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.3.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.3.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.3.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.3.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.3.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.3.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.3.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.3.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.3.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.3.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.3.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.3.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.30.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.30.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.30.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.30.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.30.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.30.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.30.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.30.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.30.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.30.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.30.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.30.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.31.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.31.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.31.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.31.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.31.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.31.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.31.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.31.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.31.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.31.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.31.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.31.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.4.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.4.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.4.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.4.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.4.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.4.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.4.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.4.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.4.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.4.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.4.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.4.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.5.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.5.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.5.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.5.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.5.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.5.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.5.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.5.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.5.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.5.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.5.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.5.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.6.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.6.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.6.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.6.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.6.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.6.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.6.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.6.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.6.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.6.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.6.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.6.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.7.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.7.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.7.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.7.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.7.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.7.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.7.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.7.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.7.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.7.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.7.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.7.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.8.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.8.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.8.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.8.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.8.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.8.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.8.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.8.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.8.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.8.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.8.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.8.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.9.attn.proj.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.9.attn.proj.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.9.attn.qkv.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.9.attn.qkv.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.9.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.9.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.9.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.9.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.9.norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.9.norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.9.norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_model.blocks.9.norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_model.patch_embed.proj.weight": "model-00001-of-00002.safetensors"
+  }
+}

processor/added_tokens.json

@@ -0,0 +1,46 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<ched>": 151674,
+  "<ecel>": 151675,
+  "<fcel>": 151676,
+  "<lcel>": 151677,
+  "<nl>": 151680,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<ucel>": 151678,
+  "<xcel>": 151679,
+  "<|MASK|>": 151669,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|md_end|>": 151673,
+  "<|md_start|>": 151672,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|paratext|>": 151686,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|ref_end|>": 151671,
+  "<|ref_start|>": 151670,
+  "<|repo_name|>": 151663,
+  "<|rotate_down|>": 151682,
+  "<|rotate_left|>": 151683,
+  "<|rotate_right|>": 151684,
+  "<|rotate_up|>": 151681,
+  "<|txt_contd|>": 151685,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

processor/chat_template.jinja

@@ -0,0 +1 @@
+{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>systemYou are a helpful assistant.<|im_end|>{% endif %}<|im_start|>{{ message['role'] }}{% if message['role'] == 'assistant' %}{% generation %}{{ message['content'][0]['text'] }}<|im_end|>{% endgeneration %}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant{% endif %}

processor/image_processing_qwen2_vl.py

@@ -0,0 +1,494 @@
+# coding=utf-8
+# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Image processor class for Qwen2-VL."""
+
+import math
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+
+from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
+from transformers.image_transforms import (
+    convert_to_rgb,
+    resize,
+    to_channel_dimension_format,
+)
+from transformers.image_utils import (
+    OPENAI_CLIP_MEAN,
+    OPENAI_CLIP_STD,
+    ChannelDimension,
+    ImageInput,
+    PILImageResampling,
+    get_image_size,
+    infer_channel_dimension_format,
+    is_scaled_image,
+    make_flat_list_of_images,
+    make_list_of_images,
+    to_numpy_array,
+    valid_images,
+    validate_preprocess_arguments,
+)
+from transformers.utils import TensorType, logging
+from transformers.video_utils import VideoInput, make_batched_videos
+
+
+logger = logging.get_logger(__name__)
+
+
+def smart_resize(
+    height: int, width: int, factor: int = 28, min_pixels: int = 56 * 56, max_pixels: int = 14 * 14 * 4 * 1280
+):
+    """Rescales the image so that the following conditions are met:
+
+    1. Both dimensions (height and width) are divisible by 'factor'.
+
+    2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+
+    3. The aspect ratio of the image is maintained as closely as possible.
+
+    """
+    if height < factor or width < factor:
+        raise ValueError(f"height:{height} and width:{width} must be larger than factor:{factor}")
+    elif max(height, width) / min(height, width) > 200:
+        raise ValueError(
+            f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}"
+        )
+    h_bar = round(height / factor) * factor
+    w_bar = round(width / factor) * factor
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = math.floor(height / beta / factor) * factor
+        w_bar = math.floor(width / beta / factor) * factor
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = math.ceil(height * beta / factor) * factor
+        w_bar = math.ceil(width * beta / factor) * factor
+    return h_bar, w_bar
+
+
+class Qwen2VLImageProcessor(BaseImageProcessor):
+    r"""
+    Constructs a Qwen2-VL image processor that dynamically resizes images based on the original images.
+
+    Args:
+        do_resize (`bool`, *optional*, defaults to `True`):
+            Whether to resize the image's (height, width) dimensions.
+        size (`Dict[str, int]`, *optional*, defaults to `{"shortest_edge": 56 * 56, "longest_edge": 28 * 28 * 1280}`):
+            Size of the image after resizing. `shortest_edge` and `longest_edge` keys must be present.
+        resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`):
+            Resampling filter to use when resizing the image.
+        do_rescale (`bool`, *optional*, defaults to `True`):
+            Whether to rescale the image by the specified scale `rescale_factor`.
+        rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):
+            Scale factor to use if rescaling the image.
+        do_normalize (`bool`, *optional*, defaults to `True`):
+            Whether to normalize the image.
+        image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`):
+            Mean to use if normalizing the image. This is a float or list of floats for each channel in the image.
+        image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`):
+            Standard deviation to use if normalizing the image. This is a float or list of floats for each channel in the image.
+        do_convert_rgb (`bool`, *optional*, defaults to `True`):
+            Whether to convert the image to RGB.
+        min_pixels (`int`, *optional*, defaults to `56 * 56`):
+            The min pixels of the image to resize the image.
+        max_pixels (`int`, *optional*, defaults to `28 * 28 * 1280`):
+            The max pixels of the image to resize the image.
+        patch_size (`int`, *optional*, defaults to 14):
+            The spatial patch size of the vision encoder.
+        temporal_patch_size (`int`, *optional*, defaults to 2):
+            The temporal patch size of the vision encoder.
+        merge_size (`int`, *optional*, defaults to 2):
+            The merge size of the vision encoder to llm encoder.
+    """
+
+    model_input_names = ["pixel_values", "image_grid_thw", "pixel_values_videos", "video_grid_thw"]
+
+    def __init__(
+        self,
+        do_resize: bool = True,
+        size: Optional[Dict[str, int]] = None,
+        resample: PILImageResampling = PILImageResampling.BICUBIC,
+        do_rescale: bool = True,
+        rescale_factor: Union[int, float] = 1 / 255,
+        do_normalize: bool = True,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        do_convert_rgb: bool = True,
+        min_pixels: Optional[int] = None,
+        max_pixels: Optional[int] = None,
+        patch_size: int = 14,
+        temporal_patch_size: int = 2,
+        merge_size: int = 2,
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        if size is not None and ("shortest_edge" not in size or "longest_edge" not in size):
+            raise ValueError("size must contain 'shortest_edge' and 'longest_edge' keys.")
+        else:
+            size = {"shortest_edge": 56 * 56, "longest_edge": 28 * 28 * 1280}
+        # backward compatibility: override size with min_pixels and max_pixels if they are provided
+        if min_pixels is not None:
+            size["shortest_edge"] = min_pixels
+        if max_pixels is not None:
+            size["longest_edge"] = max_pixels
+        self.min_pixels = size["shortest_edge"]
+        self.max_pixels = size["longest_edge"]
+        self.size = size
+
+        self.do_resize = do_resize
+        self.resample = resample
+        self.do_rescale = do_rescale
+        self.rescale_factor = rescale_factor
+        self.do_normalize = do_normalize
+        self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN
+        self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD
+
+        self.patch_size = patch_size
+        self.temporal_patch_size = temporal_patch_size
+        self.merge_size = merge_size
+        self.do_convert_rgb = do_convert_rgb
+
+    def _preprocess(
+        self,
+        images: Union[ImageInput, VideoInput],
+        do_resize: Optional[bool] = None,
+        size: Optional[Dict[str, int]] = None,
+        resample: PILImageResampling = None,
+        do_rescale: Optional[bool] = None,
+        rescale_factor: Optional[float] = None,
+        do_normalize: Optional[bool] = None,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        patch_size: Optional[int] = None,
+        temporal_patch_size: Optional[int] = None,
+        merge_size: Optional[int] = None,
+        do_convert_rgb: Optional[bool] = None,
+        data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
+    ):
+        """
+        Preprocess an image or batch of images. Copy of the `preprocess` method from `CLIPImageProcessor`.
+
+        Args:
+            images (`ImageInput`):
+                Image or batch of images to preprocess. Expects pixel values ranging from 0 to 255. If pixel values range from 0 to 1, set `do_rescale=False`.
+            vision_info (`List[Dict]`, *optional*):
+                Optional list of dictionaries containing additional information about vision inputs.
+            do_resize (`bool`, *optional*, defaults to `self.do_resize`):
+                Whether to resize the image.
+            size (`Dict[str, int]`, *optional*, defaults to `self.size`):
+                Size of the image after resizing. `shortest_edge` and `longest_edge` keys must be present.
+            resample (`PILImageResampling`, *optional*, defaults to `self.resample`):
+                Resampling filter to use if resizing the image. This can be one of the `PILImageResampling` enums.
+            do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
+                Whether to rescale the image.
+            rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
+                Scale factor to use if rescaling the image.
+            do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
+                Whether to normalize the image.
+            image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+                Mean to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image.
+            image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+                Standard deviation to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image.
+            patch_size (`int`, *optional*, defaults to `self.patch_size`):
+                The spatial patch size of the vision encoder.
+            temporal_patch_size (`int`, *optional*, defaults to `self.temporal_patch_size`):
+                The temporal patch size of the vision encoder.
+            merge_size (`int`, *optional*, defaults to `self.merge_size`):
+                The merge size of the vision encoder to llm encoder.
+            do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
+                Whether to convert the image to RGB.
+            data_format (`ChannelDimension`, *optional*, defaults to `ChannelDimension.FIRST`):
+                The channel dimension format for the output image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - Unset: Use the channel dimension format of the input image.
+            input_data_format (`ChannelDimension` or `str`, *optional*):
+                The channel dimension format for the input image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.   - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+        """
+        images = make_list_of_images(images)
+
+        if do_convert_rgb:
+            images = [convert_to_rgb(image) for image in images]
+
+        # All transformations expect numpy arrays.
+        images = [to_numpy_array(image) for image in images]
+
+        if do_rescale and is_scaled_image(images[0]):
+            logger.warning_once(
+                "It looks like you are trying to rescale already rescaled images. If the input"
+                " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again."
+            )
+        if input_data_format is None:
+            # We assume that all images have the same channel dimension format.
+            input_data_format = infer_channel_dimension_format(images[0])
+
+        height, width = get_image_size(images[0], channel_dim=input_data_format)
+        resized_height, resized_width = height, width
+        processed_images = []
+        for image in images:
+            if do_resize:
+                resized_height, resized_width = smart_resize(
+                    height,
+                    width,
+                    factor=patch_size * merge_size,
+                    min_pixels=size["shortest_edge"],
+                    max_pixels=size["longest_edge"],
+                )
+                image = resize(
+                    image, size=(resized_height, resized_width), resample=resample, input_data_format=input_data_format
+                )
+
+            if do_rescale:
+                image = self.rescale(image, scale=rescale_factor, input_data_format=input_data_format)
+
+            if do_normalize:
+                image = self.normalize(
+                    image=image, mean=image_mean, std=image_std, input_data_format=input_data_format
+                )
+
+            image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format)
+            processed_images.append(image)
+
+        patches = np.array(processed_images)
+        if data_format == ChannelDimension.LAST:
+            patches = patches.transpose(0, 3, 1, 2)
+        if patches.shape[0] % temporal_patch_size != 0:
+            repeats = np.repeat(
+                patches[-1][np.newaxis], temporal_patch_size - (patches.shape[0] % temporal_patch_size), axis=0
+            )
+            patches = np.concatenate([patches, repeats], axis=0)
+        channel = patches.shape[1]
+        grid_t = patches.shape[0] // temporal_patch_size
+        grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
+        patches = patches.reshape(
+            grid_t,
+            temporal_patch_size,
+            channel,
+            grid_h // merge_size,
+            merge_size,
+            patch_size,
+            grid_w // merge_size,
+            merge_size,
+            patch_size,
+        )
+        patches = patches.transpose(0, 3, 6, 4, 7, 2, 1, 5, 8)
+        flatten_patches = patches.reshape(
+            grid_t * grid_h * grid_w, channel * temporal_patch_size * patch_size * patch_size
+        )
+
+        return flatten_patches, (grid_t, grid_h, grid_w)
+
+    def preprocess(
+        self,
+        images: ImageInput,
+        videos: VideoInput = None,
+        do_resize: Optional[bool] = None,
+        size: Optional[Dict[str, int]] = None,
+        min_pixels: Optional[int] = None,
+        max_pixels: Optional[int] = None,
+        resample: PILImageResampling = None,
+        do_rescale: Optional[bool] = None,
+        rescale_factor: Optional[float] = None,
+        do_normalize: Optional[bool] = None,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        patch_size: Optional[int] = None,
+        temporal_patch_size: Optional[int] = None,
+        merge_size: Optional[int] = None,
+        do_convert_rgb: Optional[bool] = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+        data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
+    ):
+        """
+        Args:
+            images (`ImageInput`):
+                Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
+                passing in images with pixel values between 0 and 1, set `do_rescale=False`.
+            videos (`VideoInput`):
+                Video to preprocess. Expects a single or batch of videos with pixel values ranging from 0 to 255. If
+                passing in videos with pixel values between 0 and 1, set `do_rescale=False`.
+            do_resize (`bool`, *optional*, defaults to `self.do_resize`):
+                Whether to resize the image.
+            size (`Dict[str, int]`, *optional*, defaults to `self.size`):
+                Size of the image after resizing. Shortest edge of the image is resized to size["shortest_edge"], with
+                the longest edge resized to keep the input aspect ratio.
+            resample (`int`, *optional*, defaults to `self.resample`):
+                Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only
+                has an effect if `do_resize` is set to `True`.
+            do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
+                Whether to rescale the image.
+            rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
+                Rescale factor to rescale the image by if `do_rescale` is set to `True`.
+            do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
+                Whether to normalize the image.
+            image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+                Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
+            image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+                Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
+                `True`.
+            min_pixels (`int`, *optional*, defaults to `self.min_pixels`):
+                The min pixels of the image to resize the image.
+            max_pixels (`int`, *optional*, defaults to `self.max_pixels`):
+                The max pixels of the image to resize the image.
+            patch_size (`int`, *optional*, defaults to `self.patch_size`):
+                The spatial patch size of the vision encoder.
+            temporal_patch_size (`int`, *optional*, defaults to `self.temporal_patch_size`):
+                The temporal patch size of the vision encoder.
+            merge_size (`int`, *optional*, defaults to `self.merge_size`):
+                The merge size of the vision encoder to llm encoder.
+            do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
+                Whether to convert the image to RGB.
+            return_tensors (`str` or `TensorType`, *optional*):
+                The type of tensors to return. Can be one of:
+                - Unset: Return a list of `np.ndarray`.
+                - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.
+                - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
+                - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
+                - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.
+            data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):
+                The channel dimension format for the output image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - Unset: Use the channel dimension format of the input image.
+            input_data_format (`ChannelDimension` or `str`, *optional*):
+                The channel dimension format for the input image. If unset, the channel dimension format is inferred
+                from the input image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+
+        """
+        min_pixels = min_pixels if min_pixels is not None else self.min_pixels
+        max_pixels = max_pixels if max_pixels is not None else self.max_pixels
+
+        if size is not None:
+            if "shortest_edge" not in size or "longest_edge" not in size:
+                raise ValueError("size must contain 'shortest_edge' and 'longest_edge' keys.")
+            min_pixels = size["shortest_edge"]
+        elif min_pixels is not None and max_pixels is not None:
+            # backward compatibility: override size with min_pixels and max_pixels if they are provided
+            size = {"shortest_edge": min_pixels, "longest_edge": max_pixels}
+        else:
+            size = {**self.size}
+
+        do_resize = do_resize if do_resize is not None else self.do_resize
+
+        resample = resample if resample is not None else self.resample
+        do_rescale = do_rescale if do_rescale is not None else self.do_rescale
+        rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor
+        do_normalize = do_normalize if do_normalize is not None else self.do_normalize
+        image_mean = image_mean if image_mean is not None else self.image_mean
+        image_std = image_std if image_std is not None else self.image_std
+        patch_size = patch_size if patch_size is not None else self.patch_size
+        temporal_patch_size = temporal_patch_size if temporal_patch_size is not None else self.temporal_patch_size
+        merge_size = merge_size if merge_size is not None else self.merge_size
+        do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
+
+        if images is not None:
+            images = make_flat_list_of_images(images)
+
+        if images is not None and not valid_images(images):
+            raise ValueError(
+                "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
+                "torch.Tensor, tf.Tensor or jax.ndarray."
+            )
+
+        validate_preprocess_arguments(
+            rescale_factor=rescale_factor,
+            do_normalize=do_normalize,
+            image_mean=image_mean,
+            image_std=image_std,
+            do_resize=do_resize,
+            size=size,
+            resample=resample,
+        )
+
+        data = {}
+        if images is not None:
+            pixel_values, vision_grid_thws = [], []
+            for image in images:
+                patches, image_grid_thw = self._preprocess(
+                    image,
+                    do_resize=do_resize,
+                    size=size,
+                    resample=resample,
+                    do_rescale=do_rescale,
+                    rescale_factor=rescale_factor,
+                    do_normalize=do_normalize,
+                    image_mean=image_mean,
+                    image_std=image_std,
+                    patch_size=patch_size,
+                    temporal_patch_size=temporal_patch_size,
+                    merge_size=merge_size,
+                    data_format=data_format,
+                    do_convert_rgb=do_convert_rgb,
+                    input_data_format=input_data_format,
+                )
+                pixel_values.extend(patches)
+                vision_grid_thws.append(image_grid_thw)
+            pixel_values = np.array(pixel_values)
+            vision_grid_thws = np.array(vision_grid_thws)
+            data.update({"pixel_values": pixel_values, "image_grid_thw": vision_grid_thws})
+
+        # kept for BC only and should be removed after v5.0
+        if videos is not None:
+            logger.warning(
+                "`Qwen2VLImageProcessor` works only with image inputs and doesn't process videos anymore. "
+                "This is a deprecated behavior and will be removed in v5.0. "
+                "Your videos should be forwarded to `Qwen2VLVideoProcessor`. "
+            )
+            videos = make_batched_videos(videos)
+            pixel_values_videos, vision_grid_thws_videos = [], []
+            for images in videos:
+                patches, video_grid_thw = self._preprocess(
+                    images,
+                    do_resize=do_resize,
+                    size=size,
+                    resample=resample,
+                    do_rescale=do_rescale,
+                    rescale_factor=rescale_factor,
+                    do_normalize=do_normalize,
+                    image_mean=image_mean,
+                    image_std=image_std,
+                    patch_size=patch_size,
+                    temporal_patch_size=temporal_patch_size,
+                    merge_size=merge_size,
+                    data_format=data_format,
+                    do_convert_rgb=do_convert_rgb,
+                    input_data_format=input_data_format,
+                )
+                pixel_values_videos.extend(patches)
+                vision_grid_thws_videos.append(video_grid_thw)
+            data.update(
+                {
+                    "pixel_values_videos": np.array(pixel_values_videos),
+                    "video_grid_thw": np.array(vision_grid_thws_videos),
+                }
+            )
+
+        return BatchFeature(data=data, tensor_type=return_tensors)
+
+
+__all__ = ["Qwen2VLImageProcessor"]

processor/preprocessor_config.json

@@ -0,0 +1,32 @@
+{
+  "auto_map": {
+    "AutoImageProcessor": "image_processing_qwen2_vl.Qwen2VLImageProcessor"
+  },
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_pixels": 1605632,
+  "merge_size": 2,
+  "min_pixels": 3136,
+  "patch_size": 14,
+  "processor_class": "SmallVLMProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 12845056,
+    "shortest_edge": 3136
+  },
+  "temporal_patch_size": 2
+}

processor/special_tokens_map.json

@@ -0,0 +1,165 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|md_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|md_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<ched>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<ecel>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<fcel>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<lcel>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<ucel>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<xcel>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<nl>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|rotate_up|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|rotate_down|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|rotate_left|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|rotate_right|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|txt_contd|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|paratext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<|MASK|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

processor/tokenization_qwen2.py

@@ -0,0 +1,342 @@
+# coding=utf-8
+# Copyright 2024 The Qwen team, Alibaba Group and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tokenization classes for Qwen2."""
+
+import json
+import os
+import unicodedata
+from functools import lru_cache
+from typing import Optional, Tuple
+
+import regex as re
+
+from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {
+    "vocab_file": "vocab.json",
+    "merges_file": "merges.txt",
+}
+
+
+MAX_MODEL_INPUT_SIZES = {"qwen/qwen-tokenizer": 32768}
+
+PRETOKENIZE_REGEX = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
+
+
+@lru_cache()
+# Copied from transformers.models.gpt2.tokenization_gpt2.bytes_to_unicode
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
+    characters the bpe code barfs on.
+
+    The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
+    if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
+    decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
+    tables between utf-8 bytes and unicode strings.
+    """
+    bs = (
+        list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
+    )
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8 + n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+
+# Copied from transformers.models.gpt2.tokenization_gpt2.get_pairs
+def get_pairs(word):
+    """
+    Return set of symbol pairs in a word.
+
+    Word is represented as tuple of symbols (symbols being variable-length strings).
+    """
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
+
+class Qwen2Tokenizer(PreTrainedTokenizer):
+    """
+    Construct a Qwen2 tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+    Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
+    be encoded differently whether it is at the beginning of the sentence (without space) or not:
+
+    ```python
+    >>> from transformers import Qwen2Tokenizer
+
+    >>> tokenizer = Qwen2Tokenizer.from_pretrained("Qwen/Qwen-tokenizer")
+    >>> tokenizer("Hello world")["input_ids"]
+    [9707, 1879]
+
+    >>> tokenizer(" Hello world")["input_ids"]
+    [21927, 1879]
+    ```
+    This is expected.
+
+    You should not use GPT2Tokenizer instead, because of the different pretokenization rules.
+
+    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+    this superclass for more information regarding those methods.
+
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+        merges_file (`str`):
+            Path to the merges file.
+        errors (`str`, *optional*, defaults to `"replace"`):
+            Paradigm to follow when decoding bytes to UTF-8. See
+            [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+            token instead.
+        bos_token (`str`, *optional*):
+            The beginning of sequence token. Not applicable for this tokenizer.
+        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The end of sequence token.
+        pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The token used for padding, for example when batching sequences of different lengths.
+        clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
+            Whether or not the model should cleanup the spaces that were added when splitting the input text during the
+            tokenization process. Not applicable to this tokenizer, since tokenization does not add spaces.
+        split_special_tokens (`bool`, *optional*, defaults to `False`):
+            Whether or not the special tokens should be split during the tokenization process. The default behavior is
+            to not split special tokens. This means that if `<|endoftext|>` is the `eos_token`, then `tokenizer.tokenize("<|endoftext|>") =
+            ['<|endoftext|>`]. Otherwise, if `split_special_tokens=True`, then `tokenizer.tokenize("<|endoftext|>")` will be give `['<',
+            '|', 'endo', 'ft', 'ext', '|', '>']`. This argument is only supported for `slow` tokenizers for the moment.
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(
+        self,
+        vocab_file,
+        merges_file,
+        errors="replace",
+        unk_token="<|endoftext|>",
+        bos_token=None,
+        eos_token="<|endoftext|>",
+        pad_token="<|endoftext|>",
+        clean_up_tokenization_spaces=False,
+        split_special_tokens=False,
+        **kwargs,
+    ):
+        # Qwen vocab does not contain control tokens; added tokens need to be special
+        bos_token = (
+            AddedToken(bos_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(bos_token, str)
+            else bos_token
+        )
+        eos_token = (
+            AddedToken(eos_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(eos_token, str)
+            else eos_token
+        )
+        unk_token = (
+            AddedToken(unk_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(unk_token, str)
+            else unk_token
+        )
+        pad_token = (
+            AddedToken(pad_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(pad_token, str)
+            else pad_token
+        )
+
+        with open(vocab_file, encoding="utf-8") as vocab_handle:
+            self.encoder = json.load(vocab_handle)
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.errors = errors  # how to handle errors in decoding
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        bpe_merges = []
+        with open(merges_file, encoding="utf-8") as merges_handle:
+            for i, line in enumerate(merges_handle):
+                line = line.strip()
+                if (i == 0 and line.startswith("#version:")) or not line:
+                    continue
+                bpe_merges.append(tuple(line.split()))
+        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
+        # NOTE: the cache can grow without bound and will get really large for long running processes
+        # (esp. for texts of language that do not use space between word, e.g. Chinese); technically
+        # not a memory leak but appears as one.
+        # GPT2Tokenizer has the same problem, so let's be consistent.
+        self.cache = {}
+
+        self.pat = re.compile(PRETOKENIZE_REGEX)
+
+        if kwargs.get("add_prefix_space", False):
+            logger.warning_once(
+                f"{self.__class__.__name} does not support `add_prefix_space`, setting it to True has no effect."
+            )
+
+        super().__init__(
+            errors=errors,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,
+            unk_token=unk_token,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            split_special_tokens=split_special_tokens,
+            **kwargs,
+        )
+
+    @property
+    def vocab_size(self) -> int:
+        return len(self.encoder)
+
+    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.get_vocab
+    def get_vocab(self):
+        return dict(self.encoder, **self.added_tokens_encoder)
+
+    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.bpe
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token)
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token
+
+        while True:
+            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                except ValueError:
+                    new_word.extend(word[i:])
+                    break
+                else:
+                    new_word.extend(word[i:j])
+                    i = j
+
+                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
+                    new_word.append(first + second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = " ".join(word)
+        self.cache[token] = word
+        return word
+
+    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._tokenize
+    def _tokenize(self, text):
+        """Tokenize a string."""
+        bpe_tokens = []
+        for token in re.findall(self.pat, text):
+            token = "".join(
+                self.byte_encoder[b] for b in token.encode("utf-8")
+            )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
+            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" "))
+        return bpe_tokens
+
+    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._convert_token_to_id
+    def _convert_token_to_id(self, token):
+        """Converts a token (str) in an id using the vocab."""
+        return self.encoder.get(token, self.encoder.get(self.unk_token))
+
+    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._convert_id_to_token
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        return self.decoder.get(index)
+
+    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.convert_tokens_to_string
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (string) in a single string."""
+        text = "".join(tokens)
+        text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)
+        return text
+
+    def decode(
+        self,
+        token_ids,
+        skip_special_tokens: bool = False,
+        clean_up_tokenization_spaces: Optional[bool] = False,
+        spaces_between_special_tokens: bool = False,
+        **kwargs,
+    ) -> str:
+        # `spaces_between_special_tokens` defaults to True for _decode in slow tokenizers
+        # and cannot be configured elsewhere, but it should default to False for Qwen2Tokenizer
+        return super().decode(
+            token_ids,
+            skip_special_tokens=skip_special_tokens,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            spaces_between_special_tokens=spaces_between_special_tokens,
+            **kwargs,
+        )
+
+    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.save_vocabulary
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        if not os.path.isdir(save_directory):
+            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
+            return
+        vocab_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
+        )
+        merge_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
+        )
+
+        with open(vocab_file, "w", encoding="utf-8") as f:
+            f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")
+
+        index = 0
+        with open(merge_file, "w", encoding="utf-8") as writer:
+            writer.write("#version: 0.2\n")
+            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
+                if index != token_index:
+                    logger.warning(
+                        f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
+                        " Please check that the tokenizer is not corrupted!"
+                    )
+                    index = token_index
+                writer.write(" ".join(bpe_tokens) + "\n")
+                index += 1
+
+        return vocab_file, merge_file
+
+    def prepare_for_tokenization(self, text, **kwargs):
+        text = unicodedata.normalize("NFC", text)
+        return (text, kwargs)
+
+
+__all__ = ["Qwen2Tokenizer"]

processor/tokenization_qwen2_fast.py

@@ -0,0 +1,137 @@
+# coding=utf-8
+# Copyright 2024 The Qwen team, Alibaba Group and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tokenization classes for Qwen2."""
+
+from typing import Optional, Tuple
+
+from transformers.tokenization_utils import AddedToken
+from transformers.tokenization_utils_fast import PreTrainedTokenizerFast
+from transformers.utils import logging
+from .tokenization_qwen2 import Qwen2Tokenizer
+
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {
+    "vocab_file": "vocab.json",
+    "merges_file": "merges.txt",
+    "tokenizer_file": "tokenizer.json",
+}
+
+
+MAX_MODEL_INPUT_SIZES = {"qwen/qwen-tokenizer": 32768}
+
+
+class Qwen2TokenizerFast(PreTrainedTokenizerFast):
+    """
+    Construct a "fast" Qwen2 tokenizer (backed by HuggingFace's *tokenizers* library). Based on byte-level
+    Byte-Pair-Encoding.
+
+    Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
+    be encoded differently whether it is at the beginning of the sentence (without space) or not:
+
+    ```python
+    >>> from transformers import Qwen2TokenizerFast
+
+    >>> tokenizer = Qwen2TokenizerFast.from_pretrained("Qwen/Qwen-tokenizer")
+    >>> tokenizer("Hello world")["input_ids"]
+    [9707, 1879]
+
+    >>> tokenizer(" Hello world")["input_ids"]
+    [21927, 1879]
+    ```
+    This is expected.
+
+    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+    refer to this superclass for more information regarding those methods.
+
+    Args:
+        vocab_file (`str`, *optional*):
+            Path to the vocabulary file.
+        merges_file (`str`, *optional*):
+            Path to the merges file.
+        tokenizer_file (`str`, *optional*):
+            Path to [tokenizers](https://github.com/huggingface/tokenizers) file (generally has a .json extension) that
+            contains everything needed to load the tokenizer.
+        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+            token instead. Not applicable to this tokenizer.
+        bos_token (`str`, *optional*):
+            The beginning of sequence token. Not applicable for this tokenizer.
+        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The end of sequence token.
+        pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The token used for padding, for example when batching sequences of different lengths.
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    model_input_names = ["input_ids", "attention_mask"]
+    slow_tokenizer_class = Qwen2Tokenizer
+
+    def __init__(
+        self,
+        vocab_file=None,
+        merges_file=None,
+        tokenizer_file=None,
+        unk_token="<|endoftext|>",
+        bos_token=None,
+        eos_token="<|endoftext|>",
+        pad_token="<|endoftext|>",
+        **kwargs,
+    ):
+        # We need to at least pass vocab_file and merges_file to base class
+        # in case a slow tokenizer needs to be initialized; other can be
+        # configured through files.
+        # following GPT2TokenizerFast, also adding unk_token, bos_token, and eos_token
+
+        bos_token = (
+            AddedToken(bos_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(bos_token, str)
+            else bos_token
+        )
+        eos_token = (
+            AddedToken(eos_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(eos_token, str)
+            else eos_token
+        )
+        unk_token = (
+            AddedToken(unk_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(unk_token, str)
+            else unk_token
+        )
+        pad_token = (
+            AddedToken(pad_token, lstrip=False, rstrip=False, special=True, normalized=False)
+            if isinstance(pad_token, str)
+            else pad_token
+        )
+
+        super().__init__(
+            vocab_file=vocab_file,
+            merges_file=merges_file,
+            tokenizer_file=tokenizer_file,
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,
+            **kwargs,
+        )
+
+    # Copied from transformers.models.gpt2.tokenization_gpt2_fast.GPT2TokenizerFast.save_vocabulary
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        files = self._tokenizer.model.save(save_directory, name=filename_prefix)
+        return tuple(files)
+
+
+__all__ = ["Qwen2TokenizerFast"]

processor/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:123025f91ad2f1b25413e751928c7240888524666fc6bc10dd29c6f57efb6808
+size 11426018

processor/tokenizer_config.json

@@ -0,0 +1,399 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "add_special_table_tokens": true,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151669": {
+      "content": "<|MASK|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<|ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "<|ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<|md_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "<|md_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "<ched>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "<ecel>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151676": {
+      "content": "<fcel>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151677": {
+      "content": "<lcel>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151678": {
+      "content": "<ucel>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151679": {
+      "content": "<xcel>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151680": {
+      "content": "<nl>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151681": {
+      "content": "<|rotate_up|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151682": {
+      "content": "<|rotate_down|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151683": {
+      "content": "<|rotate_left|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151684": {
+      "content": "<|rotate_right|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151685": {
+      "content": "<|txt_contd|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151686": {
+      "content": "<|paratext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|ref_start|>",
+    "<|ref_end|>",
+    "<|md_start|>",
+    "<|md_end|>",
+    "<ched>",
+    "<ecel>",
+    "<fcel>",
+    "<lcel>",
+    "<ucel>",
+    "<xcel>",
+    "<nl>",
+    "<|rotate_up|>",
+    "<|rotate_down|>",
+    "<|rotate_left|>",
+    "<|rotate_right|>",
+    "<|txt_contd|>",
+    "<|paratext|>",
+    "<|image_pad|>",
+    "<|vision_start|>",
+    "<|vision_end|>"
+  ],
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_qwen2.Qwen2Tokenizer",
+      "tokenization_qwen2_fast.Qwen2TokenizerFast"
+    ]
+  },
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "mask_token": "<|MASK|>",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "processor_class": "SmallVLMProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

training_config.yaml

@@ -0,0 +1,99 @@
+DATA_CONFIG:
+  max_length: 16384
+MODEL_CONFIG:
+  architecture: dmllm.modeling_dmllm.DMLLM
+  language_model:
+    architecture: dmllm.modeling_sdar.SDARForCausalLM
+    attn_implementation: flash_attention_2
+    freeze: 0
+    name_or_path: ./hf_models/JetLM/SDAR-1.7B-Chat-b32/
+  pretrained_path:
+  - ./work_dirs/native_sdar_no_merger_pm2x/all_s1_9e/best/model-00001-of-00002.safetensors
+  - ./work_dirs/native_sdar_no_merger_pm2x/all_s1_9e/best/model-00002-of-00002.safetensors
+  rm_vit_merger: true
+  torch_dtype: bfloat16
+  vision_abstractor:
+    freeze: 0
+    projection_type: patch_merger2x
+  vision_model:
+    attn_implementation: flash_attention_2
+    freeze: 0
+    name_or_path: ./hf_models/shilinxu/Qwen2-VL-7B-ViT/
+  vision_model_type: qwen2vit
+  vision_output_key: null
+PROCESSOR_CONFIG:
+  chat_template: '{% for message in messages %}{% if loop.first and message[''role'']
+    != ''system'' %}<|im_start|>systemYou are a helpful assistant.<|im_end|>{% endif
+    %}<|im_start|>{{ message[''role''] }}{% if message[''role''] == ''assistant''
+    %}{% generation %}{{ message[''content''][0][''text''] }}<|im_end|>{% endgeneration
+    %}{% else %}{% for content in message[''content''] %}{% if content[''type''] ==
+    ''image'' or ''image'' in content or ''image_url'' in content %}<|vision_start|><|image_pad|><|vision_end|>{%
+    elif content[''type''] == ''video'' or ''video'' in content %}<|vision_start|><|video_pad|><|vision_end|>{%
+    elif ''text'' in content %}{{ content[''text''] }}{% endif %}{% endfor %}<|im_end|>{%
+    endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant{% endif
+    %}'
+  image_processor_config:
+    max_pixels: 1605632
+    min_pixels: 3136
+    name_or_path: ./hf_models/shilinxu/Qwen2-VL-7B-ViT/
+  image_token: <|image_pad|>
+  processor_class: navit_qwen2.processing_smallvlm.SmallVLMProcessor
+  special_tokens:
+  - <|image_pad|>
+  - <|vision_start|>
+  - <|vision_end|>
+  tokenizer_config:
+    add_special_table_tokens: true
+    name_or_path: ./hf_models/JetLM/SDAR-1.7B-Chat-b32/
+  vision_token_share_pe: false
+TRAINING_CONFIG:
+  bf16: true
+  block_generation: true
+  block_size: 32
+  custom_lr_scheduler: start,min=2e-7,2e-6
+  dataloader_drop_last: true
+  dataloader_num_workers: 8
+  dataloader_pin_memory: false
+  dataloader_prefetch_factor: 64
+  ddp_backend: null
+  deepspeed:
+    bf16:
+      enabled: auto
+    fp16:
+      enabled: auto
+      hysteresis: 2
+      initial_scale_power: 16
+      loss_scale: 0
+      loss_scale_window: 1000
+      min_loss_scale: 1
+    gradient_accumulation_steps: auto
+    gradient_clipping: auto
+    train_batch_size: auto
+    train_micro_batch_size_per_gpu: auto
+    zero_optimization:
+      allgather_bucket_size: 209715200
+      contiguous_gradients: true
+      overlap_comm: true
+      reduce_bucket_size: 209715200
+      stage: 2
+  gradient_accumulation_steps: 8
+  gradient_checkpointing: true
+  learning_rate: 2.0e-05
+  logging_steps: 1
+  lr_scheduler_type: cosine
+  max_grad_norm: 1.0
+  num_train_epochs: 9
+  output_dir: work_dirs/native_sdar_no_merger_pm2x/all_s2_9e_nolayout_custom_lr_2e-7,2e-5,2e-6
+  per_device_train_batch_size: 1
+  report_to: tensorboard
+  save_only_model: false
+  save_safetensors: true
+  save_steps: 1000
+  save_strategy: steps
+  save_total_limit: 1
+  tf32: true
+  torch_empty_cache_steps: 627
+  use_online_length_grouped_dataloader: true
+  warmup_ratio: 0.1
+  weight_decay: 0.0
+_PARAMETERS_: true