Use pretrained first 2 text layers and 2 vision blocks

config.json

@@ -4,9 +4,9 @@
     "HyperCLOVAXVisionV2ForConditionalGeneration"
   ],
   "audio_projector_type": "mlp",
-  "dtype": "float32",
+  "dtype": "bfloat16",
   "eos_token_id": 128001,
-  "hidden_size": 32,
+  "hidden_size": 5120,
   "ignore_index": -100,
   "image_end_token_id": 128057,
   "image_start_token_id": 128056,
@@ -24,22 +24,23 @@
     ],
     "attention_bias": false,
     "attention_dropout": 0.0,
-    "attention_multiplier": 0.25,
+    "attention_multiplier": 0.08838834764831845,
     "bos_token_id": 128000,
+    "dtype": "bfloat16",
     "embedding_multiplier": 1.0,
     "eos_token_id": 128001,
-    "head_dim": 16,
+    "head_dim": 128,
     "hidden_act": "silu",
-    "hidden_size": 32,
+    "hidden_size": 5120,
     "initializer_range": 0.006,
-    "intermediate_size": 64,
+    "intermediate_size": 24192,
     "logits_scaling": 1.0,
     "max_position_embeddings": 131072,
     "mlp_bias": false,
     "model_type": "hyperclovax",
-    "num_attention_heads": 2,
+    "num_attention_heads": 40,
     "num_hidden_layers": 2,
-    "num_key_value_heads": 1,
+    "num_key_value_heads": 8,
     "pad_token_id": 0,
     "pretraining_tp": 1,
     "resid_pdrop": 0.2,
@@ -71,19 +72,18 @@
       "Qwen2_5_VisionTransformerPretrainedModel"
     ],
     "depth": 2,
-    "fullatt_block_indexes": [
-      1
-    ],
+    "dtype": "bfloat16",
+    "fullatt_block_indexes": [],
     "hidden_act": "silu",
-    "hidden_size": 32,
+    "hidden_size": 1280,
     "in_channels": 3,
     "in_chans": 3,
     "initializer_range": 0.02,
-    "intermediate_size": 64,
+    "intermediate_size": 3456,
     "max_num_grids": -1,
     "model_type": "qwen2_5_vl_vision",
-    "num_heads": 2,
-    "out_hidden_size": 32,
+    "num_heads": 16,
+    "out_hidden_size": 5120,
     "patch_size": 14,
     "rope_parameters": {
       "rope_theta": 10000.0,

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e40eedd111044e0a4e7596cf6a2b6badef53d118082405163ed611c964493f6
+size 1313341592

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62b7b6cb3bb010c493216decf7b9f014cb1f3a8c6c8eccb7baf17caef85ab147
+size 1808804408

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f01efde7f673c7255a306d8130dd74faebc64fea1677c7d723333c9e5ee397f5
+size 1482301776

model.safetensors.index.json

@@ -0,0 +1,61 @@
+{
+  "metadata": {
+    "total_parameters": 2302220544,
+    "total_size": 4604441088
+  },
+  "weight_map": {
+    "model.language_model.lm_head.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.embed_tokens.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.0.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.0.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.0.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.0.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.0.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.0.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.0.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.0.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.0.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.1.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.1.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.1.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.1.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.1.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.1.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.1.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.1.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.1.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.norm.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.0.attn.proj.bias": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.0.attn.proj.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.0.attn.qkv.bias": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.0.attn.qkv.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.0.mlp.down_proj.bias": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.0.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.0.mlp.gate_proj.bias": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.0.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.0.mlp.up_proj.bias": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.0.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.0.norm1.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.0.norm2.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.1.attn.proj.bias": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.1.attn.proj.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.1.attn.qkv.bias": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.1.attn.qkv.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.1.mlp.down_proj.bias": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.1.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.1.mlp.gate_proj.bias": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.1.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.1.mlp.up_proj.bias": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.1.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.1.norm1.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.blocks.1.norm2.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.merger.ln_q.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.merger.mlp.0.bias": "model-00003-of-00003.safetensors",
+    "model.vision_model.merger.mlp.0.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.merger.mlp.2.bias": "model-00003-of-00003.safetensors",
+    "model.vision_model.merger.mlp.2.weight": "model-00003-of-00003.safetensors",
+    "model.vision_model.patch_embed.proj.weight": "model-00003-of-00003.safetensors",
+    "model.vision_projector.bias": "model-00003-of-00003.safetensors",
+    "model.vision_projector.weight": "model-00003-of-00003.safetensors"
+  }
+}