Update config.json

config.json

@@ -1,60 +1,42 @@
 {
-  "model_type": "llama",
-  "architectures": ["LlamaForCausalLM"],
-  "hidden_size": 2048,
-  "intermediate_size": 5632,
-  "max_position_embeddings": 16384,
-  "num_attention_heads": 16,
-  "num_hidden_layers": 24,
-  "num_key_value_heads": 16,
-  "vocab_size": 102400,
+  "architectures": ["MultiModalityCausalLM"],
+  "model_type": "multi_modality",
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.33.1",
-  "aligner_config": {
-    "cls": "MlpProjector",
-    "model_type": "aligner",
-    "params": {
-      "depth": 2,
-      "input_dim": 1024,
-      "n_embed": 2048,
-      "projector_type": "mlp_gelu"
-    }
+  
+  "llama_config": {
+    "architectures": ["LlamaForCausalLM"],
+    "hidden_size": 2048,
+    "intermediate_size": 5632,
+    "max_position_embeddings": 16384,
+    "num_attention_heads": 16,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 16,
+    "rms_norm_eps": 1e-6,
+    "vocab_size": 102400,
+    "torch_dtype": "bfloat16"
   },
-  "gen_aligner_config": {
-    "cls": "MlpProjector",
-    "model_type": "gen_aligner",
-    "params": {
-      "depth": 2,
-      "input_dim": 8,
-      "n_embed": 2048,
-      "projector_type": "mlp_gelu"
-    }
+
+  "clip_vision_config": {
+    "architectures": ["CLIPVisionModel"],
+    "hidden_size": 1024,
+    "image_size": 384,
+    "patch_size": 16,
+    "projection_dim": 2048,
+    "num_hidden_layers": 24,
+    "num_attention_heads": 16,
+    "intermediate_size": 4096
   },
-  "gen_head_config": {
-    "cls": "vision_head",
-    "model_type": "gen_head",
-    "params": {
-      "image_token_embed": 2048,
-      "image_token_size": 16384,
-      "n_embed": 2048
-    }
+
+  "alignment_config": {
+    "projector_type": "mlp_gelu",
+    "depth": 2,
+    "input_dim": 1024,
+    "n_embed": 2048
   },
-  "gen_vision_config": {
-    "cls": "VQ-16",
-    "model_type": "gen_vision",
-    "params": {
-      "image_token_size": 16384,
-      "n_embed": 8
-    }
-  },
-  "vision_config": {
-    "cls": "CLIPVisionTower",
-    "model_type": "vision",
-    "params": {
-      "image_size": 384,
-      "model_name": "siglip_large_patch16_384",
-      "select_feature": "same",
-      "select_layer": -1
-    }
+
+  "generation_config": {
+    "image_token_size": 16384,
+    "vq_codebook_size": 8,
+    "image_token_embed": 2048
   }
 }