Update config.json

config.json

@@ -4,12 +4,35 @@
   ],
   "model_type": "causal_lm",
   "n_ctx": 1024,
-  "n_embd": 768,
-  "n_head": 12,
-  "n_layer": 12,
-  "vocab_size": 50257,
-  "tokenizer_class": "AutoTokenizer",
+  "n_embd": 4096,
+  "n_head": 32,
+  "n_layer": 32,
+  "vocab_size": 32000,
+  "tokenizer_class": "PreTrainedTokenizerFast",
   "special_tokens_map_file": "special_tokens_map.json",
   "tokenizer_config_file": "tokenizer_config.json",
-  "name_or_path": "Ozaii/W.AI-13B-Chat"
+  "name_or_path": "Ozaii/W.AI-13B-Chat",
+  "hidden_size": 4096,
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "intermediate_size": 16384,
+  "initializer_range": 0.02,
+  "layer_norm_eps": 1e-5,
+  "max_position_embeddings": 2048,
+  "use_cache": false,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "pad_token_id": 1,
+  "gradient_checkpointing": true,
+  "task_type": "CAUSAL_LM",
+  "r": 64,
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "bias": "none",
+  "target_modules": [
+    "q_proj",
+    "o_proj",
+    "v_proj",
+    "k_proj"
+  ]
 }