Upload config.json with huggingface_hub

config.json

@@ -1,14 +1,14 @@
 {
-  "vocab_size": 8192,
-  "dim": 256,
-  "n_layers": 6,
+  "vocab_size": 16384,
+  "dim": 384,
+  "n_layers": 8,
   "n_heads": 8,
-  "max_seq_len": 384,
-  "channel_top_k": 96,
-  "token_top_k": 64,
+  "max_seq_len": 512,
+  "channel_top_k": 128,
+  "token_top_k": 96,
   "ffn_mult": 4,
   "mem_dim": 128,
-  "mem_size": 15000,
+  "mem_size": 20000,
   "mem_k": 8,
   "mem_threshold": 0.4,
   "memory_layers": [
@@ -28,16 +28,17 @@
   "modulation_strength": 0.1,
   "excitation_strength": 0.3,
   "dropout": 0.1,
-  "batch_size": 32,
+  "batch_size": 24,
   "lr": 0.0003,
-  "max_steps": 6000,
-  "warmup": 500,
+  "max_steps": 8000,
+  "warmup": 800,
   "log_every": 100,
   "eval_every": 500,
-  "patience": 6,
+  "patience": 8,
   "use_diversity": true,
   "pad_id": 0,
   "eos_id": 2,
+  "unk_id": 3,
   "architecture": "SparseMind",
   "model_name": "Cofos Code"
 }