Upload 2 files

upload 1b version

config.json

@@ -1,10 +1,10 @@
-{
-    "vocab_size":100000,
-    "n_dim": 1024,
-    "n_head": 16,
-    "n_kvhead": 4,
-    "d_ffn": 4096,
-    "m_len": 1024,
-    "n_layer": 20,
-    "norm_eps": 1e-5
-}
+{
+    "vocab_size": 100000,
+    "n_dim": 1536,
+    "n_head": 24,
+    "n_kvhead": 4,
+    "d_ffn": 6912,
+    "m_len": 2048,
+    "n_layer": 24,
+    "norm_eps": 1e-05
+}