togethercomputer
/

m2-bert-80M-32k

Model card Files Files and versions

danfu09 commited on Jan 9, 2024

Commit

b6752c3

·

1 Parent(s): 0a2b9f1

Update config.json

Files changed (1) hide show

config.json +1 -1

config.json CHANGED Viewed

@@ -30,7 +30,7 @@
     "long_conv_kernel_learning_rate": 0.001,
     "long_conv_l_max": 32768,
     "max_position_embeddings": 32768,
-    "model_type": "bert",
     "monarch_mlp_nblocks": 4,
     "num_attention_heads": 12,
     "num_hidden_layers": 12,

     "long_conv_kernel_learning_rate": 0.001,
     "long_conv_l_max": 32768,
     "max_position_embeddings": 32768,
+    "model_type": "m2_bert",
     "monarch_mlp_nblocks": 4,
     "num_attention_heads": 12,
     "num_hidden_layers": 12,