Upload TextTransformer

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "WAC_Masked_language_modeling/output_models/pytorch_model.bin",
+  "_name_or_path": "WAC_Masked_language_modeling_test/output_models/pytorch_model.bin",
   "add_cross_attentions": false,
   "architectures": [
     "TextTransformer"
@@ -8,16 +8,16 @@
   "attention_probability_dropout_probability": 0.1,
   "bos_token_id": 0,
   "dropout_probability": 0.1,
-  "embedding_size": 512,
+  "embedding_size": 128,
   "eos_token_id": 2,
   "hidden_activation_function": "gelu",
-  "hidden_size": 768,
+  "hidden_size": 256,
   "initializer_factor": 1.0,
-  "intermediate_size": 3072,
-  "layer_norm_eps": 1e-05,
+  "intermediate_size": 1024,
+  "layer_norm_eps": 1e-12,
   "mask_token_id": 4,
   "max_position_embeddings": 514,
-  "num_attention_heads": 12,
+  "num_attention_heads": 4,
   "num_hidden_layers": 12,
   "num_intermediate_layers": 1,
   "pad_token_id": 1,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:60e35fe13febd95b8a24c0896e1711aeacf1c7531f0b58dda35d91f800decc1a
-size 901321512
+oid sha256:987b0d17d593b27431b81a5a41d7288ca31df63e2977ceff09f1f26f5c2e23ba
+size 114715936