onboard pruned state dict, remove tokenizer, remove inference api, update readme

README.md

@@ -3,6 +3,7 @@ license: apache-2.0
 language:
 - en
 pipeline_tag: fill-mask
+inference: false
 ---
 
 # Monarch Mixer-BERT
@@ -14,17 +15,27 @@ Check out our [GitHub](https://github.com/HazyResearch/m2/tree/main) for instruc
 
 ## How to use
 
-Using AutoModel:
+You can load this model using Hugging Face `AutoModel`:
 ```python
 from transformers import AutoModelForMaskedLM
 mlm = AutoModelForMaskedLM.from_pretrained('alycialee/m2-bert-260m', trust_remote_code=True)
 ```
 
+This model uses the Hugging Face `bert-base-uncased tokenizer`:
+```
+from transformers import BertTokenizer
+tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+```
+
 You can use this model with a pipeline for masked language modeling:
 ```python
-from transformers import pipeline
-unmasker = pipeline('fill-mask', model='alycialee/m2-bert-260m', trust_remote_code=True)
-unmasker("Every morning, I enjoy a cup of [MASK] to start my day.")
+from transformers import AutoModelForMaskedLM, BertTokenizer, pipeline
+
+tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+mlm = AutoModelForMaskedLM.from_pretrained('alycialee/m2-bert-260m', trust_remote_code=True)
+
+unmasker = pipeline('fill-mask', model=mlm, tokenizer=tokenizer)
+unmasker('Every morning, I enjoy a cup of [MASK] to start my day.')
 ```
 
 ### Remote Code

bert_layers.py

@@ -245,6 +245,7 @@ class BertLayer(nn.Module):
             hyena_filter_dropout=config.hyena_filter_dropout,
             hyena_filter_order=config.hyena_filter_order,
             residual_long_conv=config.residual_long_conv,
+            hyena_training_additions=config.hyena_training_additions,
         )
             
         if config.use_glu_mlp:
@@ -887,4 +888,3 @@ class BertForSequenceClassification(BertPreTrainedModel):
             hidden_states=None,
             attentions=None,
         )
-

config.json

@@ -5,6 +5,7 @@
         "BertForMaskedLM"
     ],
     "attention_probs_dropout_prob": 0.0,
+    "bidirectional": true,
     "auto_map": {
         "AutoConfig": "configuration_bert.BertConfig",
         "AutoModelForMaskedLM": "bert_layers.BertForMaskedLM"
@@ -27,7 +28,6 @@
     "transformers_version": "4.28.1",
     "type_vocab_size": 2,
     "use_cache": true,
-    "vocab_size": 30522,
     "long_conv_l_max": 128,
     "long_conv_kernel_learning_rate": 1e-3,
     "hyena_lr_pos_emb": 1e-5,
@@ -35,10 +35,10 @@
     "hyena_wd": 0.1,
     "hyena_emb_dim": 5,
     "hyena_filter_order": 128,
-    "bidirectional": true, 
     "residual_long_conv": true, 
     "use_glu_mlp": true, 
     "use_monarch_mlp": true, 
     "monarch_mlp_nblocks": 4, 
-    "use_positional_encodings" : true
+    "use_positional_encodings" : true,
+    "vocab_size": 30528
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9c7f227d6e7b073df86573a94cef19a6ae1a2bc27269c8a9445db67a5c061cf
-size 956301312
+oid sha256:0155ae9c0b8923f8ea79e768fb70bb1af75af0cd4adb0b166ea288e0d8732117
+size 1036917225

special_tokens_map.json

@@ -1,7 +0,0 @@
-{
-  "cls_token": "[CLS]",
-  "mask_token": "[MASK]",
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "unk_token": "[UNK]"
-}

tokenizer_config.json

@@ -1,13 +0,0 @@
-{
-  "clean_up_tokenization_spaces": true,
-  "cls_token": "[CLS]",
-  "do_lower_case": true,
-  "mask_token": "[MASK]",
-  "model_max_length": 512,
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "strip_accents": null,
-  "tokenize_chinese_chars": true,
-  "tokenizer_class": "BertTokenizer",
-  "unk_token": "[UNK]"
-}