Config: Explicitly set `ffn_act_fn` as `silu`| Tiktoken: Fix vocab size to include special tokens (#15)

config.json

@@ -20,7 +20,10 @@
     "moe_jitter_eps": 0.01,
     "moe_loss_weight": 0.05,
     "moe_num_experts": 16,
-    "moe_top_k": 4
+    "moe_top_k": 4,
+    "ffn_act_fn": {
+      "name": "silu"
+    }
   },
   "initializer_range": 0.02,
   "max_seq_len": 32768,

tiktoken.py

@@ -247,7 +247,7 @@ class TiktokenTokenizerWrapper(PreTrainedTokenizer):
             # Get an index to add and add the item
             vocab_clone[candidate_extra_id] = index_to_add
 
-        return vocab_clone
+        return dict(vocab_clone, **self.added_tokens_encoder)
 
     def _tokenize(self, text: str) -> List[str]:
         """Returns a tokenized string."""