Update configuration_mybert.py

configuration_mybert.py

@@ -6,11 +6,11 @@ class MyBertConfig(PretrainedConfig):
 
     def __init__(
         self,
-        vocab_size=28996,                 # google-bert/bert-base-cased
+        vocab_size=28996,                 
         hidden_size=768,
         num_hidden_layers=12,
         num_attention_heads=12,
-        intermediate_size=2048,           # SwiGLU: 2/3*3072 -> param-neutral vs GELU FFN
+        intermediate_size=2048,           
         max_position_embeddings=2048,
         hidden_dropout_prob=0.0,
         attention_probs_dropout_prob=0.0,
@@ -18,7 +18,7 @@ class MyBertConfig(PretrainedConfig):
         initializer_range=0.02,
         rope_theta=10000.0,
         use_bias=False,
-        sparse_prediction=True,           # MLM head only on masked positions
+        sparse_prediction=True,           
         pad_token_id=0,
         tie_word_embeddings=True,
         **kwargs,
@@ -38,4 +38,4 @@ class MyBertConfig(PretrainedConfig):
         self.initializer_range = initializer_range
         self.rope_theta = rope_theta
         self.use_bias = use_bias
-        self.sparse_prediction = sparse_prediction
+        self.sparse_prediction = sparse_prediction