Add RoFormer-slav

README.md

@@ -71,3 +71,4 @@ model = AutoModelForMaskedLM.from_pretrained(
 ## Contact
 
 Maxim Eremeev, maeremeev@edu.hse.ru
+

config.json

@@ -22,7 +22,7 @@
   "use_cache": true,
   "vocab_size": 50000,
   "auto_map": {
-    "AutoConfig": "configuration_roformer.RoFormerConfig",
-    "AutoModelForMaskedLM": "modeling_roformer.RoFormerForMaskedLM"
+    "AutoConfig": "config.RoFormerConfig",
+    "AutoModelForMaskedLM": "model.RoFormerForMaskedLM"
   }
 }

config.py

@@ -0,0 +1,18 @@
+from transformers import RoFormerConfig
+
+
+def get_roformer_config(vocab_size: int, pad_token_id: int):
+    """
+    Returns the RoFormer configuration with parameters optimized for Ancient Russian text.
+    """
+    return RoFormerConfig(
+        vocab_size=vocab_size,
+        embedding_size=512,
+        hidden_size=512,
+        num_hidden_layers=6,
+        num_attention_heads=8,
+        intermediate_size=2048,
+        max_position_embeddings=514,
+        pad_token_id=pad_token_id,
+        rotary_value=False,
+    )

model.py

@@ -0,0 +1,11 @@
+from config import get_roformer_config
+from transformers import RoFormerForMaskedLM
+
+
+def get_model(vocab_size: int, pad_token_id: int):
+    """
+    Initializes and returns a RoFormer model for Masked Language Modeling.
+    """
+    config = get_roformer_config(vocab_size, pad_token_id)
+    model = RoFormerForMaskedLM(config)
+    return model