Create modeling_arabic-gpt.py

modeling_arabic-gpt.py

@@ -0,0 +1,130 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+import regex as re
+import collections
+import os
+import random
+from tqdm import tqdm
+from transformers import PreTrainedModel
+from transformers import PretrainedConfig
+
+from transformers import PretrainedConfig
+
+class ArabicGPTConfig(PretrainedConfig):
+    model_type = "arabic-gpt"
+
+    def __init__(self,
+                 vocab_size=32000,
+                 max_seq_len=1024,
+                 embed_dim=768,
+                 num_heads=12,
+                 num_layers=12,
+                 ff_dim=3072,
+                 dropout=0.1,
+                 **kwargs):
+        super().__init__(**kwargs)
+        self.vocab_size = vocab_size
+        self.max_seq_len = max_seq_len
+        self.embed_dim = embed_dim
+        self.num_heads = num_heads
+        self.num_layers = num_layers
+        self.ff_dim = ff_dim
+        self.dropout = dropout
+        self.tie_word_embeddings = True
+
+
+import torch
+import torch.nn as nn
+from transformers import PreTrainedModel
+
+class ArabicGPTModel(PreTrainedModel):
+    config_class = ArabicGPTConfig
+
+    def __init__(self, config: ArabicGPTConfig):
+        super().__init__(config)
+        self.model = ArabicGPT(
+            vocab_size=config.vocab_size,
+            max_seq_len=config.max_seq_len,
+            embed_dim=config.embed_dim,
+            num_heads=config.num_heads,
+            num_layers=config.num_layers,
+            ff_dim=config.ff_dim,
+            dropout=config.dropout,
+        )
+
+    def forward(self, x):
+        return self.model(x)
+
+    def generate(self, prompt_ids, max_new_tokens, temperature=1.0, top_k=50, top_p=0.9):
+        return self.model.generate(prompt_ids, max_new_tokens, temperature=1.0, top_k=50, top_p=0.9)
+
+    def get_input_embeddings(self):
+        return self.model.token_embedding
+
+    def set_input_embeddings(self, new_embeddings):
+        self.model.token_embedding = new_embeddings
+    
+    def get_output_embeddings(self):
+        return self.model.lm_head
+    
+    def tie_weights(self):
+        self.model.lm_head.weight = self.model.token_embedding.weight
+
+class ArabicGPTConfig(PretrainedConfig):
+    model_type = "arabic-gpt"
+
+    def __init__(self,
+                 vocab_size=32000,
+                 max_seq_len=1024,
+                 embed_dim=768,
+                 num_heads=12,
+                 num_layers=12,
+                 ff_dim=3072,
+                 dropout=0.1,
+                 **kwargs):
+        super().__init__(**kwargs)
+        self.vocab_size = vocab_size
+        self.max_seq_len = max_seq_len
+        self.embed_dim = embed_dim
+        self.num_heads = num_heads
+        self.num_layers = num_layers
+        self.ff_dim = ff_dim
+        self.dropout = dropout
+        self.tie_word_embeddings = True
+
+
+class ArabicGPTModel(PreTrainedModel):
+    config_class = ArabicGPTConfig
+
+    def __init__(self, config: ArabicGPTConfig):
+        super().__init__(config)
+        self.model = ArabicGPT(
+            vocab_size=config.vocab_size,
+            max_seq_len=config.max_seq_len,
+            embed_dim=config.embed_dim,
+            num_heads=config.num_heads,
+            num_layers=config.num_layers,
+            ff_dim=config.ff_dim,
+            dropout=config.dropout,
+        )
+
+    def forward(self, x):
+        return self.model(x)
+
+    def generate(self, prompt_ids, max_new_tokens, temperature=1.0, top_k=50, top_p=0.9):
+        return self.model.generate(prompt_ids, max_new_tokens, temperature=1.0, top_k=50, top_p=0.9)
+
+    def get_input_embeddings(self):
+        return self.model.token_embedding
+
+    def set_input_embeddings(self, new_embeddings):
+        self.model.token_embedding = new_embeddings
+    
+    def get_output_embeddings(self):
+        return self.model.lm_head
+    
+    def tie_weights(self):
+        self.model.lm_head.weight = self.model.token_embedding.weight
+