Update modeling.py

modeling.py

@@ -2,6 +2,7 @@ import torch
 import torch.nn as nn
 from transformers import PreTrainedModel, PretrainedConfig
 from transformers.modeling_outputs import CausalLMOutputWithPast
+from configure import RecombinationTransformerConfig
 
 class MaskedSelfAttentionLayer(nn.Module):
     def __init__(self, embed_dim, num_heads):
@@ -133,14 +134,7 @@ class RecombinationTransformerLayer(nn.Module):
         
         return x
 
-class RecombinationTransformerConfig(PretrainedConfig):
-    model_type = "RecombinationTransformer"
-    def __init__(self, embed_dim=1024, num_heads=8, num_layers=6, vocab_size=151643, **kwargs):
-        super().__init__(**kwargs)
-        self.embed_dim = embed_dim
-        self.num_heads = num_heads
-        self.num_layers = num_layers
-        self.vocab_size = vocab_size
+
 
 class RecombinationTransformerForCausalLM(PreTrainedModel):
     config_class = RecombinationTransformerConfig