kk

modelLM.py

@@ -11,7 +11,7 @@ class OBILanguageModel(PreTrainedModel):
     def __init__(self, config):
         super(OBILanguageModel,self).__init__(config)
         self.token_embedding_table = nn.Embedding(config.vocab_size, config.hidden_size)  # Use length of SentencePiece vocab
-        self.position_embedding_table = nn.Embedding(384, config.hidden_size)
+        self.position_embedding_table = nn.Embedding(config.block_size, config.hidden_size)
   
 
         self.transformer = nn.Transformer(