Upload CubeLM

CubeConfig.py

@@ -1,8 +1,4 @@
-#from transformers import PretrainedConfig
 from transformers import GPT2Config
-from cubeLM.tokenizer import vocab
-
-vocab_size = len(vocab)
 
 
 class CubeConfig(GPT2Config):
@@ -10,10 +6,10 @@ class CubeConfig(GPT2Config):
 
     def __init__(
         self,
-        vocab_size=vocab_size,
-        bos_token_id=vocab_size - 1,
-        eos_token_id=vocab_size - 1,
-        pad_token_id=vocab_size - 1,
+        vocab_size=16,
+        bos_token_id=15,
+        eos_token_id=15,
+        pad_token_id=15,
         n_positions=40,
         n_embd=512,
         n_layer=8,
@@ -29,5 +25,3 @@ class CubeConfig(GPT2Config):
         self.bos_token_id = bos_token_id
         self.eos_token_id = eos_token_id
         self.pad_token_id = pad_token_id
-
-

CubeLM.py

@@ -7,10 +7,11 @@ from transformers import (
     GPT2Model,
     GenerationMixin,
     GPT2PreTrainedModel,
-    PreTrainedModel
+    PreTrainedModel,
 )
 from transformers.utils import ModelOutput
-from train_scripts.utils import IGNORE_INDEX
+
+IGNORE_INDEX = -100
 
 
 @dataclass