ds

tokenizeConfig.py

@@ -121,6 +121,10 @@ class OBITokenizer(PreTrainedTokenizer):
         )
 
         # Save the BPE vocab
-        self.tokenizer.get_vocab().to_file(out_vocab_file)
+          # Training: Fit the tokenizer on your text data
+        trainer = trainers.BpeTrainer(special_tokens=["<unk>", "<s>", "</s>","[PAD]"])
+        self.tokenizer.train(trainer=trainer, files=[out_vocab_file])
+        # Save the trained tokenizer to a file
+        self.tokenizer.save(out_vocab_file)
 
         return (out_vocab_file,)