Upload processor

processing_action_tokenizer.py

@@ -3,10 +3,11 @@ from typing import ClassVar
 
 import numpy as np
 from scipy.fft import dct, idct
-from tokenizers import ByteLevelBPETokenizer
+from tokenizers import ByteLevelBPETokenizer, Tokenizer
 from tokenizers.trainers import BpeTrainer
 from transformers import PreTrainedTokenizerFast
 from transformers.processing_utils import ProcessorMixin
+from tokenizers.models import BPE
 
 
 class UniversalActionProcessor(ProcessorMixin):
@@ -132,6 +133,7 @@ class UniversalActionProcessor(ProcessorMixin):
 
         # Train BPE tokenizer
         bpe = ByteLevelBPETokenizer()
+        # tokenizer = Tokenizer(BPE())
 
         # Set up the entire range of possible tokens as the initial alphabet
         alphabet = [chr(i) for i in range(max_token - min_token + 1)]
@@ -140,14 +142,16 @@ class UniversalActionProcessor(ProcessorMixin):
             min_frequency=8,
             show_progress=True,
             special_tokens=[],
-            initial_alphabet=alphabet,
+            # initial_alphabet=alphabet,
             # max_token_length=10000,
             max_token_length=100,
         )
 
         # Train the inner tokenizer (don't use ByteLevelBPETokenizer.train_from_iterator()
         # because it doesn't support custom alphabets)
-        bpe._tokenizer.train_from_iterator(_token_iter(), trainer=trainer)
+        # bpe._tokenizer.train_from_iterator(_token_iter(), trainer=trainer)
+        # trainer.train_from_iterator(_token_iter(), trainer=trainer)
+        bpe.train_from_iterator(_token_iter(), vocab_size=vocab_size,)
 
         return cls(
             PreTrainedTokenizerFast(tokenizer_object=bpe, clean_up_tokenization_spaces=False),