fix processor

processing_action_tokenizer.py

@@ -3,11 +3,9 @@ from typing import ClassVar
 
 import numpy as np
 from scipy.fft import dct, idct
-from tokenizers import ByteLevelBPETokenizer, Tokenizer
-from tokenizers.trainers import BpeTrainer
+from tokenizers import ByteLevelBPETokenizer
 from transformers import PreTrainedTokenizerFast
 from transformers.processing_utils import ProcessorMixin
-from tokenizers.models import BPE
 
 
 class UniversalActionProcessor(ProcessorMixin):
@@ -133,25 +131,29 @@ class UniversalActionProcessor(ProcessorMixin):
 
         # Train BPE tokenizer
         bpe = ByteLevelBPETokenizer()
-        # tokenizer = Tokenizer(BPE())
 
         # Set up the entire range of possible tokens as the initial alphabet
-        alphabet = [chr(i) for i in range(max_token - min_token + 1)]
-        trainer = BpeTrainer(
-            vocab_size=vocab_size,
-            min_frequency=8,
-            show_progress=True,
-            special_tokens=[],
-            # initial_alphabet=alphabet,
-            # max_token_length=10000,
-            max_token_length=100,
-        )
+        # alphabet = [chr(i) for i in range(max_token - min_token + 1)]
+        # trainer = BpeTrainer(
+        #     vocab_size=vocab_size,
+        #     min_frequency=2,
+        #     show_progress=True,
+        #     special_tokens=[],
+        #     initial_alphabet=alphabet,
+        #     max_token_length=10000,
+        # )
 
         # Train the inner tokenizer (don't use ByteLevelBPETokenizer.train_from_iterator()
         # because it doesn't support custom alphabets)
         # bpe._tokenizer.train_from_iterator(_token_iter(), trainer=trainer)
         # trainer.train_from_iterator(_token_iter(), trainer=trainer)
-        bpe.train_from_iterator(_token_iter(), vocab_size=vocab_size,)
+        bpe.train_from_iterator(
+            _token_iter(), 
+            vocab_size=vocab_size, 
+            min_frequency=2,
+            special_tokens=[],
+            length=len(dct_tokens)
+        )
 
         return cls(
             PreTrainedTokenizerFast(tokenizer_object=bpe, clean_up_tokenization_spaces=False),