Upload processor

processing_action_tokenizer.py

@@ -4,8 +4,9 @@ from typing import ClassVar
 import numpy as np
 from scipy.fft import dct, idct
 from tokenizers import ByteLevelBPETokenizer
-from transformers import PreTrainedTokenizerFast
+from tokenizers.trainers import BpeTrainer
 from transformers.processing_utils import ProcessorMixin
+from transformers import PreTrainedTokenizerFast
 
 
 class UniversalActionProcessor(ProcessorMixin):
@@ -134,26 +135,27 @@ class UniversalActionProcessor(ProcessorMixin):
 
         # Set up the entire range of possible tokens as the initial alphabet
         # alphabet = [chr(i) for i in range(max_token - min_token + 1)]
-        # trainer = BpeTrainer(
-        #     vocab_size=vocab_size,
-        #     min_frequency=2,
-        #     show_progress=True,
-        #     special_tokens=[],
-        #     initial_alphabet=alphabet,
-        #     max_token_length=10000,
-        # )
+        trainer = BpeTrainer(
+            vocab_size=vocab_size,
+            min_frequency=2,
+            show_progress=True,
+            special_tokens=[],
+            # initial_alphabet=alphabet,
+            max_token_length=512,
+        )
 
         # Train the inner tokenizer (don't use ByteLevelBPETokenizer.train_from_iterator()
         # because it doesn't support custom alphabets)
-        # bpe._tokenizer.train_from_iterator(_token_iter(), trainer=trainer)
+        bpe._tokenizer.train_from_iterator(_token_iter(), trainer=trainer)
         # trainer.train_from_iterator(_token_iter(), trainer=trainer)
-        bpe.train_from_iterator(
-            _token_iter(), 
-            vocab_size=vocab_size, 
-            min_frequency=2,
-            special_tokens=[],
-            length=len(dct_tokens)
-        )
+        # bpe.train_from_iterator(
+        #     _token_iter(), 
+        #     vocab_size=vocab_size, 
+        #     min_frequency=2,
+        #     special_tokens=[],
+        #     length=len(dct_tokens),
+        #     max_token_length=512,
+        # )
 
         return cls(
             PreTrainedTokenizerFast(tokenizer_object=bpe, clean_up_tokenization_spaces=False),