Upload processor

processing_action_tokenizer.py

@@ -3,7 +3,7 @@ from typing import ClassVar
 
 import numpy as np
 from scipy.fft import dct, idct
-from tokenizers import Tokenizer, decoders, pre_tokenizers, processors
+from tokenizers import Tokenizer, pre_tokenizers
 from tokenizers.models import BPE
 from tokenizers.trainers import BpeTrainer
 from transformers import PreTrainedTokenizerFast
@@ -177,7 +177,6 @@ class UniversalActionProcessor(ProcessorMixin):
         # tokenizer.decoder = decoders.ByteLevel(
         #     add_prefix_space=False, trim_offsets=False, use_regex=False
         # )
-        # tokenizer.decoder.decode()
         # tokenizer.post_processor = processors.ByteLevel(
         #     add_prefix_space=False, trim_offsets=False, use_regex=False
         # )
@@ -191,8 +190,8 @@ class UniversalActionProcessor(ProcessorMixin):
             special_tokens=[],
             # initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
             initial_alphabet=alphabet,
-            max_token_length=256,
-            # max_token_length=10_000,
+            # max_token_length=256,
+            max_token_length=10_000,
         )
         tokenizer.train_from_iterator(
             _token_iter(), trainer=trainer, length=len(dct_tokens)