Upload processor

processing_action_tokenizer.py

@@ -3,7 +3,7 @@ from typing import ClassVar
 
 import numpy as np
 from scipy.fft import dct, idct
-from tokenizers import Tokenizer, pre_tokenizers
+from tokenizers import Tokenizer, decoders, pre_tokenizers, processors
 from tokenizers.models import BPE
 from tokenizers.trainers import BpeTrainer
 from transformers import PreTrainedTokenizerFast
@@ -171,15 +171,15 @@ class UniversalActionProcessor(ProcessorMixin):
 
         # Train BPE tokenizer
         tokenizer = Tokenizer(BPE())
-        # tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
-        #     add_prefix_space=False, use_regex=False, trim_offsets=False
-        # )
-        # tokenizer.decoder = decoders.ByteLevel(
-        #     add_prefix_space=False, trim_offsets=False, use_regex=False
-        # )
-        # tokenizer.post_processor = processors.ByteLevel(
-        #     add_prefix_space=False, trim_offsets=False, use_regex=False
-        # )
+        tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
+            add_prefix_space=False, use_regex=False, trim_offsets=False
+        )
+        tokenizer.decoder = decoders.ByteLevel(
+            add_prefix_space=False, trim_offsets=False, use_regex=False
+        )
+        tokenizer.post_processor = processors.ByteLevel(
+            add_prefix_space=False, trim_offsets=False, use_regex=False
+        )
 
         # Set up the entire range of possible tokens as the initial alphabet
         alphabet = [chr(i) for i in range(256)]