Upload processor

processing_action_tokenizer.py

@@ -171,15 +171,15 @@ class UniversalActionProcessor(ProcessorMixin):
 
         # Train BPE tokenizer
         tokenizer = Tokenizer(BPE())
-        # tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
-        #     add_prefix_space=False, use_regex=False, trim_offsets=False
-        # )
-        # tokenizer.decoder = decoders.ByteLevel(
-        #     add_prefix_space=False, trim_offsets=False, use_regex=False
-        # )
-        # tokenizer.post_processor = processors.ByteLevel(
-        #     add_prefix_space=False, trim_offsets=False, use_regex=False
-        # )
+        tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
+            add_prefix_space=False, use_regex=False, trim_offsets=False
+        )
+        tokenizer.decoder = decoders.ByteLevel(
+            add_prefix_space=False, trim_offsets=False, use_regex=False
+        )
+        tokenizer.post_processor = processors.ByteLevel(
+            add_prefix_space=False, trim_offsets=False, use_regex=False
+        )
 
         # Set up the entire range of possible tokens as the initial alphabet
         alphabet = [chr(i) for i in range(256)]