Upload processor

processing_action_tokenizer.py

@@ -171,17 +171,16 @@ class UniversalActionProcessor(ProcessorMixin):
 
         # Train BPE tokenizer
         tokenizer = Tokenizer(BPE())
-        tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
-            add_prefix_space=False, use_regex=False, trim_offsets=False
-        )
-        tokenizer.pre_tokenizer.pre_tokenize_str
-        tokenizer.decoder = decoders.ByteLevel(
-            add_prefix_space=False, trim_offsets=False, use_regex=False
-        )
-        tokenizer.dec
-        tokenizer.post_processor = processors.ByteLevel(
-            add_prefix_space=False, trim_offsets=False, use_regex=False
-        )
+        # tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
+        #     add_prefix_space=False, use_regex=False, trim_offsets=False
+        # )
+        # tokenizer.decoder = decoders.ByteLevel(
+        #     add_prefix_space=False, trim_offsets=False, use_regex=False
+        # )
+        # tokenizer.decoder.decode()
+        # tokenizer.post_processor = processors.ByteLevel(
+        #     add_prefix_space=False, trim_offsets=False, use_regex=False
+        # )
 
         # Set up the entire range of possible tokens as the initial alphabet
         alphabet = [chr(i) for i in range(256)]