Upload processor

processing_action_tokenizer.py

@@ -171,15 +171,15 @@ class UniversalActionProcessor(ProcessorMixin):
 
         # Train BPE tokenizer
         tokenizer = Tokenizer(BPE())
-        tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
-            add_prefix_space=False, use_regex=False, trim_offsets=False
-        )
-        tokenizer.decoder = decoders.ByteLevel(
-            add_prefix_space=False, trim_offsets=False, use_regex=False
-        )
-        tokenizer.post_processor = processors.ByteLevel(
-            add_prefix_space=False, trim_offsets=False, use_regex=False
-        )
+        # tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
+        #     add_prefix_space=False, use_regex=False, trim_offsets=False
+        # )
+        # tokenizer.decoder = decoders.ByteLevel(
+        #     add_prefix_space=False, trim_offsets=False, use_regex=False
+        # )
+        # tokenizer.post_processor = processors.ByteLevel(
+        #     add_prefix_space=False, trim_offsets=False, use_regex=False
+        # )
 
         # Set up the entire range of possible tokens as the initial alphabet
         alphabet = [chr(i) for i in range(256)]

processor_config.json

@@ -6,6 +6,6 @@
   "min_token": -128,
   "processor_class": "UniversalActionProcessor",
   "scale": 10.0,
-  "time_horizon": null,
+  "time_horizon": 50,
   "vocab_size": 1024
 }