Upload processor

processing_action_tokenizer.py

@@ -145,7 +145,7 @@ class UniversalActionProcessor(ProcessorMixin):
 
         # Train BPE tokenizer
         tokenizer = Tokenizer(BPE())
-        tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
+        tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False)
         tokenizer.decoder = decoders.ByteLevel()
         tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)
 
@@ -157,7 +157,7 @@ class UniversalActionProcessor(ProcessorMixin):
             show_progress=True,
             special_tokens=[],
             initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
-            max_token_length=2*action_dim,
+            max_token_length=128,
         )
         tokenizer.train_from_iterator(
             _token_iter(), trainer=trainer, length=len(dct_tokens)