Upload processor

processing_action_tokenizer.py

@@ -159,14 +159,14 @@ class UniversalActionProcessor(ProcessorMixin):
         tokenizer.decoder = decoders.ByteLevel()
         tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)
 
-        alphabet = [chr(i) for i in range(256)]
+        # alphabet = [chr(i) for i in range(256)]
         trainer = BpeTrainer(
             vocab_size=vocab_size,
             min_frequency=2,
             show_progress=True,
             special_tokens=[],
-            # initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
-            initial_alphabet=alphabet,
+            initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
+            # initial_alphabet=alphabet,
             # max_token_length=256,
             max_token_length=10_000,
         )

processor_config.json

@@ -7,5 +7,5 @@
   "processor_class": "UniversalActionProcessor",
   "scale": 10.0,
   "time_horizon": 50,
-  "vocab_size": 1024
+  "vocab_size": 2048
 }