Upload processor

processing_action_tokenizer.py

@@ -150,13 +150,14 @@ class UniversalActionProcessor(ProcessorMixin):
         tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)
 
         # Set up the entire range of possible tokens as the initial alphabet
-        # alphabet = [chr(i) for i in range(max_token - min_token + 1)]
+        alphabet = [chr(i) for i in range(max_token - min_token + 1)]
         trainer = BpeTrainer(
             vocab_size=vocab_size,
             min_frequency=2,
             show_progress=True,
             special_tokens=[],
-            initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
+            # initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
+            initial_alphabet=alphabet,
             max_token_length=10000,
         )
         tokenizer.train_from_iterator(