Upload processor

Files changed (2) hide show

processing_action_tokenizer.py CHANGED Viewed

@@ -156,8 +156,8 @@ class UniversalActionProcessor(ProcessorMixin):
             min_frequency=2,
             show_progress=True,
             special_tokens=[],
-            # initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
-            initial_alphabet=alphabet,
             max_token_length=10000,
         )
         tokenizer.train_from_iterator(

             min_frequency=2,
             show_progress=True,
             special_tokens=[],
+            initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
+            # initial_alphabet=alphabet,
             max_token_length=10000,
         )
         tokenizer.train_from_iterator(

tokenizer.json CHANGED Viewed

The diff for this file is too large to render. See raw diff