Upload processor

Files changed (2) hide show

processing_action_tokenizer.py CHANGED Viewed

@@ -158,7 +158,7 @@ class UniversalActionProcessor(ProcessorMixin):
             special_tokens=[],
             initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
             # initial_alphabet=alphabet,
-            max_token_length=10000,
         )
         tokenizer.train_from_iterator(
             _token_iter(), trainer=trainer, length=len(dct_tokens)

             special_tokens=[],
             initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
             # initial_alphabet=alphabet,
+            max_token_length=256,
         )
         tokenizer.train_from_iterator(
             _token_iter(), trainer=trainer, length=len(dct_tokens)

tokenizer.json CHANGED Viewed

The diff for this file is too large to render. See raw diff