Upload processor

Files changed (2) hide show

processing_action_tokenizer.py CHANGED Viewed

@@ -147,7 +147,7 @@ class UniversalActionProcessor(ProcessorMixin):
             show_progress=True,
             special_tokens=[],
             initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
-            max_token_length=512,
         )
         tokenizer.train_from_iterator(_token_iter(), trainer=trainer, length=len(dct_tokens))

             show_progress=True,
             special_tokens=[],
             initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
+            max_token_length=64,
         )
         tokenizer.train_from_iterator(_token_iter(), trainer=trainer, length=len(dct_tokens))

tokenizer.json CHANGED Viewed

The diff for this file is too large to render. See raw diff