Upload processor

Files changed (2) hide show

processing_action_tokenizer.py CHANGED Viewed

@@ -157,7 +157,7 @@ class UniversalActionProcessor(ProcessorMixin):
             show_progress=True,
             special_tokens=[],
             initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
-            max_token_length=128,
         )
         tokenizer.train_from_iterator(
             _token_iter(), trainer=trainer, length=len(dct_tokens)

             show_progress=True,
             special_tokens=[],
             initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
+            max_token_length=10000,
         )
         tokenizer.train_from_iterator(
             _token_iter(), trainer=trainer, length=len(dct_tokens)

tokenizer.json CHANGED Viewed

The diff for this file is too large to render. See raw diff