tobiges
/

behavior_fast

Model card Files Files and versions

tobiges commited on Nov 3, 2025

Commit

d8d9d2f

·

1 Parent(s): ebf7916

push

Files changed (1) hide show

processing_action_tokenizer.py +1 -1

processing_action_tokenizer.py CHANGED Viewed

@@ -158,7 +158,7 @@ class UniversalActionProcessor(ProcessorMixin):
             special_tokens=[],
             initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
             # initial_alphabet=alphabet,
-            max_token_length=10000,
         )
         tokenizer.train_from_iterator(
             _token_iter(), trainer=trainer, length=len(dct_tokens)

             special_tokens=[],
             initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
             # initial_alphabet=alphabet,
+            max_token_length=256,
         )
         tokenizer.train_from_iterator(
             _token_iter(), trainer=trainer, length=len(dct_tokens)