Upload processor

processing_action_tokenizer.py

@@ -2,8 +2,7 @@ import logging
 from typing import ClassVar
 
 import numpy as np
-from scipy.fft import dct
-from scipy.fft import idct
+from scipy.fft import dct, idct
 from tokenizers import ByteLevelBPETokenizer
 from tokenizers.trainers import BpeTrainer
 from transformers import PreTrainedTokenizerFast
@@ -112,6 +111,7 @@ class UniversalActionProcessor(ProcessorMixin):
         max_token = int(np.around(np.concatenate(dct_tokens) * scale).max())
         min_token = int(np.around(np.concatenate(dct_tokens) * scale).min())
         min_vocab_size = max_token - min_token
+        print(f"Min token: {min_token}, Max token: {max_token}, Min vocab size: {min_vocab_size}")
 
         assert (
             min_vocab_size <= vocab_size
@@ -137,11 +137,12 @@ class UniversalActionProcessor(ProcessorMixin):
         alphabet = [chr(i) for i in range(max_token - min_token + 1)]
         trainer = BpeTrainer(
             vocab_size=vocab_size,
-            min_frequency=2,
+            min_frequency=8,
             show_progress=True,
             special_tokens=[],
             initial_alphabet=alphabet,
-            max_token_length=10000,
+            # max_token_length=10000,
+            max_token_length=100,
         )
 
         # Train the inner tokenizer (don't use ByteLevelBPETokenizer.train_from_iterator()

processor_config.json

@@ -3,7 +3,7 @@
   "auto_map": {
     "AutoProcessor": "processing_action_tokenizer.UniversalActionProcessor"
   },
-  "min_token": -71,
+  "min_token": -501,
   "processor_class": "UniversalActionProcessor",
   "scale": 10,
   "time_horizon": null,