Upload processor

processing_action_tokenizer.py

@@ -119,6 +119,7 @@ class UniversalActionProcessor(ProcessorMixin):
         max_token = int(np.around(np.concatenate(dct_tokens) * scale).max())
         min_token = int(np.around(np.concatenate(dct_tokens) * scale).min())
         min_vocab_size = max_token - min_token
+        print(f"Min token: {min_token}, Max token: {max_token}, Min vocab size: {min_vocab_size}")
 
         assert (
             min_vocab_size <= vocab_size
@@ -129,6 +130,10 @@ class UniversalActionProcessor(ProcessorMixin):
                 f"size {vocab_size}, consider increasing vocab size"
             )
 
+        assert min_token >= -128 + 10, f"Min token {min_token} is less than -128 + 10 (for buffer space)"
+        assert max_token < 128 - 10, f"Max token {max_token} is greater than 128 - 10 (for buffer space)"
+        min_token = -128
+
         # Make token iterator for BPE training
         def _token_iter():
             while dct_tokens:
@@ -163,7 +168,7 @@ class UniversalActionProcessor(ProcessorMixin):
         tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)
 
         # Set up the entire range of possible tokens as the initial alphabet
-        alphabet = [chr(i) for i in range(max_token - min_token + 1)]
+        alphabet = [chr(i) for i in range(256)]
         trainer = BpeTrainer(
             vocab_size=vocab_size,
             min_frequency=2,

processor_config.json

@@ -3,7 +3,7 @@
   "auto_map": {
     "AutoProcessor": "processing_action_tokenizer.UniversalActionProcessor"
   },
-  "min_token": -71,
+  "min_token": -128,
   "processor_class": "UniversalActionProcessor",
   "scale": 10.0,
   "time_horizon": null,