Upload processor

processing_action_tokenizer.py

@@ -8,7 +8,14 @@ from tokenizers import ByteLevelBPETokenizer
 from tokenizers.trainers import BpeTrainer
 from transformers import PreTrainedTokenizerFast
 from transformers.processing_utils import ProcessorMixin
-
+import numpy as np
+from scipy.fft import dct, idct
+from tokenizers import Tokenizer, decoders, pre_tokenizers, processors
+from tokenizers.models import BPE
+from tokenizers.trainers import BpeTrainer
+from transformers import PreTrainedTokenizerFast
+from transformers.processing_utils import ProcessorMixin
+from tokenizers import Tokenizer
 
 class UniversalActionProcessor(ProcessorMixin):
     attributes: ClassVar[list[str]] = ["bpe_tokenizer"]
@@ -124,14 +131,36 @@ class UniversalActionProcessor(ProcessorMixin):
 
         # Make token iterator for BPE training
         def _token_iter():
-            for tokens in dct_tokens:
+            while dct_tokens:
+                tokens = dct_tokens.pop()
                 rounded_tokens = np.around(tokens * scale) - min_token
                 rounded_tokens = rounded_tokens.astype(int)
                 string = "".join(map(chr, rounded_tokens))
                 yield string
 
+        # # Train BPE tokenizer
+        # bpe = ByteLevelBPETokenizer()
+
+        # # Set up the entire range of possible tokens as the initial alphabet
+        # alphabet = [chr(i) for i in range(max_token - min_token + 1)]
+        # trainer = BpeTrainer(
+        #     vocab_size=vocab_size,
+        #     min_frequency=2,
+        #     show_progress=True,
+        #     special_tokens=[],
+        #     initial_alphabet=alphabet,
+        #     max_token_length=10000,
+        # )
+
+        # # Train the inner tokenizer (don't use ByteLevelBPETokenizer.train_from_iterator()
+        # # because it doesn't support custom alphabets)
+        # bpe._tokenizer.train_from_iterator(_token_iter(), trainer=trainer)
+
         # Train BPE tokenizer
-        bpe = ByteLevelBPETokenizer()
+        tokenizer = Tokenizer(BPE())
+        tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False)
+        tokenizer.decoder = decoders.ByteLevel()
+        tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)
 
         # Set up the entire range of possible tokens as the initial alphabet
         alphabet = [chr(i) for i in range(max_token - min_token + 1)]
@@ -140,16 +169,17 @@ class UniversalActionProcessor(ProcessorMixin):
             min_frequency=2,
             show_progress=True,
             special_tokens=[],
+            # initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
             initial_alphabet=alphabet,
-            max_token_length=10000,
+            # max_token_length=256,
+            max_token_length=10_000,
+        )
+        tokenizer.train_from_iterator(
+            _token_iter(), trainer=trainer, length=len(dct_tokens)
         )
-
-        # Train the inner tokenizer (don't use ByteLevelBPETokenizer.train_from_iterator()
-        # because it doesn't support custom alphabets)
-        bpe._tokenizer.train_from_iterator(_token_iter(), trainer=trainer)
 
         return cls(
-            PreTrainedTokenizerFast(tokenizer_object=bpe, clean_up_tokenization_spaces=False),
+            PreTrainedTokenizerFast(tokenizer_object=tokenizer, clean_up_tokenization_spaces=False),
             scale=scale,
             vocab_size=vocab_size,
             min_token=min_token,

processor_config.json

@@ -3,9 +3,9 @@
   "auto_map": {
     "AutoProcessor": "processing_action_tokenizer.UniversalActionProcessor"
   },
-  "min_token": -141,
+  "min_token": -71,
   "processor_class": "UniversalActionProcessor",
-  "scale": 20.0,
+  "scale": 10.0,
   "time_horizon": null,
   "vocab_size": 2048
 }