Upload processor

processing_action_tokenizer.py

@@ -4,11 +4,11 @@ from typing import ClassVar
 
 import numpy as np
 from scipy.fft import dct, idct
-from tokenizers import ByteLevelBPETokenizer, AddedToken, Tokenizer, decoders, pre_tokenizers, processors, trainers
+from tokenizers import Tokenizer, decoders, pre_tokenizers, processors
+from tokenizers.models import BPE
 from tokenizers.trainers import BpeTrainer
-from transformers.processing_utils import ProcessorMixin
 from transformers import PreTrainedTokenizerFast
-from tokenizers.models import BPE
+from transformers.processing_utils import ProcessorMixin
 
 
 class UniversalActionProcessor(ProcessorMixin):
@@ -42,7 +42,9 @@ class UniversalActionProcessor(ProcessorMixin):
         super().__init__(bpe_tokenizer)
 
     def __call__(self, action_chunk: np.array) -> np.array:
-        assert action_chunk.ndim <= 3, "Only 3 dimensions supported: [batch, timesteps, action_dim]"
+        assert action_chunk.ndim <= 3, (
+            "Only 3 dimensions supported: [batch, timesteps, action_dim]"
+        )
         if action_chunk.ndim == 2:
             action_chunk = action_chunk[None, ...]
 
@@ -54,7 +56,9 @@ class UniversalActionProcessor(ProcessorMixin):
         dct_coeff = np.around(dct_coeff * self.scale)
         tokens = []
         for elem in dct_coeff:
-            token_str = "".join(map(chr, np.maximum(elem.flatten() - self.min_token, 0).astype(int)))
+            token_str = "".join(
+                map(chr, np.maximum(elem.flatten() - self.min_token, 0).astype(int))
+            )
             tokens.append(self.bpe_tokenizer(token_str)["input_ids"])
         return tokens
 
@@ -65,35 +69,40 @@ class UniversalActionProcessor(ProcessorMixin):
         time_horizon: int | None = None,
         action_dim: int | None = None,
     ) -> np.array:
-        self.time_horizon = time_horizon or self.time_horizon or self.called_time_horizon
+        self.time_horizon = (
+            time_horizon or self.time_horizon or self.called_time_horizon
+        )
         self.action_dim = action_dim or self.action_dim or self.called_action_dim
 
         # Cache the time horizon and action dimension for the next call
         self.called_time_horizon = self.time_horizon
         self.called_action_dim = self.action_dim
 
-        assert (
-            self.time_horizon is not None and self.action_dim is not None
-        ), "Tokenizer not initialized, call encode() once or pass in time_horizon and action_dim."
+        assert self.time_horizon is not None and self.action_dim is not None, (
+            "Tokenizer not initialized, call encode() once or pass in time_horizon and action_dim."
+        )
 
         decoded_actions = []
         for token in tokens:
             try:
                 decoded_tokens = self.bpe_tokenizer.decode(token)
-                decoded_dct_coeff = np.array(list(map(ord, decoded_tokens))) + self.min_token
+                decoded_dct_coeff = (
+                    np.array(list(map(ord, decoded_tokens))) + self.min_token
+                )
                 decoded_dct_coeff = decoded_dct_coeff.reshape(-1, self.action_dim)
-                assert (
-                    decoded_dct_coeff.shape
-                    == (
-                        self.time_horizon,
-                        self.action_dim,
-                    )
-                ), f"Decoded DCT coefficients have shape {decoded_dct_coeff.shape}, expected ({self.time_horizon}, {self.action_dim})"
+                assert decoded_dct_coeff.shape == (
+                    self.time_horizon,
+                    self.action_dim,
+                ), (
+                    f"Decoded DCT coefficients have shape {decoded_dct_coeff.shape}, expected ({self.time_horizon}, {self.action_dim})"
+                )
             except Exception as e:
                 print(f"Error decoding tokens: {e}")
                 print(f"Tokens: {token}")
                 decoded_dct_coeff = np.zeros((self.time_horizon, self.action_dim))
-            decoded_actions.append(idct(decoded_dct_coeff / self.scale, axis=0, norm="ortho"))
+            decoded_actions.append(
+                idct(decoded_dct_coeff / self.scale, axis=0, norm="ortho")
+            )
         return np.stack(decoded_actions)
 
     @classmethod
@@ -113,11 +122,13 @@ class UniversalActionProcessor(ProcessorMixin):
         max_token = int(np.around(np.concatenate(dct_tokens) * scale).max())
         min_token = int(np.around(np.concatenate(dct_tokens) * scale).min())
         min_vocab_size = max_token - min_token
-        print(f"Min token: {min_token}, Max token: {max_token}, Min vocab size: {min_vocab_size}")
+        print(
+            f"Min token: {min_token}, Max token: {max_token}, Min vocab size: {min_vocab_size}"
+        )
 
-        assert (
-            min_vocab_size <= vocab_size
-        ), f"Vocab size {vocab_size} is too small for the range of tokens {min_vocab_size}"
+        assert min_vocab_size <= vocab_size, (
+            f"Vocab size {vocab_size} is too small for the range of tokens {min_vocab_size}"
+        )
         if min_vocab_size + 100 > vocab_size:
             logging.warning(
                 f"Initial alphabet size {min_vocab_size} is almost as large as the vocab"
@@ -133,12 +144,11 @@ class UniversalActionProcessor(ProcessorMixin):
                 yield string
 
         # Train BPE tokenizer
-        # bpe = ByteLevelBPETokenizer()
         tokenizer = Tokenizer(BPE())
-        tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()
+        tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
         tokenizer.decoder = decoders.ByteLevel()
-        tokenizer.post_processor = processors.ByteLevel()
-        
+        tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)
+
         # Set up the entire range of possible tokens as the initial alphabet
         # alphabet = [chr(i) for i in range(max_token - min_token + 1)]
         trainer = BpeTrainer(
@@ -147,24 +157,16 @@ class UniversalActionProcessor(ProcessorMixin):
             show_progress=True,
             special_tokens=[],
             initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
-            max_token_length=64,
+            max_token_length=22,
+        )
+        tokenizer.train_from_iterator(
+            _token_iter(), trainer=trainer, length=len(dct_tokens)
         )
-        tokenizer.train_from_iterator(_token_iter(), trainer=trainer, length=len(dct_tokens))
-
-        # Train the inner tokenizer (don't use ByteLevelBPETokenizer.train_from_iterator()
-        # because it doesn't support custom alphabets)
-        # bpe._tokenizer.train_from_iterator(_token_iter(), trainer=trainer, length=len(dct_tokens))
-        # trainer.train_from_iterator(_token_iter(), trainer=trainer)
-        # bpe.train_from_iterator(
-        #     _token_iter(), 
-        #     vocab_size=vocab_size, 
-        #     min_frequency=2,
-        #     special_tokens=[],
-        #     length=len(dct_tokens),
-        # )
 
         return cls(
-            PreTrainedTokenizerFast(tokenizer_object=tokenizer, clean_up_tokenization_spaces=False),
+            PreTrainedTokenizerFast(
+                tokenizer_object=tokenizer, clean_up_tokenization_spaces=False
+            ),
             scale=scale,
             vocab_size=vocab_size,
             min_token=min_token,