Upload processor

processing_action_tokenizer.py

@@ -1,13 +1,6 @@
 import logging
 from typing import ClassVar
 
-import numpy as np
-from scipy.fft import dct
-from scipy.fft import idct
-from tokenizers import ByteLevelBPETokenizer
-from tokenizers.trainers import BpeTrainer
-from transformers import PreTrainedTokenizerFast
-from transformers.processing_utils import ProcessorMixin
 import numpy as np
 from scipy.fft import dct, idct
 from tokenizers import Tokenizer, decoders, pre_tokenizers, processors
@@ -15,7 +8,7 @@ from tokenizers.models import BPE
 from tokenizers.trainers import BpeTrainer
 from transformers import PreTrainedTokenizerFast
 from transformers.processing_utils import ProcessorMixin
-from tokenizers import Tokenizer
+
 
 class UniversalActionProcessor(ProcessorMixin):
     attributes: ClassVar[list[str]] = ["bpe_tokenizer"]
@@ -48,7 +41,9 @@ class UniversalActionProcessor(ProcessorMixin):
         super().__init__(bpe_tokenizer)
 
     def __call__(self, action_chunk: np.array) -> np.array:
-        assert action_chunk.ndim <= 3, "Only 3 dimensions supported: [batch, timesteps, action_dim]"
+        assert action_chunk.ndim <= 3, (
+            "Only 3 dimensions supported: [batch, timesteps, action_dim]"
+        )
         if action_chunk.ndim == 2:
             action_chunk = action_chunk[None, ...]
 
@@ -60,7 +55,9 @@ class UniversalActionProcessor(ProcessorMixin):
         dct_coeff = np.around(dct_coeff * self.scale)
         tokens = []
         for elem in dct_coeff:
-            token_str = "".join(map(chr, np.maximum(elem.flatten() - self.min_token, 0).astype(int)))
+            token_str = "".join(
+                map(chr, np.maximum(elem.flatten() - self.min_token, 0).astype(int))
+            )
             tokens.append(self.bpe_tokenizer(token_str)["input_ids"])
         return tokens
 
@@ -71,35 +68,40 @@ class UniversalActionProcessor(ProcessorMixin):
         time_horizon: int | None = None,
         action_dim: int | None = None,
     ) -> np.array:
-        self.time_horizon = time_horizon or self.time_horizon or self.called_time_horizon
+        self.time_horizon = (
+            time_horizon or self.time_horizon or self.called_time_horizon
+        )
         self.action_dim = action_dim or self.action_dim or self.called_action_dim
 
         # Cache the time horizon and action dimension for the next call
         self.called_time_horizon = self.time_horizon
         self.called_action_dim = self.action_dim
 
-        assert (
-            self.time_horizon is not None and self.action_dim is not None
-        ), "Tokenizer not initialized, call encode() once or pass in time_horizon and action_dim."
+        assert self.time_horizon is not None and self.action_dim is not None, (
+            "Tokenizer not initialized, call encode() once or pass in time_horizon and action_dim."
+        )
 
         decoded_actions = []
         for token in tokens:
             try:
                 decoded_tokens = self.bpe_tokenizer.decode(token)
-                decoded_dct_coeff = np.array(list(map(ord, decoded_tokens))) + self.min_token
+                decoded_dct_coeff = (
+                    np.array(list(map(ord, decoded_tokens))) + self.min_token
+                )
                 decoded_dct_coeff = decoded_dct_coeff.reshape(-1, self.action_dim)
-                assert (
-                    decoded_dct_coeff.shape
-                    == (
-                        self.time_horizon,
-                        self.action_dim,
-                    )
-                ), f"Decoded DCT coefficients have shape {decoded_dct_coeff.shape}, expected ({self.time_horizon}, {self.action_dim})"
+                assert decoded_dct_coeff.shape == (
+                    self.time_horizon,
+                    self.action_dim,
+                ), (
+                    f"Decoded DCT coefficients have shape {decoded_dct_coeff.shape}, expected ({self.time_horizon}, {self.action_dim})"
+                )
             except Exception as e:
                 print(f"Error decoding tokens: {e}")
                 print(f"Tokens: {token}")
                 decoded_dct_coeff = np.zeros((self.time_horizon, self.action_dim))
-            decoded_actions.append(idct(decoded_dct_coeff / self.scale, axis=0, norm="ortho"))
+            decoded_actions.append(
+                idct(decoded_dct_coeff / self.scale, axis=0, norm="ortho")
+            )
         return np.stack(decoded_actions)
 
     @classmethod
@@ -119,19 +121,25 @@ class UniversalActionProcessor(ProcessorMixin):
         max_token = int(np.around(np.concatenate(dct_tokens) * scale).max())
         min_token = int(np.around(np.concatenate(dct_tokens) * scale).min())
         min_vocab_size = max_token - min_token
-        print(f"Min token: {min_token}, Max token: {max_token}, Min vocab size: {min_vocab_size}")
+        print(
+            f"Min token: {min_token}, Max token: {max_token}, Min vocab size: {min_vocab_size}"
+        )
 
-        assert (
-            min_vocab_size <= vocab_size
-        ), f"Vocab size {vocab_size} is too small for the range of tokens {min_vocab_size}"
+        assert min_vocab_size <= vocab_size, (
+            f"Vocab size {vocab_size} is too small for the range of tokens {min_vocab_size}"
+        )
         if min_vocab_size + 100 > vocab_size:
             logging.warning(
                 f"Initial alphabet size {min_vocab_size} is almost as large as the vocab"
                 f"size {vocab_size}, consider increasing vocab size"
             )
 
-        assert min_token >= -128 + 10, f"Min token {min_token} is less than -128 + 10 (for buffer space)"
-        assert max_token < 128 - 10, f"Max token {max_token} is greater than 128 - 10 (for buffer space)"
+        assert min_token >= -128 + 10, (
+            f"Min token {min_token} is less than -128 + 10 (for buffer space)"
+        )
+        assert max_token < 128 - 10, (
+            f"Max token {max_token} is greater than 128 - 10 (for buffer space)"
+        )
         min_token = -128
 
         # Make token iterator for BPE training
@@ -163,9 +171,17 @@ class UniversalActionProcessor(ProcessorMixin):
 
         # Train BPE tokenizer
         tokenizer = Tokenizer(BPE())
-        tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False)
-        tokenizer.decoder = decoders.ByteLevel()
-        tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)
+        tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
+            add_prefix_space=False, use_regex=False, trim_offsets=False
+        )
+        tokenizer.pre_tokenizer.pre_tokenize_str
+        tokenizer.decoder = decoders.ByteLevel(
+            add_prefix_space=False, trim_offsets=False, use_regex=False
+        )
+        tokenizer.dec
+        tokenizer.post_processor = processors.ByteLevel(
+            add_prefix_space=False, trim_offsets=False, use_regex=False
+        )
 
         # Set up the entire range of possible tokens as the initial alphabet
         alphabet = [chr(i) for i in range(256)]
@@ -176,15 +192,17 @@ class UniversalActionProcessor(ProcessorMixin):
             special_tokens=[],
             # initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
             initial_alphabet=alphabet,
-            # max_token_length=256,
-            max_token_length=10_000,
+            max_token_length=256,
+            # max_token_length=10_000,
         )
         tokenizer.train_from_iterator(
             _token_iter(), trainer=trainer, length=len(dct_tokens)
         )
 
         return cls(
-            PreTrainedTokenizerFast(tokenizer_object=tokenizer, clean_up_tokenization_spaces=False),
+            PreTrainedTokenizerFast(
+                tokenizer_object=tokenizer, clean_up_tokenization_spaces=False
+            ),
             scale=scale,
             vocab_size=vocab_size,
             min_token=min_token,