Upload processor

processing_action_tokenizer.py

@@ -1,11 +1,10 @@
 import logging
-from sre_parse import Tokenizer
 from typing import ClassVar
 
 import numpy as np
-from scipy.fft import dct, idct
-from tokenizers import Tokenizer, decoders, pre_tokenizers, processors
-from tokenizers.models import BPE
+from scipy.fft import dct
+from scipy.fft import idct
+from tokenizers import ByteLevelBPETokenizer
 from tokenizers.trainers import BpeTrainer
 from transformers import PreTrainedTokenizerFast
 from transformers.processing_utils import ProcessorMixin
@@ -42,9 +41,7 @@ class UniversalActionProcessor(ProcessorMixin):
         super().__init__(bpe_tokenizer)
 
     def __call__(self, action_chunk: np.array) -> np.array:
-        assert action_chunk.ndim <= 3, (
-            "Only 3 dimensions supported: [batch, timesteps, action_dim]"
-        )
+        assert action_chunk.ndim <= 3, "Only 3 dimensions supported: [batch, timesteps, action_dim]"
         if action_chunk.ndim == 2:
             action_chunk = action_chunk[None, ...]
 
@@ -56,9 +53,7 @@ class UniversalActionProcessor(ProcessorMixin):
         dct_coeff = np.around(dct_coeff * self.scale)
         tokens = []
         for elem in dct_coeff:
-            token_str = "".join(
-                map(chr, np.maximum(elem.flatten() - self.min_token, 0).astype(int))
-            )
+            token_str = "".join(map(chr, np.maximum(elem.flatten() - self.min_token, 0).astype(int)))
             tokens.append(self.bpe_tokenizer(token_str)["input_ids"])
         return tokens
 
@@ -69,40 +64,35 @@ class UniversalActionProcessor(ProcessorMixin):
         time_horizon: int | None = None,
         action_dim: int | None = None,
     ) -> np.array:
-        self.time_horizon = (
-            time_horizon or self.time_horizon or self.called_time_horizon
-        )
+        self.time_horizon = time_horizon or self.time_horizon or self.called_time_horizon
         self.action_dim = action_dim or self.action_dim or self.called_action_dim
 
         # Cache the time horizon and action dimension for the next call
         self.called_time_horizon = self.time_horizon
         self.called_action_dim = self.action_dim
 
-        assert self.time_horizon is not None and self.action_dim is not None, (
-            "Tokenizer not initialized, call encode() once or pass in time_horizon and action_dim."
-        )
+        assert (
+            self.time_horizon is not None and self.action_dim is not None
+        ), "Tokenizer not initialized, call encode() once or pass in time_horizon and action_dim."
 
         decoded_actions = []
         for token in tokens:
             try:
                 decoded_tokens = self.bpe_tokenizer.decode(token)
-                decoded_dct_coeff = (
-                    np.array(list(map(ord, decoded_tokens))) + self.min_token
-                )
+                decoded_dct_coeff = np.array(list(map(ord, decoded_tokens))) + self.min_token
                 decoded_dct_coeff = decoded_dct_coeff.reshape(-1, self.action_dim)
-                assert decoded_dct_coeff.shape == (
-                    self.time_horizon,
-                    self.action_dim,
-                ), (
-                    f"Decoded DCT coefficients have shape {decoded_dct_coeff.shape}, expected ({self.time_horizon}, {self.action_dim})"
-                )
+                assert (
+                    decoded_dct_coeff.shape
+                    == (
+                        self.time_horizon,
+                        self.action_dim,
+                    )
+                ), f"Decoded DCT coefficients have shape {decoded_dct_coeff.shape}, expected ({self.time_horizon}, {self.action_dim})"
             except Exception as e:
                 print(f"Error decoding tokens: {e}")
                 print(f"Tokens: {token}")
                 decoded_dct_coeff = np.zeros((self.time_horizon, self.action_dim))
-            decoded_actions.append(
-                idct(decoded_dct_coeff / self.scale, axis=0, norm="ortho")
-            )
+            decoded_actions.append(idct(decoded_dct_coeff / self.scale, axis=0, norm="ortho"))
         return np.stack(decoded_actions)
 
     @classmethod
@@ -122,13 +112,10 @@ class UniversalActionProcessor(ProcessorMixin):
         max_token = int(np.around(np.concatenate(dct_tokens) * scale).max())
         min_token = int(np.around(np.concatenate(dct_tokens) * scale).min())
         min_vocab_size = max_token - min_token
-        print(
-            f"Min token: {min_token}, Max token: {max_token}, Min vocab size: {min_vocab_size}"
-        )
 
-        assert min_vocab_size <= vocab_size, (
-            f"Vocab size {vocab_size} is too small for the range of tokens {min_vocab_size}"
-        )
+        assert (
+            min_vocab_size <= vocab_size
+        ), f"Vocab size {vocab_size} is too small for the range of tokens {min_vocab_size}"
         if min_vocab_size + 100 > vocab_size:
             logging.warning(
                 f"Initial alphabet size {min_vocab_size} is almost as large as the vocab"
@@ -144,10 +131,7 @@ class UniversalActionProcessor(ProcessorMixin):
                 yield string
 
         # Train BPE tokenizer
-        tokenizer = Tokenizer(BPE())
-        tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False)
-        tokenizer.decoder = decoders.ByteLevel()
-        tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)
+        bpe = ByteLevelBPETokenizer()
 
         # Set up the entire range of possible tokens as the initial alphabet
         alphabet = [chr(i) for i in range(max_token - min_token + 1)]
@@ -156,18 +140,16 @@ class UniversalActionProcessor(ProcessorMixin):
             min_frequency=2,
             show_progress=True,
             special_tokens=[],
-            initial_alphabet=pre_tokenizers.ByteLevel.alphabet(),
-            # initial_alphabet=alphabet,
-            max_token_length=256,
-        )
-        tokenizer.train_from_iterator(
-            _token_iter(), trainer=trainer, length=len(dct_tokens)
+            initial_alphabet=alphabet,
+            max_token_length=10000,
         )
 
+        # Train the inner tokenizer (don't use ByteLevelBPETokenizer.train_from_iterator()
+        # because it doesn't support custom alphabets)
+        bpe._tokenizer.train_from_iterator(_token_iter(), trainer=trainer)
+
         return cls(
-            PreTrainedTokenizerFast(
-                tokenizer_object=tokenizer, clean_up_tokenization_spaces=False
-            ),
+            PreTrainedTokenizerFast(tokenizer_object=bpe, clean_up_tokenization_spaces=False),
             scale=scale,
             vocab_size=vocab_size,
             min_token=min_token,