Upload tokenizer.py with huggingface_hub

tokenizer.py

@@ -4,8 +4,8 @@ from transformers import PreTrainedTokenizer
 
 class CharacterTokenizer(PreTrainedTokenizer):
     """
-    Character-level tokenizer for OCR tasks.
-    Each character becomes a separate token.
+    Character-level tokenizer for OCR tasks that follows HuggingFace conventions.
+    Each character becomes a separate token, but decoding produces continuous text.
     """
 
     def __init__(
@@ -75,7 +75,7 @@ class CharacterTokenizer(PreTrainedTokenizer):
 
         # Remove vocab_file from kwargs if it exists to avoid duplicate argument
         kwargs.pop('vocab_file', None)
-        
+
         return cls(vocab_file=vocab_file, **kwargs)
 
     @property
@@ -86,15 +86,68 @@ class CharacterTokenizer(PreTrainedTokenizer):
         return self.token_to_id
 
     def _tokenize(self, text):
+        """Tokenize text into individual characters"""
         return list(text)
 
     def _convert_token_to_id(self, token):
+        """Convert a token (character) to its ID"""
         return self.token_to_id.get(token, self.unk_token_id)
 
     def _convert_id_to_token(self, index):
+        """Convert an ID to its token (character)"""
         return self.id_to_token.get(index, self.unk_token)
 
+    def convert_tokens_to_string(self, tokens):
+        """
+        Convert a sequence of tokens to a single string.
+        This is the KEY method that HuggingFace uses for decoding!
+        For character-level tokenization, we join without spaces.
+        """
+        # Filter out special tokens
+        filtered_tokens = []
+        for token in tokens:
+            if token not in {self.pad_token, self.bos_token, self.eos_token, self.unk_token}:
+                filtered_tokens.append(token)
+        
+        # Join characters directly without spaces
+        return ''.join(filtered_tokens)
+
+    def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True, **kwargs):
+        """
+        Override decode to ensure proper character-level decoding.
+        This follows HuggingFace conventions but handles character-level properly.
+        """
+        # Convert tensor to list if needed
+        if hasattr(token_ids, 'tolist'):
+            token_ids = token_ids.tolist()
+        
+        # Convert IDs to tokens
+        tokens = [self._convert_id_to_token(id) for id in token_ids]
+        
+        # Filter special tokens if requested
+        if skip_special_tokens:
+            tokens = [token for token in tokens if token not in {
+                self.pad_token, self.bos_token, self.eos_token, self.unk_token
+            }]
+        
+        # Use our convert_tokens_to_string method
+        text = self.convert_tokens_to_string(tokens)
+        
+        # For character-level, we don't want clean_up_tokenization_spaces
+        # since we're not using word-level tokenization
+        return text
+
+    def batch_decode(self, sequences, skip_special_tokens=False, clean_up_tokenization_spaces=True, **kwargs):
+        """
+        Batch decode following HuggingFace conventions
+        """
+        return [
+            self.decode(seq, skip_special_tokens=skip_special_tokens, clean_up_tokenization_spaces=clean_up_tokenization_spaces, **kwargs)
+            for seq in sequences
+        ]
+
     def save_vocabulary(self, save_directory, filename_prefix=None):
+        """Save vocabulary following HuggingFace conventions"""
         os.makedirs(save_directory, exist_ok=True)
 
         vocab_path = os.path.join(save_directory, "vocab.json")
@@ -113,11 +166,13 @@ class CharacterTokenizer(PreTrainedTokenizer):
                 "unk_token": self.unk_token,
                 "pad_token": self.pad_token,
                 "vocab_file": "vocab.json",
+                "clean_up_tokenization_spaces": False,  # Important for character-level
             }, f, indent=2)
 
         return (vocab_path,)
 
     def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        """Build inputs with special tokens following HuggingFace conventions"""
         if token_ids_1 is None:
             return [self.bos_token_id] + token_ids_0 + [self.eos_token_id]
         else:
@@ -130,6 +185,7 @@ class CharacterTokenizer(PreTrainedTokenizer):
             )
 
     def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
+        """Create token type IDs following HuggingFace conventions"""
         return [0] * len(
             self.build_inputs_with_special_tokens(token_ids_0, token_ids_1)
         )