Update tessar_tokenizer.py

tessar_tokenizer.py

@@ -8,6 +8,9 @@ from transformers import PreTrainedTokenizerFast
 class TessarTokenizer(PreTrainedTokenizerFast):
     """
     Tessar Tokenizer implementation for Hugging Face Transformers
+    
+    This custom tokenizer extends the PreTrainedTokenizerFast with specialized 
+    configuration and tokenization methods for the Tessar model.
     """
     
     model_input_names = ['input_ids', 'attention_mask']
@@ -74,6 +77,9 @@ class TessarTokenizer(PreTrainedTokenizerFast):
         Returns:
             tuple: Paths to the saved files
         """
+        # Ensure the save directory exists
+        os.makedirs(save_directory, exist_ok=True)
+        
         # Prepare file paths
         vocab_file = os.path.join(
             save_directory, 
@@ -161,4 +167,19 @@ def load_tessar_tokenizer(pretrained_model_name_or_path: str):
     Returns:
         TessarTokenizer: Initialized tokenizer
     """
-    return TessarTokenizer.from_pretrained(pretrained_model_name_or_path)
+    return TessarTokenizer.from_pretrained(pretrained_model_name_or_path)
+
+
+# Optionally, add some example usage
+if __name__ == "__main__":
+    # Example of loading a pretrained tokenizer
+    try:
+        tokenizer = load_tessar_tokenizer("SVECTOR-CORPORATION/Tessar-largest")
+        print("Tokenizer loaded successfully!")
+        
+        # Basic tokenization example
+        text = "Hello, how are you doing today?"
+        encoded = tokenizer(text, return_tensors="pt")
+        print("Encoded Input:", encoded)
+    except Exception as e:
+        print(f"Error loading tokenizer: {e}")