Update tessar_tokenizer_example.py

tessar_tokenizer_example.py

@@ -1,38 +1,22 @@
-from tessar_tokenizer import TessarTokenizer, load_tessar_tokenizer
-
-# Example 1: Initialize a new Tessar Tokenizer
+# Standard usage with default settings
 tokenizer = TessarTokenizer.from_pretrained("SVECTOR-CORPORATION/Tessar-largest")
 
-# Example 2: Tokenize a simple text
+# Tokenize a single piece of text
 text = "Hello, how are you doing today?"
 encoded = tokenizer(text, return_tensors="pt")
-print("Encoded Input:", encoded)
 
-# Example 3: Batch tokenization
+# Batch tokenization of multiple texts
 texts = [
     "Hello, world!",
     "This is a test sentence.",
     "Tokenization is an important NLP task."
 ]
 batch_encoded = tokenizer(texts, padding=True, truncation=True, return_tensors="pt")
-print("Batch Encoded Inputs:", batch_encoded)
-
-# Example 4: Save and reload tokenizer
-save_directory = "./tessar_tokenizer"
-tokenizer.save_pretrained(save_directory)
 
-# Reload the saved tokenizer
-reloaded_tokenizer = load_tessar_tokenizer(save_directory)
-
-# Example 5: Custom tokenization with specific parameters
+# Custom tokenizer with specific settings
 custom_tokenizer = TessarTokenizer(
     do_lower_case=True,
     max_cell_length=20,
     unk_token="[UNK]",
     pad_token="[PAD]"
-)
-
-# Tokenize with custom settings
-custom_text = "A custom tokenization example"
-custom_encoded = custom_tokenizer(custom_text, return_tensors="pt")
-print("Custom Tokenizer Encoded:", custom_encoded)
+)