Create tokenizer.json

tokenizer.json

@@ -0,0 +1,19 @@
+from tokenizers import Tokenizer, models, trainers, pre_tokenizers, decoders
+
+# Create a BPE tokenizer
+tokenizer = Tokenizer(models.BPE())
+tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()
+tokenizer.decoder = decoders.ByteLevel()
+
+# Train on your text data
+trainer = trainers.BpeTrainer(
+    vocab_size=30000,
+    special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>"]
+)
+
+# Replace 'train.txt' with your text file containing all training data
+tokenizer.train(files=["train.txt"], trainer=trainer)
+
+# Save the tokenizer.json
+tokenizer.save("tokenizer.json")
+print("tokenizer.json is ready!")