Upload 3 files

aoban_tokenizer.py

@@ -0,0 +1,32 @@
+from tokenizers import Tokenizer
+from tokenizers.models import BPE
+from tokenizers.trainers import BpeTrainer
+from tokenizers.pre_tokenizers import ByteLevel
+from tokenizers.decoders import ByteLevel as ByteLevelDecoder
+from tokenizers.processors import TemplateProcessing
+
+tokenizer = Tokenizer(BPE(unk_token="<UNK>"))
+
+tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=True)
+tokenizer.decoder = ByteLevelDecoder()
+
+trainer = BpeTrainer(
+    vocab_size=5000,
+    min_frequency=1, # Lowered to combine words more easily
+    special_tokens=["<PAD>", "<UNK>", "<|endoftext|>"]
+)
+
+tokenizer.train(
+    files=["training_data.txt"],
+    trainer=trainer
+)
+
+tokenizer.post_processor = TemplateProcessing(
+    single="$A <|endoftext|>",
+    special_tokens=[
+        ("<|endoftext|>", tokenizer.token_to_id("<|endoftext|>"))
+    ]
+)
+
+tokenizer.save("aoban_tokenizer.json")
+print("Tokenizer fixed and saved.")

aoban_weights.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a9e242fc49770cc8f4a2bc5707c33c03e8039c159383aaf3a68e548bf9198b5
+size 475722284