File size: 1,904 Bytes
bbf30b7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 | from tokenizer import (
build_xonelm_tokenizer,
MultiTurnConversationFormatter,
SpecialTokenConfig,
)
def run_tokenizer_demo():
sample_corpus = [
"Once upon a time, Lily found a golden key in the garden.",
"Timmy and his dog Max played with a red ball.",
"def solve_quadratic(a, b, c): return (-b + (b**2 - 4*a*c)**0.5) / (2*a)",
"\\int_{0}^{\\infty} e^{-x^2} dx = \\frac{\\sqrt{\\pi}}{2}",
"The system latency is <= 10ms with async/await workers.",
]
tokenizer = build_xonelm_tokenizer(corpus=sample_corpus, vocab_size=1000)
print("Tokenizer Vocab Size:", len(tokenizer))
text_to_encode = "Lily solved \\alpha + \\beta == 42 async await."
encoded = tokenizer.encode(text_to_encode)
token_ids = encoded.ids if hasattr(encoded, "ids") else encoded["input_ids"]
decoded = tokenizer.decode(token_ids)
print("Single Text Tokenization")
print("Input Text :", text_to_encode)
print("Token IDs :", token_ids)
print("Decoded :", decoded)
conversation = [
{"role": "system", "content": "You are a helpful and wise AI assistant."},
{"role": "user", "content": "Can you explain how it's work?"},
{"role": "assistant", "content": "No! I can't. hehe"},
]
cfg = SpecialTokenConfig(
pad_token_id=0,
bos_token_id=1,
eos_token_id=2,
unk_token_id=3,
eod_token_id=4,
)
formatter = MultiTurnConversationFormatter(tokenizer, cfg)
formatted = formatter.format_conversation(conversation)
print("Multi-Turn ChatML Formatting")
print("Input IDs Length :", len(formatted["input_ids"]))
print("Labels Length :", len(formatted["labels"]))
print("Formatted Text :\n" + tokenizer.decode(formatted["input_ids"]))
if __name__ == "__main__":
run_tokenizer_demo() |