File size: 1,439 Bytes
708f4a3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | # Crayon v5.1.0 Simple Demo
from crayon import CrayonVocab
def main():
print("Crayon Tokenizer Demo")
print("=======================\n")
# 1. Initialize & Load Profile
# 'auto' will use GPU if available, else CPU
vocab = CrayonVocab(device="auto")
vocab.load_profile("lite")
print(f"Loaded Profile: 'lite' on {vocab.device.upper()}")
# 2. Define Input Text
text = "Hello, Crayon! This is a simple test."
# 3. Tokenize
# This converts the string into a list of integer IDs
tokens = vocab.tokenize(text)
print(f"\nInput Text: '{text}'")
print(f"Token IDs: {tokens}")
print(f"Count: {len(tokens)} tokens\n")
# 4. Analyze Each Token
# We decode each ID individually to show exactly what substring it represents
print("Token Breakdown:")
print(f"{'ID':<8} | {'Substring':<20}")
print("-" * 30)
for tid in tokens:
# We pass a list [tid] because decode expects a sequence
substring = vocab.decode([tid])
print(f"{tid:<8} | '{substring}'")
# 5. Full Decode
# Convert the list of IDs back to the original string
decoded_text = vocab.decode(tokens)
print(f"\nFull Decode check: '{decoded_text}'")
# Verification
if text == decoded_text:
print("[MATCH] Exact Match!")
else:
print("[MISMATCH] Mismatch (canonicalization might differ)")
if __name__ == "__main__":
main()
|