import sys; sys.path.insert(0, '.')
from angstrom_nano.tokenizer import AngstromNanoTokenizer

tok = AngstromNanoTokenizer()
text = "def hello(): return 42"
ids = tok.encode(text)
decoded = tok.decode(ids)
print(f"Text: {text}")
print(f"Tokens: {ids[:10]}... ({len(ids)} total)")
print(f"Decoded: {decoded}")
print(f"Match: {text == decoded}")
tok.save('angstrom_nano/tokenizer_config.json')
print("Tokenizer PASSED")