alanamarzoev/american-stories-tokenizer

American Stories Tokenizer

BPE tokenizer trained on the American Stories dataset for use with nanochat models.

Usage

import torch
from huggingface_hub import hf_hub_download

# Download
tokenizer_path = hf_hub_download(repo_id="alanamarzoev/american-stories-tokenizer", filename="token_bytes.pt")

# Load (requires nanochat)
from nanochat.tokenizer import Tokenizer
tokenizer = Tokenizer.from_file(tokenizer_path)

# Encode/decode
tokens = tokenizer.encode("Hello world")
text = tokenizer.decode(tokens)