preprocess_data.py

from transformers import AutoTokenizer

model_checkpoint = "distilbert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)

def tokenize(batch):
    return tokenizer(batch["text"], truncation=True, padding="max_length")

tokenized = data.map(tokenize, batched=True)