| from transformers import AutoTokenizer | |
| model_checkpoint = "distilbert-base-uncased" | |
| tokenizer = AutoTokenizer.from_pretrained(model_checkpoint) | |
| def tokenize(batch): | |
| return tokenizer(batch["text"], truncation=True, padding="max_length") | |
| tokenized = data.map(tokenize, batched=True) | |