File size: 2,615 Bytes
25fc268 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | # train_tokenizer.py
from tokenizers import ByteLevelBPETokenizer
from pathlib import Path
import requests
#Devansh Sinha
texts = []
#Devansh Sinha
# Download multiple books from Project Gutenberg for a larger dataset
urls = [
"https://www.gutenberg.org/files/11/11-0.txt", # Alice in Wonderland
"https://www.gutenberg.org/files/1342/1342-0.txt", # Pride and Prejudice
"https://www.gutenberg.org/files/84/84-0.txt", # Frankenstein
"https://www.gutenberg.org/files/1661/1661-0.txt", # Sherlock Holmes
"https://www.gutenberg.org/files/2701/2701-0.txt", # Moby Dick
"https://www.gutenberg.org/files/98/98-0.txt", # A Tale of Two Cities
"https://www.gutenberg.org/files/5200/5200-0.txt", # Metamorphosis
"https://www.gutenberg.org/files/2600/2600-0.txt", # War and Peace
"https://www.gutenberg.org/files/74/74-0.txt", # The Adventures of Tom Sawyer
"https://www.gutenberg.org/files/1400/1400-0.txt", # Great Expectations
]
#Devansh Sinha
for url in urls:
print(f"Downloading {url} ...")
response = requests.get(url)
if response.status_code == 200:
book_texts = [line.strip() for line in response.text.split('\n') if line.strip()]
texts.extend(book_texts)
print(f"Added {len(book_texts)} lines.")
else:
print(f"Failed to download {url}")
#Devansh Sinha
print(f"Total lines collected: {len(texts)}")
#Devansh Sinha
# Count total number of training tokens (words/wordpieces)
total_tokens = sum(len(line.split()) for line in texts)
print(f"Total number of training tokens (approximate, whitespace split): {total_tokens}")
# Save all texts to a temporary file for training
corpus_path = "corpus.txt"
with open(corpus_path, "w", encoding="utf-8") as f:
for line in texts:
f.write(line + "\n")
#Devansh Sinha
# Train a Byte Pair Encoding (BPE) tokenizer
tokenizer = ByteLevelBPETokenizer()
tokenizer.train(
files=corpus_path,
vocab_size=10000,
min_frequency=2,
special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>"]
)
#Devansh Sinha
# Save the tokenizer
save_dir = "my-10k-bpe-tokenizer"
Path(save_dir).mkdir(exist_ok=True)
tokenizer.save_model(save_dir)
#Devansh Sinha
# Save as HuggingFace tokenizer JSON for compatibility
tokenizer_json_path = str(Path(save_dir) / "tokenizer.json")
tokenizer.save(tokenizer_json_path)
print(f"Saved HuggingFace-compatible tokenizer.json to {tokenizer_json_path}")
print(f"BPE tokenizer trained and saved to {save_dir}/")
print(f"Number of tokens in vocab: {tokenizer.get_vocab_size()}")
|