File size: 5,082 Bytes
e055374 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 | # ============================================================================
# TOKENIZER TRAINING SCRIPT (OPTIMIZED FOR PYTHON 3.11)
# ============================================================================
# Script to train the Hindi BPE tokenizer on a corpus file
# Optimized for large datasets with streaming and progress tracking
from hindi_bpe_encoder import HindiBPEEncoder
import os
import sys
import time
from pathlib import Path
def train_from_file(corpus_file="my_corpus.txt", vocab_size=5000):
"""
Train the Hindi BPE tokenizer from a corpus file.
Args:
corpus_file (str): Path to the corpus file
vocab_size (int): Vocabulary size for the tokenizer
"""
print("=" * 70)
print("Hindi BPE Tokenizer Training")
print("=" * 70)
# Check if corpus file exists
if not os.path.exists(corpus_file):
print(f"❌ Error: Corpus file '{corpus_file}' not found!")
print(f" Please make sure the file exists in the current directory.")
return False
# Get file size
file_size = os.path.getsize(corpus_file)
print(f"\n📄 Corpus file: {corpus_file}")
print(f" Size: {file_size:,} bytes ({file_size / 1024 / 1024:.2f} MB)")
# Initialize encoder
print("\n🔧 Initializing Hindi BPE Encoder...")
encoder = HindiBPEEncoder()
# Check file size to determine if we should use streaming
file_size_mb = file_size / (1024 * 1024)
use_streaming = file_size_mb > 100 # Use streaming for files > 100MB
if use_streaming:
print(f"\n📖 Using streaming mode for large corpus ({file_size_mb:.1f} MB)...")
print(" This avoids loading the entire file into memory.")
# Train tokenizer with streaming
print(f"\n🎓 Training tokenizer with vocab_size={vocab_size}...")
print(" Processing in chunks - this may take several minutes...")
start_time = time.time()
try:
result = encoder.train_tokenizer(
corpus_file, # Pass file path instead of content
vocab_size=vocab_size,
use_streaming=True
)
elapsed_time = time.time() - start_time
except Exception as e:
print(f"\n❌ Error during training: {e}")
import traceback
traceback.print_exc()
return False
else:
# Load corpus into memory (for smaller files)
print(f"\n📖 Loading corpus from '{corpus_file}'...")
try:
with open(corpus_file, 'r', encoding='utf-8') as f:
corpus = f.read()
if not corpus or not corpus.strip():
print("❌ Error: Corpus file is empty!")
return False
print(f" Loaded {len(corpus):,} characters")
print(f" Estimated words: ~{len(corpus.split()):,}")
except Exception as e:
print(f"❌ Error reading corpus file: {e}")
return False
# Train tokenizer
print(f"\n🎓 Training tokenizer with vocab_size={vocab_size}...")
print(" This may take a few minutes depending on corpus size...")
start_time = time.time()
try:
result = encoder.train_tokenizer(corpus, vocab_size=vocab_size)
elapsed_time = time.time() - start_time
except Exception as e:
print(f"\n❌ Error during training: {e}")
import traceback
traceback.print_exc()
return False
print(f"\n✅ {result}")
# Show vocabulary size and performance stats
vocab_size_actual = encoder.get_vocab_size()
print(f"\n📊 Tokenizer Statistics:")
print(f" Vocabulary size: {vocab_size_actual:,}")
print(f" Tokenizer saved to: {encoder.tokenizer_path}")
print(f" Training time: {elapsed_time:.2f} seconds ({elapsed_time/60:.2f} minutes)")
if file_size_mb > 0:
print(f" Processing speed: {file_size_mb/elapsed_time:.2f} MB/s")
print("\n" + "=" * 70)
print("🎉 Training complete!")
print("=" * 70)
print("\nYou can now use the trained tokenizer in your app:")
print(" python3 app.py")
return True
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(
description="Train Hindi BPE tokenizer from corpus file"
)
parser.add_argument(
'--corpus',
type=str,
default='my_corpus.txt',
help='Path to corpus file (default: my_corpus.txt)'
)
parser.add_argument(
'--vocab-size',
type=int,
default=5000,
help='Vocabulary size (default: 5000, uses 256 base tokens so merges are always learned)'
)
args = parser.parse_args()
success = train_from_file(
corpus_file=args.corpus,
vocab_size=args.vocab_size
)
sys.exit(0 if success else 1)
|