File size: 353 Bytes
187a491 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 | #!/bin/bash
# BoccaccioAI - Fase 1: Train BPE Tokenizer
# De Lauretis Tech
set -euo pipefail
echo "=== BoccaccioAI - Training Tokenizer BPE 32K ==="
python -m src.tokenizer.train_tokenizer \
--config configs/tokenizer.yaml \
--output-dir tokenizer/ \
--vocab-size 32000 \
--corpus-size-gb 5
echo "=== Tokenizer training complete ==="
|