| # BoccaccioAI - Fase 2: Data Pipeline (Download + Filter + Tokenize) | |
| # De Lauretis Tech | |
| set -euo pipefail | |
| echo "=== BoccaccioAI - Data Pipeline ===" | |
| # Step 1: Download CulturaX Italian | |
| echo "[1/3] Downloading CulturaX IT..." | |
| python -m src.data.download \ | |
| --output-dir data/raw \ | |
| --max-size-gb 30 | |
| # Step 2: Filter and deduplicate | |
| echo "[2/3] Filtering and deduplicating..." | |
| python -m src.data.filter \ | |
| --input-dir data/raw \ | |
| --output-dir data/filtered \ | |
| --jaccard-threshold 0.85 \ | |
| --min-doc-length 200 \ | |
| --num-workers 3 | |
| # Step 3: Pre-tokenize into binary format | |
| echo "[3/3] Pre-tokenizing corpus..." | |
| python -m src.data.tokenize_corpus \ | |
| --input-dir data/filtered \ | |
| --output-dir data/tokenized/pretrain \ | |
| --tokenizer-path tokenizer/boccaccio-32k.json \ | |
| --sequence-length 2048 \ | |
| --val-split 0.005 | |
| echo "=== Data pipeline complete ===" | |