| # Reproducible data build: HuggingFace -> raw parquet/jsonl -> memmap shards. | |
| # Run once on a CPU (x86) node before training; training only reads the shards. | |
| # scripts/pretrain/build_data.sh [workers] | |
| set -euo pipefail | |
| WORKERS=${1:-16} | |
| ROOT=$(cd "$(dirname "$0")/../.." && pwd) | |
| source "$ROOT/env.sh" | |
| PY="${PYTHON:-python3}" | |
| RAW=$STOICHEIA_DATA/raw | |
| $PY "$ROOT/data/fetch_hf.py" --out "$RAW" | |
| $PY "$ROOT/data/build_shards.py" --raw "$RAW/AncientGreek/data" \ | |
| --out "$STOICHEIA_DATA/shards/v1_punct" --workers "$WORKERS" | |
| $PY "$ROOT/data/build_bronze.py" --jsonl "$RAW/SyntheticAncientGreek-CorpusCorporum/bronze.jsonl" \ | |
| --out "$STOICHEIA_DATA/shards/bronze_punct" --workers "$WORKERS" | |
| cp "$RAW/provenance.json" "$STOICHEIA_DATA/shards/provenance.json" | |
| echo "DATA BUILD DONE: $STOICHEIA_DATA/shards/{v1_punct,bronze_punct}" | |