File size: 843 Bytes
7ed86c3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 | #!/bin/bash
# Reproducible data build: HuggingFace -> raw parquet/jsonl -> memmap shards.
# Run once on a CPU (x86) node before training; training only reads the shards.
# scripts/pretrain/build_data.sh [workers]
set -euo pipefail
WORKERS=${1:-16}
ROOT=$(cd "$(dirname "$0")/../.." && pwd)
source "$ROOT/env.sh"
PY="${PYTHON:-python3}"
RAW=$STOICHEIA_DATA/raw
$PY "$ROOT/data/fetch_hf.py" --out "$RAW"
$PY "$ROOT/data/build_shards.py" --raw "$RAW/AncientGreek/data" \
--out "$STOICHEIA_DATA/shards/v1_punct" --workers "$WORKERS"
$PY "$ROOT/data/build_bronze.py" --jsonl "$RAW/SyntheticAncientGreek-CorpusCorporum/bronze.jsonl" \
--out "$STOICHEIA_DATA/shards/bronze_punct" --workers "$WORKERS"
cp "$RAW/provenance.json" "$STOICHEIA_DATA/shards/provenance.json"
echo "DATA BUILD DONE: $STOICHEIA_DATA/shards/{v1_punct,bronze_punct}"
|