| #!/bin/bash |
| set -uo pipefail |
| export HF_HOME=/scratch/hf |
| if [ -f /scratch/prep/manifests/waxal_sna_train.jsonl ] && [ -f /scratch/prep/manifests/cv_lug_train.jsonl ]; then |
| echo "DATA_OK (deja presente)"; exit 0 |
| fi |
| mkdir -p /scratch/data /scratch/submissions |
| /root/venv/bin/python << "PY" |
| from huggingface_hub import snapshot_download |
| pats = [f"data/ASR/{l}/{l}-{s}-*" for l in ("lin","lug","sna") for s in ("train","validation","test")] |
| snapshot_download(repo_id="google/WaxalNLP", repo_type="dataset", allow_patterns=pats, local_dir="/scratch/data/waxal", max_workers=16) |
| for repo, dest in [("KasuleTrevor/Afrivoice_ln_final","/scratch/data/afrivoice_ln"),("Beijuka/DigitalUmuganda_AfriVoice_shona","/scratch/data/afrivoice_sna"),("KasuleTrevor/lg_100hrs","/scratch/data/lg_100hrs")]: |
| snapshot_download(repo_id=repo, repo_type="dataset", local_dir=dest, max_workers=16) |
| print("DL OK") |
| PY |
| /root/venv/bin/python /root/prep_data.py > /root/prep3.log 2>&1 |
| grep -q "PREP_DONE" /root/prep3.log || { echo "ERREUR prep"; exit 1; } |
| /root/venv/bin/python /root/prep_lg100.py > /root/prep_lg3.log 2>&1 |
| echo "DATA_OK" |
|
|
