#!/bin/bash
set -uo pipefail
export HF_HOME=/scratch/hf
if [ -f /scratch/prep/manifests/waxal_sna_train.jsonl ] && [ -f /scratch/prep/manifests/cv_lug_train.jsonl ]; then
  echo "DATA_OK (deja presente)"; exit 0
fi
mkdir -p /scratch/data /scratch/submissions
/root/venv/bin/python << "PY"
from huggingface_hub import snapshot_download
pats = [f"data/ASR/{l}/{l}-{s}-*" for l in ("lin","lug","sna") for s in ("train","validation","test")]
snapshot_download(repo_id="google/WaxalNLP", repo_type="dataset", allow_patterns=pats, local_dir="/scratch/data/waxal", max_workers=16)
for repo, dest in [("KasuleTrevor/Afrivoice_ln_final","/scratch/data/afrivoice_ln"),("Beijuka/DigitalUmuganda_AfriVoice_shona","/scratch/data/afrivoice_sna"),("KasuleTrevor/lg_100hrs","/scratch/data/lg_100hrs")]:
    snapshot_download(repo_id=repo, repo_type="dataset", local_dir=dest, max_workers=16)
print("DL OK")
PY
/root/venv/bin/python /root/prep_data.py > /root/prep3.log 2>&1
grep -q "PREP_DONE" /root/prep3.log || { echo "ERREUR prep"; exit 1; }
/root/venv/bin/python /root/prep_lg100.py > /root/prep_lg3.log 2>&1
echo "DATA_OK"