social-data-attribution-w2/scripts/slurm/dedup/upload_unique_docs.sbatch

#!/usr/bin/env bash
#SBATCH --job-name=upload_unique
#SBATCH --cpus-per-task=4
#SBATCH --mem=32G
#SBATCH --time=24:00:00
#SBATCH --output=logs/dedup/%x_%j.out
#SBATCH --error=logs/dedup/%x_%j.err

set -euo pipefail

REPO_DIR="${SLURM_SUBMIT_DIR:-$(cd "$(dirname "$0")/../../.." && pwd)}"
cd "$REPO_DIR"
mkdir -p logs/dedup

if [ -f .venv/bin/activate ]; then
    source .venv/bin/activate
fi

export PYTHONPATH="${REPO_DIR}/src${PYTHONPATH:+:$PYTHONPATH}"
export HF_HUB_DISABLE_XET=1
export HF_TOKEN="${HF_TOKEN:-$(cat ~/.hf_token 2>/dev/null || true)}"

python3 - <<'PY'
import os
from pathlib import Path
from huggingface_hub import HfApi

work_root = Path(os.environ.get("WORK_ROOT", "/storage/ice-shared/cs7634/staff/TDA/soc-90/unique_docs_work"))
repo_id = os.environ.get("UPLOAD_REPO", "HCAI-Lab/dolma3-6t-unique")
private = os.environ.get("HF_PRIVATE", "true").lower() == "true"
api = HfApi(token=os.environ["HF_TOKEN"])
api.create_repo(repo_id, repo_type="dataset", private=private, exist_ok=True)
api.upload_folder(folder_path=work_root / "pool", path_in_repo="data/pool", repo_id=repo_id, repo_type="dataset")
api.upload_folder(folder_path=work_root / "mix_nonpool_final", path_in_repo="data/mix_nonpool", repo_id=repo_id, repo_type="dataset")
for name in ("manifest.parquet", "stats.json", "README.md"):
    path = work_root / name
    api.upload_file(path_or_fileobj=str(path), path_in_repo=name, repo_id=repo_id, repo_type="dataset")
PY