social-data-attribution-w2/scripts/slurm/dedup/preflight_materialize_docs.sbatch

#!/usr/bin/env bash
#SBATCH --job-name=preflight_materialize
#SBATCH --cpus-per-task=4
#SBATCH --mem=16G
#SBATCH --time=02:00:00
#SBATCH -C localNVMe
#SBATCH --output=logs/dedup/%x_%j.out
#SBATCH --error=logs/dedup/%x_%j.err

set -euo pipefail

REPO_DIR="${SLURM_SUBMIT_DIR:-$(cd "$(dirname "$0")/../../.." && pwd)}"
cd "$REPO_DIR"
mkdir -p logs/dedup

if [ -f .venv/bin/activate ]; then
    source .venv/bin/activate
fi

export PYTHONPATH="${REPO_DIR}/src${PYTHONPATH:+:$PYTHONPATH}"
export HF_HUB_DISABLE_XET=1
export HF_TOKEN="${HF_TOKEN:-$(cat ~/.hf_token 2>/dev/null || true)}"
PREFLIGHT_FILE="${PREFLIGHT_FILE:-${PREFLIGHT_OUT:-/storage/ice-shared/cs7634/staff/TDA/soc-90/unique_docs_work/preflight.json}}"
DOWNLOAD_DIR_DEFAULT="${DOWNLOAD_DIR:-${TMPDIR:-/tmp}}"

python3 scripts/slurm/dedup/preflight_materialize_docs.py \
    --pool-manifest "${POOL_MANIFEST:-scripts/slurm/shard_manifest_pool_shared.txt}" \
    --mix-manifest "${MIX_MANIFEST:-scripts/slurm/shard_manifest_6t.txt}" \
    --bloom-file "${BLOOM_FILE:-/storage/ice-shared/cs7634/staff/TDA/soc-90/unique_6t_ids.bloom}" \
    --output "$PREFLIGHT_FILE" \
    --download-dir "$DOWNLOAD_DIR_DEFAULT" \
    --sample-shards-per-family "${SAMPLE_SHARDS_PER_FAMILY:-8}" \
    --verbose