social-data-attribution-w2/scripts/slurm/dedup/finalize_unique_docs.sbatch

#!/usr/bin/env bash
#SBATCH --job-name=finalize_unique
#SBATCH --cpus-per-task=4
#SBATCH --mem=32G
#SBATCH --time=12:00:00
#SBATCH --output=logs/dedup/%x_%j.out
#SBATCH --error=logs/dedup/%x_%j.err

set -euo pipefail

REPO_DIR="${SLURM_SUBMIT_DIR:-$(cd "$(dirname "$0")/../../.." && pwd)}"
cd "$REPO_DIR"
mkdir -p logs/dedup

if [ -f .venv/bin/activate ]; then
    source .venv/bin/activate
fi

export PYTHONPATH="${REPO_DIR}/src${PYTHONPATH:+:$PYTHONPATH}"

MIX_RAW_DIR="${MIX_RAW_DIR:-/storage/ice-shared/cs7634/staff/TDA/soc-90/unique_docs_work/mix_nonpool_raw}"
MIX_FINAL_DIR="${MIX_FINAL_DIR:-/storage/ice-shared/cs7634/staff/TDA/soc-90/unique_docs_work/mix_nonpool_final}"
RUN_MERGE="${RUN_MERGE:-auto}"

MERGE_ARGS=()
if [ "${MERGE_OVERWRITE:-0}" = "1" ]; then
    MERGE_ARGS+=(--overwrite)
fi

if [ "$RUN_MERGE" = "1" ] || { [ "$RUN_MERGE" = "auto" ] && [ ! -f "$MIX_FINAL_DIR/merge.stats.json" ]; }; then
    python3 scripts/slurm/dedup/merge_mix_docs.py \
        --input-dir "$MIX_RAW_DIR" \
        --output-dir "$MIX_FINAL_DIR" \
        "${MERGE_ARGS[@]}"
fi

python3 scripts/slurm/dedup/finalize_unique_docs.py \
    --work-root "${WORK_ROOT:-/storage/ice-shared/cs7634/staff/TDA/soc-90/unique_docs_work}"