social-data-attribution-w2/scripts/slurm/eda/sidecar_eda_array_phoenix.sbatch

#!/usr/bin/env bash
#SBATCH --job-name=soc91_eda_array
#SBATCH -A gts-schava6-qcf
#SBATCH -q embers
#SBATCH --partition=cpu-small
#SBATCH --cpus-per-task=8
#SBATCH --mem=32G
#SBATCH --time=08:00:00
#SBATCH --array=0-0
#SBATCH --output=logs/soc91_sidecar_eda/%x_%A_%a.out
#SBATCH --error=logs/soc91_sidecar_eda/%x_%A_%a.err

set -euo pipefail

REPO_DIR="${SLURM_SUBMIT_DIR:-$(cd "$(dirname "$0")/../../.." && pwd)}"
cd "$REPO_DIR"
mkdir -p logs/soc91_sidecar_eda

export PYTHONPATH="${REPO_DIR}/src${PYTHONPATH:+:$PYTHONPATH}"
PYTHONEXEC="${PYTHONEXEC:-$REPO_DIR/.venv/bin/python}"
if [ ! -x "$PYTHONEXEC" ]; then
    echo "ERROR: Python executable not found: $PYTHONEXEC"
    exit 1
fi

R2_ENV_FILE="${R2_ENV_FILE:-$HOME/.r2_credentials}"
if [ -f "$R2_ENV_FILE" ]; then
    source "$R2_ENV_FILE"
fi
if [ -z "${R2_ACCESS_KEY_ID:-}" ] || [ -z "${R2_SECRET_ACCESS_KEY:-}" ]; then
    echo "ERROR: R2 credentials not set"
    exit 1
fi

OUTDIR="${OUTDIR:-$HOME/scratch/soc91/sidecar_eda_runs/current}"
MANIFEST_PATH="${MANIFEST_PATH:?MANIFEST_PATH is required}"
CHUNK_COUNT="${CHUNK_COUNT:-32}"
WORKERS_DIR="${WORKERS_DIR:-$OUTDIR/workers}"
WORKER_OUTPUT_DIR="$WORKERS_DIR/${SLURM_ARRAY_TASK_ID}"
mkdir -p "$WORKER_OUTPUT_DIR"

srun "$PYTHONEXEC" scripts/validation/analyze_sidecar_labels.py \
    --mode worker \
    --output-dir "$OUTDIR" \
    --worker-output-dir "$WORKER_OUTPUT_DIR" \
    --manifest-path "$MANIFEST_PATH" \
    --chunk-count "$CHUNK_COUNT" \
    --chunk-index "$SLURM_ARRAY_TASK_ID" \
    --no-write-final-outputs