social-data-attribution-w2/scripts/slurm/analysis/rq4_array.sbatch

#!/usr/bin/env bash
#SBATCH --job-name=rq4-worker
#SBATCH --partition=ice-cpu
#SBATCH --nodes=1
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=1
#SBATCH --mem=8G
#SBATCH --time=01:00:00
#SBATCH --array=0-31
#SBATCH --output=/storage/ice-shared/cs7634/staff/TDA/logs/rq4_array_%A_%a.out
#SBATCH --error=/storage/ice-shared/cs7634/staff/TDA/logs/rq4_array_%A_%a.err

set -euo pipefail

SCRIPT="/storage/ice-shared/cs7634/staff/TDA/code/rq4_bin_characterization.py"
MANIFEST="${MANIFEST:-/storage/ice-shared/cs7634/staff/TDA/soc149/working_sample_manifest.parquet}"
SHARDS_DIR="${SHARDS_DIR:-/storage/ice-shared/cs7634/staff/TDA/trackstar/shards_10k/sample_10000_docs}"
CHUNK_COUNT="${CHUNK_COUNT:-32}"
OUTDIR="${OUTDIR:-/storage/ice-shared/cs7634/staff/TDA/outputs/rq4_workers}"
WORKER_DIR="$OUTDIR/$SLURM_ARRAY_TASK_ID"

mkdir -p "$WORKER_DIR"

echo "RQ4 worker $SLURM_ARRAY_TASK_ID/$CHUNK_COUNT starting at $(date)"
echo "Node: $(hostname)"

python3 "$SCRIPT" \
    --mode worker \
    --manifest "$MANIFEST" \
    --shards-dir "$SHARDS_DIR" \
    --chunk-index "$SLURM_ARRAY_TASK_ID" \
    --chunk-count "$CHUNK_COUNT" \
    --worker-output-dir "$WORKER_DIR"

echo "Worker $SLURM_ARRAY_TASK_ID finished at $(date)"