social-data-attribution-w2/scripts/slurm/transform/check_ids.sbatch

#!/bin/bash
#SBATCH --job-name=check_ids
#SBATCH --output=logs/check_ids/%x_%j.out
#SBATCH --error=logs/check_ids/%x_%j.err
#SBATCH --account=gts-mriedl3
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=4
#SBATCH --mem=64G
#SBATCH --time=01:00:00

set -euo pipefail
mkdir -p logs/check_ids
cd "${SLURM_SUBMIT_DIR:-$(pwd)}"

COMBINED="/storage/scratch1/6/cchakraborty3/stratified_data/stratified/stratified_combined.jsonl"
ALL="/storage/scratch1/6/cchakraborty3/data_attribution/stratified_data/stratified_all.jsonl"

echo "Extracting doc_id from stratified_combined..."
jq -r '.doc_id' "$COMBINED" | sort > /tmp/combined_ids_${SLURM_JOB_ID}.txt
COMBINED_COUNT=$(wc -l < /tmp/combined_ids_${SLURM_JOB_ID}.txt)
echo "stratified_combined: ${COMBINED_COUNT} ids"

echo "Extracting id from stratified_all..."
jq -r '.id' "$ALL" | sort > /tmp/all_ids_${SLURM_JOB_ID}.txt
ALL_COUNT=$(wc -l < /tmp/all_ids_${SLURM_JOB_ID}.txt)
echo "stratified_all: ${ALL_COUNT} ids"

echo ""
echo "=== Comparison ==="

ONLY_COMBINED=$(comm -23 /tmp/combined_ids_${SLURM_JOB_ID}.txt /tmp/all_ids_${SLURM_JOB_ID}.txt | wc -l)
ONLY_ALL=$(comm -13 /tmp/combined_ids_${SLURM_JOB_ID}.txt /tmp/all_ids_${SLURM_JOB_ID}.txt | wc -l)
SHARED=$(comm -12 /tmp/combined_ids_${SLURM_JOB_ID}.txt /tmp/all_ids_${SLURM_JOB_ID}.txt | wc -l)

echo "IDs in both:              ${SHARED}"
echo "IDs only in combined:     ${ONLY_COMBINED}"
echo "IDs only in all:          ${ONLY_ALL}"

if [ "$ONLY_COMBINED" -eq 0 ] && [ "$ONLY_ALL" -eq 0 ]; then
    echo ""
    echo "PASS: ID sets match exactly."
else
    echo ""
    echo "MISMATCH: ID sets differ."
    if [ "$ONLY_COMBINED" -gt 0 ]; then
        echo ""
        echo "Sample IDs only in combined (first 10):"
        comm -23 /tmp/combined_ids_${SLURM_JOB_ID}.txt /tmp/all_ids_${SLURM_JOB_ID}.txt | head -10
    fi
    if [ "$ONLY_ALL" -gt 0 ]; then
        echo ""
        echo "Sample IDs only in all (first 10):"
        comm -13 /tmp/combined_ids_${SLURM_JOB_ID}.txt /tmp/all_ids_${SLURM_JOB_ID}.txt | head -10
    fi
fi

rm -f /tmp/combined_ids_${SLURM_JOB_ID}.txt /tmp/all_ids_${SLURM_JOB_ID}.txt