extract_site2.sh

#!/bin/bash
# Extract embeddings for Site 2 (Split 2)
# Usage: bash extract_site2.sh

set -e  # Exit on error

# Configuration
ROOT_DIR="/roshare/nlst_global/data_23Dec2024/manifest-NLST_allCT/NLST/"  # Update this to your NLST data path
OUTPUT_BASE="site2_data"
SITE_SPLITS="../subsets/site_splits"

echo "============================================"
echo "SITE 2 - Embedding Extraction"
echo "============================================"
echo ""

# Create output directories
mkdir -p ${OUTPUT_BASE}/train
mkdir -p ${OUTPUT_BASE}/test

# Extract training embeddings
echo "📦 Extracting TRAINING embeddings..."
python extract-embeddings.py \
  --root-dir ${ROOT_DIR} \
  --pid-csv ${SITE_SPLITS}/train_pid_labelsT0T7_split_2.csv \
  --output-dir ${OUTPUT_BASE}/train \
  --num-workers 8 \
  --checkpoint-interval 500

echo ""
echo "✓ Training embeddings complete!"
echo ""

# Extract test embeddings
echo "📦 Extracting TEST embeddings..."
python extract-embeddings.py \
  --root-dir ${ROOT_DIR} \
  --pid-csv ${SITE_SPLITS}/test_pid_labelsT0T7_split_2.csv \
  --output-dir ${OUTPUT_BASE}/test \
  --num-workers 8 \
  --checkpoint-interval 500

echo ""
echo "✓ Test embeddings complete!"
echo ""

# Prepare files for federated learning
echo "📋 Preparing files for federated learning..."
mkdir -p ${OUTPUT_BASE}/fl_ready

# Copy and rename embeddings
cp ${OUTPUT_BASE}/train/all_embeddings.parquet ${OUTPUT_BASE}/fl_ready/site2_embeddings_train.parquet
cp ${OUTPUT_BASE}/test/all_embeddings.parquet ${OUTPUT_BASE}/fl_ready/site2_embeddings_test.parquet

# Extract just pid and label columns from the CSV files
echo "Creating site2_labels-train.csv..."
head -n 1 ${SITE_SPLITS}/train_pid_labelsT0T7_split_2.csv | cut -d, -f1,14-20 > ${OUTPUT_BASE}/fl_ready/site2_labels-train.csv
tail -n +2 ${SITE_SPLITS}/train_pid_labelsT0T7_split_2.csv | cut -d, -f1,14-20 >> ${OUTPUT_BASE}/fl_ready/site2_labels-train.csv

echo "Creating site2_labels-test.csv..."
head -n 1 ${SITE_SPLITS}/test_pid_labelsT0T7_split_2.csv | cut -d, -f1,14-20 > ${OUTPUT_BASE}/fl_ready/site2_labels-test.csv
tail -n +2 ${SITE_SPLITS}/test_pid_labelsT0T7_split_2.csv | cut -d, -f1,14-20 >> ${OUTPUT_BASE}/fl_ready/site2_labels-test.csv

echo ""
echo "============================================"
echo "SITE 2 - COMPLETE! ✅"
echo "============================================"
echo ""
echo "FL-ready files in: ${OUTPUT_BASE}/fl_ready/"
ls -lh ${OUTPUT_BASE}/fl_ready/
echo ""
echo "Files ready for federated learning:"
echo "  ✓ site2_embeddings_train.parquet"
echo "  ✓ site2_embeddings_test.parquet"
echo "  ✓ site2_labels-train.csv"
echo "  ✓ site2_labels-test.csv"
echo ""