code/probes/cluster_scripts/run_novel_probes_v2.sh

#!/bin/bash
#SBATCH --job-name=novel_probes_v2
#SBATCH --partition=nova
#SBATCH --gres=gpu:a100:1
#SBATCH --cpus-per-task=8
#SBATCH --mem=64G
#SBATCH --time=04:00:00
#SBATCH --output=bert_v6_contrastive/scripts/novel_probes_v2_%j.out
#SBATCH --error=bert_v6_contrastive/scripts/novel_probes_v2_%j.err

echo "========================================"
echo "Novel Embedding Probes V2 — v5a vs v6"
echo "Job ID: $SLURM_JOB_ID"
echo "Node: $(hostname)"
echo "Started: $(date)"
echo "========================================"

# Activate blade environment (correct path confirmed on cluster)
source /work/ratul1/supantha/miniconda3/etc/profile.d/conda.sh
conda activate blade

cd /work/ratul1/supantha/glycan-SD-VS/bert_training_v3/v3.1_cluster_training

# Debug info
echo ""
echo "Python: $(which python3)"
echo "Python version: $(python3 --version)"
echo "Conda env: $CONDA_DEFAULT_ENV"
nvidia-smi --query-gpu=name,memory.total --format=csv,noheader 2>/dev/null || echo "No GPU info"
echo ""

# Install missing deps if needed
pip install -q scikit-learn umap-learn 2>/dev/null

# Check key files exist
echo "Checking files..."
ls -lh checkpoints_v5_bpe_topo/best_v5_bpe_topo_model.pt 2>/dev/null || echo "WARNING: v5a checkpoint not found"
ls -lh bert_v5.1_contrastive/checkpoints/best_v51_contrastive_model.pt 2>/dev/null || echo "WARNING: v6 checkpoint not found"
ls -lh bert_training_v4/data/vocabulary.json 2>/dev/null || echo "WARNING: vocabulary.json not found"
ls -lh bert_training_v4/data/multimodal_index.csv 2>/dev/null || echo "WARNING: multimodal_index.csv not found"
echo ""

echo "========== V5a — All 8 Novel Probes (Full Data) =========="
python3 -u bert_v6_contrastive/scripts/novel_probes_v2.py \
    --model v5 \
    --probe all \
    --max_samples 10000 \
    --device cuda

echo ""
echo "Generated V5a files:"
ls -la bert_v6_contrastive/analysis/novel_probes_v2_v5/ 2>/dev/null || echo "No V5 output dir found"

echo ""
echo "========== V6 — All 8 Novel Probes (Full Data) =========="
python3 -u bert_v6_contrastive/scripts/novel_probes_v2.py \
    --model v6 \
    --probe all \
    --max_samples 10000 \
    --device cuda

echo ""
echo "Generated V6 files:"
ls -la bert_v6_contrastive/analysis/novel_probes_v2_v6/ 2>/dev/null || echo "No V6 output dir found"

echo ""
echo "Completed: $(date)"