code/launch_v2_v3.sh

#!/bin/bash
# Phase 1+2+3 launcher — run inside a tmux session on the VM.
# Usage:
#   tmux new -s fundus2
#   bash launch_v2_v3.sh 2>&1 | tee v2_v3_run.log
set -e
cd ~/fundus_project
source ~/.venv/bin/activate

mkdir -p final_experiments_v2 final_experiments_v3 weights_v2 weights_v3 gradcam_v2

echo "=================================================================="
echo "Phase 1.1 — group-aware split builder (perceptual hashing)"
echo "=================================================================="
python comparison_experiment/build_grouped_split.py \
    --original-dir "Database/Original_Dataset" \
    --augmented-dir "Database/Augmented_Dataset" \
    --output holdout_split_augmented.json \
    --hamming-threshold 8 \
    --n-folds 5

echo "=================================================================="
echo "Phase 1.2 — v2 training (6 CNNs/CLIP w/ CLAHE + RandAug + Sampler + MixUp + TTA)"
echo "=================================================================="
python comparison_experiment/run_v2_experiments.py \
    --manifest holdout_split_augmented.json \
    --out-dir final_experiments_v2 \
    --weights-dir weights_v2 \
    --epochs 60 \
    --folds 5 \
    --batch-size 32 \
    --workers 4 \
    --patience 12 \
    --skip-cv

# (Skip CV in first pass to keep total wall < 18h; CV happens on best models in Phase 3.)

echo "=================================================================="
echo "Phase 2 — Foundation backbones (DINOv2, Swin, RETFound)"
echo "=================================================================="
# RETFound weights gated on HuggingFace; skip unless a non-empty file exists locally.
FM_MODELS="dinov2_l swin_b"
if [ -s weights_retfound.pth ]; then
    echo "RETFound weights found, including in run."
    FM_MODELS="$FM_MODELS retfound"
else
    echo "RETFound weights missing or empty -> skipping RETFound (HF gated)."
fi

python comparison_experiment/run_foundation_models.py \
    --manifest holdout_split_augmented.json \
    --out-dir final_experiments_v3 \
    --weights-dir weights_v3 \
    --retfound-weights weights_retfound.pth \
    --models $FM_MODELS \
    --batch-size 24 \
    --workers 4 \
    --lp-epochs 20 \
    --ft-epochs 15 \
    --patience 8

echo "=================================================================="
echo "Phase 3 — Ensemble + extended statistics"
echo "=================================================================="
# Combine v2 + v3 predictions in one directory
mkdir -p final_experiments_all
cp final_experiments_v2/*_test_preds.json final_experiments_all/ 2>/dev/null || true
cp final_experiments_v2/*_test.json       final_experiments_all/ 2>/dev/null || true
cp final_experiments_v3/*_test_preds.json final_experiments_all/ 2>/dev/null || true
cp final_experiments_v3/*_test.json       final_experiments_all/ 2>/dev/null || true

python comparison_experiment/ensemble_and_stats.py \
    --results-dir final_experiments_all \
    --out final_experiments_all/ensemble_report.json

echo "=================================================================="
echo "Phase 1.3 — Grad-CAM v2 (best model)"
echo "=================================================================="
python comparison_experiment/generate_all_gradcam.py \
    --manifest holdout_split_augmented.json \
    --weights-dir weights_v2 \
    --out-dir gradcam_v2 || echo "GradCAM optional step failed; continuing"

echo "All done. Outputs:"
ls -la final_experiments_v2 final_experiments_v3 final_experiments_all gradcam_v2 2>/dev/null