source/scripts/spark_campaign.sh

#!/usr/bin/env bash
set -euo pipefail

campaign_tag="${1:?usage: scripts/spark_campaign.sh CAMPAIGN_TAG [CONFIG]}"
config_path="${2:-configs/main.yaml}"
project_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$project_dir"

python_bin="${BGC_PYTHON:?set BGC_PYTHON to a CUDA-enabled Python executable}"
dependency_dir="${BGC_EXTRA_PYTHONPATH:-$project_dir/.deps}"
export PYTHONPATH="$dependency_dir:$project_dir/src${PYTHONPATH:+:$PYTHONPATH}"

run_bgc() {
  "$python_bin" -c 'from bgc_retrieval.cli import main; main()' "$@"
}

run_step() {
  local name="$1"
  shift
  local started=$SECONDS
  printf '[%s] START %s\n' "$(date --iso-8601=seconds)" "$name"
  "$@"
  printf '[%s] DONE  %s (%ss)\n' "$(date --iso-8601=seconds)" "$name" "$((SECONDS - started))"
}

"$python_bin" -c 'import torch; assert torch.cuda.is_available(), "CUDA GPU is required"; print(torch.__version__, torch.cuda.get_device_name(0))'
printf 'CONFIG %s\n' "$config_path"

if [[ ! -f data/manifests/legacy_manifest.json ]]; then
  run_step audit run_bgc audit --config "$config_path"
fi
if [[ ! -f data/manifests/silver_split.csv ]]; then
  run_step splits run_bgc build-splits --config "$config_path"
fi
if [[ ! -f data/external/processed/external_atlas.csv ]]; then
  run_step external-preparation "$python_bin" scripts/prepare_external_benchmark.py
fi
if [[ ! -f data/external/processed/external_esm2.h5 ]]; then
  run_step external-esm2 "$python_bin" scripts/embed_external_proteins.py --resume
fi

run_step phase1 run_bgc train --config "$config_path" --stage phase1 \
  --run-id "${campaign_tag}-phase1"
run_step phase2-main run_bgc train --config "$config_path" --stage phase2 \
  --run-id "${campaign_tag}-main" \
  --phase1-checkpoint "artifacts/${campaign_tag}-phase1/phase1_best.pt"
run_step phase2-no-phase1 run_bgc train --config "$config_path" --stage phase2 \
  --run-id "${campaign_tag}-no-phase1"
run_step internal-main run_bgc evaluate --config "$config_path" \
  --checkpoint "artifacts/${campaign_tag}-main/phase2_best.pt" \
  --run-id "${campaign_tag}-main-evaluation"
run_step internal-no-phase1 run_bgc evaluate --config "$config_path" \
  --checkpoint "artifacts/${campaign_tag}-no-phase1/phase2_best.pt" \
  --run-id "${campaign_tag}-no-phase1-evaluation"
run_step external-main "$python_bin" scripts/evaluate_external.py --config "$config_path" \
  --checkpoint "artifacts/${campaign_tag}-main/phase2_best.pt" \
  --run-id "${campaign_tag}-main-external" \
  --bootstrap-samples "${EXTERNAL_BOOTSTRAP_SAMPLES:-1000}"
run_step external-no-phase1 "$python_bin" scripts/evaluate_external.py --config "$config_path" \
  --checkpoint "artifacts/${campaign_tag}-no-phase1/phase2_best.pt" \
  --run-id "${campaign_tag}-no-phase1-external" \
  --bootstrap-samples "${EXTERNAL_BOOTSTRAP_SAMPLES:-1000}"

printf '[%s] CAMPAIGN COMPLETE: %s\n' "$(date --iso-8601=seconds)" "$campaign_tag"