#!/usr/bin/env bash
# Usage:
#   ./deploy_all.sh              launch STT, LLM, TTS in parallel
#   ./deploy_all.sh restart-clean   kill everything first, then launch fresh
set -uo pipefail

DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
MODE="${1:-}"

if [ "$MODE" = "restart-clean" ]; then
  echo "[all] killing existing servers..."
  pkill -f "sglang.launch_server" 2>/dev/null || true
  pkill -f "vllm serve"           2>/dev/null || true
  pkill -f "sgl-omni serve"       2>/dev/null || true
  # Omni/SGLang leave multiprocessing workers that keep VRAM after the parent dies.
  pkill -f "sglang::scheduler"    2>/dev/null || true
  pkill -f "sglang::detokenizer"  2>/dev/null || true
  pkill -f "VLLM::EngineCore"     2>/dev/null || true
  sleep 5
fi

# Layout: TTS alone on GPU1; STT then LLM share GPU0 (STT reserves first).
# Sequential installs stay disk-safe on small overlays; HF cache → /dev/shm.
echo "[all] bootstrapping envs sequentially (disk-safe)..."
FAILED=0
for script in start_tts.sh start_stt.sh start_llm.sh; do
  name="${script#start_}"; name="${name%.sh}"
  echo "[all] starting $name..."
  if ! bash "$DIR/$script"; then
    echo "[all] $name FAILED to start"
    FAILED=1
  fi
done

if [ "$FAILED" -eq 1 ]; then
  echo "[all] one or more services failed. check ~/agent-logs/*.log"
  exit 1
fi

check () {
  local port=$1 label=$2
  if curl -sf "http://localhost:${port}/health" > /dev/null 2>&1; then
    echo "[all] $label: UP"
  else
    echo "[all] $label: DOWN"
    FAILED=1
  fi
}

check 8002 "STT"
check 8000 "LLM"
check 8003 "TTS"

if [ "$FAILED" -eq 1 ]; then
  exit 1
fi

nvidia-smi --query-gpu=index,memory.used,memory.total --format=csv
echo "[all] deploy complete."